def upgrade_v3_npc_cleanup():
status.maintenance('Cleaning up Calico 2 policy controller')
resources = [('Deployment', 'kube-system', 'calico-policy-controller'),
('ClusterRoleBinding', None, 'calico-policy-controller'),
('ClusterRole', None, 'calico-policy-controller'),
('ServiceAccount', 'kube-system', 'calico-policy-controller')]
for kind, namespace, name in resources:
args = ['delete', '--ignore-not-found', kind, name]
if namespace:
args += ['-n', namespace]
try:
kubectl(*args)
except CalledProcessError:
log('Failed to cleanup %s %s %s' % (kind, namespace, name))
return
leader_set({'calico-v3-npc-cleanup-needed': None})
def deploy_network_policy_controller():
''' Deploy the Calico network policy controller. '''
status.maintenance('Deploying network policy controller.')
etcd = endpoint_from_flag('etcd.available')
context = {
'connection_string': etcd.get_connection_string(),
'etcd_key_path': ETCD_KEY_PATH,
'etcd_cert_path': ETCD_CERT_PATH,
'etcd_ca_path': ETCD_CA_PATH,
'calico_policy_image': charm_config('calico-policy-image'),
'etcd_cert_last_modified': os.path.getmtime(ETCD_CERT_PATH)
}
render('policy-controller.yaml', '/tmp/policy-controller.yaml', context)
try:
kubectl('apply', '-f', '/tmp/policy-controller.yaml')
set_state('calico.npc.deployed')
except CalledProcessError as e:
status.waiting('Waiting for kubernetes')
log(str(e))
def configure_node():
status.maintenance('Configuring Calico node')
node_name = gethostname()
as_number = get_unit_as_number()
route_reflector_cluster_id = get_route_reflector_cluster_id()
try:
node = calicoctl_get('node', node_name)
node['spec']['bgp']['asNumber'] = as_number
node['spec']['bgp']['routeReflectorClusterID'] = \
route_reflector_cluster_id
calicoctl_apply(node)
except CalledProcessError:
log(traceback.format_exc())
status.waiting('Waiting to retry Calico node configuration')
return
set_state('calico.node.configured')
def install_flannel_service():
''' Install the flannel service. '''
status.maintenance('Installing flannel service.')
# keep track of our etcd connections so we can detect when it changes later
etcd = endpoint_from_flag('etcd.tls.available')
etcd_connections = etcd.get_connection_string()
data_changed('flannel_etcd_connections', etcd_connections)
data_changed('flannel_etcd_cert', etcd.get_client_credentials())
iface = config('iface') or get_bind_address_interface()
context = {
'iface': iface,
'connection_string': etcd_connections,
'cert_path': ETCD_PATH
}
render('flannel.service', '/lib/systemd/system/flannel.service', context)
service('enable', 'flannel')
set_state('flannel.service.installed')
remove_state('flannel.service.started')
def _remove_docker_network_bridge():
"""
By default docker uses the docker0 bridge for container networking.
This method removes the default docker bridge, and reconfigures the
DOCKER_OPTS to use the SDN networking bridge.
:return: None
"""
status.maintenance("Reconfiguring container runtime network bridge.")
host.service_stop("docker")
apt_install(["bridge-utils"], fatal=True)
# cmd = "ifconfig docker0 down"
# ifconfig doesn't always work. use native linux networking commands to
# mark the bridge as inactive.
check_call(["ip", "link", "set", "docker0", "down"])
check_call(["brctl", "delbr", "docker0"])
# Render the config and restart docker.
recycle_daemon()
def configure_cni():
''' Configure Calico CNI. '''
status.maintenance('Configuring Calico CNI')
cni = endpoint_from_flag('cni.connected')
etcd = endpoint_from_flag('etcd.available')
os.makedirs('/etc/cni/net.d', exist_ok=True)
ip_versions = {net.version for net in get_networks(charm_config('cidr'))}
context = {
'connection_string': etcd.get_connection_string(),
'etcd_key_path': ETCD_KEY_PATH,
'etcd_cert_path': ETCD_CERT_PATH,
'etcd_ca_path': ETCD_CA_PATH,
'kubeconfig_path': '/opt/calicoctl/kubeconfig',
'mtu': get_mtu(),
'assign_ipv4': 'true' if 4 in ip_versions else 'false',
'assign_ipv6': 'true' if 6 in ip_versions else 'false',
}
render('10-calico.conflist', '/etc/cni/net.d/10-calico.conflist', context)
config = charm_config()
cni.set_config(cidr=config['cidr'], cni_conf_file='10-calico.conflist')
set_state('calico.cni.configured')
def install_from_custom_apt():
"""
Install docker from custom repository.
:return: None or False
"""
status.maintenance("Installing Docker from custom repository.")
repo_string = config("docker_runtime_repo")
key_url = config("docker_runtime_key_url")
package_name = config("docker_runtime_package")
if not repo_string:
message = "`docker_runtime_repo` must be set"
hookenv.log(message)
status.blocked(message)
return False
if not key_url:
message = "`docker_runtime_key_url` must be set"
hookenv.log(message)
status.blocked(message)
return False
if not package_name:
message = "`docker_runtime_package` must be set"
hookenv.log(message)
status.blocked(message)
return False
lsb = host.lsb_release()
format_dictionary = {"ARCH": arch(), "CODE": lsb["DISTRIB_CODENAME"]}
add_apt_key_url(key_url)
write_docker_sources([repo_string.format(**format_dictionary)])
apt_update()
apt_install([package_name])
return True
def pull_calicoctl_image():
status.maintenance('Pulling calicoctl image')
registry = hookenv.config('registry') or DEFAULT_REGISTRY
encoded_creds = hookenv.config('registry-credentials')
creds = b64decode(encoded_creds).decode('utf-8')
if creds:
creds = json.loads(creds)
images = {
os.path.join(registry, hookenv.config('calico-node-image')):
resource_get('calico-node-image'),
os.path.join(registry, hookenv.config('calicoctl-image')):
resource_get('calicoctl-image')
}
for name, path in images.items():
if not path or os.path.getsize(path) == 0:
status.maintenance('Pulling {} image'.format(name))
if not creds or not creds.get('auths') or \
registry not in creds.get('auths'):
CTL.pull(name, )
else:
auth = creds['auths'][registry]['auth']
username, password = b64decode(auth).decode('utf-8').split(':')
CTL.pull(name, username=username, password=password)
else:
status.maintenance('Loading {} image'.format(name))
unzipped = '/tmp/calico-node-image.tar'
with gzip.open(path, 'rb') as f_in:
with open(unzipped, 'wb') as f_out:
f_out.write(f_in.read())
CTL.load(unzipped)
set_state('calico.image.pulled')
def configure_bgp_globals():
status.maintenance('Configuring BGP globals')
config = charm_config()
try:
try:
bgp_config = calicoctl_get('bgpconfig', 'default')
except CalledProcessError as e:
if b'resource does not exist' in e.output:
log('default BGPConfiguration does not exist')
bgp_config = {
'apiVersion': 'projectcalico.org/v3',
'kind': 'BGPConfiguration',
'metadata': {
'name': 'default'
},
'spec': {}
}
else:
raise
spec = bgp_config['spec']
spec['asNumber'] = config['global-as-number']
spec['nodeToNodeMeshEnabled'] = config['node-to-node-mesh']
spec['serviceClusterIPs'] = [{
'cidr': cidr
} for cidr in config['bgp-service-cluster-ips'].split()]
spec['serviceExternalIPs'] = [{
'cidr': cidr
} for cidr in config['bgp-service-external-ips'].split()]
spec['serviceLoadBalancerIPs'] = [{
'cidr': cidr
} for cidr in config['bgp-service-loadbalancer-ips'].split()]
calicoctl_apply(bgp_config)
except CalledProcessError:
log(traceback.format_exc())
status.waiting('Waiting to retry BGP global configuration')
return
set_state('calico.bgp.globals.configured')
def configure_nvidia():
"""
Based on charm config, install and configure Nivida drivers.
:return: None
"""
status.maintenance('Installing Nvidia drivers.')
dist = host.lsb_release()
os_release_id = dist['DISTRIB_ID'].lower()
os_release_version_id = dist['DISTRIB_RELEASE']
os_release_version_id_no_dot = os_release_version_id.replace('.', '')
proxies = {"http": config('http_proxy'), "https": config('https_proxy')}
key_urls = config('nvidia_apt_key_urls').split()
for key_url in key_urls:
formatted_key_url = key_url.format(
id=os_release_id,
version_id=os_release_version_id,
version_id_no_dot=os_release_version_id_no_dot)
gpg_key = requests.get(formatted_key_url, proxies=proxies).text
import_key(gpg_key)
sources = config('nvidia_apt_sources').splitlines()
formatted_sources = [
source.format(id=os_release_id,
version_id=os_release_version_id,
version_id_no_dot=os_release_version_id_no_dot)
for source in sources
]
with open('/etc/apt/sources.list.d/nvidia.list', 'w') as f:
f.write('\n'.join(formatted_sources))
apt_update()
packages = config('nvidia_apt_packages').split()
apt_install(packages, fatal=True)
set_state('containerd.nvidia.ready')
config_changed()
def install_flannel_binaries():
''' Unpack the Flannel binaries. '''
try:
resource_name = 'flannel-{}'.format(arch())
archive = resource_get(resource_name)
except Exception:
message = 'Error fetching the flannel resource.'
log(message)
status.blocked(message)
return
if not archive:
message = 'Missing flannel resource.'
log(message)
status.blocked(message)
return
filesize = os.stat(archive).st_size
if filesize < 1000000:
message = 'Incomplete flannel resource'
log(message)
status.blocked(message)
return
status.maintenance('Unpacking flannel resource.')
charm_dir = os.getenv('CHARM_DIR')
unpack_path = os.path.join(charm_dir, 'files', 'flannel')
os.makedirs(unpack_path, exist_ok=True)
cmd = ['tar', 'xfz', archive, '-C', unpack_path]
log(cmd)
check_call(cmd)
apps = [
{'name': 'flanneld', 'path': '/usr/local/bin'},
{'name': 'etcdctl', 'path': '/usr/local/bin'}
]
for app in apps:
unpacked = os.path.join(unpack_path, app['name'])
app_path = os.path.join(app['path'], app['name'])
install = ['install', '-v', '-D', unpacked, app_path]
check_call(install)
set_state('flannel.binaries.installed')
def install_calico_service():
''' Install the calico-node systemd service. '''
status.maintenance('Installing calico-node service.')
etcd = endpoint_from_flag('etcd.available')
service_path = os.path.join(os.sep, 'lib', 'systemd', 'system',
'calico-node.service')
render(
'calico-node.service',
service_path,
{
'connection_string': etcd.get_connection_string(),
'etcd_key_path': ETCD_KEY_PATH,
'etcd_ca_path': ETCD_CA_PATH,
'etcd_cert_path': ETCD_CERT_PATH,
'nodename': gethostname(),
# specify IP so calico doesn't grab a silly one from, say, lxdbr0
'ip': get_bind_address(),
'calico_node_image': charm_config('calico-node-image')
})
check_call(['systemctl', 'daemon-reload'])
service_restart('calico-node')
service('enable', 'calico-node')
set_state('calico.service.installed')
def configure_cni():
''' Configure Calico CNI. '''
status.maintenance('Configuring Calico CNI')
try:
subnet = get_flannel_subnet()
except FlannelSubnetNotFound:
hookenv.log(traceback.format_exc())
status.waiting('Waiting for Flannel')
return
os.makedirs('/etc/cni/net.d', exist_ok=True)
cni = endpoint_from_flag('cni.configured')
etcd = endpoint_from_flag('etcd.available')
cni_config = cni.get_config()
context = {
'connection_string': etcd.get_connection_string(),
'etcd_key_path': ETCD_KEY_PATH,
'etcd_cert_path': ETCD_CERT_PATH,
'etcd_ca_path': ETCD_CA_PATH,
'kubeconfig_path': cni_config['kubeconfig_path'],
'subnet': subnet
}
render('10-canal.conflist', '/etc/cni/net.d/10-canal.conflist', context)
cni.set_config(cidr=config('cidr'), cni_conf_file='10-canal.conflist')
set_state('canal.cni.configured')
def container_sdn_setup(sdn):
"""
Receive the information from the SDN plugin, and render the docker
engine options.
:param sdn: SDNPluginProvider
:return: None
"""
sdn_config = sdn.get_sdn_config()
bind_ip = sdn_config["subnet"]
mtu = sdn_config["mtu"]
if data_changed("bip", bind_ip) or data_changed("mtu", mtu):
status.maintenance("Configuring container runtime with SDN.")
opts = DockerOpts()
# This is a great way to misconfigure a docker daemon. Remove the
# existing bind ip and mtu values of the SDN
if opts.exists("bip"):
opts.pop("bip")
if opts.exists("mtu"):
opts.pop("mtu")
opts.add("bip", bind_ip)
opts.add("mtu", mtu)
_remove_docker_network_bridge()
set_state("docker.sdn.configured")
def install_queued():
'''Installs queued deb packages.
Removes the apt.queued_installs flag and sets the apt.installed flag.
On failure, sets the unit's workload status to 'blocked' and returns
False. Package installs remain queued.
On success, sets the apt.installed.{packagename} flag for each
installed package and returns True.
'''
store = unitdata.kv()
queue = sorted((options, package)
for package, options in store.getrange('apt.install_queue.',
strip=True).items())
installed = set()
for options, batch in itertools.groupby(queue, lambda x: x[0]):
packages = [b[1] for b in batch]
try:
status.maintenance('Installing {}'.format(','.join(packages)))
fetch.apt_install(packages, options, fatal=True)
store.unsetrange(packages, prefix='apt.install_queue.')
installed.update(packages)
except subprocess.CalledProcessError:
status.blocked('Unable to install packages {}'.format(
','.join(packages)))
return False # Without setting reactive flag.
for package in installed:
reactive.set_flag('apt.installed.{}'.format(package))
reactive.clear_flag('apt.queued_installs')
reset_application_version()
return True
def configure_bgp_peers():
status.maintenance('Configuring BGP peers')
peers = []
# Global BGP peers
config = charm_config()
peers += yaml.safe_load(config['global-bgp-peers'])
# Subnet-scoped BGP peers
subnet_bgp_peers = yaml.safe_load(config['subnet-bgp-peers'])
subnets = filter_local_subnets(subnet_bgp_peers)
for subnet in subnets:
peers += subnet_bgp_peers[str(subnet)]
# Unit-scoped BGP peers
unit_id = get_unit_id()
unit_bgp_peers = yaml.safe_load(config['unit-bgp-peers'])
if unit_id in unit_bgp_peers:
peers += unit_bgp_peers[unit_id]
# Give names to peers
safe_unit_name = local_unit().replace('/', '-')
named_peers = {
# name must consist of lower case alphanumeric characters, '-' or '.'
'%s-%s-%s' %
(safe_unit_name, peer['address'].replace(':', '-'), peer['as-number']):
peer
for peer in peers
}
try:
node_name = gethostname()
for peer_name, peer in named_peers.items():
peer_def = {
'apiVersion': 'projectcalico.org/v3',
'kind': 'BGPPeer',
'metadata': {
'name': peer_name,
},
'spec': {
'node': node_name,
'peerIP': peer['address'],
'asNumber': peer['as-number']
}
}
calicoctl_apply(peer_def)
# Delete unrecognized peers
existing_peers = calicoctl_get('bgppeers')['items']
existing_peers = [peer['metadata']['name'] for peer in existing_peers]
peers_to_delete = [
peer for peer in existing_peers
if peer.startswith(safe_unit_name +
'-') and peer not in named_peers
]
for peer in peers_to_delete:
calicoctl('delete', 'bgppeer', peer)
except CalledProcessError:
log(traceback.format_exc())
status.waiting('Waiting to retry BGP peer configuration')
return
set_state('calico.bgp.peers.configured')
def install_calico_binaries():
''' Unpack the Calico binaries. '''
# on intel, the resource is called 'calico'; other arches have a suffix
architecture = arch()
if architecture == "amd64":
resource_name = 'calico'
else:
resource_name = 'calico-{}'.format(architecture)
try:
archive = resource_get(resource_name)
except Exception:
message = 'Error fetching the calico resource.'
log(message)
status.blocked(message)
return
if not archive:
message = 'Missing calico resource.'
log(message)
status.blocked(message)
return
filesize = os.stat(archive).st_size
if filesize < 1000000:
message = 'Incomplete calico resource'
log(message)
status.blocked(message)
return
status.maintenance('Unpacking calico resource.')
charm_dir = os.getenv('CHARM_DIR')
unpack_path = os.path.join(charm_dir, 'files', 'calico')
os.makedirs(unpack_path, exist_ok=True)
cmd = ['tar', 'xfz', archive, '-C', unpack_path]
log(cmd)
check_call(cmd)
apps = [
{
'name': 'calicoctl',
'path': CALICOCTL_PATH
},
{
'name': 'calico',
'path': '/opt/cni/bin'
},
{
'name': 'calico-ipam',
'path': '/opt/cni/bin'
},
]
for app in apps:
unpacked = os.path.join(unpack_path, app['name'])
app_path = os.path.join(app['path'], app['name'])
install = ['install', '-v', '-D', unpacked, app_path]
check_call(install)
calicoctl_path = '/usr/local/bin/calicoctl'
render('calicoctl', calicoctl_path, {})
os.chmod(calicoctl_path, 0o775)
set_state('calico.binaries.installed')
def install_kafka():
status.maintenance('Installing Kafka')
# Check if mimimum amount of brokers are available
min_brokers = config().get('broker-count')
broker_count = 1
if min_brokers > 1 and is_flag_set('endpoint.broker.joined'):
kafka_peers = endpoint_from_flag('endpoint.broker.joined')
broker_count = kafka_peers.kafka_broker_count()
if broker_count != min_brokers:
status.blocked(
"Waiting for {} units to start bootstrapping.".format(min_brokers))
return
# Install Java
status.maintenance('Installing Java')
install_java()
# Unpack Kafka files and setup user/group
status.maintenance('Unpacking Kafka files')
filename = resource_get('apache-kafka')
filepath = filename and Path(filename)
if filepath and filepath.exists() and filepath.stat().st_size:
tar = tarfile.open(filepath, "r:gz")
tar.extractall("/usr/lib")
tar.close()
distconfig = utils.DistConfig("{}/files/setup.yaml".format(charm_dir()))
distconfig.add_users()
distconfig.add_dirs()
if not os.path.exists('/usr/lib/kafka'):
# Assumes that there is only 1 kafka_* dir
kafka_dir = glob.glob('/usr/lib/kafka_*')[0]
os.symlink(kafka_dir, '/usr/lib/kafka')
if not os.path.exists('/usr/lib/kafka/logs'):
os.makedirs('/usr/lib/kafka/logs')
os.symlink('/usr/lib/kafka/logs', '/var/log/kafka')
os.chmod('/var/log/kafka', 0o775)
shutil.chown('/var/log/kafka', user='******', group='kafka')
# Create server.properties
status.maintenance('Creating Kafka config')
zookeepers = endpoint_from_flag('zookeeper.ready')
zoo_brokers = []
for zoo in zookeepers.zookeepers():
zoo_brokers.append("{}:{}".format(zoo['host'], zoo['port']))
render(source="server.properties.j2",
target='/usr/lib/kafka/config/server.properties',
context={
'broker_count': min_brokers,
'transaction_min_isr':
1 if min_brokers == 1 else min_brokers - 1,
'zookeeper_brokers': ",".join(zoo_brokers),
})
# Create systemd service
render(source='kafka.service.j2',
target='/etc/systemd/system/kafka.service',
context={
'java_home': java_home(),
'jmx': 1 if config().get('enable-jmx') else 0,
})
# Start systemd service
status.maintenance('Starting Kafka services')
try:
check_call(['systemctl', 'daemon-reload'])
check_call(['systemctl', 'start', 'kafka.service'])
check_call(['systemctl', 'enable', 'kafka.service'])
except CalledProcessError as e:
log(e)
status.blocked('Could not start Kafka services')
return
open_port(9092)
if config().get('enable-jmx'):
open_port(9999)
status.active('Ready')
set_flag('kafka.installed')
def install_elasticsearch():
"""
Apt layer will take care of this.
"""
status.maintenance('Preparing to install.')
def install_logrotate():
status.maintenance('installing logrotate')
apt_install(['logrotate'])
set_flag('logrotate.installed')
set_flag('logrotate.init')
def deploy_network_policy_controller():
''' Deploy the Calico network policy controller. '''
status.maintenance('Applying registry credentials secret')
# FIXME: We're just stealing a server key and cert from a random
# worker. What should really go here?
key_path = '/root/cdk/server.key'
cert_path = '/root/cdk/server.crt'
if not os.path.exists(key_path) or not os.path.exists(cert_path):
msg = 'Waiting for cert generation'
log(msg)
status.waiting(msg)
return
etcd = endpoint_from_flag('etcd.available')
encoded_creds = hookenv.config('registry-credentials')
registry = hookenv.config('registry')
etcd_cert_hash = get_etcd_cert_hash()
apiserver_ips = get_apiserver_ips()
templates = []
if encoded_creds:
templates.append(('cnx-pull-secret.yaml', {
'credentials': encoded_creds
}))
templates += [('calico-config.yaml', {
'etcd_endpoints': etcd.get_connection_string()
}),
('calico-etcd-secrets.yaml', {
'etcd_key': read_file_to_base64(ETCD_KEY_PATH),
'etcd_cert': read_file_to_base64(ETCD_CERT_PATH),
'etcd_ca': read_file_to_base64(ETCD_CA_PATH)
}),
('calico-kube-controllers.yaml', {
'registry': registry,
'etcd_cert_hash': etcd_cert_hash
}),
('cnx-manager-tls-secret.yaml', {
'key': read_file_to_base64(key_path),
'cert': read_file_to_base64(cert_path)
}),
('cnx-etcd.yaml', {
'registry': registry,
'etcd_cert_hash': etcd_cert_hash
}), ('cnx-policy.yaml', {})]
# elasticsearch-operator junk
# elasticsearch-operator requires vm.max_map_count>=262144 on the host
if hookenv.config('enable-elasticsearch-operator'):
check_call(['sysctl', 'vm.max_map_count=262144'])
templates += [('elasticsearch-operator.yaml', {
'registry': registry
}),
('monitor-calico.yaml', {
'apiserver_ips': json.dumps(apiserver_ips),
'registry': registry
})]
for template, context in templates:
status.maintenance('Applying ' + template)
dest = '/tmp/' + template
render(template, dest, context)
try:
kubectl('apply', '-f', dest)
except CalledProcessError:
msg = 'Waiting to retry applying ' + template
log(msg)
status.waiting(msg)
return
license_key_b64 = hookenv.config('license-key')
license_key = b64decode(license_key_b64).decode('utf-8')
license_key_path = '/tmp/license-key.yaml'
with open(license_key_path, 'w') as f:
f.write(license_key)
try:
calicoctl('apply', '-f', license_key_path)
except CalledProcessError:
msg = 'Waiting to retry applying license-key'
log(msg)
status.waiting(msg)
return
db.set('tigera.apiserver_ips_used', apiserver_ips)
set_state('calico.npc.deployed')
def configure_master_cni():
status.maintenance('Configuring Calico CNI')
cni = endpoint_from_flag('cni.is-master')
config = charm_config()
cni.set_config(cidr=config['cidr'], cni_conf_file='10-calico.conflist')
set_state('calico.cni.configured')
