def enable_gpu():
"""Enable GPU usage on this node.
"""
if get_version('kubelet') < (1, 9):
hookenv.status_set(
'active',
'Upgrade to snap channel >= 1.9/stable to enable GPU support.')
return
hookenv.log('Enabling gpu mode')
try:
# Not sure why this is necessary, but if you don't run this, k8s will
# think that the node has 0 gpus (as shown by the output of
# `kubectl get nodes -o yaml`
check_call(['nvidia-smi'])
except CalledProcessError as cpe:
hookenv.log('Unable to communicate with the NVIDIA driver.')
hookenv.log(cpe)
return
set_label('gpu', 'true')
set_label('cuda', 'true')
set_state('kubernetes-worker.gpu.enabled')
set_state('kubernetes-worker.restart-needed')
def set_cloud_pending():
k8s_version = get_version('kubelet')
k8s_1_11 = k8s_version >= (1, 11)
k8s_1_12 = k8s_version >= (1, 12)
vsphere_joined = is_state('endpoint.vsphere.joined')
azure_joined = is_state('endpoint.azure.joined')
if (vsphere_joined and not k8s_1_12) or (azure_joined and not k8s_1_11):
set_state('kubernetes-worker.cloud.blocked')
else:
remove_state('kubernetes-worker.cloud.blocked')
set_state('kubernetes-worker.cloud.pending')
def render_and_launch_ingress():
''' Launch the Kubernetes ingress controller & default backend (404) '''
config = hookenv.config()
# need to test this in case we get in
# here from a config change to the image
if not config.get('ingress'):
return
context = {}
context['arch'] = arch()
addon_path = '/root/cdk/addons/{}'
context['juju_application'] = hookenv.service_name()
# If present, workers will get the ingress containers from the configured
# registry. Otherwise, we'll set an appropriate upstream image registry.
registry_location = get_registry_location()
context['defaultbackend_image'] = config.get('default-backend-image')
if (context['defaultbackend_image'] == ""
or context['defaultbackend_image'] == "auto"):
if registry_location:
backend_registry = registry_location
else:
backend_registry = 'k8s.gcr.io'
if context['arch'] == 's390x':
context['defaultbackend_image'] = \
"{}/defaultbackend-s390x:1.4".format(backend_registry)
elif context['arch'] == 'arm64':
context['defaultbackend_image'] = \
"{}/defaultbackend-arm64:1.5".format(backend_registry)
else:
context['defaultbackend_image'] = \
"{}/defaultbackend-amd64:1.5".format(backend_registry)
# Render the ingress daemon set controller manifest
context['ssl_chain_completion'] = config.get(
'ingress-ssl-chain-completion')
context['ingress_image'] = config.get('nginx-image')
if context['ingress_image'] == "" or context['ingress_image'] == "auto":
if registry_location:
nginx_registry = registry_location
else:
nginx_registry = 'quay.io'
images = {
'amd64':
'kubernetes-ingress-controller/nginx-ingress-controller-amd64:0.22.0', # noqa
'arm64':
'kubernetes-ingress-controller/nginx-ingress-controller-arm64:0.22.0', # noqa
's390x':
'kubernetes-ingress-controller/nginx-ingress-controller-s390x:0.20.0', # noqa
'ppc64el':
'kubernetes-ingress-controller/nginx-ingress-controller-ppc64le:0.20.0', # noqa
}
context['ingress_image'] = '{}/{}'.format(
nginx_registry, images.get(context['arch'], images['amd64']))
if get_version('kubelet') < (1, 9):
context['daemonset_api_version'] = 'extensions/v1beta1'
else:
context['daemonset_api_version'] = 'apps/v1beta2'
manifest = addon_path.format('ingress-daemon-set.yaml')
render('ingress-daemon-set.yaml', manifest, context)
hookenv.log('Creating the ingress daemon set.')
try:
kubectl('apply', '-f', manifest)
except CalledProcessError as e:
hookenv.log(e)
hookenv.log(
'Failed to create ingress controller. Will attempt again next update.'
) # noqa
hookenv.close_port(80)
hookenv.close_port(443)
return
# Render the default http backend (404) deployment manifest
# needs to happen after ingress-daemon-set since that sets up the namespace
manifest = addon_path.format('default-http-backend.yaml')
render('default-http-backend.yaml', manifest, context)
hookenv.log('Creating the default http backend.')
try:
kubectl('apply', '-f', manifest)
except CalledProcessError as e:
hookenv.log(e)
hookenv.log(
'Failed to create default-http-backend. Will attempt again next update.'
) # noqa
hookenv.close_port(80)
hookenv.close_port(443)
return
set_state('kubernetes-worker.ingress.available')
hookenv.open_port(80)
hookenv.open_port(443)
def configure_kubelet(dns, ingress_ip):
kubelet_opts = {}
kubelet_opts['require-kubeconfig'] = 'true'
kubelet_opts['kubeconfig'] = kubeconfig_path
kubelet_opts['network-plugin'] = 'cni'
kubelet_opts['v'] = '0'
kubelet_opts['logtostderr'] = 'true'
kubelet_opts['node-ip'] = ingress_ip
endpoint_config = \
endpoint_from_flag('endpoint.container-runtime.available').get_config()
kubelet_opts['container-runtime'] = endpoint_config['runtime']
if kubelet_opts['container-runtime'] == 'remote':
kubelet_opts['container-runtime-endpoint'] = endpoint_config['socket']
kubelet_cloud_config_path = cloud_config_path('kubelet')
if is_state('endpoint.aws.ready'):
kubelet_opts['cloud-provider'] = 'aws'
elif is_state('endpoint.gcp.ready'):
kubelet_opts['cloud-provider'] = 'gce'
kubelet_opts['cloud-config'] = str(kubelet_cloud_config_path)
elif is_state('endpoint.openstack.ready'):
kubelet_opts['cloud-provider'] = 'openstack'
kubelet_opts['cloud-config'] = str(kubelet_cloud_config_path)
elif is_state('endpoint.vsphere.joined'):
# vsphere just needs to be joined on the worker (vs 'ready')
kubelet_opts['cloud-provider'] = 'vsphere'
# NB: vsphere maps node product-id to its uuid (no config file needed).
uuid_file = '/sys/class/dmi/id/product_uuid'
with open(uuid_file, 'r') as f:
uuid = f.read().strip()
kubelet_opts['provider-id'] = 'vsphere://{}'.format(uuid)
elif is_state('endpoint.azure.ready'):
azure = endpoint_from_flag('endpoint.azure.ready')
kubelet_opts['cloud-provider'] = 'azure'
kubelet_opts['cloud-config'] = str(kubelet_cloud_config_path)
kubelet_opts['provider-id'] = azure.vm_id
if get_version('kubelet') >= (1, 10):
# Put together the KubeletConfiguration data
kubelet_config = {
'apiVersion': 'kubelet.config.k8s.io/v1beta1',
'kind': 'KubeletConfiguration',
'address': '0.0.0.0',
'authentication': {
'anonymous': {
'enabled': False
},
'x509': {
'clientCAFile': str(ca_crt_path)
}
},
'clusterDomain': dns['domain'],
'failSwapOn': False,
'port': 10250,
'tlsCertFile': str(server_crt_path),
'tlsPrivateKeyFile': str(server_key_path)
}
if dns['enable-kube-dns']:
kubelet_config['clusterDNS'] = [dns['sdn-ip']]
if is_state('kubernetes-worker.gpu.enabled'):
kubelet_config['featureGates'] = {'DevicePlugins': True}
# Workaround for DNS on bionic
# https://github.com/juju-solutions/bundle-canonical-kubernetes/issues/655
resolv_path = os.path.realpath('/etc/resolv.conf')
if resolv_path == '/run/systemd/resolve/stub-resolv.conf':
kubelet_config['resolvConf'] = '/run/systemd/resolve/resolv.conf'
# Add kubelet-extra-config. This needs to happen last so that it
# overrides any config provided by the charm.
kubelet_extra_config = hookenv.config('kubelet-extra-config')
kubelet_extra_config = yaml.safe_load(kubelet_extra_config)
merge_kubelet_extra_config(kubelet_config, kubelet_extra_config)
# Render the file and configure Kubelet to use it
os.makedirs('/root/cdk/kubelet', exist_ok=True)
with open('/root/cdk/kubelet/config.yaml', 'w') as f:
f.write('# Generated by kubernetes-worker charm, do not edit\n')
yaml.dump(kubelet_config, f)
kubelet_opts['config'] = '/root/cdk/kubelet/config.yaml'
else:
# NOTE: This is for 1.9. Once we've dropped 1.9 support, we can remove
# this whole block and the parent if statement.
kubelet_opts['address'] = '0.0.0.0'
kubelet_opts['anonymous-auth'] = 'false'
kubelet_opts['client-ca-file'] = str(ca_crt_path)
kubelet_opts['cluster-domain'] = dns['domain']
kubelet_opts['fail-swap-on'] = 'false'
kubelet_opts['port'] = '10250'
kubelet_opts['tls-cert-file'] = str(server_crt_path)
kubelet_opts['tls-private-key-file'] = str(server_key_path)
if dns['enable-kube-dns']:
kubelet_opts['cluster-dns'] = dns['sdn-ip']
if is_state('kubernetes-worker.gpu.enabled'):
kubelet_opts['feature-gates'] = 'DevicePlugins=true'
# Workaround for DNS on bionic, for k8s 1.9
# https://github.com/juju-solutions/bundle-canonical-kubernetes/issues/655
resolv_path = os.path.realpath('/etc/resolv.conf')
if resolv_path == '/run/systemd/resolve/stub-resolv.conf':
kubelet_opts['resolv-conf'] = '/run/systemd/resolve/resolv.conf'
if get_version('kubelet') >= (1, 11):
kubelet_opts['dynamic-config-dir'] = '/root/cdk/kubelet/dynamic-config'
# If present, ensure kubelet gets the pause container from the configured
# registry. When not present, kubelet uses a default image location
# (currently k8s.gcr.io/pause:3.1).
registry_location = get_registry_location()
if registry_location:
kubelet_opts['pod-infra-container-image'] = \
'{}/pause-{}:3.1'.format(registry_location, arch())
configure_kubernetes_service(configure_prefix, 'kubelet', kubelet_opts,
'kubelet-extra-args')
def render_and_launch_ingress():
"""Launch the Kubernetes ingress controller & default backend (404)"""
config = hookenv.config()
# need to test this in case we get in
# here from a config change to the image
if not config.get("ingress"):
return
context = {}
context["arch"] = arch()
addon_path = "/root/cdk/addons/{}"
context["juju_application"] = hookenv.service_name()
# If present, workers will get the ingress containers from the configured
# registry. Otherwise, we'll set an appropriate upstream image registry.
registry_location = get_registry_location()
context["defaultbackend_image"] = config.get("default-backend-image")
if (
context["defaultbackend_image"] == ""
or context["defaultbackend_image"] == "auto"
):
if registry_location:
backend_registry = registry_location
else:
backend_registry = "k8s.gcr.io"
if context["arch"] == "s390x":
context["defaultbackend_image"] = "{}/defaultbackend-s390x:1.4".format(
backend_registry
)
elif context["arch"] == "ppc64el":
context["defaultbackend_image"] = "{}/defaultbackend-ppc64le:1.5".format(
backend_registry
)
else:
context["defaultbackend_image"] = "{}/defaultbackend-{}:1.5".format(
backend_registry, context["arch"]
)
# Render the ingress daemon set controller manifest
context["ssl_chain_completion"] = config.get("ingress-ssl-chain-completion")
context["enable_ssl_passthrough"] = config.get("ingress-ssl-passthrough")
context["default_ssl_certificate_option"] = None
if config.get("ingress-default-ssl-certificate") and config.get(
"ingress-default-ssl-key"
):
context["default_ssl_certificate"] = b64encode(
config.get("ingress-default-ssl-certificate").encode("utf-8")
).decode("utf-8")
context["default_ssl_key"] = b64encode(
config.get("ingress-default-ssl-key").encode("utf-8")
).decode("utf-8")
default_certificate_option = (
"- --default-ssl-certificate=" "$(POD_NAMESPACE)/default-ssl-certificate"
)
context["default_ssl_certificate_option"] = default_certificate_option
context["ingress_image"] = config.get("nginx-image")
if context["ingress_image"] == "" or context["ingress_image"] == "auto":
if context["arch"] == "ppc64el":
# multi-arch image doesn't include ppc64le, have to use an older version
image = "nginx-ingress-controller-ppc64le"
context["ingress_uid"] = "33"
context["ingress_image"] = "/".join(
[
registry_location or "quay.io",
"kubernetes-ingress-controller/{}:0.20.0".format(image),
]
)
else:
context["ingress_uid"] = "101"
context["ingress_image"] = "/".join(
[
registry_location or "us.gcr.io",
"k8s-artifacts-prod/ingress-nginx/controller:v1.2.0",
]
)
kubelet_version = get_version("kubelet")
if kubelet_version < (1, 9):
context["daemonset_api_version"] = "extensions/v1beta1"
context["deployment_api_version"] = "extensions/v1beta1"
elif kubelet_version < (1, 16):
context["daemonset_api_version"] = "apps/v1beta2"
context["deployment_api_version"] = "extensions/v1beta1"
else:
context["daemonset_api_version"] = "apps/v1"
context["deployment_api_version"] = "apps/v1"
context["use_forwarded_headers"] = (
"true" if config.get("ingress-use-forwarded-headers") else "false"
)
manifest = addon_path.format("ingress-daemon-set.yaml")
render("ingress-daemon-set.yaml", manifest, context)
hookenv.log("Creating the ingress daemon set.")
try:
kubectl("apply", "-f", manifest)
except CalledProcessError as e:
hookenv.log(e)
hookenv.log(
"Failed to create ingress controller. Will attempt again next update."
) # noqa
hookenv.close_port(80)
hookenv.close_port(443)
return
# Render the default http backend (404) deployment manifest
# needs to happen after ingress-daemon-set since that sets up the namespace
manifest = addon_path.format("default-http-backend.yaml")
render("default-http-backend.yaml", manifest, context)
hookenv.log("Creating the default http backend.")
try:
kubectl("apply", "-f", manifest)
except CalledProcessError as e:
hookenv.log(e)
hookenv.log(
"Failed to create default-http-backend. Will attempt again next update."
) # noqa
hookenv.close_port(80)
hookenv.close_port(443)
return
set_state("kubernetes-worker.ingress.available")
hookenv.open_port(80)
hookenv.open_port(443)