def enable_gpu(): """Enable GPU usage on this node. """ if get_version('kubelet') < (1, 9): hookenv.status_set( 'active', 'Upgrade to snap channel >= 1.9/stable to enable GPU support.') return hookenv.log('Enabling gpu mode') try: # Not sure why this is necessary, but if you don't run this, k8s will # think that the node has 0 gpus (as shown by the output of # `kubectl get nodes -o yaml` check_call(['nvidia-smi']) except CalledProcessError as cpe: hookenv.log('Unable to communicate with the NVIDIA driver.') hookenv.log(cpe) return set_label('gpu', 'true') set_label('cuda', 'true') set_state('kubernetes-worker.gpu.enabled') set_state('kubernetes-worker.restart-needed')
def set_cloud_pending(): k8s_version = get_version('kubelet') k8s_1_11 = k8s_version >= (1, 11) k8s_1_12 = k8s_version >= (1, 12) vsphere_joined = is_state('endpoint.vsphere.joined') azure_joined = is_state('endpoint.azure.joined') if (vsphere_joined and not k8s_1_12) or (azure_joined and not k8s_1_11): set_state('kubernetes-worker.cloud.blocked') else: remove_state('kubernetes-worker.cloud.blocked') set_state('kubernetes-worker.cloud.pending')
def render_and_launch_ingress(): ''' Launch the Kubernetes ingress controller & default backend (404) ''' config = hookenv.config() # need to test this in case we get in # here from a config change to the image if not config.get('ingress'): return context = {} context['arch'] = arch() addon_path = '/root/cdk/addons/{}' context['juju_application'] = hookenv.service_name() # If present, workers will get the ingress containers from the configured # registry. Otherwise, we'll set an appropriate upstream image registry. registry_location = get_registry_location() context['defaultbackend_image'] = config.get('default-backend-image') if (context['defaultbackend_image'] == "" or context['defaultbackend_image'] == "auto"): if registry_location: backend_registry = registry_location else: backend_registry = 'k8s.gcr.io' if context['arch'] == 's390x': context['defaultbackend_image'] = \ "{}/defaultbackend-s390x:1.4".format(backend_registry) elif context['arch'] == 'arm64': context['defaultbackend_image'] = \ "{}/defaultbackend-arm64:1.5".format(backend_registry) else: context['defaultbackend_image'] = \ "{}/defaultbackend-amd64:1.5".format(backend_registry) # Render the ingress daemon set controller manifest context['ssl_chain_completion'] = config.get( 'ingress-ssl-chain-completion') context['ingress_image'] = config.get('nginx-image') if context['ingress_image'] == "" or context['ingress_image'] == "auto": if registry_location: nginx_registry = registry_location else: nginx_registry = 'quay.io' images = { 'amd64': 'kubernetes-ingress-controller/nginx-ingress-controller-amd64:0.22.0', # noqa 'arm64': 'kubernetes-ingress-controller/nginx-ingress-controller-arm64:0.22.0', # noqa 's390x': 'kubernetes-ingress-controller/nginx-ingress-controller-s390x:0.20.0', # noqa 'ppc64el': 'kubernetes-ingress-controller/nginx-ingress-controller-ppc64le:0.20.0', # noqa } context['ingress_image'] = '{}/{}'.format( nginx_registry, images.get(context['arch'], images['amd64'])) if get_version('kubelet') < (1, 9): context['daemonset_api_version'] = 'extensions/v1beta1' else: context['daemonset_api_version'] = 'apps/v1beta2' manifest = addon_path.format('ingress-daemon-set.yaml') render('ingress-daemon-set.yaml', manifest, context) hookenv.log('Creating the ingress daemon set.') try: kubectl('apply', '-f', manifest) except CalledProcessError as e: hookenv.log(e) hookenv.log( 'Failed to create ingress controller. Will attempt again next update.' ) # noqa hookenv.close_port(80) hookenv.close_port(443) return # Render the default http backend (404) deployment manifest # needs to happen after ingress-daemon-set since that sets up the namespace manifest = addon_path.format('default-http-backend.yaml') render('default-http-backend.yaml', manifest, context) hookenv.log('Creating the default http backend.') try: kubectl('apply', '-f', manifest) except CalledProcessError as e: hookenv.log(e) hookenv.log( 'Failed to create default-http-backend. Will attempt again next update.' ) # noqa hookenv.close_port(80) hookenv.close_port(443) return set_state('kubernetes-worker.ingress.available') hookenv.open_port(80) hookenv.open_port(443)
def configure_kubelet(dns, ingress_ip): kubelet_opts = {} kubelet_opts['require-kubeconfig'] = 'true' kubelet_opts['kubeconfig'] = kubeconfig_path kubelet_opts['network-plugin'] = 'cni' kubelet_opts['v'] = '0' kubelet_opts['logtostderr'] = 'true' kubelet_opts['node-ip'] = ingress_ip endpoint_config = \ endpoint_from_flag('endpoint.container-runtime.available').get_config() kubelet_opts['container-runtime'] = endpoint_config['runtime'] if kubelet_opts['container-runtime'] == 'remote': kubelet_opts['container-runtime-endpoint'] = endpoint_config['socket'] kubelet_cloud_config_path = cloud_config_path('kubelet') if is_state('endpoint.aws.ready'): kubelet_opts['cloud-provider'] = 'aws' elif is_state('endpoint.gcp.ready'): kubelet_opts['cloud-provider'] = 'gce' kubelet_opts['cloud-config'] = str(kubelet_cloud_config_path) elif is_state('endpoint.openstack.ready'): kubelet_opts['cloud-provider'] = 'openstack' kubelet_opts['cloud-config'] = str(kubelet_cloud_config_path) elif is_state('endpoint.vsphere.joined'): # vsphere just needs to be joined on the worker (vs 'ready') kubelet_opts['cloud-provider'] = 'vsphere' # NB: vsphere maps node product-id to its uuid (no config file needed). uuid_file = '/sys/class/dmi/id/product_uuid' with open(uuid_file, 'r') as f: uuid = f.read().strip() kubelet_opts['provider-id'] = 'vsphere://{}'.format(uuid) elif is_state('endpoint.azure.ready'): azure = endpoint_from_flag('endpoint.azure.ready') kubelet_opts['cloud-provider'] = 'azure' kubelet_opts['cloud-config'] = str(kubelet_cloud_config_path) kubelet_opts['provider-id'] = azure.vm_id if get_version('kubelet') >= (1, 10): # Put together the KubeletConfiguration data kubelet_config = { 'apiVersion': 'kubelet.config.k8s.io/v1beta1', 'kind': 'KubeletConfiguration', 'address': '0.0.0.0', 'authentication': { 'anonymous': { 'enabled': False }, 'x509': { 'clientCAFile': str(ca_crt_path) } }, 'clusterDomain': dns['domain'], 'failSwapOn': False, 'port': 10250, 'tlsCertFile': str(server_crt_path), 'tlsPrivateKeyFile': str(server_key_path) } if dns['enable-kube-dns']: kubelet_config['clusterDNS'] = [dns['sdn-ip']] if is_state('kubernetes-worker.gpu.enabled'): kubelet_config['featureGates'] = {'DevicePlugins': True} # Workaround for DNS on bionic # https://github.com/juju-solutions/bundle-canonical-kubernetes/issues/655 resolv_path = os.path.realpath('/etc/resolv.conf') if resolv_path == '/run/systemd/resolve/stub-resolv.conf': kubelet_config['resolvConf'] = '/run/systemd/resolve/resolv.conf' # Add kubelet-extra-config. This needs to happen last so that it # overrides any config provided by the charm. kubelet_extra_config = hookenv.config('kubelet-extra-config') kubelet_extra_config = yaml.safe_load(kubelet_extra_config) merge_kubelet_extra_config(kubelet_config, kubelet_extra_config) # Render the file and configure Kubelet to use it os.makedirs('/root/cdk/kubelet', exist_ok=True) with open('/root/cdk/kubelet/config.yaml', 'w') as f: f.write('# Generated by kubernetes-worker charm, do not edit\n') yaml.dump(kubelet_config, f) kubelet_opts['config'] = '/root/cdk/kubelet/config.yaml' else: # NOTE: This is for 1.9. Once we've dropped 1.9 support, we can remove # this whole block and the parent if statement. kubelet_opts['address'] = '0.0.0.0' kubelet_opts['anonymous-auth'] = 'false' kubelet_opts['client-ca-file'] = str(ca_crt_path) kubelet_opts['cluster-domain'] = dns['domain'] kubelet_opts['fail-swap-on'] = 'false' kubelet_opts['port'] = '10250' kubelet_opts['tls-cert-file'] = str(server_crt_path) kubelet_opts['tls-private-key-file'] = str(server_key_path) if dns['enable-kube-dns']: kubelet_opts['cluster-dns'] = dns['sdn-ip'] if is_state('kubernetes-worker.gpu.enabled'): kubelet_opts['feature-gates'] = 'DevicePlugins=true' # Workaround for DNS on bionic, for k8s 1.9 # https://github.com/juju-solutions/bundle-canonical-kubernetes/issues/655 resolv_path = os.path.realpath('/etc/resolv.conf') if resolv_path == '/run/systemd/resolve/stub-resolv.conf': kubelet_opts['resolv-conf'] = '/run/systemd/resolve/resolv.conf' if get_version('kubelet') >= (1, 11): kubelet_opts['dynamic-config-dir'] = '/root/cdk/kubelet/dynamic-config' # If present, ensure kubelet gets the pause container from the configured # registry. When not present, kubelet uses a default image location # (currently k8s.gcr.io/pause:3.1). registry_location = get_registry_location() if registry_location: kubelet_opts['pod-infra-container-image'] = \ '{}/pause-{}:3.1'.format(registry_location, arch()) configure_kubernetes_service(configure_prefix, 'kubelet', kubelet_opts, 'kubelet-extra-args')
def render_and_launch_ingress(): """Launch the Kubernetes ingress controller & default backend (404)""" config = hookenv.config() # need to test this in case we get in # here from a config change to the image if not config.get("ingress"): return context = {} context["arch"] = arch() addon_path = "/root/cdk/addons/{}" context["juju_application"] = hookenv.service_name() # If present, workers will get the ingress containers from the configured # registry. Otherwise, we'll set an appropriate upstream image registry. registry_location = get_registry_location() context["defaultbackend_image"] = config.get("default-backend-image") if ( context["defaultbackend_image"] == "" or context["defaultbackend_image"] == "auto" ): if registry_location: backend_registry = registry_location else: backend_registry = "k8s.gcr.io" if context["arch"] == "s390x": context["defaultbackend_image"] = "{}/defaultbackend-s390x:1.4".format( backend_registry ) elif context["arch"] == "ppc64el": context["defaultbackend_image"] = "{}/defaultbackend-ppc64le:1.5".format( backend_registry ) else: context["defaultbackend_image"] = "{}/defaultbackend-{}:1.5".format( backend_registry, context["arch"] ) # Render the ingress daemon set controller manifest context["ssl_chain_completion"] = config.get("ingress-ssl-chain-completion") context["enable_ssl_passthrough"] = config.get("ingress-ssl-passthrough") context["default_ssl_certificate_option"] = None if config.get("ingress-default-ssl-certificate") and config.get( "ingress-default-ssl-key" ): context["default_ssl_certificate"] = b64encode( config.get("ingress-default-ssl-certificate").encode("utf-8") ).decode("utf-8") context["default_ssl_key"] = b64encode( config.get("ingress-default-ssl-key").encode("utf-8") ).decode("utf-8") default_certificate_option = ( "- --default-ssl-certificate=" "$(POD_NAMESPACE)/default-ssl-certificate" ) context["default_ssl_certificate_option"] = default_certificate_option context["ingress_image"] = config.get("nginx-image") if context["ingress_image"] == "" or context["ingress_image"] == "auto": if context["arch"] == "ppc64el": # multi-arch image doesn't include ppc64le, have to use an older version image = "nginx-ingress-controller-ppc64le" context["ingress_uid"] = "33" context["ingress_image"] = "/".join( [ registry_location or "quay.io", "kubernetes-ingress-controller/{}:0.20.0".format(image), ] ) else: context["ingress_uid"] = "101" context["ingress_image"] = "/".join( [ registry_location or "us.gcr.io", "k8s-artifacts-prod/ingress-nginx/controller:v1.2.0", ] ) kubelet_version = get_version("kubelet") if kubelet_version < (1, 9): context["daemonset_api_version"] = "extensions/v1beta1" context["deployment_api_version"] = "extensions/v1beta1" elif kubelet_version < (1, 16): context["daemonset_api_version"] = "apps/v1beta2" context["deployment_api_version"] = "extensions/v1beta1" else: context["daemonset_api_version"] = "apps/v1" context["deployment_api_version"] = "apps/v1" context["use_forwarded_headers"] = ( "true" if config.get("ingress-use-forwarded-headers") else "false" ) manifest = addon_path.format("ingress-daemon-set.yaml") render("ingress-daemon-set.yaml", manifest, context) hookenv.log("Creating the ingress daemon set.") try: kubectl("apply", "-f", manifest) except CalledProcessError as e: hookenv.log(e) hookenv.log( "Failed to create ingress controller. Will attempt again next update." ) # noqa hookenv.close_port(80) hookenv.close_port(443) return # Render the default http backend (404) deployment manifest # needs to happen after ingress-daemon-set since that sets up the namespace manifest = addon_path.format("default-http-backend.yaml") render("default-http-backend.yaml", manifest, context) hookenv.log("Creating the default http backend.") try: kubectl("apply", "-f", manifest) except CalledProcessError as e: hookenv.log(e) hookenv.log( "Failed to create default-http-backend. Will attempt again next update." ) # noqa hookenv.close_port(80) hookenv.close_port(443) return set_state("kubernetes-worker.ingress.available") hookenv.open_port(80) hookenv.open_port(443)