def get_params(self, context, cluster_template, cluster, **kwargs): LOG.info("########## get_params") extra_params = kwargs.pop('extra_params', {}) label_list = [ 'minikube_version', 'kubectl_version', 'occm_container_infra_prefix', 'etcd_version' ] for label in label_list: label_value = cluster.labels.get(label) if label_value: extra_params[label] = label_value # handover the ca_key ca_cert = cert_manager.get_cluster_ca_certificate(cluster, context) if six.PY3 and isinstance(ca_cert.get_private_key_passphrase(), six.text_type): extra_params['ca_key'] = x509.decrypt_key( ca_cert.get_private_key(), ca_cert.get_private_key_passphrase().encode()).decode( ).replace("\n", "\\n") else: extra_params['ca_key'] = x509.decrypt_key( ca_cert.get_private_key(), ca_cert.get_private_key_passphrase()).replace("\n", "\\n") return super(AtomicMinikubeTemplateDefinition, self).get_params(context, cluster_template, cluster, extra_params=extra_params, **kwargs)
def get_ca_certificate(self, context, cluster): ca_cert = cert_manager.get_cluster_ca_certificate(cluster, context=context) certificate = objects.Certificate.from_object_cluster(cluster) if six.PY3 and isinstance(ca_cert.get_certificate(), six.binary_type): certificate.pem = ca_cert.get_certificate().decode() else: certificate.pem = ca_cert.get_certificate() return certificate
def get_params(self, context, cluster_template, cluster, **kwargs): extra_params = kwargs.pop('extra_params', {}) extra_params['username'] = context.user_name osc = self.get_osc(context) extra_params['region_name'] = osc.cinder_region_name() # set docker_volume_type # use the configuration default if None provided docker_volume_type = cluster.labels.get( 'docker_volume_type', CONF.cinder.default_docker_volume_type) extra_params['docker_volume_type'] = docker_volume_type extra_params['nodes_affinity_policy'] = \ CONF.cluster.nodes_affinity_policy if cluster_template.network_driver == 'flannel': extra_params["pods_network_cidr"] = \ cluster.labels.get('flannel_network_cidr', '10.100.0.0/16') if cluster_template.network_driver == 'calico': extra_params["pods_network_cidr"] = \ cluster.labels.get('calico_ipv4pool', '10.100.0.0/16') label_list = [ 'coredns_tag', 'kube_tag', 'container_infra_prefix', 'availability_zone', 'calico_tag', 'calico_kube_controllers_tag', 'calico_ipv4pool', 'calico_ipv4pool_ipip', 'etcd_tag', 'flannel_tag' ] labels = self._get_relevant_labels(cluster, kwargs) for label in label_list: label_value = labels.get(label) if label_value: extra_params[label] = label_value cert_manager_api = cluster.labels.get('cert_manager_api') if strutils.bool_from_string(cert_manager_api): extra_params['cert_manager_api'] = cert_manager_api ca_cert = cert_manager.get_cluster_ca_certificate(cluster) extra_params['ca_key'] = x509.decrypt_key( ca_cert.get_private_key(), ca_cert.get_private_key_passphrase()).replace("\n", "\\n") plain_openstack_ca = utils.get_openstack_ca() encoded_openstack_ca = base64.b64encode(plain_openstack_ca.encode()) extra_params['openstack_ca_coreos'] = encoded_openstack_ca.decode() return super(CoreOSK8sTemplateDefinition, self).get_params(context, cluster_template, cluster, extra_params=extra_params, **kwargs)
def test_get_cluster_ca_certificate(self): mock_cluster = mock.MagicMock() mock_cluster.uuid = "mock_cluster_uuid" mock_ca_cert = mock.MagicMock() self.CertManager.get_cert.return_value = mock_ca_cert cluster_ca_cert = cert_manager.get_cluster_ca_certificate(mock_cluster) self.CertManager.get_cert.assert_called_once_with( mock_cluster.ca_cert_ref, resource_ref=mock_cluster.uuid, context=None) self.assertEqual(mock_ca_cert, cluster_ca_cert)
def get_params(self, context, cluster_template, cluster, **kwargs): extra_params = kwargs.pop('extra_params', {}) extra_params['username'] = context.user_name osc = self.get_osc(context) extra_params['region_name'] = osc.cinder_region_name() # set docker_volume_type # use the configuration default if None provided docker_volume_type = cluster.labels.get( 'docker_volume_type', CONF.cinder.default_docker_volume_type) extra_params['docker_volume_type'] = docker_volume_type extra_params['nodes_affinity_policy'] = \ CONF.cluster.nodes_affinity_policy if cluster_template.network_driver == 'flannel': extra_params["pods_network_cidr"] = \ cluster.labels.get('flannel_network_cidr', '10.100.0.0/16') if cluster_template.network_driver == 'calico': extra_params["pods_network_cidr"] = \ cluster.labels.get('calico_ipv4pool', '192.168.0.0/16') label_list = ['coredns_tag', 'kube_tag', 'container_infra_prefix', 'availability_zone', 'calico_tag', 'calico_cni_tag', 'calico_kube_controllers_tag', 'calico_ipv4pool', 'etcd_tag', 'flannel_tag'] for label in label_list: label_value = cluster.labels.get(label) if label_value: extra_params[label] = label_value cert_manager_api = cluster.labels.get('cert_manager_api') if strutils.bool_from_string(cert_manager_api): extra_params['cert_manager_api'] = cert_manager_api ca_cert = cert_manager.get_cluster_ca_certificate(cluster) extra_params['ca_key'] = x509.decrypt_key( ca_cert.get_private_key(), ca_cert.get_private_key_passphrase()).replace("\n", "\\n") plain_openstack_ca = utils.get_openstack_ca() encoded_openstack_ca = base64.b64encode(plain_openstack_ca.encode()) extra_params['openstack_ca_coreos'] = encoded_openstack_ca.decode() return super(CoreOSK8sTemplateDefinition, self).get_params(context, cluster_template, cluster, extra_params=extra_params, **kwargs)
def _set_cert_manager_params(self, cluster, extra_params): cert_manager_api = cluster.labels.get('cert_manager_api') if strutils.bool_from_string(cert_manager_api): extra_params['cert_manager_api'] = cert_manager_api ca_cert = cert_manager.get_cluster_ca_certificate(cluster) if six.PY3 and isinstance(ca_cert.get_private_key_passphrase(), six.text_type): extra_params['ca_key'] = x509.decrypt_key( ca_cert.get_private_key(), ca_cert.get_private_key_passphrase().encode()).decode( ).replace("\n", "\\n") else: extra_params['ca_key'] = x509.decrypt_key( ca_cert.get_private_key(), ca_cert.get_private_key_passphrase()).replace("\n", "\\n")
def _set_cert_manager_params(self, cluster, extra_params): cert_manager_api = cluster.labels.get('cert_manager_api') if strutils.bool_from_string(cert_manager_api): extra_params['cert_manager_api'] = cert_manager_api ca_cert = cert_manager.get_cluster_ca_certificate(cluster) if six.PY3 and isinstance(ca_cert.get_private_key_passphrase(), six.text_type): extra_params['ca_key'] = x509.decrypt_key( ca_cert.get_private_key(), ca_cert.get_private_key_passphrase().encode() ).decode().replace("\n", "\\n") else: extra_params['ca_key'] = x509.decrypt_key( ca_cert.get_private_key(), ca_cert.get_private_key_passphrase()).replace("\n", "\\n")
def _get_kubeconfig(self, context, cluster, ca_cert_encoded=None): """Prepare kubeconfig file to interact with the new cluster.""" # TODO(lxkong): This kubeconfig file should be stored in a shared place # between magnum controllers. cluster_cache_dir = os.path.join( CONF.cluster.temp_cache_dir, cluster.uuid ) fileutils.ensure_tree(cluster_cache_dir) kubeconfig_path = os.path.join(cluster_cache_dir, "kubeconfig") if os.path.exists(kubeconfig_path): return kubeconfig_path if ca_cert_encoded is None: ca_cert = cert_manager.get_cluster_ca_certificate( cluster, context=context ) ca_cert_encoded = base64.b64encode(ca_cert.get_certificate()) admin_cert = cert_manager.get_cluster_magnum_cert( cluster, context=context ) admin_cert_encoded = base64.b64encode(admin_cert.get_certificate()) admin_key_encoded = base64.b64encode( admin_cert.get_decrypted_private_key()) split_ret = netutils.urlsplit(cluster.api_address) external_apiserver_address = split_ret.netloc.split(":")[0] kubeconfig_params = { "cluster_id": cluster.uuid, "apiserver_address": external_apiserver_address, "ca_cert": ca_cert_encoded, "cert": admin_cert_encoded, "key": admin_key_encoded, "user": "******", } kubeconfig_template = self.jinja_env.get_template("kubeconfig.j2") kubeconfig = kubeconfig_template.render(kubeconfig_params) with open(kubeconfig_path, 'w') as fd: fd.write(kubeconfig) return kubeconfig_path
def get_params(self, context, cluster_template, cluster, **kwargs): extra_params = kwargs.pop('extra_params', {}) extra_params['username'] = context.user_name osc = self.get_osc(context) extra_params['region_name'] = osc.cinder_region_name() # set docker_volume_type # use the configuration default if None provided docker_volume_type = cluster.labels.get( 'docker_volume_type', CONF.cinder.default_docker_volume_type) extra_params['docker_volume_type'] = docker_volume_type extra_params['nodes_affinity_policy'] = \ CONF.cluster.nodes_affinity_policy label_list = [ 'kube_tag', 'container_infra_prefix', 'availability_zone' ] for label in label_list: label_value = cluster.labels.get(label) if label_value: extra_params[label] = label_value cert_manager_api = cluster.labels.get('cert_manager_api') if strutils.bool_from_string(cert_manager_api): extra_params['cert_manager_api'] = cert_manager_api ca_cert = cert_manager.get_cluster_ca_certificate(cluster) extra_params['ca_key'] = x509.decrypt_key( ca_cert.get_private_key(), ca_cert.get_private_key_passphrase()).replace("\n", "\\n") return super(K8sFedoraTemplateDefinition, self).get_params(context, cluster_template, cluster, extra_params=extra_params, **kwargs)
def get_params(self, context, cluster_template, cluster, **kwargs): extra_params = kwargs.pop('extra_params', {}) extra_params['username'] = context.user_name osc = self.get_osc(context) extra_params['region_name'] = osc.cinder_region_name() # set docker_volume_type # use the configuration default if None provided docker_volume_type = cluster.labels.get( 'docker_volume_type', CONF.cinder.default_docker_volume_type) extra_params['docker_volume_type'] = docker_volume_type extra_params['nodes_affinity_policy'] = \ CONF.cluster.nodes_affinity_policy if cluster_template.network_driver == 'flannel': extra_params["pods_network_cidr"] = \ cluster.labels.get('flannel_network_cidr', '10.100.0.0/16') if cluster_template.network_driver == 'calico': extra_params["pods_network_cidr"] = \ cluster.labels.get('calico_ipv4pool', '192.168.0.0/16') # check cloud provider and cinder options. If cinder is selected, # the cloud provider needs to be enabled. cloud_provider_enabled = cluster.labels.get('cloud_provider_enabled', 'true').lower() if (cluster_template.volume_driver == 'cinder' and cloud_provider_enabled == 'false'): raise exception.InvalidParameterValue( _('"cinder" volume driver needs "cloud_provider_enabled" label ' 'to be true or unset.')) label_list = [ 'kube_tag', 'container_infra_prefix', 'availability_zone', 'cgroup_driver', 'calico_tag', 'calico_cni_tag', 'calico_kube_controllers_tag', 'calico_ipv4pool', 'etcd_tag', 'flannel_tag', 'cloud_provider_enabled', 'cloud_provider_tag', 'prometheus_tag', 'grafana_tag', 'heat_container_agent_tag', 'keystone_auth_enabled', 'k8s_keystone_auth_tag', 'tiller_enabled', 'tiller_tag', 'tiller_namespace' ] for label in label_list: label_value = cluster.labels.get(label) if label_value: extra_params[label] = label_value csr_keys = x509.generate_csr_and_key(u"Kubernetes Service Account") extra_params['kube_service_account_key'] = \ csr_keys["public_key"].replace("\n", "\\n") extra_params['kube_service_account_private_key'] = \ csr_keys["private_key"].replace("\n", "\\n") cert_manager_api = cluster.labels.get('cert_manager_api') if strutils.bool_from_string(cert_manager_api): extra_params['cert_manager_api'] = cert_manager_api ca_cert = cert_manager.get_cluster_ca_certificate(cluster) if six.PY3 and isinstance(ca_cert.get_private_key_passphrase(), six.text_type): extra_params['ca_key'] = x509.decrypt_key( ca_cert.get_private_key(), ca_cert.get_private_key_passphrase().encode()).decode( ).replace("\n", "\\n") else: extra_params['ca_key'] = x509.decrypt_key( ca_cert.get_private_key(), ca_cert.get_private_key_passphrase()).replace("\n", "\\n") extra_params['project_id'] = cluster.project_id return super(K8sFedoraTemplateDefinition, self).get_params(context, cluster_template, cluster, extra_params=extra_params, **kwargs)
def create_cluster(self, context, cluster, cluster_create_timeout): LOG.info("Starting to create cluster %s", cluster.uuid) cluster_template = conductor_utils.retrieve_cluster_template( context, cluster ) cluser_service_ip_range = cluster.labels.get( 'service_cluster_ip_range', '10.97.0.0/16' ) if cluster_template.network_driver == 'flannel': cluser_pod_ip_range = cluster.labels.get( 'flannel_network_cidr', '10.100.0.0/16' ) if cluster_template.network_driver == 'calico': cluser_pod_ip_range = cluster.labels.get( 'calico_ipv4pool', '192.168.0.0/16' ) port_info = self._create_vip_port(context, cluster, cluster_template) # This address should be internal IP that other services could # communicate with. self.apiserver_address = port_info["private_ip"] external_apiserver_address = port_info.get("public_ip", port_info["private_ip"]) # The master address is always the private VIP address. cluster.api_address = 'https://%s:6443' % external_apiserver_address master_ng = cluster.default_ng_master setattr(master_ng, "node_addresses", [self.apiserver_address]) master_ng.save() self.public_network_id = ( cluster_template.external_network_id or "public") if not uuidutils.is_uuid_like(self.public_network_id): self.public_network_id = neutron.get_network_id( context, self.public_network_id ) ca_cert = cert_manager.get_cluster_ca_certificate( cluster, context=context ) ca_cert_encoded = base64.b64encode(ca_cert.get_certificate()) ca_key_encoded = base64.b64encode(ca_cert.get_decrypted_private_key()) cloud_provider_enabled = strutils.bool_from_string( cluster.labels.get("cloud_provider_enabled", "true") ) ca_cert_encoded_str = ca_cert_encoded.decode('utf-8') ca_cert_encoded_str = ca_cert_encoded_str.replace("'","") ca_key_encoded_str = ca_key_encoded.decode('utf-8') ca_key_encoded_str = ca_key_encoded_str.replace("'","") params = { "namespace": cluster.uuid, "vip_port_ip": self.apiserver_address, "vip_external_ip": external_apiserver_address, "vip_port_id": port_info["port_id"], "service_ip_range": cluser_service_ip_range, "pod_ip_range": cluser_pod_ip_range, "ca_cert": ca_cert_encoded_str, "ca_key": ca_key_encoded_str, "subnet_id": cluster_template.fixed_subnet, "public_network_id": self.public_network_id, "cloud_provider_enabled": cloud_provider_enabled, "kube_version": cluster.labels.get("kube_tag", "v1.14.3"), "cloud_provider_tag": cluster.labels.get("cloud_provider_tag", "v1.15.0") } # Keystone related info. osc = clients.OpenStackClients(context) params['trustee_user_id'] = cluster.trustee_user_id params['trustee_password'] = cluster.trustee_password if CONF.trust.cluster_user_trust: params['trust_id'] = cluster.trust_id else: params['trust_id'] = "" kwargs = { 'service_type': 'identity', 'interface': CONF.trust.trustee_keystone_interface, 'version': 3 } if CONF.trust.trustee_keystone_region_name: kwargs['region_name'] = CONF.trust.trustee_keystone_region_name params['auth_url'] = osc.url_for(**kwargs).rstrip('/') _apply_manifest = functools.partial(self._apply_manifest, params) LOG.info("Creating namespace for cluster %s", cluster.uuid) _apply_manifest('namespace.yaml.j2') # Create Secret for the new cluster CA and the kube services, the CA # could be referenced by various cluster components. LOG.info("Creating Secrets for cluster %s", cluster.uuid) _apply_manifest('secrets.yaml.j2') # TODO: Wait for all the certificates are ready # etcd Service and StatefulSet LOG.info("Creating etcd service for cluster %s", cluster.uuid) _apply_manifest('etcd.yaml.j2') # apiserver Service and Deployment LOG.info("Creating kube-apiserver for cluster %s", cluster.uuid) _apply_manifest('kube-apiserver.yaml.j2') # Deploy kube-controller-manager LOG.info("Creating kube-controller-manager for cluster %s", cluster.uuid) _apply_manifest('kube-controllermgr.yaml.j2') # Deploy kube-scheduler LOG.info("Creating kube-scheduler for cluster %s", cluster.uuid) _apply_manifest('kube-scheduler.yaml.j2') kubeconfig_path = self._get_kubeconfig( context, cluster, ca_cert_encoded=ca_cert_encoded ) LOG.info( "Kubeconfig created for cluster %s, path: %s", cluster.uuid, kubeconfig_path ) cluster_kubectl = kubectl.KubeCtl( bin="/usr/bin/kubectl", global_flags="--kubeconfig %s" % kubeconfig_path ) LOG.info( "Waiting for all the components up and running for " "cluster %s", cluster.uuid ) self._wait_for_apiserver(cluster.uuid, cluster_kubectl) if cloud_provider_enabled: # Deploy openstack-cloud-controller-manager LOG.info("Creating openstack-cloud-controller-manager for " "cluster %s", cluster.uuid) # Create RBAC for openstack-cloud-controller-manager in the # cluster. _apply_manifest( "openstack-cloud-controller-manager-in-cluster.yaml.j2", cluster_kubectl ) _apply_manifest('openstack-cloud-controller-manager.yaml.j2') # Create bootstrap token and the bootstrap RBAC in the new cluster LOG.info( "Creating bootstrap token and RBAC in the cluster %s", cluster.uuid ) expiration = timeutils.utcnow() + datetime.timedelta(days=1) # For bootstrap token, refer to # https://kubernetes.io/docs/reference/access-authn-authz/bootstrap-tokens/ token_id = self._generate_random_string(6) token_secret = self._generate_random_string(16) bootstrap_params = { "token_id": token_id, "token_secret": token_secret, "expiration": expiration.strftime('%Y-%m-%dT%H:%M:%SZ'), } bootstrap_template = self.jinja_env.get_template('bootstrap.yaml.j2') bootstrap_body = bootstrap_template.render(bootstrap_params) cluster_kubectl.apply(definition=bootstrap_body) self.bootstrap_token = "%s.%s" % (token_id, token_secret) # Grant privilege to 'kubernetes' user so that apiserver can access # to kubelet for operations like logs, exec, etc. # The user name here must be the same with apiserver CN in # secrets.yaml.j2 cluster_kubectl.execute( "create clusterrolebinding kube-apiserver --clusterrole " "cluster-admin --user kubernetes" ) # Starts to create VMs and bootstrap kubelet LOG.info("Creating worker nodes for cluster %s", cluster.uuid) super(Driver, self).create_cluster( context, cluster, cluster_create_timeout )
def get_ca_certificate(self, context, cluster): ca_cert = cert_manager.get_cluster_ca_certificate(cluster, context=context) certificate = objects.Certificate.from_object_cluster(cluster) certificate.pem = ca_cert.get_certificate() return certificate
def get_params(self, context, cluster_template, cluster, **kwargs): # Add all the params from the cluster's nodegroups self.add_nodegroup_params(cluster) extra_params = kwargs.pop('extra_params', {}) extra_params["bootstrap_token"] = self.bootstrap_token extra_params["apiserver_address"] = self.apiserver_address extra_params['kubernetes_version'] = cluster.labels.get( 'kube_tag', 'v1.13.2').lstrip("v") extra_params['external_network'] = self.public_network_id cluser_service_ip_range = cluster.labels.get( 'service_cluster_ip_range', '10.96.0.0/12') service_ip_net = netaddr.IPNetwork(cluser_service_ip_range) extra_params['cluster_dns_service_ip'] = service_ip_net[10] # Certs ca_cert = cert_manager.get_cluster_ca_certificate( cluster, context=context ) ca_cert_content = ca_cert.get_certificate() ca_key_content = ca_cert.get_decrypted_private_key() extra_params['ca_content'] = ca_cert_content extra_params['cakey_content'] = ca_key_content # CNI if cluster_template.network_driver == 'flannel': extra_params["pods_network_cidr"] = cluster.labels.get( 'flannel_network_cidr', '10.100.0.0/16') if cluster_template.network_driver == 'calico': extra_params["pods_network_cidr"] = cluster.labels.get( 'calico_ipv4pool', '192.168.0.0/16') # Keystone related osc = clients.OpenStackClients(context) extra_params['trustee_user_id'] = cluster.trustee_user_id extra_params['trustee_password'] = cluster.trustee_password if CONF.trust.cluster_user_trust: extra_params['trust_id'] = cluster.trust_id else: extra_params['trust_id'] = "" svc_kwargs = { 'service_type': 'identity', 'interface': CONF.trust.trustee_keystone_interface, 'version': 3 } if CONF.trust.trustee_keystone_region_name: svc_kwargs['region_name'] = CONF.trust.trustee_keystone_region_name extra_params['auth_url'] = osc.url_for(**svc_kwargs).rstrip('/') # Labels extra_params['cloud_provider_enabled'] = strutils.bool_from_string( cluster.labels.get("cloud_provider_enabled", "true") ) return super(UbuntuK8sTemplateDefinition, self).get_params( context, cluster_template, cluster, extra_params=extra_params, **kwargs )