def get_params(self, context, cluster_template, cluster, **kwargs):
LOG.info("########## get_params")
extra_params = kwargs.pop('extra_params', {})
label_list = [
'minikube_version', 'kubectl_version',
'occm_container_infra_prefix', 'etcd_version'
]
for label in label_list:
label_value = cluster.labels.get(label)
if label_value:
extra_params[label] = label_value
# handover the ca_key
ca_cert = cert_manager.get_cluster_ca_certificate(cluster, context)
if six.PY3 and isinstance(ca_cert.get_private_key_passphrase(),
six.text_type):
extra_params['ca_key'] = x509.decrypt_key(
ca_cert.get_private_key(),
ca_cert.get_private_key_passphrase().encode()).decode(
).replace("\n", "\\n")
else:
extra_params['ca_key'] = x509.decrypt_key(
ca_cert.get_private_key(),
ca_cert.get_private_key_passphrase()).replace("\n", "\\n")
return super(AtomicMinikubeTemplateDefinition,
self).get_params(context,
cluster_template,
cluster,
extra_params=extra_params,
**kwargs)
def get_ca_certificate(self, context, cluster):
ca_cert = cert_manager.get_cluster_ca_certificate(cluster,
context=context)
certificate = objects.Certificate.from_object_cluster(cluster)
if six.PY3 and isinstance(ca_cert.get_certificate(), six.binary_type):
certificate.pem = ca_cert.get_certificate().decode()
else:
certificate.pem = ca_cert.get_certificate()
return certificate
def get_ca_certificate(self, context, cluster):
ca_cert = cert_manager.get_cluster_ca_certificate(cluster,
context=context)
certificate = objects.Certificate.from_object_cluster(cluster)
if six.PY3 and isinstance(ca_cert.get_certificate(), six.binary_type):
certificate.pem = ca_cert.get_certificate().decode()
else:
certificate.pem = ca_cert.get_certificate()
return certificate
def get_params(self, context, cluster_template, cluster, **kwargs):
extra_params = kwargs.pop('extra_params', {})
extra_params['username'] = context.user_name
osc = self.get_osc(context)
extra_params['region_name'] = osc.cinder_region_name()
# set docker_volume_type
# use the configuration default if None provided
docker_volume_type = cluster.labels.get(
'docker_volume_type', CONF.cinder.default_docker_volume_type)
extra_params['docker_volume_type'] = docker_volume_type
extra_params['nodes_affinity_policy'] = \
CONF.cluster.nodes_affinity_policy
if cluster_template.network_driver == 'flannel':
extra_params["pods_network_cidr"] = \
cluster.labels.get('flannel_network_cidr', '10.100.0.0/16')
if cluster_template.network_driver == 'calico':
extra_params["pods_network_cidr"] = \
cluster.labels.get('calico_ipv4pool', '10.100.0.0/16')
label_list = [
'coredns_tag', 'kube_tag', 'container_infra_prefix',
'availability_zone', 'calico_tag', 'calico_kube_controllers_tag',
'calico_ipv4pool', 'calico_ipv4pool_ipip', 'etcd_tag',
'flannel_tag'
]
labels = self._get_relevant_labels(cluster, kwargs)
for label in label_list:
label_value = labels.get(label)
if label_value:
extra_params[label] = label_value
cert_manager_api = cluster.labels.get('cert_manager_api')
if strutils.bool_from_string(cert_manager_api):
extra_params['cert_manager_api'] = cert_manager_api
ca_cert = cert_manager.get_cluster_ca_certificate(cluster)
extra_params['ca_key'] = x509.decrypt_key(
ca_cert.get_private_key(),
ca_cert.get_private_key_passphrase()).replace("\n", "\\n")
plain_openstack_ca = utils.get_openstack_ca()
encoded_openstack_ca = base64.b64encode(plain_openstack_ca.encode())
extra_params['openstack_ca_coreos'] = encoded_openstack_ca.decode()
return super(CoreOSK8sTemplateDefinition,
self).get_params(context,
cluster_template,
cluster,
extra_params=extra_params,
**kwargs)
def test_get_cluster_ca_certificate(self):
mock_cluster = mock.MagicMock()
mock_cluster.uuid = "mock_cluster_uuid"
mock_ca_cert = mock.MagicMock()
self.CertManager.get_cert.return_value = mock_ca_cert
cluster_ca_cert = cert_manager.get_cluster_ca_certificate(mock_cluster)
self.CertManager.get_cert.assert_called_once_with(
mock_cluster.ca_cert_ref, resource_ref=mock_cluster.uuid,
context=None)
self.assertEqual(mock_ca_cert, cluster_ca_cert)
def test_get_cluster_ca_certificate(self):
mock_cluster = mock.MagicMock()
mock_cluster.uuid = "mock_cluster_uuid"
mock_ca_cert = mock.MagicMock()
self.CertManager.get_cert.return_value = mock_ca_cert
cluster_ca_cert = cert_manager.get_cluster_ca_certificate(mock_cluster)
self.CertManager.get_cert.assert_called_once_with(
mock_cluster.ca_cert_ref, resource_ref=mock_cluster.uuid,
context=None)
self.assertEqual(mock_ca_cert, cluster_ca_cert)
def get_params(self, context, cluster_template, cluster, **kwargs):
extra_params = kwargs.pop('extra_params', {})
extra_params['username'] = context.user_name
osc = self.get_osc(context)
extra_params['region_name'] = osc.cinder_region_name()
# set docker_volume_type
# use the configuration default if None provided
docker_volume_type = cluster.labels.get(
'docker_volume_type', CONF.cinder.default_docker_volume_type)
extra_params['docker_volume_type'] = docker_volume_type
extra_params['nodes_affinity_policy'] = \
CONF.cluster.nodes_affinity_policy
if cluster_template.network_driver == 'flannel':
extra_params["pods_network_cidr"] = \
cluster.labels.get('flannel_network_cidr', '10.100.0.0/16')
if cluster_template.network_driver == 'calico':
extra_params["pods_network_cidr"] = \
cluster.labels.get('calico_ipv4pool', '192.168.0.0/16')
label_list = ['coredns_tag',
'kube_tag', 'container_infra_prefix',
'availability_zone',
'calico_tag', 'calico_cni_tag',
'calico_kube_controllers_tag', 'calico_ipv4pool',
'etcd_tag', 'flannel_tag']
for label in label_list:
label_value = cluster.labels.get(label)
if label_value:
extra_params[label] = label_value
cert_manager_api = cluster.labels.get('cert_manager_api')
if strutils.bool_from_string(cert_manager_api):
extra_params['cert_manager_api'] = cert_manager_api
ca_cert = cert_manager.get_cluster_ca_certificate(cluster)
extra_params['ca_key'] = x509.decrypt_key(
ca_cert.get_private_key(),
ca_cert.get_private_key_passphrase()).replace("\n", "\\n")
plain_openstack_ca = utils.get_openstack_ca()
encoded_openstack_ca = base64.b64encode(plain_openstack_ca.encode())
extra_params['openstack_ca_coreos'] = encoded_openstack_ca.decode()
return super(CoreOSK8sTemplateDefinition,
self).get_params(context, cluster_template, cluster,
extra_params=extra_params,
**kwargs)
def _set_cert_manager_params(self, cluster, extra_params):
cert_manager_api = cluster.labels.get('cert_manager_api')
if strutils.bool_from_string(cert_manager_api):
extra_params['cert_manager_api'] = cert_manager_api
ca_cert = cert_manager.get_cluster_ca_certificate(cluster)
if six.PY3 and isinstance(ca_cert.get_private_key_passphrase(),
six.text_type):
extra_params['ca_key'] = x509.decrypt_key(
ca_cert.get_private_key(),
ca_cert.get_private_key_passphrase().encode()).decode(
).replace("\n", "\\n")
else:
extra_params['ca_key'] = x509.decrypt_key(
ca_cert.get_private_key(),
ca_cert.get_private_key_passphrase()).replace("\n", "\\n")
def _set_cert_manager_params(self, cluster, extra_params):
cert_manager_api = cluster.labels.get('cert_manager_api')
if strutils.bool_from_string(cert_manager_api):
extra_params['cert_manager_api'] = cert_manager_api
ca_cert = cert_manager.get_cluster_ca_certificate(cluster)
if six.PY3 and isinstance(ca_cert.get_private_key_passphrase(),
six.text_type):
extra_params['ca_key'] = x509.decrypt_key(
ca_cert.get_private_key(),
ca_cert.get_private_key_passphrase().encode()
).decode().replace("\n", "\\n")
else:
extra_params['ca_key'] = x509.decrypt_key(
ca_cert.get_private_key(),
ca_cert.get_private_key_passphrase()).replace("\n", "\\n")
def _get_kubeconfig(self, context, cluster, ca_cert_encoded=None):
"""Prepare kubeconfig file to interact with the new cluster."""
# TODO(lxkong): This kubeconfig file should be stored in a shared place
# between magnum controllers.
cluster_cache_dir = os.path.join(
CONF.cluster.temp_cache_dir, cluster.uuid
)
fileutils.ensure_tree(cluster_cache_dir)
kubeconfig_path = os.path.join(cluster_cache_dir, "kubeconfig")
if os.path.exists(kubeconfig_path):
return kubeconfig_path
if ca_cert_encoded is None:
ca_cert = cert_manager.get_cluster_ca_certificate(
cluster,
context=context
)
ca_cert_encoded = base64.b64encode(ca_cert.get_certificate())
admin_cert = cert_manager.get_cluster_magnum_cert(
cluster,
context=context
)
admin_cert_encoded = base64.b64encode(admin_cert.get_certificate())
admin_key_encoded = base64.b64encode(
admin_cert.get_decrypted_private_key())
split_ret = netutils.urlsplit(cluster.api_address)
external_apiserver_address = split_ret.netloc.split(":")[0]
kubeconfig_params = {
"cluster_id": cluster.uuid,
"apiserver_address": external_apiserver_address,
"ca_cert": ca_cert_encoded,
"cert": admin_cert_encoded,
"key": admin_key_encoded,
"user": "******",
}
kubeconfig_template = self.jinja_env.get_template("kubeconfig.j2")
kubeconfig = kubeconfig_template.render(kubeconfig_params)
with open(kubeconfig_path, 'w') as fd:
fd.write(kubeconfig)
return kubeconfig_path
def get_params(self, context, cluster_template, cluster, **kwargs):
extra_params = kwargs.pop('extra_params', {})
extra_params['username'] = context.user_name
osc = self.get_osc(context)
extra_params['region_name'] = osc.cinder_region_name()
# set docker_volume_type
# use the configuration default if None provided
docker_volume_type = cluster.labels.get(
'docker_volume_type', CONF.cinder.default_docker_volume_type)
extra_params['docker_volume_type'] = docker_volume_type
extra_params['nodes_affinity_policy'] = \
CONF.cluster.nodes_affinity_policy
label_list = [
'kube_tag', 'container_infra_prefix', 'availability_zone'
]
for label in label_list:
label_value = cluster.labels.get(label)
if label_value:
extra_params[label] = label_value
cert_manager_api = cluster.labels.get('cert_manager_api')
if strutils.bool_from_string(cert_manager_api):
extra_params['cert_manager_api'] = cert_manager_api
ca_cert = cert_manager.get_cluster_ca_certificate(cluster)
extra_params['ca_key'] = x509.decrypt_key(
ca_cert.get_private_key(),
ca_cert.get_private_key_passphrase()).replace("\n", "\\n")
return super(K8sFedoraTemplateDefinition,
self).get_params(context,
cluster_template,
cluster,
extra_params=extra_params,
**kwargs)
def get_params(self, context, cluster_template, cluster, **kwargs):
extra_params = kwargs.pop('extra_params', {})
extra_params['username'] = context.user_name
osc = self.get_osc(context)
extra_params['region_name'] = osc.cinder_region_name()
# set docker_volume_type
# use the configuration default if None provided
docker_volume_type = cluster.labels.get(
'docker_volume_type', CONF.cinder.default_docker_volume_type)
extra_params['docker_volume_type'] = docker_volume_type
extra_params['nodes_affinity_policy'] = \
CONF.cluster.nodes_affinity_policy
if cluster_template.network_driver == 'flannel':
extra_params["pods_network_cidr"] = \
cluster.labels.get('flannel_network_cidr', '10.100.0.0/16')
if cluster_template.network_driver == 'calico':
extra_params["pods_network_cidr"] = \
cluster.labels.get('calico_ipv4pool', '192.168.0.0/16')
# check cloud provider and cinder options. If cinder is selected,
# the cloud provider needs to be enabled.
cloud_provider_enabled = cluster.labels.get('cloud_provider_enabled',
'true').lower()
if (cluster_template.volume_driver == 'cinder'
and cloud_provider_enabled == 'false'):
raise exception.InvalidParameterValue(
_('"cinder" volume driver needs "cloud_provider_enabled" label '
'to be true or unset.'))
label_list = [
'kube_tag', 'container_infra_prefix', 'availability_zone',
'cgroup_driver', 'calico_tag', 'calico_cni_tag',
'calico_kube_controllers_tag', 'calico_ipv4pool', 'etcd_tag',
'flannel_tag', 'cloud_provider_enabled', 'cloud_provider_tag',
'prometheus_tag', 'grafana_tag', 'heat_container_agent_tag',
'keystone_auth_enabled', 'k8s_keystone_auth_tag', 'tiller_enabled',
'tiller_tag', 'tiller_namespace'
]
for label in label_list:
label_value = cluster.labels.get(label)
if label_value:
extra_params[label] = label_value
csr_keys = x509.generate_csr_and_key(u"Kubernetes Service Account")
extra_params['kube_service_account_key'] = \
csr_keys["public_key"].replace("\n", "\\n")
extra_params['kube_service_account_private_key'] = \
csr_keys["private_key"].replace("\n", "\\n")
cert_manager_api = cluster.labels.get('cert_manager_api')
if strutils.bool_from_string(cert_manager_api):
extra_params['cert_manager_api'] = cert_manager_api
ca_cert = cert_manager.get_cluster_ca_certificate(cluster)
if six.PY3 and isinstance(ca_cert.get_private_key_passphrase(),
six.text_type):
extra_params['ca_key'] = x509.decrypt_key(
ca_cert.get_private_key(),
ca_cert.get_private_key_passphrase().encode()).decode(
).replace("\n", "\\n")
else:
extra_params['ca_key'] = x509.decrypt_key(
ca_cert.get_private_key(),
ca_cert.get_private_key_passphrase()).replace("\n", "\\n")
extra_params['project_id'] = cluster.project_id
return super(K8sFedoraTemplateDefinition,
self).get_params(context,
cluster_template,
cluster,
extra_params=extra_params,
**kwargs)
def create_cluster(self, context, cluster, cluster_create_timeout):
LOG.info("Starting to create cluster %s", cluster.uuid)
cluster_template = conductor_utils.retrieve_cluster_template(
context,
cluster
)
cluser_service_ip_range = cluster.labels.get(
'service_cluster_ip_range', '10.97.0.0/16'
)
if cluster_template.network_driver == 'flannel':
cluser_pod_ip_range = cluster.labels.get(
'flannel_network_cidr', '10.100.0.0/16'
)
if cluster_template.network_driver == 'calico':
cluser_pod_ip_range = cluster.labels.get(
'calico_ipv4pool', '192.168.0.0/16'
)
port_info = self._create_vip_port(context, cluster, cluster_template)
# This address should be internal IP that other services could
# communicate with.
self.apiserver_address = port_info["private_ip"]
external_apiserver_address = port_info.get("public_ip",
port_info["private_ip"])
# The master address is always the private VIP address.
cluster.api_address = 'https://%s:6443' % external_apiserver_address
master_ng = cluster.default_ng_master
setattr(master_ng, "node_addresses", [self.apiserver_address])
master_ng.save()
self.public_network_id = (
cluster_template.external_network_id or "public")
if not uuidutils.is_uuid_like(self.public_network_id):
self.public_network_id = neutron.get_network_id(
context,
self.public_network_id
)
ca_cert = cert_manager.get_cluster_ca_certificate(
cluster,
context=context
)
ca_cert_encoded = base64.b64encode(ca_cert.get_certificate())
ca_key_encoded = base64.b64encode(ca_cert.get_decrypted_private_key())
cloud_provider_enabled = strutils.bool_from_string(
cluster.labels.get("cloud_provider_enabled", "true")
)
ca_cert_encoded_str = ca_cert_encoded.decode('utf-8')
ca_cert_encoded_str = ca_cert_encoded_str.replace("'","")
ca_key_encoded_str = ca_key_encoded.decode('utf-8')
ca_key_encoded_str = ca_key_encoded_str.replace("'","")
params = {
"namespace": cluster.uuid,
"vip_port_ip": self.apiserver_address,
"vip_external_ip": external_apiserver_address,
"vip_port_id": port_info["port_id"],
"service_ip_range": cluser_service_ip_range,
"pod_ip_range": cluser_pod_ip_range,
"ca_cert": ca_cert_encoded_str,
"ca_key": ca_key_encoded_str,
"subnet_id": cluster_template.fixed_subnet,
"public_network_id": self.public_network_id,
"cloud_provider_enabled": cloud_provider_enabled,
"kube_version": cluster.labels.get("kube_tag", "v1.14.3"),
"cloud_provider_tag": cluster.labels.get("cloud_provider_tag",
"v1.15.0")
}
# Keystone related info.
osc = clients.OpenStackClients(context)
params['trustee_user_id'] = cluster.trustee_user_id
params['trustee_password'] = cluster.trustee_password
if CONF.trust.cluster_user_trust:
params['trust_id'] = cluster.trust_id
else:
params['trust_id'] = ""
kwargs = {
'service_type': 'identity',
'interface': CONF.trust.trustee_keystone_interface,
'version': 3
}
if CONF.trust.trustee_keystone_region_name:
kwargs['region_name'] = CONF.trust.trustee_keystone_region_name
params['auth_url'] = osc.url_for(**kwargs).rstrip('/')
_apply_manifest = functools.partial(self._apply_manifest, params)
LOG.info("Creating namespace for cluster %s", cluster.uuid)
_apply_manifest('namespace.yaml.j2')
# Create Secret for the new cluster CA and the kube services, the CA
# could be referenced by various cluster components.
LOG.info("Creating Secrets for cluster %s", cluster.uuid)
_apply_manifest('secrets.yaml.j2')
# TODO: Wait for all the certificates are ready
# etcd Service and StatefulSet
LOG.info("Creating etcd service for cluster %s", cluster.uuid)
_apply_manifest('etcd.yaml.j2')
# apiserver Service and Deployment
LOG.info("Creating kube-apiserver for cluster %s", cluster.uuid)
_apply_manifest('kube-apiserver.yaml.j2')
# Deploy kube-controller-manager
LOG.info("Creating kube-controller-manager for cluster %s",
cluster.uuid)
_apply_manifest('kube-controllermgr.yaml.j2')
# Deploy kube-scheduler
LOG.info("Creating kube-scheduler for cluster %s", cluster.uuid)
_apply_manifest('kube-scheduler.yaml.j2')
kubeconfig_path = self._get_kubeconfig(
context, cluster,
ca_cert_encoded=ca_cert_encoded
)
LOG.info(
"Kubeconfig created for cluster %s, path: %s",
cluster.uuid, kubeconfig_path
)
cluster_kubectl = kubectl.KubeCtl(
bin="/usr/bin/kubectl",
global_flags="--kubeconfig %s" % kubeconfig_path
)
LOG.info(
"Waiting for all the components up and running for "
"cluster %s", cluster.uuid
)
self._wait_for_apiserver(cluster.uuid, cluster_kubectl)
if cloud_provider_enabled:
# Deploy openstack-cloud-controller-manager
LOG.info("Creating openstack-cloud-controller-manager for "
"cluster %s", cluster.uuid)
# Create RBAC for openstack-cloud-controller-manager in the
# cluster.
_apply_manifest(
"openstack-cloud-controller-manager-in-cluster.yaml.j2",
cluster_kubectl
)
_apply_manifest('openstack-cloud-controller-manager.yaml.j2')
# Create bootstrap token and the bootstrap RBAC in the new cluster
LOG.info(
"Creating bootstrap token and RBAC in the cluster %s",
cluster.uuid
)
expiration = timeutils.utcnow() + datetime.timedelta(days=1)
# For bootstrap token, refer to
# https://kubernetes.io/docs/reference/access-authn-authz/bootstrap-tokens/
token_id = self._generate_random_string(6)
token_secret = self._generate_random_string(16)
bootstrap_params = {
"token_id": token_id,
"token_secret": token_secret,
"expiration": expiration.strftime('%Y-%m-%dT%H:%M:%SZ'),
}
bootstrap_template = self.jinja_env.get_template('bootstrap.yaml.j2')
bootstrap_body = bootstrap_template.render(bootstrap_params)
cluster_kubectl.apply(definition=bootstrap_body)
self.bootstrap_token = "%s.%s" % (token_id, token_secret)
# Grant privilege to 'kubernetes' user so that apiserver can access
# to kubelet for operations like logs, exec, etc.
# The user name here must be the same with apiserver CN in
# secrets.yaml.j2
cluster_kubectl.execute(
"create clusterrolebinding kube-apiserver --clusterrole "
"cluster-admin --user kubernetes"
)
# Starts to create VMs and bootstrap kubelet
LOG.info("Creating worker nodes for cluster %s", cluster.uuid)
super(Driver, self).create_cluster(
context, cluster, cluster_create_timeout
)
def get_ca_certificate(self, context, cluster):
ca_cert = cert_manager.get_cluster_ca_certificate(cluster,
context=context)
certificate = objects.Certificate.from_object_cluster(cluster)
certificate.pem = ca_cert.get_certificate()
return certificate
def get_params(self, context, cluster_template, cluster, **kwargs):
# Add all the params from the cluster's nodegroups
self.add_nodegroup_params(cluster)
extra_params = kwargs.pop('extra_params', {})
extra_params["bootstrap_token"] = self.bootstrap_token
extra_params["apiserver_address"] = self.apiserver_address
extra_params['kubernetes_version'] = cluster.labels.get(
'kube_tag', 'v1.13.2').lstrip("v")
extra_params['external_network'] = self.public_network_id
cluser_service_ip_range = cluster.labels.get(
'service_cluster_ip_range', '10.96.0.0/12')
service_ip_net = netaddr.IPNetwork(cluser_service_ip_range)
extra_params['cluster_dns_service_ip'] = service_ip_net[10]
# Certs
ca_cert = cert_manager.get_cluster_ca_certificate(
cluster,
context=context
)
ca_cert_content = ca_cert.get_certificate()
ca_key_content = ca_cert.get_decrypted_private_key()
extra_params['ca_content'] = ca_cert_content
extra_params['cakey_content'] = ca_key_content
# CNI
if cluster_template.network_driver == 'flannel':
extra_params["pods_network_cidr"] = cluster.labels.get(
'flannel_network_cidr', '10.100.0.0/16')
if cluster_template.network_driver == 'calico':
extra_params["pods_network_cidr"] = cluster.labels.get(
'calico_ipv4pool', '192.168.0.0/16')
# Keystone related
osc = clients.OpenStackClients(context)
extra_params['trustee_user_id'] = cluster.trustee_user_id
extra_params['trustee_password'] = cluster.trustee_password
if CONF.trust.cluster_user_trust:
extra_params['trust_id'] = cluster.trust_id
else:
extra_params['trust_id'] = ""
svc_kwargs = {
'service_type': 'identity',
'interface': CONF.trust.trustee_keystone_interface,
'version': 3
}
if CONF.trust.trustee_keystone_region_name:
svc_kwargs['region_name'] = CONF.trust.trustee_keystone_region_name
extra_params['auth_url'] = osc.url_for(**svc_kwargs).rstrip('/')
# Labels
extra_params['cloud_provider_enabled'] = strutils.bool_from_string(
cluster.labels.get("cloud_provider_enabled", "true")
)
return super(UbuntuK8sTemplateDefinition, self).get_params(
context, cluster_template, cluster, extra_params=extra_params,
**kwargs
)
def get_ca_certificate(self, context, cluster):
ca_cert = cert_manager.get_cluster_ca_certificate(cluster,
context=context)
certificate = objects.Certificate.from_object_cluster(cluster)
certificate.pem = ca_cert.get_certificate()
return certificate