def _resize_stack(self, context, cluster, resize_manager,
node_count, nodes_to_remove, nodegroup=None,
rollback=False):
definition = self.get_template_definition()
osc = clients.OpenStackClients(context)
# Find what changed checking the stack params
# against the ones in the template_def.
stack = osc.heat().stacks.get(nodegroup.stack_id,
resolve_outputs=True)
stack_params = stack.parameters
definition.add_nodegroup_params(cluster, nodegroups=[nodegroup])
heat_params = definition.get_stack_diff(context, stack_params, cluster)
LOG.debug('Updating stack with these params: %s', heat_params)
scale_params = definition.get_scale_params(context,
cluster,
resize_manager,
nodes_to_remove)
heat_params.update(scale_params)
fields = {
'parameters': heat_params,
'existing': True,
'disable_rollback': not rollback
}
osc = clients.OpenStackClients(context)
osc.heat().stacks.update(nodegroup.stack_id, **fields)
def delete_cluster(self, context, cluster):
LOG.info("Starting to delete cluster %s", cluster.uuid)
self.pre_delete_cluster(context, cluster)
c_template = conductor_utils.retrieve_cluster_template(
context,
cluster
)
# NOTE: The fake fields are only for the yaml file integrity and do not
# affect the deletion
params = {
"namespace": cluster.uuid,
"cloud_provider_tag": "fake",
"kube_version": "fake",
}
_delete_manifest = functools.partial(self._delete_manifest, params)
LOG.info("Deleting components for cluster %s", cluster.uuid)
for tmpl in [
"openstack-cloud-controller-manager.yaml.j2",
"kube-scheduler.yaml.j2", "kube-controllermgr.yaml.j2",
"kube-apiserver.yaml.j2", "etcd.yaml.j2",
"secrets.yaml.j2", "namespace.yaml.j2"
]:
_delete_manifest(tmpl)
# Delete floating ip if needed.
if (self._master_lb_fip_enabled(cluster, c_template) and
cluster.api_address):
network_client = clients.OpenStackClients(context).neutron()
ip = netutils.urlsplit(cluster.api_address).netloc.split(":")[0]
fips = network_client.list_floatingips(floating_ip_address=ip)
for fip in fips['floatingips']:
LOG.info("Deleting floating ip %s for cluster %s",
fip["floating_ip_address"], cluster.uuid)
network_client.delete_floatingip(fip['id'])
# Delete VIP port
LOG.info("Deleting ports for cluster %s", cluster.uuid)
tag = {"magnum": cluster.uuid}
tags = [jsonutils.dumps(tag)]
neutron.delete_port_by_tags(context, tags)
# Delete Heat stack.
if cluster.stack_id:
LOG.info("Deleting Heat stack %s for cluster %s",
cluster.stack_id, cluster.uuid)
self._delete_stack(
context, clients.OpenStackClients(context), cluster
)
def delete_port_by_tags(context, tags):
n_client = clients.OpenStackClients(context).neutron()
filter = {'tags': tags}
ports = n_client.list_ports(**filter)
for port in ports.get("ports", []):
n_client.delete_port(port.get("id"))
def _collect_fault_info(self, context, bay):
"""Collect fault info from heat resources of given bay
and store them into bay.bay_faults.
"""
osc = clients.OpenStackClients(context)
filters = {'status': 'FAILED'}
try:
failed_resources = osc.heat().resources.list(bay.stack_id,
nested_depth=2,
filters=filters)
except Exception as e:
failed_resources = []
LOG.warning(
"Failed to retrieve failed resources for "
"bay %(bay)s from Heat stack %(stack)s "
"due to error: %(e)s", {
'bay': bay.uuid,
'stack': bay.stack_id,
'e': e
},
exc_info=True)
return {
res.resource_name: res.resource_status_reason
for res in failed_resources
}
def update_cluster(self,
context,
cluster,
scale_manager=None,
rollback=False):
self._update_stack(context, clients.OpenStackClients(context), cluster,
scale_manager, rollback)
def bay_create(self, context, bay, bay_create_timeout):
LOG.debug('bay_heat bay_create')
osc = clients.OpenStackClients(context)
bay.uuid = uuid.uuid4()
try:
# Create trustee/trust and set them to bay
trust_manager.create_trustee_and_trust(osc, bay)
# Generate certificate and set the cert reference to bay
cert_manager.generate_certificates_to_bay(bay, context=context)
conductor_utils.notify_about_bay_operation(
context, taxonomy.ACTION_CREATE, taxonomy.OUTCOME_PENDING)
created_stack = _create_stack(context, osc, bay,
bay_create_timeout)
except Exception as e:
cert_manager.delete_certificates_from_bay(bay, context=context)
trust_manager.delete_trustee_and_trust(osc, context, bay)
conductor_utils.notify_about_bay_operation(
context, taxonomy.ACTION_CREATE, taxonomy.OUTCOME_FAILURE)
if isinstance(e, exc.HTTPBadRequest):
e = exception.InvalidParameterValue(message=six.text_type(e))
raise e
bay.stack_id = created_stack['stack']['id']
bay.status = bay_status.CREATE_IN_PROGRESS
bay.create()
self._poll_and_check(osc, bay)
return bay
def delete_floatingip(context, fix_port_id, cluster):
"""Deletes the floating IP associated with the fix_port_id.
Only delete the floating IP if it's created and associated with the
the load balancers that corresponding to the services and ingresses in
Kubernetes cluster.
This method only works with the Kubernetes cluster with
cloud-provider-openstack controller manager deployed.
"""
pattern = (r'Floating IP for Kubernetes .+ from cluster %s$' %
cluster.uuid)
try:
n_client = clients.OpenStackClients(context).neutron()
fips = n_client.list_floatingips(port_id=fix_port_id)
if len(fips["floatingips"]) == 0:
return
# Liberty Neutron doesn't support description field, although Liberty
# is no longer supported, we write good code here.
desc = fips["floatingips"][0].get("description", "")
id = fips["floatingips"][0]["id"]
if re.match(pattern, desc):
LOG.debug("Deleting floating ip %s for cluster %s", id,
cluster.uuid)
n_client.delete_floatingip(id)
except Exception as e:
raise exception.PreDeletionFailed(cluster_uuid=cluster.uuid,
msg=str(e))
def _get_random_volume_type(context):
c_client = clients.OpenStackClients(context).cinder()
volume_types = c_client.volume_types.list()
if volume_types:
return volume_types[0].name
else:
raise exception.VolumeTypeNotFound()
def bay_update(self, context, bay):
LOG.debug('bay_heat bay_update')
osc = clients.OpenStackClients(context)
stack = osc.heat().stacks.get(bay.stack_id)
if (stack.stack_status != bay_status.CREATE_COMPLETE
and stack.stack_status != bay_status.UPDATE_COMPLETE):
operation = _('Updating a bay when stack status is '
'"%s"') % stack.stack_status
raise exception.NotSupported(operation=operation)
delta = set(bay.obj_what_changed())
if 'node_count' in delta:
delta.remove('node_count')
manager = scale_manager.ScaleManager(context, osc, bay)
_update_stack(context, osc, bay, manager)
self._poll_and_check(osc, bay)
if delta:
raise exception.InvalidParameterValue(
err=("cannot change bay property(ies) %s." % ", ".join(delta)))
bay.save()
return bay
def _get_cluster_stacks(self, clusters, sid_to_cluster_mapping,
cluster_stack_ids):
stacks = []
_clusters = clusters
_sid_to_cluster_mapping = sid_to_cluster_mapping
_cluster_stack_ids = cluster_stack_ids
for cluster in _clusters:
try:
# Create client with cluster's trustee user context
bosc = clients.OpenStackClients(
context.make_cluster_context(cluster))
stack = bosc.heat().stacks.get(cluster.stack_id)
stacks.append(stack)
# No need to do anything in this case
except heat_exc.HTTPNotFound:
pass
except Exception as e:
# Any other exception means we do not perform any
# action on this cluster in the current sync run, so remove
# it from all records.
LOG.warning(
_LW("Exception while attempting to retrieve "
"Heat stack %(stack_id)s for cluster %(cluster_id)s. "
"Traceback follows."), {
'stack_id': cluster.stack_id,
'cluster_id': cluster.id
})
LOG.warning(e)
_sid_to_cluster_mapping.pop(cluster.stack_id)
_cluster_stack_ids.remove(cluster.stack_id)
_clusters.remove(cluster)
return [stacks, _clusters, _cluster_stack_ids, _sid_to_cluster_mapping]
def get_params(self, context, baymodel, bay, **kwargs):
extra_params = kwargs.pop('extra_params', {})
label_list = ['flannel_network_cidr', 'flannel_use_vxlan',
'flannel_network_subnetlen']
scale_mgr = kwargs.pop('scale_manager', None)
if scale_mgr:
hosts = self.get_output('kube_minions')
extra_params['minions_to_remove'] = (
scale_mgr.get_removal_nodes(hosts))
extra_params['discovery_url'] = self.get_discovery_url(bay)
# Kubernetes backend code is still using v2 API
extra_params['auth_url'] = context.auth_url.replace("v3", "v2")
extra_params['username'] = context.user_name
extra_params['tenant_name'] = context.tenant
osc = clients.OpenStackClients(context)
extra_params['user_token'] = self._get_user_token(context, osc, bay)
extra_params['magnum_url'] = osc.magnum_url()
if baymodel.tls_disabled:
extra_params['loadbalancing_protocol'] = 'HTTP'
extra_params['kubernetes_port'] = 8080
for label in label_list:
extra_params[label] = baymodel.labels.get(label)
return super(AtomicK8sTemplateDefinition,
self).get_params(context, baymodel, bay,
extra_params=extra_params,
**kwargs)
def _collect_fault_info(self, context, cluster):
"""Collect fault info from heat resources of given cluster
and store them into cluster.faults.
"""
osc = clients.OpenStackClients(context)
filters = {'status': 'FAILED'}
try:
failed_resources = osc.heat().resources.list(cluster.stack_id,
nested_depth=2,
filters=filters)
except Exception as e:
failed_resources = []
LOG.warning(_LW("Failed to retrieve failed resources for "
"cluster %(cluster)s from Heat stack "
"%(stack)s due to error: %(e)s"), {
'cluster': cluster.uuid,
'stack': cluster.stack_id,
'e': e
},
exc_info=True)
return {
res.resource_name: res.resource_status_reason
for res in failed_resources
}
def cluster_update(self, context, cluster, rollback=False):
LOG.debug('cluster_heat cluster_update')
osc = clients.OpenStackClients(context)
stack = osc.heat().stacks.get(cluster.stack_id)
allow_update_status = (fields.ClusterStatus.CREATE_COMPLETE,
fields.ClusterStatus.UPDATE_COMPLETE,
fields.ClusterStatus.RESUME_COMPLETE,
fields.ClusterStatus.RESTORE_COMPLETE,
fields.ClusterStatus.ROLLBACK_COMPLETE,
fields.ClusterStatus.SNAPSHOT_COMPLETE,
fields.ClusterStatus.CHECK_COMPLETE,
fields.ClusterStatus.ADOPT_COMPLETE)
if stack.stack_status not in allow_update_status:
conductor_utils.notify_about_cluster_operation(
context, taxonomy.ACTION_UPDATE, taxonomy.OUTCOME_FAILURE)
operation = _('Updating a cluster when stack status is '
'"%s"') % stack.stack_status
raise exception.NotSupported(operation=operation)
delta = cluster.obj_what_changed()
if not delta:
return cluster
manager = scale_manager.ScaleManager(context, osc, cluster)
conductor_utils.notify_about_cluster_operation(
context, taxonomy.ACTION_UPDATE, taxonomy.OUTCOME_PENDING)
_update_stack(context, osc, cluster, manager, rollback)
self._poll_and_check(osc, cluster)
return cluster
def cluster_create(self, context, cluster, create_timeout):
LOG.debug('cluster_heat cluster_create')
osc = clients.OpenStackClients(context)
try:
# Create trustee/trust and set them to cluster
trust_manager.create_trustee_and_trust(osc, cluster)
# Generate certificate and set the cert reference to cluster
cert_manager.generate_certificates_to_cluster(cluster,
context=context)
conductor_utils.notify_about_cluster_operation(
context, taxonomy.ACTION_CREATE, taxonomy.OUTCOME_PENDING)
created_stack = _create_stack(context, osc, cluster,
create_timeout)
except Exception as e:
cluster.status = fields.ClusterStatus.CREATE_FAILED
cluster.status_reason = six.text_type(e)
cluster.create()
conductor_utils.notify_about_cluster_operation(
context, taxonomy.ACTION_CREATE, taxonomy.OUTCOME_FAILURE)
if isinstance(e, exc.HTTPBadRequest):
e = exception.InvalidParameterValue(message=six.text_type(e))
raise e
raise
cluster.stack_id = created_stack['stack']['id']
cluster.status = fields.ClusterStatus.CREATE_IN_PROGRESS
cluster.create()
self._poll_and_check(osc, cluster)
return cluster
def sync_bay_status(self, ctx):
try:
LOG.debug('Starting to sync up bay status')
osc = clients.OpenStackClients(ctx)
status = [bay_status.CREATE_IN_PROGRESS,
bay_status.UPDATE_IN_PROGRESS,
bay_status.DELETE_IN_PROGRESS]
filters = {'status': status}
bays = objects.Bay.list(ctx, filters=filters)
if not bays:
return
sid_to_bay_mapping = {bay.stack_id: bay for bay in bays}
bay_stack_ids = sid_to_bay_mapping.keys()
stacks = osc.heat().stacks.list(global_tenant=True,
filters={'id': bay_stack_ids})
sid_to_stack_mapping = {s.id: s for s in stacks}
# intersection of bays magnum has and heat has
for sid in (six.viewkeys(sid_to_bay_mapping) &
six.viewkeys(sid_to_stack_mapping)):
stack = sid_to_stack_mapping[sid]
bay = sid_to_bay_mapping[sid]
self._sync_existing_bay(bay, stack)
# the stacks that magnum has but heat doesn't have
for sid in (six.viewkeys(sid_to_bay_mapping) -
six.viewkeys(sid_to_stack_mapping)):
bay = sid_to_bay_mapping[sid]
self._sync_missing_heat_stack(bay)
except Exception as e:
LOG.warning(_LW(
"Ignore error [%s] when syncing up bay status."
), e, exc_info=True)
def bay_delete(self, context, uuid):
LOG.debug('bay_heat bay_delete')
osc = clients.OpenStackClients(context)
bay = objects.Bay.get_by_uuid(context, uuid)
stack_id = bay.stack_id
# NOTE(sdake): This will execute a stack_delete operation. This will
# Ignore HTTPNotFound exceptions (stack wasn't present). In the case
# that Heat couldn't find the stack representing the bay, likely a user
# has deleted the stack outside the context of Magnum. Therefore the
# contents of the bay are forever lost.
#
# If the exception is unhandled, the original exception will be raised.
try:
osc.heat().stacks.delete(stack_id)
except exc.HTTPNotFound:
LOG.info(
_LI('The stack %s was not be found during bay'
' deletion.') % stack_id)
try:
cert_manager.delete_certificates_from_bay(bay)
bay.destroy()
except exception.BayNotFound:
LOG.info(_LI('The bay %s has been deleted by others.') % uuid)
return None
except Exception:
raise
self._poll_and_check(osc, bay)
return None
def rotate_ca_certificate(self, context, cluster):
cluster_template = conductor_utils.retrieve_cluster_template(context,
cluster)
if cluster_template.cluster_distro not in ["fedora-coreos"]:
raise exception.NotSupported("Rotating the CA certificate is "
"not supported for cluster with "
"cluster_distro: %s." %
cluster_template.cluster_distro)
osc = clients.OpenStackClients(context)
rollback = True
heat_params = {}
csr_keys = x509.generate_csr_and_key(u"Kubernetes Service Account")
heat_params['kube_service_account_key'] = \
csr_keys["public_key"].replace("\n", "\\n")
heat_params['kube_service_account_private_key'] = \
csr_keys["private_key"].replace("\n", "\\n")
fields = {
'existing': True,
'parameters': heat_params,
'disable_rollback': not rollback
}
osc.heat().stacks.update(cluster.stack_id, **fields)
def post(self, baymodel):
"""Create a new baymodel.
:param baymodel: a baymodel within the request body.
"""
baymodel_dict = baymodel.as_dict()
context = pecan.request.context
cli = clients.OpenStackClients(context)
attr_validator.validate_keypair(cli, baymodel_dict['keypair_id'])
image_data = attr_validator.validate_image(cli,
baymodel_dict['image_id'])
baymodel_dict['cluster_distro'] = image_data['os_distro']
baymodel_dict['project_id'] = context.project_id
baymodel_dict['user_id'] = context.user_id
# check permissions for making baymodel public
if baymodel_dict['public']:
if not policy.enforce(context, "baymodel:publish", None,
do_raise=False):
raise exception.BaymodelPublishDenied()
new_baymodel = objects.BayModel(context, **baymodel_dict)
new_baymodel.create()
# Set the HTTP Location Header
pecan.response.location = link.build_url('baymodels',
new_baymodel.uuid)
return BayModel.convert_with_links(new_baymodel)
def _validate_project_id(project_id):
try:
context = pecan.request.context
osc = clients.OpenStackClients(context)
osc.keystone().domain_admin_client.projects.get(project_id)
except ka_exception.http.NotFound:
raise exception.ProjectNotFound(name='project_id', id=project_id)
def test_url_for(self, mock_keystone):
obj = clients.OpenStackClients(None)
obj.url_for(service_type='fake_service', endpoint_type='fake_endpoint')
mock_cat = mock_keystone.return_value.client.service_catalog
mock_cat.url_for.assert_called_once_with(service_type='fake_service',
endpoint_type='fake_endpoint')
def get_params(self, context, baymodel, bay, **kwargs):
extra_params = kwargs.pop('extra_params', {})
# HACK(apmelton) - This uses the user's bearer token, ideally
# it should be replaced with an actual trust token with only
# access to do what the template needs it to do.
osc = clients.OpenStackClients(context)
extra_params['auth_url'] = context.auth_url
extra_params['username'] = context.user_name
extra_params['tenant_name'] = context.tenant
extra_params['domain_name'] = context.domain_name
extra_params['region_name'] = osc.cinder_region_name()
label_list = [
'rexray_preempt', 'mesos_slave_isolation',
'mesos_slave_image_providers', 'mesos_slave_work_dir',
'mesos_slave_executor_environment_variables'
]
for label in label_list:
extra_params[label] = baymodel.labels.get(label)
return super(UbuntuMesosTemplateDefinition,
self).get_params(context,
baymodel,
bay,
extra_params=extra_params,
**kwargs)
def test_url_for(self, mock_keystone):
obj = clients.OpenStackClients(None)
obj.url_for(service_type='fake_service', interface='fake_endpoint')
mock_endpoint = mock_keystone.return_value.session.get_endpoint
mock_endpoint.assert_called_once_with(service_type='fake_service',
interface='fake_endpoint')
def check_keypair_exists(self, context, keypair):
"""Checks the existence of the keypair"""
cli = clients.OpenStackClients(context)
try:
cli.nova().keypairs.get(keypair)
except nova_exc.NotFound:
raise exception.KeyPairNotFound(keypair=keypair)
def create_nodegroup(self, context, cluster, nodegroup):
stack = self._create_stack(context,
clients.OpenStackClients(context),
cluster,
cluster.create_timeout,
nodegroup=nodegroup)
nodegroup.stack_id = stack['stack']['id']
def bay_update(self, context, bay):
LOG.debug('bay_heat bay_update')
osc = clients.OpenStackClients(context)
stack = osc.heat().stacks.get(bay.stack_id)
allow_update_status = (bay_status.CREATE_COMPLETE,
bay_status.UPDATE_COMPLETE,
bay_status.RESUME_COMPLETE,
bay_status.RESTORE_COMPLETE,
bay_status.ROLLBACK_COMPLETE,
bay_status.SNAPSHOT_COMPLETE,
bay_status.CHECK_COMPLETE,
bay_status.ADOPT_COMPLETE)
if stack.stack_status not in allow_update_status:
conductor_utils.notify_about_bay_operation(
context, taxonomy.ACTION_UPDATE, taxonomy.OUTCOME_FAILURE)
operation = _('Updating a bay when stack status is '
'"%s"') % stack.stack_status
raise exception.NotSupported(operation=operation)
delta = bay.obj_what_changed()
if not delta:
return bay
manager = scale_manager.ScaleManager(context, osc, bay)
conductor_utils.notify_about_bay_operation(context,
taxonomy.ACTION_UPDATE,
taxonomy.OUTCOME_PENDING)
_update_stack(context, osc, bay, manager)
self._poll_and_check(osc, bay)
return bay
def _update_stack(self,
context,
cluster,
scale_manager=None,
rollback=False):
# update worked properly only for scaling nodes up and down
# before nodegroups. Maintain this logic until we deprecate
# and remove the command.
# Fixed behaviour Id84e5d878b21c908021e631514c2c58b3fe8b8b0
nodegroup = cluster.default_ng_worker
definition = self.get_template_definition()
scale_params = definition.get_scale_params(context,
cluster,
nodegroup.node_count,
scale_manager,
nodes_to_remove=None)
fields = {
'parameters': scale_params,
'existing': True,
'disable_rollback': not rollback
}
LOG.info('Updating cluster %s stack %s with these params: %s',
cluster.uuid, nodegroup.stack_id, scale_params)
osc = clients.OpenStackClients(context)
osc.heat().stacks.update(nodegroup.stack_id, **fields)
def post(self, baymodel):
"""Create a new Baymodel.
:param baymodel: a Baymodel within the request body.
"""
context = pecan.request.context
policy.enforce(context, 'baymodel:create',
action='baymodel:create')
baymodel_dict = baymodel.as_dict()
cli = clients.OpenStackClients(context)
attr_validator.validate_os_resources(context, baymodel_dict)
image_data = attr_validator.validate_image(cli,
baymodel_dict['image_id'])
baymodel_dict['cluster_distro'] = image_data['os_distro']
baymodel_dict['project_id'] = context.project_id
baymodel_dict['user_id'] = context.user_id
# check permissions for making baymodel public
if baymodel_dict['public']:
if not policy.enforce(context, "baymodel:publish", None,
do_raise=False):
raise exception.ClusterTemplatePublishDenied()
# NOTE(yuywz): We will generate a random human-readable name for
# baymodel if the name is not specified by user.
arg_name = baymodel_dict.get('name')
name = arg_name or self._generate_name_for_baymodel(context)
baymodel_dict['name'] = name
new_baymodel = objects.ClusterTemplate(context, **baymodel_dict)
new_baymodel.create()
# Set the HTTP Location Header
pecan.response.location = link.build_url('baymodels',
new_baymodel.uuid)
return BayModel.convert_with_links(new_baymodel)
def _resize_stack(self,
context,
cluster,
resize_manager,
node_count,
nodes_to_remove,
nodegroup=None,
rollback=False):
definition = self.get_template_definition()
scale_params = definition.get_scale_params(
context,
cluster,
nodegroup.node_count,
resize_manager,
nodes_to_remove=nodes_to_remove)
fields = {
'parameters': scale_params,
'existing': True,
'disable_rollback': not rollback
}
LOG.info('Resizing cluster %s stack %s with these params: %s',
cluster.uuid, nodegroup.stack_id, scale_params)
osc = clients.OpenStackClients(context)
osc.heat().stacks.update(nodegroup.stack_id, **fields)
def create_cluster(self, context, cluster, cluster_create_timeout):
stack = self._create_stack(context, clients.OpenStackClients(context),
cluster, cluster_create_timeout)
# TODO(randall): keeping this for now to reduce/eliminate data
# migration. Should probably come up with something more generic in
# the future once actual non-heat-based drivers are implemented.
cluster.stack_id = stack['stack']['id']
def _update_stack(self,
context,
cluster,
scale_manager=None,
rollback=False):
definition = self.get_template_definition()
osc = clients.OpenStackClients(context)
heat_params = {}
# Find what changed checking the stack params
# against the ones in the template_def.
stack = osc.heat().stacks.get(cluster.stack_id, resolve_outputs=True)
stack_params = stack.parameters
definition.add_nodegroup_params(cluster)
heat_params = definition.get_stack_diff(context, stack_params, cluster)
scale_params = definition.get_scale_params(context, cluster,
scale_manager)
heat_params.update(scale_params)
fields = {
'parameters': heat_params,
'existing': True,
'disable_rollback': not rollback
}
LOG.info('Updating cluster %s stack %s with these params: %s',
cluster.uuid, cluster.stack_id, heat_params)
osc.heat().stacks.update(cluster.stack_id, **fields)