def scale_cluster(self, cluster, node_group_id_map): ctx = context.ctx() cluster = g.change_cluster_status(cluster, "Scaling") instance_ids = self._scale_cluster_instances(cluster, node_group_id_map) self._update_rollback_strategy(cluster, instance_ids=instance_ids) cluster = conductor.cluster_get(ctx, cluster) g.clean_cluster_from_empty_ng(cluster) cluster = conductor.cluster_get(ctx, cluster) instances = g.get_instances(cluster, instance_ids) self._await_active(cluster, instances) self._assign_floating_ips(instances) self._await_networks(cluster, instances) cluster = conductor.cluster_get(ctx, cluster) volumes.attach_to_instances( g.get_instances(cluster, instance_ids)) # we should be here with valid cluster: if instances creation # was not successful all extra-instances will be removed above if instance_ids: self._configure_instances(cluster) self._update_rollback_strategy(cluster) return instance_ids
def scale_cluster(self, cluster, node_group_id_map): ctx = context.ctx() cluster = g.change_cluster_status(cluster, "Scaling") instance_ids = self._scale_cluster_instances(cluster, node_group_id_map) self._update_rollback_strategy(cluster, instance_ids=instance_ids) cluster = conductor.cluster_get(ctx, cluster) g.clean_cluster_from_empty_ng(cluster) cluster = conductor.cluster_get(ctx, cluster) instances = g.get_instances(cluster, instance_ids) self._await_active(cluster, instances) self._assign_floating_ips(instances) self._await_networks(cluster, instances) cluster = conductor.cluster_get(ctx, cluster) volumes.attach_to_instances(g.get_instances(cluster, instance_ids)) # we should be here with valid cluster: if instances creation # was not successful all extra-instances will be removed above if instance_ids: self._configure_instances(cluster) self._update_rollback_strategy(cluster) return instance_ids
def scale_cluster(self, cluster, node_group_id_map): ctx = context.ctx() instance_ids = [] try: instance_ids = self._scale_cluster_instances(cluster, node_group_id_map) cluster = conductor.cluster_get(ctx, cluster) g.clean_cluster_from_empty_ng(cluster) cluster = conductor.cluster_get(ctx, cluster) instances = g.get_instances(cluster, instance_ids) self._await_active(cluster, instances) if not g.check_cluster_exists(cluster): LOG.info(g.format_cluster_deleted_message(cluster)) return [] self._assign_floating_ips(instances) self._await_networks(cluster, instances) if not g.check_cluster_exists(cluster): LOG.info(g.format_cluster_deleted_message(cluster)) return [] cluster = conductor.cluster_get(ctx, cluster) volumes.attach_to_instances( g.get_instances(cluster, instance_ids)) except Exception as ex: with excutils.save_and_reraise_exception(): if not g.check_cluster_exists(cluster): LOG.info(g.format_cluster_deleted_message(cluster)) return [] self._log_operation_exception( "Can't scale cluster '%s' (reason: %s)", cluster, ex) cluster = conductor.cluster_get(ctx, cluster) self._rollback_cluster_scaling( cluster, g.get_instances(cluster, instance_ids), ex) instance_ids = [] cluster = conductor.cluster_get(ctx, cluster) g.clean_cluster_from_empty_ng(cluster) cluster = conductor.cluster_update(ctx, cluster, {"status": "Active"}) LOG.info(g.format_cluster_status(cluster)) # we should be here with valid cluster: if instances creation # was not successful all extra-instances will be removed above if instance_ids: self._configure_instances(cluster) return instance_ids
def _rollback_cluster_scaling(self, cluster, instances, ex): """Attempt to rollback cluster scaling.""" for i in instances: self._shutdown_instance(i) cluster = conductor.cluster_get(context.ctx(), cluster) g.clean_cluster_from_empty_ng(cluster)
def test_clean_cluster_from_empty_ng(self): ctx = context.ctx() cluster = self._make_sample() ng = cluster.node_groups[0] ng_len = len(cluster.node_groups) self.api.node_group_update(ctx, ng, {'count': 0}) cluster = self.api.cluster_get(ctx, cluster.id) general.clean_cluster_from_empty_ng(cluster) cluster = self.api.cluster_get(ctx, cluster.id) self.assertEqual(ng_len - 1, len(cluster.node_groups))
def _rollback_cluster_scaling(self, cluster, instances, ex): """Attempt to rollback cluster scaling.""" LOG.info(_LI("Cluster '%(name)s' scaling rollback " "(reason: %(reason)s)"), {'name': cluster.name, 'reason': ex}) for i in instances: self._shutdown_instance(i) cluster = conductor.cluster_get(context.ctx(), cluster) g.clean_cluster_from_empty_ng(cluster)
def scale_cluster(self, cluster, node_group_id_map): ctx = context.ctx() instance_ids = [] try: instance_ids = self._scale_cluster_instances(cluster, node_group_id_map) cluster = conductor.cluster_get(ctx, cluster) g.clean_cluster_from_empty_ng(cluster) cluster = conductor.cluster_get(ctx, cluster) instances = g.get_instances(cluster, instance_ids) self._await_active(cluster, instances) self._assign_floating_ips(instances) self._await_networks(cluster, instances) cluster = conductor.cluster_get(ctx, cluster) volumes.attach_to_instances( g.get_instances(cluster, instance_ids)) except Exception as ex: with excutils.save_and_reraise_exception(): self._log_operation_exception( "Can't scale cluster '%s' (reason: %s)", cluster, ex) cluster = conductor.cluster_get(ctx, cluster) self._rollback_cluster_scaling( cluster, g.get_instances(cluster, instance_ids), ex) instance_ids = [] cluster = conductor.cluster_get(ctx, cluster) g.clean_cluster_from_empty_ng(cluster) if cluster.status == 'Decommissioning': cluster = conductor.cluster_update(ctx, cluster, {"status": "Error"}) else: cluster = conductor.cluster_update(ctx, cluster, {"status": "Active"}) LOG.info(g.format_cluster_status(cluster)) # we should be here with valid cluster: if instances creation # was not successful all extra-instances will be removed above if instance_ids: self._configure_instances(cluster) return instance_ids
def scale_cluster(id, data): ctx = context.ctx() cluster = conductor.cluster_get(ctx, id) plugin = plugin_base.PLUGINS.get_plugin(cluster.plugin_name) existing_node_groups = data.get('resize_node_groups', []) additional_node_groups = data.get('add_node_groups', []) #the next map is the main object we will work with #to_be_enlarged : {node_group_id: desired_amount_of_instances} to_be_enlarged = {} for ng in existing_node_groups: ng_id = g.find(cluster.node_groups, name=ng['name'])['id'] to_be_enlarged.update({ng_id: ng['count']}) additional = construct_ngs_for_scaling(cluster, additional_node_groups) cluster = conductor.cluster_get(ctx, cluster) # update nodegroup image usernames for nodegroup in cluster.node_groups: if additional.get(nodegroup.id): image_username = INFRA.get_node_group_image_username(nodegroup) conductor.node_group_update(ctx, nodegroup, {"image_username": image_username}) cluster = conductor.cluster_get(ctx, cluster) try: cluster = conductor.cluster_update(ctx, cluster, {"status": "Validating"}) LOG.info(g.format_cluster_status(cluster)) plugin.validate_scaling(cluster, to_be_enlarged, additional) except Exception: with excutils.save_and_reraise_exception(): g.clean_cluster_from_empty_ng(cluster) cluster = conductor.cluster_update(ctx, cluster, {"status": "Active"}) LOG.info(g.format_cluster_status(cluster)) # If we are here validation is successful. # So let's update to_be_enlarged map: to_be_enlarged.update(additional) for node_group in cluster.node_groups: if node_group.id not in to_be_enlarged: to_be_enlarged[node_group.id] = node_group.count context.spawn("cluster-scaling-%s" % id, _provision_scaled_cluster, id, to_be_enlarged) return conductor.cluster_get(ctx, id)
def scale_cluster(id, data): ctx = context.ctx() cluster = conductor.cluster_get(ctx, id) plugin = plugin_base.PLUGINS.get_plugin(cluster.plugin_name) existing_node_groups = data.get('resize_node_groups', []) additional_node_groups = data.get('add_node_groups', []) #the next map is the main object we will work with #to_be_enlarged : {node_group_id: desired_amount_of_instances} to_be_enlarged = {} for ng in existing_node_groups: ng_id = g.find(cluster.node_groups, name=ng['name'])['id'] to_be_enlarged.update({ng_id: ng['count']}) additional = construct_ngs_for_scaling(cluster, additional_node_groups) cluster = conductor.cluster_get(ctx, cluster) # update nodegroup image usernames for nodegroup in cluster.node_groups: if additional.get(nodegroup.id): image_username = INFRA.get_node_group_image_username(nodegroup) conductor.node_group_update( ctx, nodegroup, {"image_username": image_username}) cluster = conductor.cluster_get(ctx, cluster) try: cluster = conductor.cluster_update(ctx, cluster, {"status": "Validating"}) LOG.info(g.format_cluster_status(cluster)) plugin.validate_scaling(cluster, to_be_enlarged, additional) except Exception: with excutils.save_and_reraise_exception(): g.clean_cluster_from_empty_ng(cluster) cluster = conductor.cluster_update(ctx, cluster, {"status": "Active"}) LOG.info(g.format_cluster_status(cluster)) # If we are here validation is successful. # So let's update to_be_enlarged map: to_be_enlarged.update(additional) for node_group in cluster.node_groups: if node_group.id not in to_be_enlarged: to_be_enlarged[node_group.id] = node_group.count context.spawn("cluster-scaling-%s" % id, _provision_scaled_cluster, id, to_be_enlarged) return conductor.cluster_get(ctx, id)
def scale_cluster(self, cluster, target_count): ctx = context.ctx() rollback_count = self._get_ng_counts(cluster) self._update_rollback_strategy(cluster, rollback_count=rollback_count, target_count=target_count) launcher = _ScaleLauncher() launcher.launch_instances(cluster, target_count) cluster = conductor.cluster_get(ctx, cluster) g.clean_cluster_from_empty_ng(cluster) self._update_rollback_strategy(cluster) return launcher.inst_ids
def scale_cluster(self, cluster, target_count): ctx = context.ctx() rollback_count = self._get_ng_counts(cluster) self._update_rollback_strategy(cluster, rollback_count=rollback_count, target_count=target_count) inst_ids = self._launch_instances( cluster, target_count, SCALE_STAGES, update_stack=True, disable_rollback=False) cluster = conductor.cluster_get(ctx, cluster) g.clean_cluster_from_empty_ng(cluster) self._update_rollback_strategy(cluster) return inst_ids
def scale_cluster(self, cluster, target_count): ctx = context.ctx() rollback_count = self._get_ng_counts(cluster) launcher = _ScaleLauncher() try: launcher.launch_instances(ctx, cluster, target_count) except Exception as ex: with excutils.save_and_reraise_exception(): if not g.check_cluster_exists(cluster): LOG.info(g.format_cluster_deleted_message(cluster)) return self._log_operation_exception( "Can't scale cluster '%s' (reason: %s)", cluster, ex) cluster = conductor.cluster_get(ctx, cluster) try: self._rollback_cluster_scaling( ctx, cluster, rollback_count, target_count) except Exception: if not g.check_cluster_exists(cluster): LOG.info(g.format_cluster_deleted_message(cluster)) return # if something fails during the rollback, we stop # doing anything further cluster = conductor.cluster_update(ctx, cluster, {"status": "Error"}) LOG.info(g.format_cluster_status(cluster)) LOG.error("Unable to complete rollback, aborting") raise cluster = conductor.cluster_update(ctx, cluster, {"status": "Active"}) LOG.info(g.format_cluster_status(cluster)) LOG.warn( "Rollback successful. Throwing off an initial exception.") finally: cluster = conductor.cluster_get(ctx, cluster) g.clean_cluster_from_empty_ng(cluster) return launcher.inst_ids
def scale_cluster(self, cluster, target_count): ctx = context.ctx() rollback_count = self._get_ng_counts(cluster) self._update_rollback_strategy(cluster, rollback_count=rollback_count, target_count=target_count) inst_ids = self._launch_instances(cluster, target_count, SCALE_STAGES, update_stack=True, disable_rollback=False) cluster = conductor.cluster_get(ctx, cluster) g.clean_cluster_from_empty_ng(cluster) self._update_rollback_strategy(cluster) return inst_ids
def scale_cluster(id, data): context.set_current_cluster_id(id) ctx = context.ctx() cluster = conductor.cluster_get(ctx, id) plugin = plugin_base.PLUGINS.get_plugin(cluster.plugin_name) existing_node_groups = data.get('resize_node_groups', []) additional_node_groups = data.get('add_node_groups', []) # the next map is the main object we will work with # to_be_enlarged : {node_group_id: desired_amount_of_instances} to_be_enlarged = {} for ng in existing_node_groups: ng_id = g.find(cluster.node_groups, name=ng['name'])['id'] to_be_enlarged.update({ng_id: ng['count']}) additional = construct_ngs_for_scaling(cluster, additional_node_groups) cluster = conductor.cluster_get(ctx, cluster) _add_ports_for_auto_sg(ctx, cluster, plugin) try: cluster = g.change_cluster_status(cluster, "Validating") quotas.check_scaling(cluster, to_be_enlarged, additional) plugin.recommend_configs(cluster) plugin.validate_scaling(cluster, to_be_enlarged, additional) except Exception as e: with excutils.save_and_reraise_exception(): g.clean_cluster_from_empty_ng(cluster) g.change_cluster_status(cluster, "Active", six.text_type(e)) # If we are here validation is successful. # So let's update to_be_enlarged map: to_be_enlarged.update(additional) for node_group in cluster.node_groups: if node_group.id not in to_be_enlarged: to_be_enlarged[node_group.id] = node_group.count OPS.provision_scaled_cluster(id, to_be_enlarged) return cluster
def scale_cluster(self, cluster, target_count): ctx = context.ctx() rollback_count = self._get_ng_counts(cluster) launcher = _ScaleLauncher() try: launcher.launch_instances(ctx, cluster, target_count) except Exception as ex: with excutils.save_and_reraise_exception(): self._log_operation_exception( "Can't scale cluster '%s' (reason: %s)", cluster, ex) cluster = conductor.cluster_get(ctx, cluster) try: self._rollback_cluster_scaling( ctx, cluster, rollback_count, target_count) except Exception: # if something fails during the rollback, we stop # doing anything further cluster = conductor.cluster_update(ctx, cluster, {"status": "Error"}) LOG.info(g.format_cluster_status(cluster)) LOG.error("Unable to complete rollback, aborting") raise cluster = conductor.cluster_update(ctx, cluster, {"status": "Active"}) LOG.info(g.format_cluster_status(cluster)) LOG.warn( "Rollback successful. Throwing off an initial exception.") finally: cluster = conductor.cluster_get(ctx, cluster) g.clean_cluster_from_empty_ng(cluster) return launcher.inst_ids