def _get_delete_candidates(self, cluster_id, action): deletion = action.data.get('deletion', None) # No deletion field in action.data which means no scaling # policy or deletion policy is attached. if deletion is None: candidates = None if action.action == consts.CLUSTER_DEL_NODES: # Get candidates from action.input candidates = action.inputs.get('candidates', []) count = len(candidates) elif action.action == consts.CLUSTER_RESIZE: # Calculate deletion count based on action input db_cluster = db_api.cluster_get(action.context, cluster_id) scaleutils.parse_resize_params(action, db_cluster) if 'deletion' not in action.data: return [] else: count = action.data['deletion']['count'] else: # action.action == consts.CLUSTER_SCALE_IN count = 1 else: count = deletion.get('count', 0) candidates = deletion.get('candidates', None) # Still no candidates available, pick count of nodes randomly if candidates is None: nodes = db_api.node_get_all_by_cluster(action.context, cluster_id=cluster_id) if count > len(nodes): count = len(nodes) candidates = scaleutils.nodes_by_random(nodes, count) return candidates
def do_scale_in(self): """Handler for the CLUSTER_SCALE_IN action. :returns: A tuple containing the result and the corresponding reason. """ # We use policy data if any, deletion policy and scaling policy might # be attached. pd = self.data.get('deletion', None) grace_period = 0 if pd: grace_period = pd.get('grace_period', 0) candidates = pd.get('candidates', []) # if scaling policy is attached, get 'count' from action data count = len(candidates) or pd['count'] else: # If no scaling policy is attached, use the input count directly candidates = [] value = self.inputs.get('count', 1) success, count = utils.get_positive_int(value) if not success: reason = _('Invalid count (%s) for scaling in.') % value return self.RES_ERROR, reason # check provided params against current properties # desired is checked when strict is True curr_size = no.Node.count_by_cluster(self.context, self.target) if count > curr_size: msg = _("Triming count (%(count)s) to current " "cluster size (%(curr)s) for scaling in") LOG.warning(msg, {'count': count, 'curr': curr_size}) count = curr_size new_size = curr_size - count result = scaleutils.check_size_params(self.entity, new_size, None, None, True) if result: return self.RES_ERROR, result self.entity.set_status(self.context, consts.CS_RESIZING, _('Cluster scale in started.'), desired_capacity=new_size) # Choose victims randomly if len(candidates) == 0: candidates = scaleutils.nodes_by_random(self.entity.nodes, count) # self._sleep(grace_period) result, reason = self._delete_nodes(candidates) if result == self.RES_OK: reason = _('Cluster scaling succeeded.') self.entity.eval_status(self.context, consts.CLUSTER_SCALE_IN) return result, reason
def test_nodes_by_random(self, mock_filter): good_nodes = [ mock.Mock(id='N11', created_at=110), mock.Mock(id='N15', created_at=150), mock.Mock(id='N12', created_at=120), mock.Mock(id='N13', created_at=130), mock.Mock(id='N14', created_at=None), ] mock_filter.return_value = (['N1', 'N2'], good_nodes) nodes = mock.Mock() res = su.nodes_by_random(nodes, 1) self.assertEqual(['N1'], res) res = su.nodes_by_random(nodes, 2) self.assertEqual(['N1', 'N2'], res) res = su.nodes_by_random(nodes, 5) self.assertIn('N1', res) self.assertIn('N2', res) self.assertEqual(5, len(res))
def test_nodes_by_random(self, mock_filter): good_nodes = [ mock.Mock(id='N11', created_at=110), mock.Mock(id='N15', created_at=150), mock.Mock(id='N12', created_at=120), mock.Mock(id='N13', created_at=130), mock.Mock(id='N14', created_at=None), ] mock_filter.return_value = (['N1', 'N2'], good_nodes) nodes = mock.Mock() res = su.nodes_by_random(nodes, 1) self.assertEqual(['N1'], res) res = su.nodes_by_random(nodes, 2) self.assertEqual(['N1', 'N2'], res) res = su.nodes_by_random(nodes, 5) self.assertIn('N1', res) self.assertIn('N2', res) self.assertEqual(5, len(res))
def do_resize(self): """Handler for the CLUSTER_RESIZE action. :returns: A tuple containing the result and the corresponding reason. """ self.cluster.set_status(self.context, self.cluster.RESIZING, 'Cluster resize started.') node_list = self.cluster.nodes current_size = len(node_list) count, desired, candidates = self._get_action_data(current_size) grace_period = None # if policy is attached to the cluster, use policy data directly, # or parse resize params to get action data. if count == 0: result, reason = scaleutils.parse_resize_params(self, self.cluster) if result != self.RES_OK: status_reason = _('Cluster resizing failed: %s') % reason self.cluster.set_status(self.context, self.cluster.ACTIVE, status_reason) return result, reason count, desired, candidates = self._get_action_data(current_size) elif 'deletion' in self.data: grace_period = self.data['deletion'].get('grace_period', None) if candidates is not None and len(candidates) == 0: # Choose victims randomly candidates = scaleutils.nodes_by_random(self.cluster.nodes, count) # delete nodes if necessary if desired < current_size: if grace_period is not None: self._wait_before_deletion(grace_period) result, reason = self._delete_nodes(candidates) # Create new nodes if desired_capacity increased else: result, reason = self._create_nodes(count) if result != self.RES_OK: self.cluster.set_status(self.context, self.cluster.WARNING, reason) return result, reason reason = _('Cluster resize succeeded.') kwargs = {'desired_capacity': desired} min_size = self.inputs.get(consts.ADJUSTMENT_MIN_SIZE, None) max_size = self.inputs.get(consts.ADJUSTMENT_MAX_SIZE, None) if min_size is not None: kwargs['min_size'] = min_size if max_size is not None: kwargs['max_size'] = max_size self.cluster.set_status(self.context, self.cluster.ACTIVE, reason, **kwargs) return self.RES_OK, reason
def do_resize(self): """Handler for the CLUSTER_RESIZE action. :returns: A tuple containing the result and the corresponding reason. """ self.cluster.set_status(self.context, self.cluster.RESIZING, 'Cluster resize started.') node_list = self.cluster.nodes current_size = len(node_list) count, desired, candidates = self._get_action_data(current_size) grace_period = None # if policy is attached to the cluster, use policy data directly, # or parse resize params to get action data. if count == 0: result, reason = scaleutils.parse_resize_params(self, self.cluster) if result != self.RES_OK: status_reason = _('Cluster resizing failed: %s') % reason self.cluster.set_status(self.context, self.cluster.ACTIVE, status_reason) return result, reason count, desired, candidates = self._get_action_data(current_size) elif 'deletion' in self.data: grace_period = self.data['deletion'].get('grace_period', None) if candidates is not None and len(candidates) == 0: # Choose victims randomly candidates = scaleutils.nodes_by_random(self.cluster.nodes, count) # delete nodes if necessary if desired < current_size: if grace_period is not None: self._wait_before_deletion(grace_period) result, reason = self._delete_nodes(candidates) # Create new nodes if desired_capacity increased else: result, reason = self._create_nodes(count) if result != self.RES_OK: self.cluster.set_status(self.context, self.cluster.WARNING, reason) return result, reason reason = _('Cluster resize succeeded.') kwargs = {'desired_capacity': desired} min_size = self.inputs.get(consts.ADJUSTMENT_MIN_SIZE, None) max_size = self.inputs.get(consts.ADJUSTMENT_MAX_SIZE, None) if min_size is not None: kwargs['min_size'] = min_size if max_size is not None: kwargs['max_size'] = max_size self.cluster.set_status(self.context, self.cluster.ACTIVE, reason, **kwargs) return self.RES_OK, reason
def _check_capacity(self): cluster = self.entity current = len(cluster.nodes) desired = cluster.desired_capacity if current < desired: count = desired - current self._create_nodes(count) if current > desired: count = current - desired nodes = no.Node.get_all_by_cluster(self.context, cluster.id) candidates = scaleutils.nodes_by_random(nodes, count) self._delete_nodes(candidates)
def _get_delete_candidates(self, cluster_id, action): deletion = action.data.get('deletion', None) # No deletion field in action.data which means no scaling # policy or deletion policy is attached. candidates = None if deletion is None: if action.action == consts.NODE_DELETE: candidates = [action.entity.id] count = 1 elif action.action == consts.CLUSTER_DEL_NODES: # Get candidates from action.input candidates = action.inputs.get('candidates', []) count = len(candidates) elif action.action == consts.CLUSTER_RESIZE: # Calculate deletion count based on action input cluster = action.entity current = len(cluster.nodes) scaleutils.parse_resize_params(action, cluster, current) if 'deletion' not in action.data: return [] else: count = action.data['deletion']['count'] else: # action.action == consts.CLUSTER_SCALE_IN count = 1 elif action.action == consts.CLUSTER_REPLACE_NODES: candidates = list(action.inputs['candidates'].keys()) count = len(candidates) else: count = deletion.get('count', 0) candidates = deletion.get('candidates', None) # Still no candidates available, pick count of nodes randomly # apply to CLUSTER_RESIZE/CLUSTER_SCALE_IN if candidates is None: if count == 0: return [] nodes = action.entity.nodes if count > len(nodes): count = len(nodes) candidates = scaleutils.nodes_by_random(nodes, count) deletion_data = action.data.get('deletion', {}) deletion_data.update({ 'count': len(candidates), 'candidates': candidates }) action.data.update({'deletion': deletion_data}) return candidates
def _victims_by_regions(self, cluster, regions): victims = [] for region in sorted(regions.keys()): count = regions[region] nodes = cluster.nodes_by_region(region) if self.criteria == self.RANDOM: candidates = scaleutils.nodes_by_random(nodes, count) elif self.criteria == self.OLDEST_PROFILE_FIRST: candidates = scaleutils.nodes_by_profile_age(nodes, count) elif self.criteria == self.OLDEST_FIRST: candidates = scaleutils.nodes_by_age(nodes, count, True) else: candidates = scaleutils.nodes_by_age(nodes, count, False) victims.extend(candidates) return victims
def _victims_by_regions(self, cluster, regions): victims = [] for region in sorted(regions.keys()): count = regions[region] nodes = cluster.nodes_by_region(region) if self.criteria == self.RANDOM: candidates = scaleutils.nodes_by_random(nodes, count) elif self.criteria == self.OLDEST_PROFILE_FIRST: candidates = scaleutils.nodes_by_profile_age(nodes, count) elif self.criteria == self.OLDEST_FIRST: candidates = scaleutils.nodes_by_age(nodes, count, True) else: candidates = scaleutils.nodes_by_age(nodes, count, False) victims.extend(candidates) return victims
def _victims_by_zones(self, cluster, zones): victims = [] for zone in sorted(zones.keys()): count = zones[zone] nodes = cluster.nodes_by_zone(zone) if self.criteria == self.RANDOM: candidates = scaleutils.nodes_by_random(nodes, count) elif self.criteria == self.OLDEST_PROFILE_FIRST: candidates = scaleutils.nodes_by_profile_age(nodes, count) elif self.criteria == self.OLDEST_FIRST: candidates = scaleutils.nodes_by_age(nodes, count, True) else: candidates = scaleutils.nodes_by_age(nodes, count, False) victims.extend(candidates) return victims
def _victims_by_zones(self, cluster, zones): victims = [] for zone in sorted(zones.keys()): count = zones[zone] nodes = cluster.nodes_by_zone(zone) if self.criteria == self.RANDOM: candidates = scaleutils.nodes_by_random(nodes, count) elif self.criteria == self.OLDEST_PROFILE_FIRST: candidates = scaleutils.nodes_by_profile_age(nodes, count) elif self.criteria == self.OLDEST_FIRST: candidates = scaleutils.nodes_by_age(nodes, count, True) else: candidates = scaleutils.nodes_by_age(nodes, count, False) victims.extend(candidates) return victims
def do_resize(self): """Handler for the CLUSTER_RESIZE action. :returns: A tuple containing the result and the corresponding reason. """ # if no policy decision(s) found, use policy inputs directly, # Note the 'parse_resize_params' function is capable of calculating # desired capacity and handling best effort scaling. It also verifies # that the inputs are valid curr_capacity = no.Node.count_by_cluster(self.context, self.entity.id) if 'creation' not in self.data and 'deletion' not in self.data: result, reason = scaleutils.parse_resize_params( self, self.entity, curr_capacity) if result != self.RES_OK: return result, reason # action input consolidated to action data now reason = 'Cluster resize succeeded.' if 'deletion' in self.data: count = self.data['deletion']['count'] candidates = self.data['deletion'].get('candidates', []) # Choose victims randomly if not already picked if not candidates: node_list = self.entity.nodes candidates = scaleutils.nodes_by_random(node_list, count) self._update_cluster_size(curr_capacity - count) grace_period = self.data['deletion'].get('grace_period', 0) self._sleep(grace_period) result, new_reason = self._delete_nodes(candidates) else: # 'creation' in self.data: count = self.data['creation']['count'] self._update_cluster_size(curr_capacity + count) result, new_reason = self._create_nodes(count) if result != self.RES_OK: reason = new_reason self.entity.eval_status(self.context, consts.CLUSTER_RESIZE) return result, reason
def _get_delete_candidates(self, cluster_id, action): deletion = action.data.get('deletion', None) # No deletion field in action.data which means no scaling # policy or deletion policy is attached. candidates = None if deletion is None: if action.action == consts.CLUSTER_DEL_NODES: # Get candidates from action.input candidates = action.inputs.get('candidates', []) count = len(candidates) elif action.action == consts.CLUSTER_RESIZE: # Calculate deletion count based on action input db_cluster = db_api.cluster_get(action.context, cluster_id) scaleutils.parse_resize_params(action, db_cluster) if 'deletion' not in action.data: return [] else: count = action.data['deletion']['count'] else: # action.action == consts.CLUSTER_SCALE_IN count = 1 else: count = deletion.get('count', 0) candidates = deletion.get('candidates', None) # Still no candidates available, pick count of nodes randomly if candidates is None: if count == 0: return [] nodes = db_api.node_get_all_by_cluster(action.context, cluster_id=cluster_id) if count > len(nodes): count = len(nodes) candidates = scaleutils.nodes_by_random(nodes, count) deletion_data = action.data.get('deletion', {}) deletion_data.update({ 'count': len(candidates), 'candidates': candidates }) action.data.update({'deletion': deletion_data}) return candidates
def pre_op(self, cluster_id, action): """Choose victims that can be deleted. :param cluster_id: ID of the cluster to be handled. :param action: The action object that triggered this policy. """ victims = action.inputs.get('candidates', []) if len(victims) > 0: self._update_action(action, victims) return db_cluster = None regions = None zones = None deletion = action.data.get('deletion', {}) if deletion: # there are policy decisions count = deletion['count'] regions = deletion.get('regions', None) zones = deletion.get('zones', None) # No policy decision, check action itself: SCALE_IN elif action.action == consts.CLUSTER_SCALE_IN: count = action.inputs.get('count', 1) # No policy decision, check action itself: RESIZE else: db_cluster = db_api.cluster_get(action.context, cluster_id) scaleutils.parse_resize_params(action, db_cluster) if 'deletion' not in action.data: return count = action.data['deletion']['count'] cluster = cluster_mod.Cluster.load(action.context, cluster=db_cluster, cluster_id=cluster_id) # Cross-region if regions: victims = self._victims_by_regions(cluster, regions) self._update_action(action, victims) return # Cross-AZ if zones: victims = self._victims_by_zones(cluster, zones) self._update_action(action, victims) return if count > len(cluster.nodes): count = len(cluster.nodes) if self.criteria == self.RANDOM: victims = scaleutils.nodes_by_random(cluster.nodes, count) elif self.criteria == self.OLDEST_PROFILE_FIRST: victims = scaleutils.nodes_by_profile_age(cluster.nodes, count) elif self.criteria == self.OLDEST_FIRST: victims = scaleutils.nodes_by_age(cluster.nodes, count, True) else: victims = scaleutils.nodes_by_age(cluster.nodes, count, False) self._update_action(action, victims) return
def pre_op(self, cluster_id, action): """Choose victims that can be deleted. :param cluster_id: ID of the cluster to be handled. :param action: The action object that triggered this policy. """ victims = action.inputs.get('candidates', []) if len(victims) > 0: self._update_action(action, victims) return if action.action == consts.NODE_DELETE: self._update_action(action, [action.entity.id]) return cluster = action.entity regions = None zones = None hooks_data = self.hooks action.data.update({'status': base.CHECK_OK, 'reason': _('lifecycle hook parameters saved'), 'hooks': hooks_data}) action.store(action.context) deletion = action.data.get('deletion', {}) if deletion: # there are policy decisions count = deletion['count'] regions = deletion.get('regions', None) zones = deletion.get('zones', None) # No policy decision, check action itself: SCALE_IN elif action.action == consts.CLUSTER_SCALE_IN: count = action.inputs.get('count', 1) # No policy decision, check action itself: RESIZE else: current = len(cluster.nodes) res, reason = su.parse_resize_params(action, cluster, current) if res == base.CHECK_ERROR: action.data['status'] = base.CHECK_ERROR action.data['reason'] = reason LOG.error(reason) return if 'deletion' not in action.data: return count = action.data['deletion']['count'] # Cross-region if regions: victims = self._victims_by_regions(cluster, regions) self._update_action(action, victims) return # Cross-AZ if zones: victims = self._victims_by_zones(cluster, zones) self._update_action(action, victims) return if count > len(cluster.nodes): count = len(cluster.nodes) if self.criteria == self.RANDOM: victims = su.nodes_by_random(cluster.nodes, count) elif self.criteria == self.OLDEST_PROFILE_FIRST: victims = su.nodes_by_profile_age(cluster.nodes, count) elif self.criteria == self.OLDEST_FIRST: victims = su.nodes_by_age(cluster.nodes, count, True) else: victims = su.nodes_by_age(cluster.nodes, count, False) self._update_action(action, victims) return
def do_scale_in(self): """Handler for the CLUSTER_SCALE_IN action. :returns: A tuple containing the result and the corresponding reason. """ self.cluster.set_status(self.context, self.cluster.RESIZING, 'Cluster scale in started.') # We use policy data if any, deletion policy and scaling policy might # be attached. pd = self.data.get('deletion', None) grace_period = 0 if pd: grace_period = pd.get('grace_period', 0) candidates = pd.get('candidates', []) # if scaling policy is attached, get 'count' from action data count = len(candidates) or pd['count'] else: # If no scaling policy is attached, use the input count directly candidates = [] count = self.inputs.get('count', 1) try: count = utils.parse_int_param('count', count, allow_zero=False) except exception.InvalidParameter: reason = _('Invalid count (%s) for scaling in.') % count status_reason = _('Cluster scaling failed: %s') % reason self.cluster.set_status(self.context, self.cluster.ACTIVE, status_reason) return self.RES_ERROR, reason # check provided params against current properties # desired is checked when strict is True curr_size = len(self.cluster.nodes) if count > curr_size: LOG.warning(_('Triming count (%(count)s) to current cluster size ' '(%(curr)s) for scaling in'), {'count': count, 'curr': curr_size}) count = curr_size new_size = curr_size - count result = scaleutils.check_size_params(self.cluster, new_size, None, None, True) if result: status_reason = _('Cluster scaling failed: %s') % result self.cluster.set_status(self.context, self.cluster.ACTIVE, status_reason) return self.RES_ERROR, result # Choose victims randomly if len(candidates) == 0: candidates = scaleutils.nodes_by_random(self.cluster.nodes, count) self._sleep(grace_period) # The policy data may contain destroy flag and grace period option result, reason = self._delete_nodes(candidates) if result == self.RES_OK: reason = _('Cluster scaling succeeded.') self.cluster.set_status(self.context, self.cluster.ACTIVE, reason, desired_capacity=new_size) elif result in [self.RES_CANCEL, self.RES_TIMEOUT, self.RES_ERROR]: self.cluster.set_status(self.context, self.cluster.ERROR, reason, desired_capacity=new_size) else: # RES_RETRY pass return result, reason
def do_scale_in(self): """Handler for the CLUSTER_SCALE_IN action. :returns: A tuple containing the result and the corresponding reason. """ self.cluster.set_status(self.context, self.cluster.RESIZING, 'Cluster scale in started.') # We use policy data if any, deletion policy and scaling policy might # be attached. pd = self.data.get('deletion', None) grace_period = None if pd is not None: grace_period = pd.get('grace_period', 0) candidates = pd.get('candidates', []) # if scaling policy is attached, get 'count' from action data count = len(candidates) or pd['count'] else: # If no scaling policy is attached, use the input count directly candidates = [] count = self.inputs.get('count', 1) try: count = utils.parse_int_param('count', count, allow_zero=False) except exception.InvalidParameter: reason = _('Invalid count (%s) for scaling in.') % count status_reason = _('Cluster scaling failed: %s') % reason self.cluster.set_status(self.context, self.cluster.ACTIVE, status_reason) return self.RES_ERROR, reason # check provided params against current properties # desired is checked when strict is True curr_size = len(self.cluster.nodes) if count > curr_size: LOG.warning( _('Triming count (%(count)s) to current cluster size ' '(%(curr)s) for scaling in'), { 'count': count, 'curr': curr_size }) count = curr_size new_size = curr_size - count result = scaleutils.check_size_params(self.cluster, new_size, None, None, True) if result: status_reason = _('Cluster scaling failed: %s') % result self.cluster.set_status(self.context, self.cluster.ACTIVE, status_reason) return self.RES_ERROR, result # Choose victims randomly if len(candidates) == 0: candidates = scaleutils.nodes_by_random(self.cluster.nodes, count) if grace_period is not None: self._wait_before_deletion(grace_period) # The policy data may contain destroy flag and grace period option result, reason = self._delete_nodes(candidates) if result == self.RES_OK: reason = _('Cluster scaling succeeded.') self.cluster.set_status(self.context, self.cluster.ACTIVE, reason, desired_capacity=new_size) elif result in [self.RES_CANCEL, self.RES_TIMEOUT, self.RES_ERROR]: self.cluster.set_status(self.context, self.cluster.ERROR, reason, desired_capacity=new_size) else: # RES_RETRY pass return result, reason
def pre_op(self, cluster_id, action): """Choose victims that can be deleted. :param cluster_id: ID of the cluster to be handled. :param action: The action object that triggered this policy. """ victims = action.inputs.get('candidates', []) if len(victims) > 0: self._update_action(action, victims) return if action.action == consts.NODE_DELETE: self._update_action(action, [action.node.id]) return db_cluster = None regions = None zones = None deletion = action.data.get('deletion', {}) if deletion: # there are policy decisions count = deletion['count'] regions = deletion.get('regions', None) zones = deletion.get('zones', None) # No policy decision, check action itself: SCALE_IN elif action.action == consts.CLUSTER_SCALE_IN: count = action.inputs.get('count', 1) # No policy decision, check action itself: RESIZE else: db_cluster = co.Cluster.get(action.context, cluster_id) current = no.Node.count_by_cluster(action.context, cluster_id) res, reason = scaleutils.parse_resize_params(action, db_cluster, current) if res == base.CHECK_ERROR: action.data['status'] = base.CHECK_ERROR action.data['reason'] = reason LOG.error(reason) return if 'deletion' not in action.data: return count = action.data['deletion']['count'] cluster = cm.Cluster.load(action.context, dbcluster=db_cluster, cluster_id=cluster_id) # Cross-region if regions: victims = self._victims_by_regions(cluster, regions) self._update_action(action, victims) return # Cross-AZ if zones: victims = self._victims_by_zones(cluster, zones) self._update_action(action, victims) return if count > len(cluster.nodes): count = len(cluster.nodes) if self.criteria == self.RANDOM: victims = scaleutils.nodes_by_random(cluster.nodes, count) elif self.criteria == self.OLDEST_PROFILE_FIRST: victims = scaleutils.nodes_by_profile_age(cluster.nodes, count) elif self.criteria == self.OLDEST_FIRST: victims = scaleutils.nodes_by_age(cluster.nodes, count, True) else: victims = scaleutils.nodes_by_age(cluster.nodes, count, False) self._update_action(action, victims) return