def test_pending_server_delete(self): """ When a pending job is cancelled, it is deleted from the job list. When the server finishes building, then ``execute_launch_config`` is called to remove the job from pending job list. It then notices that pending job_id is not there in job list and calls ``execute_delete_server`` to delete the server. """ self.supervisor.execute_delete_server.return_value = succeed(None) s = GroupState('tenant', 'group', 'name', {}, {'1': {}}, None, {}, False) def fake_modify_state(callback, *args, **kwargs): callback(self.group, s, *args, **kwargs) self.group.modify_state.side_effect = fake_modify_state supervisor.execute_launch_config(self.log, '1', self.fake_state, 'launch', self.group, 1) s.remove_job('1') self.execute_config_deferreds[0].callback({'id': 's1'}) # first bind is system='otter.job.launch', second is job_id='1' self.del_job.assert_called_once_with( matches(IsInstance(self.log.__class__)), '1', self.group, {'id': 's1'}, self.supervisor) self.del_job.return_value.start.assert_called_once_with()
def test_job_failure(self): """ ``execute_launch_config`` sets it up so that when a job fails, it is removed from pending. It is also lgoged. """ s = GroupState('tenant', 'group', 'name', {}, {'1': {}}, None, {}, False) written = [] # modify state writes on callback, doesn't write on error def fake_modify_state(callback, *args, **kwargs): d = maybeDeferred(callback, self.group, s, *args, **kwargs) d.addCallback(written.append) return d self.group.modify_state.side_effect = fake_modify_state supervisor.execute_launch_config(self.log, '1', self.fake_state, 'launch', self.group, 1) f = Failure(Exception('meh')) self.execute_config_deferreds[0].errback(f) # job is removed and no active servers added self.assertEqual(s, GroupState('tenant', 'group', 'name', {}, {}, None, {}, False)) # state is written self.assertEqual(len(written), 1) self.assertEqual(written[0], s) self.log.err.assert_called_with(f, 'Launching server failed', system="otter.job.launch", image_ref="Unable to pull image ref.", flavor_ref="Unable to pull flavor ref.", job_id='1')
def test_add_job_called_with_new_jobs(self): """ ``execute_launch_config`` calls ``add_job`` on the state for every job that has been started """ supervisor.execute_launch_config(self.log, '1', self.fake_state, 'launch', self.group, 3) self.fake_state.add_job.assert_has_calls( [mock.call(str(i)) for i in (1, 2, 3)]) self.assertEqual(self.fake_state.add_job.call_count, 3)
def test_positive_delta_execute_config_called_delta_times(self): """ If delta > 0, ``execute_launch_config`` calls ``supervisor.execute_config`` delta times. """ supervisor.execute_launch_config(self.log, '1', self.fake_state, 'launch', self.group, 5) self.assertEqual(self.supervisor.execute_config.mock_calls, [mock.call(matches(IsInstance(self.log.__class__)), '1', self.group, 'launch')] * 5)
def test_modify_state_failure_logged(self): """ If the job succeeded but modifying the state fails, that error is logged. """ self.group.modify_state.side_effect = AssertionError supervisor.execute_launch_config(self.log, '1', self.fake_state, 'launch', self.group, 1) self.execute_config_deferreds[0].callback({'id': 's1'}) self.log.err.assert_called_once_with( CheckFailure(AssertionError), system="otter.job.launch", image_ref="Unable to pull image ref.", flavor_ref="Unable to pull flavor ref.", job_id='1')
def test_on_job_completion_modify_state_called(self): """ ``execute_launch_config`` sets it up so that the group's ``modify_state``state is called with the result as an arg whenever a job finishes, whether successfully or not """ supervisor.execute_launch_config(self.log, '1', self.fake_state, 'launch', self.group, 3) self.execute_config_deferreds[0].callback({'id': '1'}) # job id 1 self.execute_config_deferreds[1].errback(Exception('meh')) # job id 2 self.execute_config_deferreds[2].callback({'id': '3'}) # job id 3 self.assertEqual(self.group.modify_state.call_count, 3)
def converge(log, transaction_id, config, scaling_group, state, launch_config, policy, config_value=config_value): """ Apply a policy's change to a scaling group, and attempt to make the resulting state a reality. This does no cooldown checking. This is done by dispatching to the appropriate orchestration backend for the scaling group; currently only direct nova interaction is supported. :param log: A bound log for logging :param str transaction_id: the transaction id :param dict config: the scaling group config :param otter.models.interface.IScalingGroup scaling_group: the scaling group object :param otter.models.interface.GroupState state: the group state :param dict launch_config: the scaling group launch config :param dict policy: the policy configuration dictionary :return: a ``Deferred`` that fires with the updated :class:`otter.models.interface.GroupState` if successful. If no changes are to be made to the group, None will synchronously be returned. """ if tenant_is_enabled(scaling_group.tenant_id, config_value): # For convergence tenants, find delta based on group's desired # capacity delta = apply_delta(log, state.desired, state, config, policy) if delta == 0: # No change in servers. Return None synchronously return None else: return defer.succeed(state) # For non-convergence tenants, the value used for desired-capacity is # the sum of active+pending, which is 0, so the delta ends up being # the min entities due to constraint calculation. delta = calculate_delta(log, state, config, policy) execute_log = log.bind(server_delta=delta) if delta == 0: execute_log.msg("no change in servers") return None elif delta > 0: execute_log.msg("executing launch configs") deferred = execute_launch_config(execute_log, transaction_id, state, launch_config, scaling_group, delta) else: # delta < 0 (scale down) execute_log.msg("scaling down") deferred = exec_scale_down(execute_log, transaction_id, state, scaling_group, -delta) deferred.addCallback(_do_convergence_audit_log, log, delta, state) return deferred
def test_no_jobs_started(self): """ If delta == 0, ``execute_launch_config`` does not create any job. It also logs """ d = execute_launch_config(self.log, "tid", self.state, "launch", "group", 0) self.assertIsNone(self.successResultOf(d)) self.log.msg.assert_called_once_with("Launching {delta} servers.", delta=0) self.assertEqual(len(self.jobs), 0)
def test_job_success(self): """ ``execute_launch_config`` sets it up so that when a job succeeds, it is removed from pending and the server is added to active. It is also logged. """ s = GroupState('tenant', 'group', 'name', {}, {'1': {}}, None, {}, False) def fake_modify_state(callback, *args, **kwargs): callback(self.group, s, *args, **kwargs) self.group.modify_state.side_effect = fake_modify_state supervisor.execute_launch_config(self.log, '1', self.fake_state, 'launch', self.group, 1) self.execute_config_deferreds[0].callback({'id': 's1'}) self.assertEqual(s.pending, {}) # job removed self.assertIn('s1', s.active) # active server added
def test_propagates_add_job_failures(self): """ ``execute_launch_config`` fails if ``add_job`` raises an error """ self.fake_state.add_job.side_effect = AssertionError d = supervisor.execute_launch_config(self.log, '1', self.fake_state, 'launch', self.group, 1) failure = self.failureResultOf(d) self.assertTrue(failure.check(AssertionError))
def converge(log, transaction_id, config, scaling_group, state, launch_config, policy, config_value=config_value): """ Apply a policy's change to a scaling group, and attempt to make the resulting state a reality. This does no cooldown checking. This is done by dispatching to the appropriate orchestration backend for the scaling group; currently only direct nova interaction is supported. :param log: A bound log for logging :param str transaction_id: the transaction id :param dict config: the scaling group config :param otter.models.interface.IScalingGroup scaling_group: the scaling group object :param otter.models.interface.GroupState state: the group state :param dict launch_config: the scaling group launch config :param dict policy: the policy configuration dictionary :return: a ``Deferred`` that fires with the updated :class:`otter.models.interface.GroupState` if successful. If no changes are to be made to the group, None will synchronously be returned. """ if tenant_is_enabled(scaling_group.tenant_id, config_value): # For convergence tenants, find delta based on group's desired # capacity delta = apply_delta(log, state.desired, state, config, policy) if delta == 0: # No change in servers. Return None synchronously return None else: return defer.succeed(state) # For non-convergence tenants, the value used for desired-capacity is # the sum of active+pending, which is 0, so the delta ends up being # the min entities due to constraint calculation. delta = calculate_delta(log, state, config, policy) execute_log = log.bind(server_delta=delta) if delta == 0: execute_log.msg("no change in servers") return None elif delta > 0: execute_log.msg("executing launch configs") deferred = execute_launch_config( execute_log, transaction_id, state, launch_config, scaling_group, delta) else: # delta < 0 (scale down) execute_log.msg("scaling down") deferred = exec_scale_down(execute_log, transaction_id, state, scaling_group, -delta) deferred.addCallback(_do_convergence_audit_log, log, delta, state) return deferred
def test_delta_jobs_started(self): """ If delta > 0, ``execute_launch_config`` creates and starts delta jobs. It adds the jobs to the state and its completion deferreds to supervisor's pool. It also logs """ d = execute_launch_config(self.log, "tid", self.state, "launch", "group", 3) self.assertIsNone(self.successResultOf(d)) self.log.msg.assert_called_once_with("Launching {delta} servers.", delta=3) self.assertEqual(len(self.jobs), 3) for job in self.jobs: self.assertEqual(job.args, (self.log, "tid", "group", self.supervisor)) self.assertEqual(job.launch, "launch") self.assertIn(job.job_id, self.state.pending) self.assertIn(job.d, self.supervisor.deferred_pool)
def converge(log, transaction_id, config, scaling_group, state, launch_config, policy): """ Apply a policy's change to a scaling group, and attempt to make the resulting state a reality. This does no cooldown checking. This is done by dispatching to the appropriate orchestration backend for the scaling group; currently only direct nova interaction is supported. :param log: A bound log for logging :param str transaction_id: the transaction id :param dict config: the scaling group config :param otter.models.interface.IScalingGroup scaling_group: the scaling group object :param otter.models.interface.GroupState state: the group state :param dict launch_config: the scaling group launch config :param dict policy: the policy configuration dictionary :return: a ``Deferred`` that fires with the updated :class:`otter.models.interface.GroupState` if successful. If no changes are to be made to the group, None will synchronously be returned. """ delta = calculate_delta(log, state, config, policy) execute_log = log.bind(server_delta=delta) if delta == 0: execute_log.msg("no change in servers") return None elif delta > 0: execute_log.msg("executing launch configs") deferred = execute_launch_config(execute_log, transaction_id, state, launch_config, scaling_group, delta) else: # delta < 0 (scale down) execute_log.msg("scaling down") deferred = exec_scale_down(execute_log, transaction_id, state, scaling_group, -delta) deferred.addCallback(_do_convergence_audit_log, log, delta, state) return deferred
def test_positive_delta_execute_config_failures_propagated(self): """ ``execute_launch_config`` fails if ``execute_config`` fails for any one case, and propagates the first ``execute_config`` error. """ class ExecuteException(Exception): pass def fake_execute(*args, **kwargs): if len(self.execute_config_deferreds) > 1: return fail(ExecuteException('no more!')) d = Deferred() self.execute_config_deferreds.append(d) return succeed(( str(len(self.execute_config_deferreds)), d)) self.supervisor.execute_config.side_effect = fake_execute d = supervisor.execute_launch_config(self.log, '1', self.fake_state, 'launch', self.group, 3) failure = self.failureResultOf(d) self.assertTrue(failure.check(ExecuteException))