def test_pending_server_delete(self):
"""
When a pending job is cancelled, it is deleted from the job list. When
the server finishes building, then ``execute_launch_config`` is called
to remove the job from pending job list. It then notices that pending
job_id is not there in job list and calls ``execute_delete_server``
to delete the server.
"""
self.supervisor.execute_delete_server.return_value = succeed(None)
s = GroupState('tenant', 'group', 'name', {}, {'1': {}}, None, {}, False)
def fake_modify_state(callback, *args, **kwargs):
callback(self.group, s, *args, **kwargs)
self.group.modify_state.side_effect = fake_modify_state
supervisor.execute_launch_config(self.log, '1', self.fake_state,
'launch', self.group, 1)
s.remove_job('1')
self.execute_config_deferreds[0].callback({'id': 's1'})
# first bind is system='otter.job.launch', second is job_id='1'
self.del_job.assert_called_once_with(
matches(IsInstance(self.log.__class__)), '1', self.group,
{'id': 's1'}, self.supervisor)
self.del_job.return_value.start.assert_called_once_with()
def test_job_failure(self):
"""
``execute_launch_config`` sets it up so that when a job fails, it is
removed from pending. It is also lgoged.
"""
s = GroupState('tenant', 'group', 'name', {}, {'1': {}}, None, {}, False)
written = []
# modify state writes on callback, doesn't write on error
def fake_modify_state(callback, *args, **kwargs):
d = maybeDeferred(callback, self.group, s, *args, **kwargs)
d.addCallback(written.append)
return d
self.group.modify_state.side_effect = fake_modify_state
supervisor.execute_launch_config(self.log, '1', self.fake_state,
'launch', self.group, 1)
f = Failure(Exception('meh'))
self.execute_config_deferreds[0].errback(f)
# job is removed and no active servers added
self.assertEqual(s, GroupState('tenant', 'group', 'name', {}, {}, None, {},
False))
# state is written
self.assertEqual(len(written), 1)
self.assertEqual(written[0], s)
self.log.err.assert_called_with(f, 'Launching server failed',
system="otter.job.launch",
image_ref="Unable to pull image ref.",
flavor_ref="Unable to pull flavor ref.",
job_id='1')
def test_add_job_called_with_new_jobs(self):
"""
``execute_launch_config`` calls ``add_job`` on the state for every job
that has been started
"""
supervisor.execute_launch_config(self.log, '1', self.fake_state,
'launch', self.group, 3)
self.fake_state.add_job.assert_has_calls(
[mock.call(str(i)) for i in (1, 2, 3)])
self.assertEqual(self.fake_state.add_job.call_count, 3)
def test_positive_delta_execute_config_called_delta_times(self):
"""
If delta > 0, ``execute_launch_config`` calls
``supervisor.execute_config`` delta times.
"""
supervisor.execute_launch_config(self.log, '1', self.fake_state,
'launch', self.group, 5)
self.assertEqual(self.supervisor.execute_config.mock_calls,
[mock.call(matches(IsInstance(self.log.__class__)), '1',
self.group, 'launch')] * 5)
def test_modify_state_failure_logged(self):
"""
If the job succeeded but modifying the state fails, that error is
logged.
"""
self.group.modify_state.side_effect = AssertionError
supervisor.execute_launch_config(self.log, '1', self.fake_state,
'launch', self.group, 1)
self.execute_config_deferreds[0].callback({'id': 's1'})
self.log.err.assert_called_once_with(
CheckFailure(AssertionError), system="otter.job.launch",
image_ref="Unable to pull image ref.",
flavor_ref="Unable to pull flavor ref.", job_id='1')
def test_on_job_completion_modify_state_called(self):
"""
``execute_launch_config`` sets it up so that the group's
``modify_state``state is called with the result as an arg whenever a
job finishes, whether successfully or not
"""
supervisor.execute_launch_config(self.log, '1', self.fake_state,
'launch', self.group, 3)
self.execute_config_deferreds[0].callback({'id': '1'}) # job id 1
self.execute_config_deferreds[1].errback(Exception('meh')) # job id 2
self.execute_config_deferreds[2].callback({'id': '3'}) # job id 3
self.assertEqual(self.group.modify_state.call_count, 3)
def converge(log,
transaction_id,
config,
scaling_group,
state,
launch_config,
policy,
config_value=config_value):
"""
Apply a policy's change to a scaling group, and attempt to make the
resulting state a reality. This does no cooldown checking.
This is done by dispatching to the appropriate orchestration backend for
the scaling group; currently only direct nova interaction is supported.
:param log: A bound log for logging
:param str transaction_id: the transaction id
:param dict config: the scaling group config
:param otter.models.interface.IScalingGroup scaling_group: the scaling
group object
:param otter.models.interface.GroupState state: the group state
:param dict launch_config: the scaling group launch config
:param dict policy: the policy configuration dictionary
:return: a ``Deferred`` that fires with the updated
:class:`otter.models.interface.GroupState` if successful. If no changes
are to be made to the group, None will synchronously be returned.
"""
if tenant_is_enabled(scaling_group.tenant_id, config_value):
# For convergence tenants, find delta based on group's desired
# capacity
delta = apply_delta(log, state.desired, state, config, policy)
if delta == 0:
# No change in servers. Return None synchronously
return None
else:
return defer.succeed(state)
# For non-convergence tenants, the value used for desired-capacity is
# the sum of active+pending, which is 0, so the delta ends up being
# the min entities due to constraint calculation.
delta = calculate_delta(log, state, config, policy)
execute_log = log.bind(server_delta=delta)
if delta == 0:
execute_log.msg("no change in servers")
return None
elif delta > 0:
execute_log.msg("executing launch configs")
deferred = execute_launch_config(execute_log, transaction_id, state,
launch_config, scaling_group, delta)
else:
# delta < 0 (scale down)
execute_log.msg("scaling down")
deferred = exec_scale_down(execute_log, transaction_id, state,
scaling_group, -delta)
deferred.addCallback(_do_convergence_audit_log, log, delta, state)
return deferred
def test_no_jobs_started(self):
"""
If delta == 0, ``execute_launch_config`` does not create any job. It also logs
"""
d = execute_launch_config(self.log, "tid", self.state, "launch", "group", 0)
self.assertIsNone(self.successResultOf(d))
self.log.msg.assert_called_once_with("Launching {delta} servers.", delta=0)
self.assertEqual(len(self.jobs), 0)
def test_job_success(self):
"""
``execute_launch_config`` sets it up so that when a job succeeds, it is
removed from pending and the server is added to active. It is also
logged.
"""
s = GroupState('tenant', 'group', 'name', {}, {'1': {}}, None, {}, False)
def fake_modify_state(callback, *args, **kwargs):
callback(self.group, s, *args, **kwargs)
self.group.modify_state.side_effect = fake_modify_state
supervisor.execute_launch_config(self.log, '1', self.fake_state,
'launch', self.group, 1)
self.execute_config_deferreds[0].callback({'id': 's1'})
self.assertEqual(s.pending, {}) # job removed
self.assertIn('s1', s.active) # active server added
def test_propagates_add_job_failures(self):
"""
``execute_launch_config`` fails if ``add_job`` raises an error
"""
self.fake_state.add_job.side_effect = AssertionError
d = supervisor.execute_launch_config(self.log, '1', self.fake_state,
'launch', self.group, 1)
failure = self.failureResultOf(d)
self.assertTrue(failure.check(AssertionError))
def converge(log, transaction_id, config, scaling_group, state, launch_config,
policy, config_value=config_value):
"""
Apply a policy's change to a scaling group, and attempt to make the
resulting state a reality. This does no cooldown checking.
This is done by dispatching to the appropriate orchestration backend for
the scaling group; currently only direct nova interaction is supported.
:param log: A bound log for logging
:param str transaction_id: the transaction id
:param dict config: the scaling group config
:param otter.models.interface.IScalingGroup scaling_group: the scaling
group object
:param otter.models.interface.GroupState state: the group state
:param dict launch_config: the scaling group launch config
:param dict policy: the policy configuration dictionary
:return: a ``Deferred`` that fires with the updated
:class:`otter.models.interface.GroupState` if successful. If no changes
are to be made to the group, None will synchronously be returned.
"""
if tenant_is_enabled(scaling_group.tenant_id, config_value):
# For convergence tenants, find delta based on group's desired
# capacity
delta = apply_delta(log, state.desired, state, config, policy)
if delta == 0:
# No change in servers. Return None synchronously
return None
else:
return defer.succeed(state)
# For non-convergence tenants, the value used for desired-capacity is
# the sum of active+pending, which is 0, so the delta ends up being
# the min entities due to constraint calculation.
delta = calculate_delta(log, state, config, policy)
execute_log = log.bind(server_delta=delta)
if delta == 0:
execute_log.msg("no change in servers")
return None
elif delta > 0:
execute_log.msg("executing launch configs")
deferred = execute_launch_config(
execute_log, transaction_id, state, launch_config,
scaling_group, delta)
else:
# delta < 0 (scale down)
execute_log.msg("scaling down")
deferred = exec_scale_down(execute_log, transaction_id, state,
scaling_group, -delta)
deferred.addCallback(_do_convergence_audit_log, log, delta, state)
return deferred
def test_delta_jobs_started(self):
"""
If delta > 0, ``execute_launch_config`` creates and starts delta jobs.
It adds the jobs to the state and its completion deferreds to supervisor's pool.
It also logs
"""
d = execute_launch_config(self.log, "tid", self.state, "launch", "group", 3)
self.assertIsNone(self.successResultOf(d))
self.log.msg.assert_called_once_with("Launching {delta} servers.", delta=3)
self.assertEqual(len(self.jobs), 3)
for job in self.jobs:
self.assertEqual(job.args, (self.log, "tid", "group", self.supervisor))
self.assertEqual(job.launch, "launch")
self.assertIn(job.job_id, self.state.pending)
self.assertIn(job.d, self.supervisor.deferred_pool)
def converge(log, transaction_id, config, scaling_group, state, launch_config,
policy):
"""
Apply a policy's change to a scaling group, and attempt to make the
resulting state a reality. This does no cooldown checking.
This is done by dispatching to the appropriate orchestration backend for
the scaling group; currently only direct nova interaction is supported.
:param log: A bound log for logging
:param str transaction_id: the transaction id
:param dict config: the scaling group config
:param otter.models.interface.IScalingGroup scaling_group: the scaling
group object
:param otter.models.interface.GroupState state: the group state
:param dict launch_config: the scaling group launch config
:param dict policy: the policy configuration dictionary
:return: a ``Deferred`` that fires with the updated
:class:`otter.models.interface.GroupState` if successful. If no changes
are to be made to the group, None will synchronously be returned.
"""
delta = calculate_delta(log, state, config, policy)
execute_log = log.bind(server_delta=delta)
if delta == 0:
execute_log.msg("no change in servers")
return None
elif delta > 0:
execute_log.msg("executing launch configs")
deferred = execute_launch_config(execute_log, transaction_id, state,
launch_config, scaling_group,
delta)
else:
# delta < 0 (scale down)
execute_log.msg("scaling down")
deferred = exec_scale_down(execute_log, transaction_id, state,
scaling_group, -delta)
deferred.addCallback(_do_convergence_audit_log, log, delta, state)
return deferred
def test_positive_delta_execute_config_failures_propagated(self):
"""
``execute_launch_config`` fails if ``execute_config`` fails for any one
case, and propagates the first ``execute_config`` error.
"""
class ExecuteException(Exception):
pass
def fake_execute(*args, **kwargs):
if len(self.execute_config_deferreds) > 1:
return fail(ExecuteException('no more!'))
d = Deferred()
self.execute_config_deferreds.append(d)
return succeed((
str(len(self.execute_config_deferreds)), d))
self.supervisor.execute_config.side_effect = fake_execute
d = supervisor.execute_launch_config(self.log, '1', self.fake_state,
'launch', self.group, 3)
failure = self.failureResultOf(d)
self.assertTrue(failure.check(ExecuteException))
