def test_remove_active_success(self):
"""
If the server ID is in the active list, ``remove_active`` removes it.
"""
state = GroupState('tid', 'gid', {'1': {}}, {}, None, {}, True)
state.remove_active('1')
self.assertEqual(state.active, {})
def test_remove_job_success(self):
"""
If the job ID is in the pending list, ``remove_job`` removes it.
"""
state = GroupState('tid', 'gid', {}, {'1': {}}, None, {}, True)
state.remove_job('1')
self.assertEqual(state.pending, {})
def test_remove_active_success(self):
"""
If the server ID is in the active list, ``remove_active`` removes it.
"""
state = GroupState('tid', 'gid', 'name', {'1': {}}, {}, None, {}, True)
state.remove_active('1')
self.assertEqual(state.active, {})
def test_remove_job_success(self):
"""
If the job ID is in the pending list, ``remove_job`` removes it.
"""
state = GroupState('tid', 'gid', 'name', {}, {'1': {}}, None, {}, True)
state.remove_job('1')
self.assertEqual(state.pending, {})
def test_pending_server_delete(self):
"""
When a pending job is cancelled, it is deleted from the job list. When
the server finishes building, then ``execute_launch_config`` is called
to remove the job from pending job list. It then notices that pending
job_id is not there in job list and calls ``execute_delete_server``
to delete the server.
"""
self.supervisor.execute_delete_server.return_value = succeed(None)
s = GroupState('tenant', 'group', 'name', {}, {'1': {}}, None, {}, False)
def fake_modify_state(callback, *args, **kwargs):
callback(self.group, s, *args, **kwargs)
self.group.modify_state.side_effect = fake_modify_state
supervisor.execute_launch_config(self.log, '1', self.fake_state,
'launch', self.group, 1)
s.remove_job('1')
self.execute_config_deferreds[0].callback({'id': 's1'})
# first bind is system='otter.job.launch', second is job_id='1'
self.del_job.assert_called_once_with(
matches(IsInstance(self.log.__class__)), '1', self.group,
{'id': 's1'}, self.supervisor)
self.del_job.return_value.start.assert_called_once_with()
def test_add_job_success(self):
"""
If the job ID is not in the pending list, ``add_job`` adds it along with
the creation time.
"""
state = GroupState('tid', 'gid', 'name', {}, {}, None, {}, True,
now=lambda: 'datetime')
state.add_job('1')
self.assertEqual(state.pending, {'1': {'created': 'datetime'}})
def test_add_active_success_preserves_creation_time(self):
"""
If the server ID is not in the active list, ``add_active`` adds it along
with server info, and does not change the server info's creation time.
"""
state = GroupState('tid', 'gid', 'name', {}, {}, None, {}, True,
now=lambda: 'other_now')
state.add_active('1', {'stuff': 'here', 'created': 'now'})
self.assertEqual(state.active,
{'1': {'stuff': 'here', 'created': 'now'}})
def test_mark_executed_updates_policy_and_group(self):
"""
Marking executed updates the policy touched and group touched to the
same time.
"""
t = ['0']
state = GroupState('tid', 'gid', 'name', {}, {}, 'date', {}, True, now=t.pop)
state.mark_executed('pid')
self.assertEqual(state.group_touched, '0')
self.assertEqual(state.policy_touched, {'pid': '0'})
def test_mark_executed_updates_policy_and_group(self):
"""
Marking executed updates the policy touched and group touched to the
same time.
"""
t = ['0']
state = GroupState('tid', 'gid', {}, {}, 'date', {}, True, now=t.pop)
state.mark_executed('pid')
self.assertEqual(state.group_touched, '0')
self.assertEqual(state.policy_touched, {'pid': '0'})
def test_add_active_success_adds_creation_time(self):
"""
If the server ID is not in the active list, ``add_active`` adds it along
with server info, and adds the creation time to server info that
does not already have it.
"""
state = GroupState('tid', 'gid', 'name', {}, {}, None, {}, True,
now=lambda: 'datetime')
state.add_active('1', {'stuff': 'here'})
self.assertEqual(state.active,
{'1': {'stuff': 'here', 'created': 'datetime'}})
def test_two_states_are_equal_if_all_vars_are_equal(self):
"""
Two groups with the same parameters (even if now is different) are
equal
"""
self.assertEqual(
GroupState('tid', 'gid', {'1': {}}, {'2': {}}, 'date', {}, True),
GroupState('tid',
'gid', {'1': {}}, {'2': {}},
'date', {},
True,
now=lambda: 'meh'))
def test_add_job_success(self):
"""
If the job ID is not in the pending list, ``add_job`` adds it along with
the creation time.
"""
state = GroupState('tid',
'gid', {}, {},
None, {},
True,
now=lambda: 'datetime')
state.add_job('1')
self.assertEqual(state.pending, {'1': {'created': 'datetime'}})
def test_get_capacity(self):
"""
Getting capacity returns a dictionary with the desired capacity,
active capacity, and pending capacity
"""
state = GroupState('tid', 'gid', 'name',
{str(i): {} for i in range(5)},
{str(i): {} for i in range(6)},
'date', {}, True, now='0')
self.assertEqual(state.get_capacity(), {
'desired_capacity': 11,
'pending_capacity': 6,
'current_capacity': 5
})
def test_add_active_success_preserves_creation_time(self):
"""
If the server ID is not in the active list, ``add_active`` adds it along
with server info, and does not change the server info's creation time.
"""
state = GroupState('tid',
'gid', {}, {},
None, {},
True,
now=lambda: 'other_now')
state.add_active('1', {'stuff': 'here', 'created': 'now'})
self.assertEqual(state.active,
{'1': {
'stuff': 'here',
'created': 'now'
}})
def test_list_group_returns_valid_schema(self, *args):
"""
``list_all_scaling_groups`` produces a repsonse has the correct schema
so long as format returns the right value
"""
self.mock_store.list_scaling_group_states.return_value = defer.succeed(
[GroupState('11111', '1', {}, {1: {}}, None, {}, False)])
body = self.assert_status_code(200)
resp = json.loads(body)
validate(resp, rest_schemas.list_groups_response)
self.assertEqual(
resp, {
"groups": [{
'active': [],
'activeCapacity': 0,
'pendingCapacity': 1,
'desiredCapacity': 1,
'paused': False,
'id': '1',
'links': [{
'href': 'hey',
'rel': 'self'
}]
}],
"groups_links": []
})
def test_job_completion_success_job_deleted_audit_logged(self):
"""
If the job succeeded, but the job ID is no longer in pending, it is
audit logged as a "server.deletable" event.
"""
self.state = GroupState(
"tenant", "group", "name", {}, {}, None, {}, False, ScalingGroupStatus.ACTIVE, desired=0
)
self.job.start(self.mock_launch)
self.completion_deferred.callback({"id": "yay"})
self.successResultOf(self.completion_deferred)
self.log.msg.assert_called_once_with(
("A pending server that is no longer needed is now active, " "and hence deletable. Deleting said server."),
event_type="server.deletable",
server_id="yay",
job_id=self.job_id,
audit_log=True,
system="otter.job.launch",
image_ref="imageID",
flavor_ref="1",
current_active=0,
current_pending=0,
current_desired=0,
)
def _unmarshal_state(state_dict):
return GroupState(state_dict['tenantId'], state_dict['groupId'],
_jsonloads_data(state_dict["active"]),
_jsonloads_data(state_dict["pending"]),
state_dict["groupTouched"],
_jsonloads_data(state_dict["policyTouched"]),
bool(ord(state_dict["paused"])))
def setUp(self):
"""
Replace the store every time with a clean one.
"""
store = MockScalingGroupCollection()
self.mock_log = mock.MagicMock()
manifest = self.successResultOf(
store.create_scaling_group(self.mock_log, self.tenant_id,
config()[0],
launch_server_config()[0]))
self.group_id = manifest['id']
set_store(store)
self.policies_url = '/v1.0/{tenant}/groups/{group}/policies/'.format(
tenant=self.tenant_id, group=self.group_id)
controller_patcher = mock.patch('otter.rest.policies.controller')
self.mock_controller = controller_patcher.start()
self.mock_controller.maybe_execute_scaling_policy.return_value = defer.succeed(
GroupState(self.tenant_id, self.group_id, {}, {}, 'date', {},
False))
self.addCleanup(controller_patcher.stop)
set_config_data({'url_root': 'http://127.0.0.1'})
self.addCleanup(set_config_data, {})
def test_group_touched_is_min_if_None(self):
"""
If a group_touched of None is provided, groupTouched is
'0001-01-01T00:00:00Z'
"""
state = GroupState('tid', 'gid', {}, {}, None, {}, False)
self.assertEqual(state.group_touched, '0001-01-01T00:00:00Z')
def setUp(self):
"""
Mock a fake supervisor, and also a fake log and group.
"""
self.transaction_id = 'transaction_id'
self.job_id = 'job_id'
patch(self, 'otter.supervisor.generate_job_id',
return_value=self.job_id)
self.state = GroupState('tenant', 'group', 'name', {}, {}, None, {},
False, ScalingGroupStatus.ACTIVE)
self.group = mock_group(self.state, 'tenant', 'group')
self.supervisor = iMock(ISupervisor)
self.supervisor.deferred_pool = DeferredPool()
self.completion_deferred = Deferred()
self.supervisor.execute_config.return_value = self.completion_deferred
self.log = mock_log()
self.job = supervisor._Job(self.log, self.transaction_id, self.group,
self.supervisor)
self.del_job = patch(self, 'otter.supervisor._DeleteJob')
self.mock_launch = {'type': 'launch_server',
'args': {'server': {'imageRef': 'imageID',
'flavorRef': '1'}}}
def test_repr_str(self):
"""
repr(GroupState) returns something human readable
"""
state = GroupState('tid', 'gid', {'1': {}}, {}, 'date', {}, True)
self.assertEqual(
repr(state), "GroupState(tid, gid, {'1': {}}, {}, date, {}, True)")
def test_view_state_returns_empty_state(self):
"""
``view_state`` a group state with empty info
"""
result = self.successResultOf(self.group.view_state())
self.assertEqual(
result, GroupState(self.tenant_id, '1', {}, {}, None, {}, False))
def test_add_active_success_adds_creation_time(self):
"""
If the server ID is not in the active list, ``add_active`` adds it along
with server info, and adds the creation time to server info that
does not already have it.
"""
state = GroupState('tid',
'gid', {}, {},
None, {},
True,
now=lambda: 'datetime')
state.add_active('1', {'stuff': 'here'})
self.assertEqual(state.active,
{'1': {
'stuff': 'here',
'created': 'datetime'
}})
def test_add_active_fails(self):
"""
If the server ID is in the active list, ``add_active`` raises an
AssertionError.
"""
state = GroupState('tid', 'gid', {'1': {}}, {}, None, {}, True)
self.assertRaises(AssertionError, state.add_active, '1', {'1': '2'})
self.assertEqual(state.active, {'1': {}})
def test_add_job_fails(self):
"""
If the job ID is in the pending list, ``add_job`` raises an
AssertionError.
"""
state = GroupState('tid', 'gid', {}, {'1': {}}, None, {}, True)
self.assertRaises(AssertionError, state.add_job, '1')
self.assertEqual(state.pending, {'1': {}})
def test_remove_active_fails(self):
"""
If the server ID is not in the active list, ``remove_active`` raises an
AssertionError.
"""
state = GroupState('tid', 'gid', {}, {}, None, {}, True)
self.assertRaises(AssertionError, state.remove_active, '1')
self.assertEqual(state.active, {})
def setUp(self):
"""
Sample group, collection and dispatcher
"""
self.log = mock_log().bind(base_log=True)
self.state = GroupState('tid', 'gid', 'g', {}, {}, None, {}, True,
ScalingGroupStatus.ACTIVE)
self.group = mock_group(self.state)
def _set_group_id(manifest):
self.group_id = manifest['id']
self.policies_url = (
'/v1.0/{tenant}/groups/{group}/policies/'.format(
tenant=self.tenant_id, group=self.group_id))
self.mock_controller.maybe_execute_scaling_policy.return_value = defer.succeed(
GroupState(self.tenant_id, self.group_id, {}, {}, 'date', {},
False))
def test_remove_job_fails(self):
"""
If the job ID is not in the pending list, ``remove_job`` raises an
AssertionError.
"""
state = GroupState('tid', 'gid', 'name', {}, {}, None, {}, True,
ScalingGroupStatus.ACTIVE)
self.assertRaises(AssertionError, state.remove_job, '1')
self.assertEqual(state.pending, {})
def test_default_desired_capacity_is_zero(self):
"""
If no desired capacity is passed, the default value is zero.
"""
self.assertEqual(
GroupState('tid', 'gid', 'name', {
'1': {}
}, {
'2': {}
}, 'date', {}, True, ScalingGroupStatus.ACTIVE).desired, 0)
def setUp(self):
self.patch(sh, "trigger_convergence", intent_func("tg"))
self.state = GroupState("tid",
"gid",
'group-name', {}, {},
None, {},
False,
ScalingGroupStatus.ACTIVE,
desired=2)
self.manifest = {"state": self.state}
def test_two_states_are_unequal_if_vars_different(self):
"""
Two groups with any different parameters are unequal
"""
args = ('tid', 'gid', {}, {}, 'date', {}, True)
def perterb(args, index):
copy = [arg for arg in args]
if isinstance(copy[index], str):
copy[index] += '_'
elif isinstance(copy[index], bool):
copy[index] = not copy[index]
else: # it's a dict
copy[index] = {'1': {}}
return copy
for i in range(len(args)):
self.assertNotEqual(GroupState(*args),
GroupState(*(perterb(args, i))))
def test_list_group_formats_gets_and_formats_all_states(self, mock_format):
"""
``list_all_scaling_groups`` translates a list of IScalingGroup to a
list of states that are all formatted
"""
states = [
GroupState('11111', '2', {}, {}, None, {}, False),
GroupState('11111', '2', {}, {}, None, {}, False)
]
self.mock_store.list_scaling_group_states.return_value = defer.succeed(
states)
self.assert_status_code(200)
self.mock_store.list_scaling_group_states.assert_called_once_with(
mock.ANY, '11111')
mock_format.assert_has_calls([mock.call(state) for state in states])
self.assertEqual(len(mock_format.mock_calls), 2)
def test_get_capacity(self):
"""
Getting capacity returns a dictionary with the desired capacity,
active capacity, and pending capacity
"""
state = GroupState('tid',
'gid',
'name', {str(i): {}
for i in range(5)},
{str(i): {}
for i in range(6)},
'date', {},
True,
ScalingGroupStatus.ACTIVE,
now='0')
self.assertEqual(state.get_capacity(), {
'desired_capacity': 11,
'pending_capacity': 6,
'current_capacity': 5
})
def test_a_state_is_not_equal_to_something_else(self):
"""
The classes of the two objects have to be the same.
"""
_GroupState = namedtuple('_GroupState', [
'tenant_id', 'group_id', 'active', 'pending', 'group_touched',
'policy_touched', 'paused'
])
self.assertNotEqual(
_GroupState('tid', 'gid', {'1': {}}, {'2': {}}, 'date', {}, True),
GroupState('tid', 'gid', {'1': {}}, {'2': {}}, 'date', {}, True))
def setUp(self):
"""
Fake supervisor, group and state
"""
self.tid = 'trans_id'
self.log = mock_log()
self.group = iMock(IScalingGroup, tenant_id='tenant', uuid='group')
self.state = GroupState('tid', 'gid', 'g', {'s0': {'id': 's0'}}, {},
None, None, None, desired=1)
self.supervisor = FakeSupervisor()
set_supervisor(self.supervisor)
self.addCleanup(set_supervisor, None)
def setUp(self):
"""
Fake supervisor, group and state
"""
self.tid = 'trans_id'
self.log = mock_log()
self.state = GroupState('tid', 'gid', 'g', {'s0': {'id': 's0'}}, {},
None, None, None, desired=1)
self.group = mock_group(self.state)
self.gen_jobid = patch(self, 'otter.supervisor.generate_job_id', return_value='jid')
self.supervisor = FakeSupervisor()
set_supervisor(self.supervisor)
self.addCleanup(set_supervisor, None)
def test_modify_state(self):
"""
``modify_state`` saves the new state returned by the function if the
tenant ids and group ids match
"""
new_state = GroupState(self.tenant_id, self.group_id, {1: {}}, {},
'date', {}, True)
def modifier(group, state):
return new_state
self.group.modify_state(modifier)
self.assertEqual(self.group.state, new_state)
def test_repr_str(self):
"""
repr(GroupState) returns something human readable
"""
state = GroupState('tid',
'gid',
'name', {'1': {}}, {},
'date', {},
True,
ScalingGroupStatus.ACTIVE,
desired=5)
self.assertEqual(
repr(state),
"GroupState(tid, gid, name, 5, {'1': {}}, {}, date, {}, True, "
"<ScalingGroupStatus=ACTIVE>)")
def setUp(self):
"""
Fake supervisor, group and state
"""
self.tid = "trans_id"
self.log = mock_log()
self.state = GroupState(
"tid", "gid", "g", {"s0": {"id": "s0"}}, {}, None, None, None, ScalingGroupStatus.ACTIVE, desired=1
)
self.group = mock_group(self.state)
self.gen_jobid = patch(self, "otter.supervisor.generate_job_id", return_value="jid")
self.supervisor = FakeSupervisor()
set_supervisor(self.supervisor)
self.addCleanup(set_supervisor, None)
self.group.view_config.return_value = succeed({"minEntities": 0})
self.group.view_launch_config.return_value = succeed("launch")
def setUp(self):
"""
Fake supervisor, group and state
"""
self.tid = 'trans_id'
self.log = mock_log()
self.state = GroupState('tid', 'gid', 'g', {'s0': {'id': 's0'}}, {},
None, None, None, ScalingGroupStatus.ACTIVE,
desired=1)
self.group = mock_group(self.state)
self.gen_jobid = patch(self, 'otter.supervisor.generate_job_id',
return_value='jid')
self.supervisor = FakeSupervisor()
set_supervisor(self.supervisor)
self.addCleanup(set_supervisor, None)
self.group.view_config.return_value = succeed({'minEntities': 0})
self.group.view_launch_config.return_value = succeed('launch')
def setUp(self):
"""
Mock a fake supervisor, and also a fake log and group.
"""
self.transaction_id = "transaction_id"
self.job_id = "job_id"
patch(self, "otter.supervisor.generate_job_id", return_value=self.job_id)
self.state = GroupState("tenant", "group", "name", {}, {}, None, {}, False, ScalingGroupStatus.ACTIVE)
self.group = mock_group(self.state, "tenant", "group")
self.supervisor = iMock(ISupervisor)
self.supervisor.deferred_pool = DeferredPool()
self.completion_deferred = Deferred()
self.supervisor.execute_config.return_value = self.completion_deferred
self.log = mock_log()
self.job = supervisor._Job(self.log, self.transaction_id, self.group, self.supervisor)
self.del_job = patch(self, "otter.supervisor._DeleteJob")
self.mock_launch = {"type": "launch_server", "args": {"server": {"imageRef": "imageID", "flavorRef": "1"}}}
def __init__(self, log, tenant_id, uuid, collection, creation=None):
"""
Creates a MockScalingGroup object. If the actual scaling group should
be created, a creation argument is provided containing the config, the
launch config, and optional scaling policies.
"""
self.log = log.bind(system=self.__class__.__name__)
self.tenant_id = tenant_id
self.uuid = uuid
self.state = GroupState(self.tenant_id, self.uuid, {}, {}, None, {},
False)
self._collection = collection
if creation is not None:
self.error = None
self.config = {
'name': "",
'cooldown': 0,
'minEntities': 0,
'maxEntities': None, # no upper limit
'metadata': {}
}
self.update_config(creation['config'], partial_update=True)
self.launch = creation['launch']
self.policies = {}
if creation['policies']:
self.create_policies(creation['policies'])
self.webhooks = defaultdict(dict)
else:
self.error = NoSuchScalingGroupError(tenant_id, uuid)
self.config = None
self.launch = None
self.policies = None
self.webhooks = None
class PrivateJobHelperTestCase(SynchronousTestCase):
"""
Tests for the private helper class `_Job`
"""
def setUp(self):
"""
Mock a fake supervisor, and also a fake log and group.
"""
self.transaction_id = "transaction_id"
self.job_id = "job_id"
patch(self, "otter.supervisor.generate_job_id", return_value=self.job_id)
self.state = GroupState("tenant", "group", "name", {}, {}, None, {}, False, ScalingGroupStatus.ACTIVE)
self.group = mock_group(self.state, "tenant", "group")
self.supervisor = iMock(ISupervisor)
self.supervisor.deferred_pool = DeferredPool()
self.completion_deferred = Deferred()
self.supervisor.execute_config.return_value = self.completion_deferred
self.log = mock_log()
self.job = supervisor._Job(self.log, self.transaction_id, self.group, self.supervisor)
self.del_job = patch(self, "otter.supervisor._DeleteJob")
self.mock_launch = {"type": "launch_server", "args": {"server": {"imageRef": "imageID", "flavorRef": "1"}}}
def test_start_binds_invalid_image_ref_to_log(self):
"""
`start` binds the image ID to a string that says that we were unable
to find the image id in the logs, if the image ref could not be found
"""
del self.mock_launch["args"]["server"]["imageRef"]
self.job.start(self.mock_launch)
self.assertEqual(
self.job.log,
matches(
IsBoundWith(
system="otter.job.launch", image_ref="Unable to pull image ref.", flavor_ref="1", job_id="job_id"
)
),
)
def test_start_binds_invalid_flavor_ref_to_log(self):
"""
`start` binds the flavor ID to a string that says that we were unable
to find the flavor id in the logs, if the flavor ref could not be found
"""
del self.mock_launch["args"]["server"]["flavorRef"]
self.job.start(self.mock_launch)
self.assertEqual(
self.job.log,
matches(
IsBoundWith(
system="otter.job.launch",
image_ref="imageID",
flavor_ref="Unable to pull flavor ref.",
job_id="job_id",
)
),
)
def test_start_calls_supervisor(self):
"""
`start` calls the supervisor's `execute_config` method with
log bound with imageRef and flavorRef from launch config
"""
self.job.start(self.mock_launch)
self.supervisor.execute_config.assert_called_once_with(
matches(IsBoundWith(system="otter.job.launch", image_ref="imageID", flavor_ref="1", job_id="job_id")),
self.transaction_id,
self.group,
self.mock_launch,
)
def test_modify_state_called_on_job_completion_success(self):
"""
If the job succeeded, and modify_state is called
"""
self.job.start("launch")
self.assertEqual(self.group.modify_state.call_count, 0)
self.completion_deferred.callback({"id": "blob"})
self.assertEqual(self.group.modify_state.call_count, 1)
def test_modify_state_called_on_job_completion_failure(self):
"""
If the job failed, modify_state is called
"""
self.job.start("launch")
self.assertEqual(self.group.modify_state.call_count, 0)
self.completion_deferred.errback(Exception("e"))
self.assertEqual(self.group.modify_state.call_count, 1)
def test_job_completion_success_job_marked_as_active(self):
"""
If the job succeeded, and the job ID is still in pending, it is removed
and added to active.
"""
self.state.add_job(self.job_id)
self.job.start("launch")
self.completion_deferred.callback({"id": "active"})
self.assertIsNone(self.successResultOf(self.completion_deferred))
self.assertEqual(self.group.modify_state_values, [self.state])
self.assertEqual(self.state.pending, {})
self.assertEqual(self.state.active, {"active": matches(ContainsDict({"id": Equals("active")}))})
def test_job_completion_success_audit_logged(self):
"""
If the job succeeded, and the job ID is still in pending, it is audit
logged as a "server.active" event, and the new state after the server
has been moved to active is logged.
"""
self.state.add_job(self.job_id)
self.state.desired = 1
self.job.start(self.mock_launch)
self.completion_deferred.callback({"id": "yay"})
self.successResultOf(self.completion_deferred)
self.log.msg.assert_called_once_with(
"Server is active.",
event_type="server.active",
server_id="yay",
job_id=self.job_id,
audit_log=True,
system="otter.job.launch",
image_ref="imageID",
flavor_ref="1",
current_active=1,
current_pending=0,
current_desired=1,
)
def test_job_completion_success_job_deleted_pending(self):
"""
If the job succeeded, but the job ID is no longer in pending, the
server is deleted and the state not changed. No error is logged.
"""
d = Deferred()
self.del_job.return_value.start.return_value = d
self.job.start("launch")
self.completion_deferred.callback({"id": "active"})
self.assertIsNone(self.successResultOf(self.completion_deferred))
self.assertEqual(self.group.modify_state_values, [self.state])
self.assertEqual(self.state.pending, {})
self.assertEqual(self.state.active, {})
self.del_job.assert_called_once_with(
matches(IsInstance(self.log.__class__)), self.transaction_id, self.group, {"id": "active"}, self.supervisor
)
self.assertIn(d, self.supervisor.deferred_pool)
self.assertEqual(self.log.err.call_count, 0)
def test_job_completion_success_job_deleted_audit_logged(self):
"""
If the job succeeded, but the job ID is no longer in pending, it is
audit logged as a "server.deletable" event.
"""
self.state = GroupState(
"tenant", "group", "name", {}, {}, None, {}, False, ScalingGroupStatus.ACTIVE, desired=0
)
self.job.start(self.mock_launch)
self.completion_deferred.callback({"id": "yay"})
self.successResultOf(self.completion_deferred)
self.log.msg.assert_called_once_with(
("A pending server that is no longer needed is now active, " "and hence deletable. Deleting said server."),
event_type="server.deletable",
server_id="yay",
job_id=self.job_id,
audit_log=True,
system="otter.job.launch",
image_ref="imageID",
flavor_ref="1",
current_active=0,
current_pending=0,
current_desired=0,
)
def test_job_completion_failure_job_removed(self):
"""
If the job failed, the job ID is removed from the pending state. The
failure is logged.
"""
self.state.add_job(self.job_id)
self.job.start(self.mock_launch)
self.completion_deferred.errback(DummyException("e"))
self.assertIsNone(self.successResultOf(self.completion_deferred))
self.assertEqual(self.group.modify_state_values, [self.state])
self.assertEqual(self.state.pending, {})
self.assertEqual(self.state.active, {})
self.log.err.assert_called_once_with(
CheckFailure(DummyException),
"Launching server failed",
system="otter.job.launch",
image_ref="imageID",
job_id=self.job_id,
flavor_ref="1",
current_active=0,
current_pending=0,
current_desired=0,
)
def test_job_completion_failure_job_deleted_pending(self):
"""
If the job failed, but the job ID is no longer in pending, the job id
is not removed (and hence no error occurs). The only error logged is
the failure. Nothing else in the state changes.
"""
self.job.start(self.mock_launch)
self.completion_deferred.errback(DummyException("e"))
self.assertIsNone(self.successResultOf(self.completion_deferred))
self.assertEqual(self.group.modify_state_values, [self.state])
self.assertEqual(self.state.pending, {})
self.assertEqual(self.state.active, {})
self.log.err.assert_called_with(
CheckFailure(DummyException),
"Launching server failed",
system="otter.job.launch",
image_ref="imageID",
job_id=self.job_id,
flavor_ref="1",
current_active=0,
current_pending=0,
current_desired=0,
)
def test_job_completion_success_NoSuchScalingGroupError(self):
"""
If a job is completed successfully, but `modify_state` fails with a
`NoSuchScalingGroupError`, then the group has been deleted and so the
server is deleted
"""
self.group.modify_state.side_effect = lambda *args, **kw: fail(NoSuchScalingGroupError("tenant", "group"))
d = Deferred()
self.del_job.return_value.start.return_value = d
self.job.start("launch")
self.completion_deferred.callback({"id": "active"})
self.del_job.assert_called_once_with(
matches(IsInstance(self.log.__class__)), self.transaction_id, self.group, {"id": "active"}, self.supervisor
)
self.del_job.return_value.start.assert_called_once_with()
self.assertIn(d, self.supervisor.deferred_pool)
def test_job_completion_success_NoSuchScalingGroupError_audit_logged(self):
"""
If the job succeeded, but the job ID is no longer in pending, it is
audit logged as a "server.deletable" event.
"""
self.group.modify_state.side_effect = lambda *args, **kw: fail(NoSuchScalingGroupError("tenant", "group"))
self.job.start(self.mock_launch)
self.completion_deferred.callback({"id": "yay"})
self.successResultOf(self.completion_deferred)
self.log.msg.assert_called_once_with(
(
"A pending server belonging to a deleted scaling group "
"({scaling_group_id}) is now active, and hence deletable. "
"Deleting said server."
),
event_type="server.deletable",
server_id="yay",
job_id=self.job_id,
audit_log=True,
system="otter.job.launch",
image_ref="imageID",
flavor_ref="1",
)
def test_job_completion_failure_NoSuchScalingGroupError(self):
"""
If a job fails, but `modify_state` fails with a
`NoSuchScalingGroupError`, then the group has been deleted and the
failure can be ignored (not logged)
"""
self.group.modify_state.side_effect = lambda *args, **kw: fail(NoSuchScalingGroupError("tenant", "group"))
self.job.start("launch")
self.completion_deferred.callback({"id": "active"})
self.assertEqual(self.log.err.call_count, 0)
def test_modify_state_failure_logged(self):
"""
If `modify_state` fails with a non-`NoSuchScalingGroupError`, the error
is logged
"""
self.group.modify_state.side_effect = lambda *args, **kw: fail(DummyException("e"))
self.job.start(self.mock_launch)
self.completion_deferred.callback({"id": "active"})
self.log.err.assert_called_once_with(
CheckFailure(DummyException),
system="otter.job.launch",
image_ref="imageID",
flavor_ref="1",
job_id=self.job_id,
)
class RemoveServerTests(SynchronousTestCase):
"""
Tests for :func:`otter.supervisor.remove_server_from_group`
"""
def setUp(self):
"""
Fake supervisor, group and state
"""
self.tid = "trans_id"
self.log = mock_log()
self.state = GroupState(
"tid", "gid", "g", {"s0": {"id": "s0"}}, {}, None, None, None, ScalingGroupStatus.ACTIVE, desired=1
)
self.group = mock_group(self.state)
self.gen_jobid = patch(self, "otter.supervisor.generate_job_id", return_value="jid")
self.supervisor = FakeSupervisor()
set_supervisor(self.supervisor)
self.addCleanup(set_supervisor, None)
self.group.view_config.return_value = succeed({"minEntities": 0})
self.group.view_launch_config.return_value = succeed("launch")
def _remove_server(self, replace=True, purge=True, server_id="s0"):
"""
Try to remove a server from the group.
"""
d = remove_server_from_group(self.log, self.tid, server_id, replace, purge, self.group, self.state)
return d
def _assert_server_in_group_state(self, state):
"""
Assert that the server is still in the group state.
"""
self.assertEqual(state.active, {"s0": {"id": "s0"}})
def _assert_server_not_in_group_state(self, state):
"""
Assert that the server is not in the group state.
"""
self.assertNotIn("s0", state.active)
def _assert_delete_scheduled(self):
"""
Assert that the server was scheduled for deletion.
"""
self.assertEqual(
self.supervisor.del_calls[-1],
(matches(IsBoundWith(server_id="s0", system="otter.job.delete")), self.tid, self.group, {"id": "s0"}),
)
def _assert_delete_not_scheduled(self):
"""
Assert that the server was scheduled for deletion.
"""
self.assertEqual(self.supervisor.del_calls, [])
def _assert_create_scheduled(self, state):
"""
Assert that a new server is being created. Specifically, checks
that the id is now pending, and that the supervisor has a
create server call.
"""
self.assertIn("jid", state.pending)
self.assertEqual(
self.supervisor.exec_calls[-1],
(
matches(IsBoundWith(image_ref=mock.ANY, flavor_ref=mock.ANY, system="otter.job.launch", job_id="jid")),
self.tid,
self.group,
"launch",
),
)
def _assert_create_not_scheduled(self, state):
"""
Assert that a new server is not being created. Specifically,
checks that the id does not exist in pending, and no creation
calls were issued.
"""
self.assertNotIn("jid", state.pending)
self.assertEqual(self.supervisor.exec_calls, [])
def _assert_metadata_scrubbing_scheduled(self, expected_server_id="s0"):
"""
Assert that otter-specific metadata scrubbing was scheduled.
"""
_, txn_id, tenant_id, server_id = self.supervisor.scrub_calls[-1]
self.assertEqual(txn_id, self.tid)
self.assertEqual(tenant_id, "tenant")
self.assertEqual(server_id, expected_server_id)
def _assert_metadata_scrubbing_not_scheduled(self):
"""
Asserts that no metadata scrubbing was scheduled.
"""
self.assertEqual(len(self.supervisor.scrub_calls), 0)
def test_server_not_found(self):
"""
If specific server is not in the group, :class:`ServerNotFoundError`
is raised.
"""
self.assertRaises(ServerNotFoundError, self._remove_server, server_id="BOGUS")
self._assert_server_in_group_state(self.state)
self._assert_create_not_scheduled(self.state)
self._assert_delete_not_scheduled()
self._assert_metadata_scrubbing_not_scheduled()
self.assertEqual(self.state.desired, 1)
def test_not_deleted_below_min(self):
"""
:class:`CannotDeleteServerBelowMinError` is raised if the current
(active + pending) number of servers is already the minimum.
"""
self.state.add_job("j1")
self.group.view_config.return_value = succeed({"minEntities": 2})
d = self._remove_server(replace=False, purge=True)
self.failureResultOf(d, CannotDeleteServerBelowMinError)
self._assert_server_in_group_state(self.state)
self._assert_delete_not_scheduled()
self._assert_create_not_scheduled(self.state)
self._assert_metadata_scrubbing_not_scheduled()
self.assertEqual(self.state.desired, 1)
def test_replaced_and_removed(self):
"""
Server is removed, purged and replaced.
"""
d = self._remove_server(replace=True, purge=True)
state = self.successResultOf(d)
self._assert_server_not_in_group_state(state)
self._assert_delete_scheduled()
self._assert_create_scheduled(state)
self.assertEqual(self.state.desired, 1)
def test_not_replaced(self):
"""
Server is removed and purged from Nova, but not replaced. The
desired is reduced by 1.
"""
d = self._remove_server(replace=False, purge=True)
state = self.successResultOf(d)
self._assert_server_not_in_group_state(state)
self._assert_delete_scheduled()
self._assert_create_not_scheduled(state)
self._assert_metadata_scrubbing_not_scheduled()
self.assertEqual(state.desired, 0)
def test_not_replaced_and_not_purged(self):
"""
The server is removed, but not replaced and not purged.
"""
d = self._remove_server(replace=False, purge=False)
state = self.successResultOf(d)
self._assert_server_not_in_group_state(state)
self._assert_delete_not_scheduled()
self._assert_create_not_scheduled(state)
self._assert_metadata_scrubbing_scheduled()
self.assertEqual(state.desired, 0)
def test_replaced_but_not_purged(self):
"""
The server is removed, replaced, but not purged.
"""
d = self._remove_server(replace=True, purge=False)
state = self.successResultOf(d)
self._assert_server_not_in_group_state(state)
self._assert_delete_not_scheduled()
self._assert_create_scheduled(state)
self._assert_metadata_scrubbing_scheduled()
self.assertEqual(state.desired, 1)
def sample_group_state(tid='tid', gid='gid'):
""" GroupState object for test """
return GroupState(tid, gid, 'g', {}, {}, None, {}, False,
ScalingGroupStatus.ACTIVE)
def modifier(group, state):
return GroupState(self.tenant_id, 'meh', {}, {}, 'date', {}, True)
class RemoveServerTests(SynchronousTestCase):
"""
Tests for :func:`otter.supervisor.remove_server_from_group`
"""
def setUp(self):
"""
Fake supervisor, group and state
"""
self.tid = 'trans_id'
self.log = mock_log()
self.group = iMock(IScalingGroup, tenant_id='tenant', uuid='group')
self.state = GroupState('tid', 'gid', 'g', {'s0': {'id': 's0'}}, {},
None, None, None, desired=1)
self.supervisor = FakeSupervisor()
set_supervisor(self.supervisor)
self.addCleanup(set_supervisor, None)
def test_server_not_found(self):
"""
If specific server is not in the group `ServerNotFoundError` is raised
"""
self.assertRaises(
ServerNotFoundError, remove_server_from_group, self.log,
self.tid, 's2', True, self.group, self.state)
# no server launched or deleted
self.assertEqual(self.supervisor.exec_calls, [])
self.assertEqual(self.supervisor.del_calls, [])
# desired & active/pending not changed
self.assertEqual(self.state.desired, 1)
self.assertEqual(self.state.active, {'s0': {'id': 's0'}})
self.assertEqual(self.state.pending, {})
def _check_removed(self, state):
self.assertNotIn('s0', state.active)
self.assertEqual(self.supervisor.del_calls[-1],
(matches(IsBoundWith(server_id='s0', system='otter.job.delete')),
self.tid, self.group, {'id': 's0'}))
def test_replaced_and_removed(self):
"""
Server is removed and replaced by creating new
"""
self.group.view_launch_config.return_value = succeed('launch')
d = remove_server_from_group(self.log, self.tid, 's0', True, self.group, self.state)
state = self.successResultOf(d)
# server removed?
self._check_removed(state)
# new server added?
self.assertIn(1, state.pending)
self.assertEqual(self.supervisor.exec_calls[-1],
(matches(IsBoundWith(image_ref=mock.ANY, flavor_ref=mock.ANY,
system='otter.job.launch')),
self.tid, self.group, 'launch'))
# desired not changed
self.assertEqual(self.state.desired, 1)
def test_not_replaced_removed(self):
"""
Server is removed, not replaced and desired is reduced by 1
"""
self.group.view_config.return_value = succeed({'minEntities': 0})
d = remove_server_from_group(self.log, self.tid, 's0', False, self.group, self.state)
state = self.successResultOf(d)
# server removed?
self._check_removed(state)
# desired reduced and no server launched?
self.assertEqual(state.desired, 0)
self.assertEqual(len(state.pending), 0)
self.assertEqual(len(self.supervisor.exec_calls), 0)
def test_not_replaced_below_min(self):
"""
`CannotDeleteServerBelowMinError` is raised if current (active + pending) == min servers
"""
self.state.add_job('j1')
self.group.view_config.return_value = succeed({'minEntities': 2})
d = remove_server_from_group(self.log, self.tid, 's0', False, self.group, self.state)
self.failureResultOf(d, CannotDeleteServerBelowMinError)
# server is not deleted
self.assertIn('s0', self.state.active)
self.assertEqual(self.supervisor.del_calls, [])
# server is not launched
self.assertEqual(self.state.pending, matches(KeysEqual('j1')))
self.assertEqual(len(self.supervisor.exec_calls), 0)
# desired & active not changed
self.assertEqual(self.state.desired, 1)
self.assertEqual(self.state.active, {'s0': {'id': 's0'}})
