def test__cleanup_post_failed(self, send_notification, post_deploy):
"""Cleanup marks release as failed if post deploy fails"""
post_deploy.return_value = False
f = FSM(state_id)
f.ch = mock.Mock(pika.channel.Channel)
f.conn = mock.Mock(pika.connection.Connection)
f.reply_queue = temp_queue
_update_state = {
'$set': {
'ended': UTCNOW,
'failed': True
}
}
with mock.patch.object(f, 'update_state', mock.Mock()) as (
us):
with mock.patch('recore.fsm.dt') as (
dt):
with mock.patch('recore.amqp.CONF') as notif_conf:
notif_conf = NOTIFICATION_CONF
dt.now.return_value = UTCNOW
f._cleanup()
# update state set the ended item in the state doc.
us.assert_called_with(_update_state)
f.conn.close.assert_called_once_with()
f.ch.queue_delete.assert_called_once_with(queue=temp_queue)
# At the very end a notification should go out no matter what
self.assertEqual(send_notification.call_count, 1)
assert send_notification.call_args[0][4] == 'failed'
post_deploy.assert_called_once()
def test_step_notification_started_no_notifications(self, send_notification, on_started, dequeue_step, setup):
"""Per-step notifications don't happen if no notifications are defined"""
f = FSM(state_id)
msg_started = {'status': 'started'}
consume_iter = [
(mock.Mock(name="method_mocked"),
mock.Mock(name="properties_mocked"),
json.dumps(msg_started))
]
# Pre-test scaffolding. Hard-code some mocked out
# attributes/variables because we're skipping the usual
# initialization steps.
f.conn = mock.Mock(pika.connection.Connection)
publish = mock.Mock()
channel = mock.Mock()
channel.consume.return_value = iter(consume_iter)
channel.basic_publish = publish
f.ch = channel
f.active_sequence = {'hosts': ['localhost']}
f.group = 'testgroup'
f.dynamic = {}
f.active_step = new_notify_step()
# Run the method now. It should terminate when it reaches the
# end of _run() with a call to a mocked out on_started()
with mock.patch('recore.amqp.MQ_CONF') as mq_conf:
mq_conf = MQ_CONF
f._run()
self.assertEqual(send_notification.call_count, 0)
def test__cleanup_failed(self):
"""Cleanup fails if update_state raises"""
f = FSM(state_id)
f.ch = mock.Mock(pika.channel.Channel)
f.conn = mock.Mock(pika.connection.Connection)
with mock.patch.object(f, 'update_state',
mock.Mock(side_effect=Exception("derp"))) as (
us_exception):
with self.assertRaises(Exception):
f._cleanup()
def test__setup_failed_pre_deploy_check(self, send_notification, move_remaining, amqp_conf):
"""Setup fails with an existing state document and a failed pre-deploy check"""
f = FSM(TEST_PBID, state_id)
# An AMQP connection hasn't been made yet
amqp_conf.get.return_value = PRE_DEPLOY_CONF['PRE_DEPLOY_CHECK']
msg_started = {'status': 'completed', 'data': {'exists': False}}
consume_iter = [
(mock.Mock(name="method_mocked"),
mock.Mock(name="properties_mocked"),
json.dumps(msg_started))
]
f.conn = mock.Mock(pika.connection.Connection)
publish = mock.Mock()
channel = mock.Mock()
channel.consume.return_value = iter(consume_iter)
channel.basic_publish = publish
f.ch = channel
with mock.patch('recore.mongo.database') as (
mongo.database):
mongo.database = mock.MagicMock(pymongo.database.Database)
mongo.database.__getitem__.return_value = mock.MagicMock(pymongo.collection.Collection)
with mock.patch('recore.mongo.lookup_state') as (
mongo.lookup_state):
mongo.lookup_state.return_value = _state
with mock.patch('recore.amqp.MQ_CONF') as mq_conf:
mq_conf = MQ_CONF
set_field = mock.MagicMock()
filter = mock.MagicMock(return_value=set_field)
f.filter = filter
f._setup()
assert f.group == _state['group']
# No matter where a release fails, 'move_remaining_to_skipped' will be called
move_remaining.assert_called_once_with()
# the first run/pre-deploy steps will record the failed state
assert f.initialized == False
# The starting phase notification will be sent
assert send_notification.call_count == 1
assert send_notification.call_args[0][4] == 'started'
# After first_run finishes self.failed should be True
with mock.patch('recore.amqp.MQ_CONF') as mq_conf:
mq_conf = MQ_CONF
f._cleanup()
assert send_notification.call_count == 2
assert send_notification.call_args[0][4] == 'failed'
assert f.failed == True
def test_step_notification_started_two_transports(self, send_notification, on_started, dequeue_step, setup):
"""Per-step notifications happen for all defined transports
Tests for the case where multiple notification transports (irc, email, etc) are defined"""
f = FSM(TEST_PBID, state_id)
msg_started = {'status': 'started'}
consume_iter = [
(mock.Mock(name="method_mocked"),
mock.Mock(name="properties_mocked"),
json.dumps(msg_started))
]
# Pre-test scaffolding. Hard-code some mocked out
# attributes/variables because we're skipping the usual
# initialization steps.
f.conn = mock.Mock(pika.connection.Connection)
publish = mock.Mock()
channel = mock.Mock()
channel.consume.return_value = iter(consume_iter)
channel.basic_publish = publish
f.ch = channel
f.active_sequence = {'hosts': ['localhost']}
f.group = 'testgroup'
f.dynamic = {}
_step = {
"service:Restart": {
"service": "megafrobber",
"notify": {
"started": {
"irc": ['#achannel'],
"email": ['*****@*****.**']
}
}
}
}
f.active_step = _step
# Run the method now. It should terminate when it reaches the
# end of _run() with a call to a mocked out on_started()
with mock.patch('recore.amqp.MQ_CONF') as mq_conf:
mq_conf = MQ_CONF
set_field = mock.MagicMock()
filter = mock.MagicMock(return_value=set_field)
f.filter = filter
f._run()
self.assertEqual(send_notification.call_count, 2)
def test_step_notification_failed(self, send_notification, move_remaining, setup, updatestate):
"""Per-step notifications work when a step fails
Tests for the case where only one notification transport (irc, email, etc) is defined"""
f = FSM(TEST_PBID, state_id)
msg_failed = {'status': 'failed'}
consume_iter = [
(mock.Mock(name="method_mocked"),
mock.Mock(name="properties_mocked"),
json.dumps(msg_failed))
]
# Pre-test scaffolding. Hard-code some mocked out
# attributes/variables because we're skipping the usual
# initialization steps.
f.conn = mock.Mock(pika.connection.Connection)
f.executed = []
f.execution = []
f.state_coll = {}
f.post_deploy_action = []
publish = mock.Mock()
channel = mock.Mock()
channel.consume.return_value = iter(consume_iter)
channel.basic_publish = publish
f.ch = channel
f.active_sequence = {'hosts': ['localhost']}
f.group = 'testgroup'
f.dynamic = {}
f.active_step = new_notify_step('failed')
set_field = mock.MagicMock()
filter = mock.MagicMock(return_value=set_field)
f.filter = filter
# Run the ended method with a body having 'status' as failed
f.on_ended(channel,
mock.Mock(name="method_mocked"),
mock.Mock(name="header_mocked"),
json.dumps(msg_failed))
self.assertEqual(send_notification.call_count, 1)
self.assertEqual(send_notification.call_args[0][1], 'notify.irc')
self.assertEqual(send_notification.call_args[0][2], state_id)
self.assertEqual(send_notification.call_args[0][3], ['#achannel'])
self.assertEqual(send_notification.call_args[0][4], 'failed')
def test__cleanup_failed(self, send_notification, post_deploy):
"""Cleanup fails if update_state raises"""
f = FSM(state_id)
f.ch = mock.Mock(pika.channel.Channel)
f.conn = mock.Mock(pika.connection.Connection)
f.failed = True # Testing the fail notification too
with mock.patch.object(f, 'update_state',
mock.Mock(side_effect=Exception("derp"))) as (
us_exception):
with self.assertRaises(Exception):
with mock.patch('recore.amqp.CONF') as notif_conf:
notif_conf = NOTIFICATION_CONF
f._cleanup()
# At the very end a notification should go out no matter what
self.assertEqual(send_notification.call_count, 1)
assert send_notification.call_args[0][4] == 'failed'
post_deploy.assert_called_once()
def test_post_deploy_passed(self, send_notification, move_active, run, skipped):
"""Post-deploy action passes"""
f = FSM(TEST_PBID, state_id)
msg_completed = {'status': 'started'}
consume_iter = [
(mock.Mock(name="method_mocked"),
mock.Mock(name="properties_mocked"),
json.dumps(msg_completed))
]
# Pre-test scaffolding. Hard-code some mocked out
# attributes/variables because we're skipping the usual
# initialization steps.
f.conn = mock.Mock(pika.connection.Connection)
publish = mock.Mock()
channel = mock.Mock()
channel.consume.return_value = iter(consume_iter)
channel.basic_publish = publish
f.ch = channel
f.active_sequence = {'hosts': ['localhost']}
f.group = 'testgroup'
f.dynamic = {}
f.active_step = new_notify_step('failed')
f.post_deploy_action = [
{
"NAME": "Update dates",
"COMMAND": "servicenow",
"SUBCOMMAND": "updatedates",
"PARAMETERS": {
"foo": "bar"
}
}
]
with mock.patch('recore.amqp.MQ_CONF') as mq_conf:
mq_conf = MQ_CONF
set_field = mock.MagicMock()
filter = mock.MagicMock(return_value=set_field)
f.filter = filter
self.assertEqual(f._post_deploy_action(), True)
def test_step_notification_started(self, send_notification, on_started, dequeue_step, setup):
"""Per-step notifications work when starting a step
Tests for the case where only one notification transport (irc, email, etc) is defined"""
f = FSM(TEST_PBID, state_id)
msg_started = {'status': 'started'}
consume_iter = [
(mock.Mock(name="method_mocked"),
mock.Mock(name="properties_mocked"),
json.dumps(msg_started))
]
# Pre-test scaffolding. Hard-code some mocked out
# attributes/variables because we're skipping the usual
# initialization steps.
f.conn = mock.Mock(pika.connection.Connection)
publish = mock.Mock()
channel = mock.Mock()
channel.consume.return_value = iter(consume_iter)
channel.basic_publish = publish
f.ch = channel
f.active_sequence = {'hosts': ['localhost']}
f.group = 'testgroup'
f.dynamic = {}
f.active_step = new_notify_step('started')
# Run the method now. It should terminate when it reaches the
# end of _run() with a call to a mocked out on_started()
with mock.patch('recore.amqp.MQ_CONF') as mq_conf:
mq_conf = MQ_CONF
set_field = mock.MagicMock()
filter = mock.MagicMock(return_value=set_field)
f.filter = filter
f._run()
self.assertEqual(send_notification.call_count, 1)
self.assertEqual(send_notification.call_args[0][1], 'notify.irc')
self.assertEqual(send_notification.call_args[0][2], state_id)
self.assertEqual(send_notification.call_args[0][3], ['#achannel'])
self.assertEqual(send_notification.call_args[0][4], 'started')
def test_on_started(self, ended, pdc):
"""Once started, the FSM waits for a response, and then calls on_ended"""
pdc.return_value = True
with nested(
mock.patch('recore.mongo.lookup_state'),
mock.patch('recore.mongo.database')
) as (lookup_state, database):
lookup_state.return_value = _state.copy()
# {
# 'group': 'PROJECT',
# 'dynamic': {},
# 'completed_steps': [],
# 'active_step': 'active_step',
# 'remaining_steps': [],
# }
f = FSM(TEST_PBID, state_id)
f.reply_queue = temp_queue
f.group = 'GROUP'
consume_iter = [
(mock.Mock(name="method_mocked"),
mock.Mock(name="properties_mocked"),
json.dumps(msg_started))
]
publish = mock.Mock()
channel = mock.Mock()
channel.consume.return_value = iter(consume_iter)
channel.basic_publish = publish
f.ch = channel
f.conn = mock.Mock(pika.connection.Connection)
set_field = mock.MagicMock()
filter = mock.MagicMock(return_value=set_field)
f.filter = filter
f._setup()
f.on_started(f.ch, *consume_iter[0])
f.ch.basic_ack.assert_called_once_with(consume_iter[0][0].delivery_tag)
f.ch.cancel.assert_called_once_with()
f.on_ended.assert_called_once_with(f.ch, *consume_iter[0])
def test__setup(self, send_notification):
"""Setup works with an existing state document"""
f = FSM(TEST_PBID, state_id)
# An AMQP connection hasn't been made yet
msg_started = {'status': 'completed', 'data': {'exists': True}}
consume_iter = [
(mock.Mock(name="method_mocked"),
mock.Mock(name="properties_mocked"),
json.dumps(msg_started))
]
f.conn = mock.Mock(pika.connection.Connection)
publish = mock.Mock()
channel = mock.Mock()
channel.consume.return_value = iter(consume_iter)
channel.basic_publish = publish
f.ch = channel
with mock.patch('recore.mongo.database') as (
mongo.database):
mongo.database = mock.MagicMock(pymongo.database.Database)
mongo.database.__getitem__.return_value = mock.MagicMock(pymongo.collection.Collection)
with mock.patch('recore.mongo.lookup_state') as (
mongo.lookup_state):
mongo.lookup_state.return_value = _state
with mock.patch('recore.amqp.CONF') as notif_conf:
notif_conf = NOTIFICATION_CONF
set_field = mock.MagicMock()
filter = mock.MagicMock(return_value=set_field)
f.filter = filter
f._setup()
assert f.group == _state['group']
# At the very end a notification should go out no matter what
assert send_notification.call_count == 1
assert send_notification.call_args[0][4] == 'started'
def test_step_notification_failed_before_started_received(self, send_notification, move_active, run, skipped):
"""Per-step notifications happen if a step fails when the worker attempts to start it"""
f = FSM(state_id)
msg_failed = {'status': 'failed'}
consume_iter = [
(mock.Mock(name="method_mocked"),
mock.Mock(name="properties_mocked"),
json.dumps(msg_failed))
]
# Pre-test scaffolding. Hard-code some mocked out
# attributes/variables because we're skipping the usual
# initialization steps.
f.conn = mock.Mock(pika.connection.Connection)
publish = mock.Mock()
channel = mock.Mock()
channel.consume.return_value = iter(consume_iter)
channel.basic_publish = publish
f.ch = channel
f.active_sequence = {'hosts': ['localhost']}
f.group = 'testgroup'
f.dynamic = {}
f.active_step = new_notify_step('failed')
# Run the ended method with a body having 'status' as completed
f.on_started(channel,
mock.Mock(name="method_mocked"),
mock.Mock(name="header_mocked"),
json.dumps(msg_failed))
self.assertEqual(send_notification.call_count, 1)
self.assertEqual(send_notification.call_args[0][1], 'notify.irc')
self.assertEqual(send_notification.call_args[0][2], state_id)
self.assertEqual(send_notification.call_args[0][3], ['#achannel'])
self.assertEqual(send_notification.call_args[0][4], 'failed')
def test__cleanup(self):
"""Cleanup erases the needful"""
f = FSM(state_id)
f.ch = mock.Mock(pika.channel.Channel)
f.conn = mock.Mock(pika.connection.Connection)
f.reply_queue = temp_queue
_update_state = {
'$set': {
'ended': UTCNOW
}
}
with mock.patch.object(f, 'update_state', mock.Mock()) as (
us):
with mock.patch('recore.fsm.dt') as (
dt):
dt.now.return_value = UTCNOW
f._cleanup()
# update state set the ended item in the state doc.
us.assert_called_with(_update_state)
f.conn.close.assert_called_once_with()
f.ch.queue_delete.assert_called_once_with(queue=temp_queue)
def test__cleanup(self, send_notification, post_deploy):
"""Cleanup erases the needful"""
f = FSM(TEST_PBID, state_id)
f.ch = mock.Mock(pika.channel.Channel)
f.conn = mock.Mock(pika.connection.Connection)
f.reply_queue = temp_queue
_update_state = {
'$set': {
'ended': UTCNOW,
'failed': False
}
}
with mock.patch.object(f, 'update_state', mock.Mock()) as (
us):
with mock.patch('recore.fsm.dt') as (
dt):
dt.utcnow.return_value = UTCNOW
with mock.patch('recore.amqp.CONF') as notif_conf:
notif_conf = NOTIFICATION_CONF
set_field = mock.MagicMock()
filter = mock.MagicMock(return_value=set_field)
f.filter = filter
f._cleanup()
# update state set the ended item in the state doc.
us.assert_called_with(_update_state)
f.conn.close.assert_called_once_with()
f.ch.queue_delete.assert_called_once_with(queue=temp_queue)
# At the very end a notification should go out no matter what
self.assertEqual(send_notification.call_count, 1)
assert send_notification.call_args[0][4] == 'completed'
post_deploy.assert_called_once()
