def test_process_job_should_exponentially_increase_timeout(self):
status = MockImageStatus('ERROR')
job = copy.deepcopy(self.job)
self._do_test_process_job_should_update_image_error(status, job=job)
self._reset_mocks()
new_now = timeutils.utcnow() + datetime.timedelta(minutes=120)
timeutils.clear_time_override()
timeutils.set_time_override(new_now)
job['status'] = 'ERROR'
job['retry_count'] = 2
job['hard_timeout'] = timeutils.strtime(
at=(new_now + datetime.timedelta(minutes=120)))
self._do_test_process_job_should_update_image_error(status,
include_create=False, include_queued=False, is_retry=True, job=job)
def job_fixture(self, instance_id, **kwargs):
now = timeutils.utcnow()
timeout = timeutils.strtime(now + datetime.timedelta(hours=1))
hard_timeout = timeutils.strtime(now + datetime.timedelta(hours=4))
fixture = {
'id': 'JOB_1',
'schedule_id': 'SCH_1',
'tenant': 'TENANT1',
'worker_id': 'WORKER_1',
'action': 'snapshot',
'status': 'QUEUED',
'timeout': timeout,
'hard_timeout': hard_timeout,
'retry_count': 0,
'metadata': {
'instance_id': instance_id,
'value': 'my_instance',
},
}
if kwargs:
fixture.update(kwargs)
return fixture
def job_fixture(self, instance_id, **kwargs):
now = timeutils.utcnow()
timeout = timeutils.strtime(now + datetime.timedelta(hours=1))
hard_timeout = timeutils.strtime(now + datetime.timedelta(hours=4))
fixture = {
'id': 'JOB_1',
'schedule_id': 'SCH_1',
'tenant': 'TENANT1',
'worker_id': 'WORKER_1',
'action': 'snapshot',
'status': 'QUEUED',
'timeout': timeout,
'hard_timeout': hard_timeout,
'retry_count': 0,
'metadata': {
'instance_id': instance_id,
'value': 'my_instance',
},
}
if kwargs:
fixture.update(kwargs)
return fixture
def test_process_job_should_exponentially_increase_timeout(self):
status = MockImageStatus('ERROR')
job = copy.deepcopy(self.job)
self._do_test_process_job_should_update_image_error(status, job=job)
self._reset_mocks()
new_now = timeutils.utcnow() + datetime.timedelta(minutes=120)
timeutils.clear_time_override()
timeutils.set_time_override(new_now)
job['status'] = 'ERROR'
job['retry_count'] = 2
job['hard_timeout'] = timeutils.strtime(
at=(new_now + datetime.timedelta(minutes=120)))
self._do_test_process_job_should_update_image_error(
status, include_create=False, include_queued=False, is_retry=True,
job=job)
def test_notifications_for_cancelled_job_on_hard_timeout_reached(self):
server = self.server_instance_fixture("INSTANCE_ID", "test")
now = timeutils.utcnow()
expired_hard_timeout = now - datetime.timedelta(hours=4)
job = self.job_fixture(
server.id, hard_timeout=timeutils.strtime(expired_hard_timeout))
with TestableSnapshotProcessor(job, server, []) as processor:
processor.process_job(job)
self.assertEqual('HARD_TIMED_OUT', job['status'])
expected_notifications = [
('qonos.job.run.start', 'INFO', 'QUEUED'),
('qonos.job.failed', 'ERROR', 'HARD_TIMED_OUT')
]
self.assert_job_notification_events(processor,
expected_notifications)
def test_notifications_for_cancelled_job_on_hard_timeout_reached(self):
server = self.server_instance_fixture("INSTANCE_ID", "test")
now = timeutils.utcnow()
expired_hard_timeout = now - datetime.timedelta(hours=4)
job = self.job_fixture(server.id,
hard_timeout=timeutils.strtime(
expired_hard_timeout))
with TestableSnapshotProcessor(job, server, []) as processor:
processor.process_job(job)
self.assertEqual('HARD_TIMED_OUT', job['status'])
expected_notifications = [
('qonos.job.run.start', 'INFO', 'QUEUED'),
('qonos.job.failed', 'ERROR', 'HARD_TIMED_OUT')]
self.assert_job_notification_events(processor,
expected_notifications)
def test_process_job_should_fail_if_hard_timed_out(self):
mox.Reset(self.worker)
self.mox.StubOutWithMock(utils, 'generate_notification')
now = timeutils.utcnow()
self.job['hard_timeout'] = timeutils.strtime(at=now)
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
self.worker.update_job(fakes.JOB_ID,
'HARD_TIMED_OUT',
timeout=None,
error_message=mox.IsA(str))\
.AndReturn({'status': 'HARD_TIMED_OUT',
'timeout': self.job['timeout']})
expected_payload = {
'job': {
'status': 'HARD_TIMED_OUT',
'hard_timeout': self.job['hard_timeout'],
'created_at': self.job['created_at'],
'modified_at': self.job['modified_at'],
'retry_count': 1,
'schedule_id': '33333333-3333-3333-3333-33333333',
'worker_id': '11111111-1111-1111-1111-11111111',
'timeout': self.job['timeout'],
'action': 'snapshot',
'id': '22222222-2222-2222-2222-22222222',
'tenant': '44444444-4444-4444-4444-44444444',
'metadata': {
'instance_id': '55555555-5555-5555-5555-55555555'
}
}
}
utils.generate_notification(None, 'qonos.job.failed', expected_payload,
mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_retention_for_hard_timedout_job(self):
server = self.server_instance_fixture("INSTANCE_ID", "test",
retention=2)
now = timeutils.utcnow()
expired_hard_timeout = now - datetime.timedelta(hours=4)
expired_hard_timeout = timeutils.strtime(expired_hard_timeout)
job = self.job_fixture(server.id, hard_timeout=expired_hard_timeout)
existing_snapshot_images = [
self.image_fixture('OLD_IMAGE_01', 'ACTIVE', server.id),
self.image_fixture('OLD_IMAGE_02', 'ACTIVE', server.id),
self.image_fixture('OLD_IMAGE_03', 'ACTIVE', server.id),
]
current_image = self.image_fixture('IMAGE_ID', 'ACTIVE', server.id)
all_instance_images = [current_image] + existing_snapshot_images
with TestableSnapshotProcessor(job, server,
[current_image]) as processor:
processor.nova_client.images.list = mock.Mock(
mock.ANY, return_value=all_instance_images)
processor.process_job(job)
self.assertEqual(2, processor.nova_client.images.delete.call_count)
processor.nova_client.images.delete.has_calls(
[mock.call('OLD_IMAGE_02'), mock.call('OLD_IMAGE_01')])
self.assert_update_job_statuses(processor, ['HARD_TIMED_OUT'])
self.assertEqual('HARD_TIMED_OUT', job['status'])
error_msg = ('Job %(job_id)s has reached/exceeded its'
' hard timeout: %(hard_timeout)s.' %
{'job_id': job['id'],
'hard_timeout': job['hard_timeout']})
expected_status_values = {
'status': 'HARD_TIMED_OUT',
'error_message': error_msg
}
self.assert_job_status_values(processor, expected_status_values)
def test_process_job_should_fail_if_hard_timed_out(self):
mox.Reset(self.worker)
self.mox.StubOutWithMock(utils, 'generate_notification')
now = timeutils.utcnow()
self.job['hard_timeout'] = timeutils.strtime(at=now)
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
self.worker.update_job(fakes.JOB_ID,
'HARD_TIMED_OUT',
timeout=None,
error_message=mox.IsA(str))\
.AndReturn({'status': 'HARD_TIMED_OUT',
'timeout': self.job['timeout']})
expected_payload = {'job':
{'status': 'HARD_TIMED_OUT',
'hard_timeout': self.job['hard_timeout'],
'created_at': self.job['created_at'],
'modified_at': self.job['modified_at'],
'retry_count': 1,
'schedule_id': '33333333-3333-3333-3333-33333333',
'worker_id': '11111111-1111-1111-1111-11111111',
'timeout': self.job['timeout'],
'action': 'snapshot',
'id': '22222222-2222-2222-2222-22222222',
'tenant': '44444444-4444-4444-4444-44444444',
'metadata': {'instance_id':
'55555555-5555-5555-5555-55555555'}}}
utils.generate_notification(None, 'qonos.job.failed', expected_payload,
mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_cancel_if_job_hard_timed_out(self):
server = self.server_instance_fixture("INSTANCE_ID", "test")
now = timeutils.utcnow()
expired_hard_timeout = now - datetime.timedelta(hours=4)
job = self.job_fixture(
server.id, hard_timeout=timeutils.strtime(expired_hard_timeout))
with TestableSnapshotProcessor(job, server, []) as processor:
processor.process_job(job)
self.assert_update_job_statuses(processor, ['HARD_TIMED_OUT'])
self.assertEqual('HARD_TIMED_OUT', job['status'])
error_msg = ('Job %(job_id)s has reached/exceeded its'
' hard timeout: %(hard_timeout)s.' % {
'job_id': job['id'],
'hard_timeout': job['hard_timeout']
})
expected_status_values = {
'status': 'HARD_TIMED_OUT',
'error_message': error_msg
}
self.assert_job_status_values(processor, expected_status_values)
def test_process_job_should_cancel_if_job_hard_timed_out(self):
server = self.server_instance_fixture("INSTANCE_ID", "test")
now = timeutils.utcnow()
expired_hard_timeout = now - datetime.timedelta(hours=4)
job = self.job_fixture(server.id,
hard_timeout=timeutils.strtime(
expired_hard_timeout))
with TestableSnapshotProcessor(job, server, []) as processor:
processor.process_job(job)
self.assert_update_job_statuses(processor, ['HARD_TIMED_OUT'])
self.assertEqual('HARD_TIMED_OUT', job['status'])
error_msg = ('Job %(job_id)s has reached/exceeded its'
' hard timeout: %(hard_timeout)s.' %
{'job_id': job['id'],
'hard_timeout': job['hard_timeout']})
expected_status_values = {
'status': 'HARD_TIMED_OUT',
'error_message': error_msg
}
self.assert_job_status_values(processor, expected_status_values)
"""
Fakes For Worker tests.
"""
import datetime
from qonos.common import timeutils
WORKER_ID = '11111111-1111-1111-1111-11111111'
JOB_ID = '22222222-2222-2222-2222-22222222'
SCHEDULE_ID = '33333333-3333-3333-3333-33333333'
TENANT = '44444444-4444-4444-4444-44444444'
INSTANCE_ID = '55555555-5555-5555-5555-55555555'
BASE_TIME_T = timeutils.utcnow()
BASE_TIME = timeutils.strtime(at=BASE_TIME_T)
TIMEOUT = timeutils.strtime(BASE_TIME_T + datetime.timedelta(hours=1))
HARD_TIMEOUT = timeutils.strtime(BASE_TIME_T + datetime.timedelta(hours=2))
JOB = {
'job': {
'id': JOB_ID,
'created_at': BASE_TIME,
'modified_at': BASE_TIME,
'schedule_id': SCHEDULE_ID,
'tenant': TENANT,
'worker_id': WORKER_ID,
'status': 'QUEUED',
'action': 'snapshot',
'retry_count': 1,
'timeout': TIMEOUT,
"""
Fakes For Worker tests.
"""
import datetime
from qonos.common import timeutils
WORKER_ID = '11111111-1111-1111-1111-11111111'
JOB_ID = '22222222-2222-2222-2222-22222222'
SCHEDULE_ID = '33333333-3333-3333-3333-33333333'
TENANT = '44444444-4444-4444-4444-44444444'
INSTANCE_ID = '55555555-5555-5555-5555-55555555'
BASE_TIME_T = timeutils.utcnow()
BASE_TIME = timeutils.strtime(at=BASE_TIME_T)
TIMEOUT = timeutils.strtime(BASE_TIME_T + datetime.timedelta(hours=1))
HARD_TIMEOUT = timeutils.strtime(BASE_TIME_T + datetime.timedelta(hours=2))
JOB = {
'job': {
'id': JOB_ID,
'created_at': BASE_TIME,
'modified_at': BASE_TIME,
'schedule_id': SCHEDULE_ID,
'tenant': TENANT,
'worker_id': WORKER_ID,
'status': 'QUEUED',
'action': 'snapshot',
'retry_count': 1,
'timeout': TIMEOUT,