def create(self, request, body):
if (body is None or body.get('job') is None or
body['job'].get('schedule_id') is None):
raise webob.exc.HTTPBadRequest()
job = body['job']
try:
schedule = self.db_api.schedule_get_by_id(job['schedule_id'])
except exception.NotFound:
raise webob.exc.HTTPNotFound()
# Check integrity of schedule and update next run
expected_next_run = job.get('next_run')
if expected_next_run:
expected_next_run = timeutils.parse_isotime(job.get('next_run'))
next_run = api_utils.schedule_to_next_run(schedule, timeutils.utcnow())
try:
self.db_api.schedule_test_and_set_next_run(schedule['id'],
expected_next_run, next_run)
except exception.NotFound:
msg = _("Specified next run does not match the current next run"
" value. This could mean schedule has either changed"
"or has already been scheduled since you last expected.")
raise webob.exc.HTTPConflict(explanation=msg)
# Update schedule last_scheduled
values = {}
values['last_scheduled'] = timeutils.utcnow()
self.db_api.schedule_update(schedule['id'], values)
# Create job
values = {}
values.update(job)
values['tenant'] = schedule['tenant']
values['action'] = schedule['action']
values['status'] = 'QUEUED'
job_metadata = []
for metadata in schedule['schedule_metadata']:
job_metadata.append({
'key': metadata['key'],
'value': metadata['value']
})
values['job_metadata'] = job_metadata
job_action = values['action']
if not 'timeout' in values:
values['timeout'] = api_utils.get_new_timeout_by_action(job_action)
values['hard_timeout'] = \
api_utils.get_new_timeout_by_action(job_action)
job = self.db_api.job_create(values)
utils.serialize_datetimes(job)
api_utils.serialize_job_metadata(job)
job = {'job': job}
utils.generate_notification(None, 'qonos.job.create', job, 'INFO')
return job
def test_process_job_should_succeed_after_multiple_tries(self):
timeutils.set_time_override()
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('QUEUED'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._simple_prepare_worker_mock()
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_not_create_on_job_error_image_ok(self):
timeutils.set_time_override()
self.job['metadata']['image_id'] = IMAGE_ID
self.job['status'] = 'ERROR'
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._simple_prepare_worker_mock(skip_metadata_update=True)
self.worker.update_job(fakes.JOB_ID,
'DONE',
timeout=None,
error_message=None)
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, mox.IsA(str), mox.IsA(dict),
mox.IsA(str)).MultipleTimes()
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_doesnt_delete_images_on_retention_error(self):
timeutils.set_time_override()
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndRaise(Exception())
mock_server = MockServer(retention=None)
self._init_worker_mock()
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
self.worker.get_qonos_client().AndReturn(self.qonos_client)
self.qonos_client.delete_schedule(mox.IsA(str))
self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None,
error_message=None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_deletes_images_more_than_retention(self):
timeutils.set_time_override()
instance_id = self.job["metadata"]["instance_id"]
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("ACTIVE"))
mock_retention = MockRetention(3)
self.nova_client.rax_scheduled_images_python_novaclient_ext.get(mox.IsA(str)).AndReturn(mock_retention)
mock_server = MockServer(instance_id=instance_id)
image_list = self._create_images_list(mock_server.id, 5)
self.nova_client.images.list(detailed=True).AndReturn(image_list)
# The image list happens to be in descending created order
self.nova_client.images.delete(image_list[-2].id)
self.nova_client.images.delete(image_list[-1].id)
self._init_worker_mock()
self.worker.update_job(fakes.JOB_ID, "DONE", timeout=None, error_message=None)
self.mox.StubOutWithMock(utils, "generate_notification")
utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str))
utils.generate_notification(None, "qonos.job.run.end", mox.IsA(dict), mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_not_create_on_job_error_image_ok(self):
timeutils.set_time_override()
self.job['metadata']['image_id'] = IMAGE_ID
self.job['status'] = 'ERROR'
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._simple_prepare_worker_mock(skip_metadata_update=True)
self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None,
error_message=None)
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, mox.IsA(str), mox.IsA(dict),
mox.IsA(str)).MultipleTimes()
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_doesnt_delete_images_from_another_instance(self):
timeutils.set_time_override()
instance_id = self.job["metadata"]["instance_id"]
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("ACTIVE"))
mock_retention = MockRetention(3)
self.nova_client.rax_scheduled_images_python_novaclient_ext.get(mox.IsA(str)).AndReturn(mock_retention)
mock_server = MockServer(instance_id=instance_id)
image_list = self._create_images_list(mock_server.id, 5)
to_delete = image_list[3:]
image_list.extend(self._create_images_list(uuidutils.generate_uuid(), 3))
self.nova_client.images.list(detailed=True).AndReturn(image_list)
# The image list happens to be in descending created order
self.nova_client.images.delete(to_delete[0].id)
self.nova_client.images.delete(to_delete[1].id)
self._simple_prepare_worker_mock()
self.mox.StubOutWithMock(utils, "generate_notification")
utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str))
utils.generate_notification(None, "qonos.job.run.end", mox.IsA(dict), mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_cancel_if_schedule_deleted(self):
self._init_qonos_client()
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
response = {'status': 'CANCELLED', 'timeout': self.job['timeout']}
self.worker.update_job(fakes.JOB_ID, 'CANCELLED', timeout=None,
error_message=mox.IsA(str)).AndReturn(response)
expected_payload = {'job':
{'status': 'CANCELLED',
'hard_timeout': self.job['hard_timeout'],
'created_at': self.job['created_at'],
'modified_at': self.job['modified_at'],
'retry_count': 1,
'schedule_id': '33333333-3333-3333-3333-33333333',
'worker_id': '11111111-1111-1111-1111-11111111',
'timeout': self.job['timeout'],
'action': 'snapshot',
'id': '22222222-2222-2222-2222-22222222',
'tenant': '44444444-4444-4444-4444-44444444',
'metadata': {'instance_id':
'55555555-5555-5555-5555-55555555'}}}
utils.generate_notification(None, 'qonos.job.failed', expected_payload,
mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_generate_notification(self):
notification = {}
def fake_publisher_id(service, host=None):
if not host:
host = 'localhost'
return "%s.%s" % (service, host)
def fake_notify(context, publisher_id, event_type, priority, payload):
notification['context'] = context
notification['publisher_id'] = publisher_id
notification['event_type'] = event_type
notification['priority'] = priority
notification['payload'] = payload
self.stubs.Set(notifier_api, 'publisher_id', fake_publisher_id)
self.stubs.Set(notifier_api, 'notify', fake_notify)
payload = {'id': 'fake-id'}
self.assertEqual(notification, {})
self.config(host='localhost')
utils.generate_notification(None, 'qonos.fake.event', payload)
expected_notification = {
'context': None,
'publisher_id': 'qonos.localhost',
'event_type': 'qonos.fake.event',
'priority': 'INFO',
'payload': {'id': 'fake-id'}
}
self.assertEqual(notification, expected_notification)
def test_process_job_should_cancel_if_schedule_deleted(self):
self._init_qonos_client()
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
response = {'status': 'CANCELLED', 'timeout': self.job['timeout']}
self.worker.update_job(fakes.JOB_ID, 'CANCELLED', timeout=None,
error_message=mox.IsA(str)).AndReturn(response)
expected_payload = {'job': {'status': 'CANCELLED',
'hard_timeout': self.job['hard_timeout'],
'created_at': self.job['created_at'],
'modified_at': self.job['modified_at'],
'retry_count': 1,
'schedule_id': '33333333-3333-3333-3333-33333333',
'worker_id': '11111111-1111-1111-1111-11111111',
'timeout': self.job['timeout'],
'action': 'snapshot',
'id': '22222222-2222-2222-2222-22222222',
'tenant': '44444444-4444-4444-4444-44444444',
'metadata': {'instance_id':
'55555555-5555-5555-5555-55555555'}}}
utils.generate_notification(None, 'qonos.job.failed', expected_payload,
mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_update_status_timestamp_no_retries(self):
base_time = timeutils.utcnow()
time_seq = [
base_time,
base_time,
base_time,
base_time + datetime.timedelta(minutes=5, seconds=5),
base_time + datetime.timedelta(minutes=60, seconds=5),
base_time + datetime.timedelta(minutes=120, seconds=5),
base_time + datetime.timedelta(minutes=180, seconds=5),
base_time + datetime.timedelta(minutes=240, seconds=5),
]
timeutils.set_time_override_seq(time_seq)
job = copy.deepcopy(self.job)
job['timeout'] = base_time + datetime.timedelta(minutes=60)
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(
mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('QUEUED'))
self.nova_client.images.get(IMAGE_ID).MultipleTimes().AndReturn(
MockImageStatus('SAVING'))
self._init_worker_mock()
self.worker.update_job(fakes.JOB_ID,
'PROCESSING',
timeout=mox.IsA(datetime.datetime),
error_message=None)
self.worker.update_job(fakes.JOB_ID,
'PROCESSING',
timeout=mox.IsA(datetime.datetime),
error_message=None)
self.worker.update_job(fakes.JOB_ID,
'PROCESSING',
timeout=mox.IsA(datetime.datetime),
error_message=None)
self.worker.update_job(fakes.JOB_ID,
'TIMED_OUT',
timeout=None,
error_message=None)
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict),
mox.IsA(str)).MultipleTimes()
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(job)
self.mox.VerifyAll()
def _do_test_process_job_should_update_image_error(self, error_status):
base_time = timeutils.utcnow()
time_seq = [
base_time,
base_time,
base_time + datetime.timedelta(seconds=305),
base_time + datetime.timedelta(seconds=605),
base_time + datetime.timedelta(seconds=905),
base_time + datetime.timedelta(seconds=1205),
base_time + datetime.timedelta(seconds=1505),
]
timeutils.set_time_override_seq(time_seq)
job = copy.deepcopy(self.job)
job['timeout'] = base_time + datetime.timedelta(minutes=60)
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('QUEUED'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
error_status)
self._init_worker_mock()
self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'ERROR', timeout=None,
error_message=mox.IsA(str))
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(job)
self.mox.VerifyAll()
def test_process_job_should_cancel_if_schedule_deleted(self):
self._init_qonos_client()
self.mox.StubOutWithMock(utils, "generate_notification")
utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str))
self.worker.update_job(fakes.JOB_ID, "CANCELLED", timeout=None, error_message=mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_update_status_and_timestamp(self):
base_time = timeutils.utcnow()
time_seq = [
base_time,
base_time,
base_time + datetime.timedelta(seconds=305),
base_time + datetime.timedelta(minutes=60, seconds=5),
base_time + datetime.timedelta(minutes=60, seconds=305),
]
timeutils.set_time_override_seq(time_seq)
job = copy.deepcopy(self.job)
job['timeout'] = base_time + datetime.timedelta(minutes=60)
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('QUEUED'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._init_worker_mock()
self.worker.update_job(fakes.JOB_ID, 'PROCESSING',
timeout=mox.IsA(datetime.datetime),
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
error_message=None)
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
self.worker.get_qonos_client().AndReturn(self.qonos_client)
self.qonos_client.delete_schedule(mox.IsA(str))
self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None,
error_message=None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(job)
self.mox.VerifyAll()
def test_process_job_should_cancel_if_no_instance_id(self):
self._init_qonos_client(schedule=MockSchedule())
self.mox.StubOutWithMock(utils, "generate_notification")
utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str))
self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=mox.IsA(datetime.datetime), error_message=None)
self.worker.update_job(fakes.JOB_ID, "CANCELLED", timeout=None, error_message=mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
del self.job["metadata"]["instance_id"]
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_fail_if_it_reached_max_retry_count(self):
mox.Reset(self.worker)
self.mox.StubOutWithMock(utils, 'generate_notification')
max_retry_count = 2
self.config(max_retry=max_retry_count, group='snapshot_worker')
self.job['retry_count'] = max_retry_count + 1
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
self.worker.update_job(fakes.JOB_ID,
'MAX_RETRIED',
timeout=None,
error_message=mox.IsA(str))\
.AndReturn({'status': 'MAX_RETRIED',
'timeout': self.job['timeout']})
expected_payload = {
'job': {
'status': 'MAX_RETRIED',
'hard_timeout': self.job['hard_timeout'],
'created_at': self.job['created_at'],
'modified_at': self.job['modified_at'],
'retry_count': self.job['retry_count'],
'schedule_id': '33333333-3333-3333-3333-33333333',
'worker_id': '11111111-1111-1111-1111-11111111',
'timeout': self.job['timeout'],
'action': 'snapshot',
'id': '22222222-2222-2222-2222-22222222',
'tenant': '44444444-4444-4444-4444-44444444',
'metadata': {
'instance_id': '55555555-5555-5555-5555-55555555'
}
}
}
utils.generate_notification(None, 'qonos.job.failed', expected_payload,
mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_cancel_if_instance_not_found(self):
self._init_qonos_client(schedule=MockSchedule())
self.mox.StubOutWithMock(utils, "generate_notification")
utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str))
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndRaise(
exceptions.NotFound("404")
)
self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=mox.IsA(datetime.datetime), error_message=None)
self.worker.update_job(fakes.JOB_ID, "CANCELLED", timeout=None, error_message=mox.IsA(str))
self.worker.get_qonos_client().AndReturn(self.qonos_client)
self.qonos_client.delete_schedule(mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_fail_if_hard_timed_out(self):
mox.Reset(self.worker)
self.mox.StubOutWithMock(utils, 'generate_notification')
now = timeutils.utcnow()
self.job['hard_timeout'] = timeutils.strtime(at=now)
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
self.worker.update_job(fakes.JOB_ID,
'HARD_TIMED_OUT',
timeout=None,
error_message=mox.IsA(str))\
.AndReturn({'status': 'HARD_TIMED_OUT',
'timeout': self.job['timeout']})
expected_payload = {
'job': {
'status': 'HARD_TIMED_OUT',
'hard_timeout': self.job['hard_timeout'],
'created_at': self.job['created_at'],
'modified_at': self.job['modified_at'],
'retry_count': 1,
'schedule_id': '33333333-3333-3333-3333-33333333',
'worker_id': '11111111-1111-1111-1111-11111111',
'timeout': self.job['timeout'],
'action': 'snapshot',
'id': '22222222-2222-2222-2222-22222222',
'tenant': '44444444-4444-4444-4444-44444444',
'metadata': {
'instance_id': '55555555-5555-5555-5555-55555555'
}
}
}
utils.generate_notification(None, 'qonos.job.failed', expected_payload,
mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_fail_if_it_reached_max_retry_count(self):
mox.Reset(self.worker)
self.mox.StubOutWithMock(utils, 'generate_notification')
max_retry_count = 2
self.config(max_retry=max_retry_count, group='snapshot_worker')
self.job['retry_count'] = max_retry_count + 1
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
self.worker.update_job(fakes.JOB_ID,
'MAX_RETRIED',
timeout=None,
error_message=mox.IsA(str))\
.AndReturn({'status': 'MAX_RETRIED',
'timeout': self.job['timeout']})
expected_payload = {'job':
{'status': 'MAX_RETRIED',
'hard_timeout': self.job['hard_timeout'],
'created_at': self.job['created_at'],
'modified_at': self.job['modified_at'],
'retry_count': self.job['retry_count'],
'schedule_id': '33333333-3333-3333-3333-33333333',
'worker_id': '11111111-1111-1111-1111-11111111',
'timeout': self.job['timeout'],
'action': 'snapshot',
'id': '22222222-2222-2222-2222-22222222',
'tenant': '44444444-4444-4444-4444-44444444',
'metadata': {'instance_id':
'55555555-5555-5555-5555-55555555'}}}
utils.generate_notification(None, 'qonos.job.failed', expected_payload,
mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_deletes_images_more_than_retention(self):
timeutils.set_time_override()
instance_id = self.job['metadata']['instance_id']
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(
mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention(3)
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
mock_server = MockServer(instance_id=instance_id)
image_list = self._create_images_list(mock_server.id, 5)
self.nova_client.images.list(detailed=True).AndReturn(image_list)
# The image list happens to be in descending created order
self.nova_client.images.delete(image_list[-2].id)
self.nova_client.images.delete(image_list[-1].id)
self._init_worker_mock()
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict),
mox.IsA(str)).MultipleTimes()
self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None,
error_message=None)
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_doesnt_delete_images_on_retention_error(self):
timeutils.set_time_override()
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(
mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndRaise(Exception())
mock_server = MockServer(retention=None)
self._init_worker_mock()
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict),
mox.IsA(str)).MultipleTimes()
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
self.worker.update_job(fakes.JOB_ID,
'DONE',
timeout=None,
error_message=None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_fail_if_hard_timed_out(self):
mox.Reset(self.worker)
self.mox.StubOutWithMock(utils, 'generate_notification')
now = timeutils.utcnow()
self.job['hard_timeout'] = timeutils.strtime(at=now)
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
self.worker.update_job(fakes.JOB_ID,
'HARD_TIMED_OUT',
timeout=None,
error_message=mox.IsA(str))\
.AndReturn({'status': 'HARD_TIMED_OUT',
'timeout': self.job['timeout']})
expected_payload = {'job':
{'status': 'HARD_TIMED_OUT',
'hard_timeout': self.job['hard_timeout'],
'created_at': self.job['created_at'],
'modified_at': self.job['modified_at'],
'retry_count': 1,
'schedule_id': '33333333-3333-3333-3333-33333333',
'worker_id': '11111111-1111-1111-1111-11111111',
'timeout': self.job['timeout'],
'action': 'snapshot',
'id': '22222222-2222-2222-2222-22222222',
'tenant': '44444444-4444-4444-4444-44444444',
'metadata': {'instance_id':
'55555555-5555-5555-5555-55555555'}}}
utils.generate_notification(None, 'qonos.job.failed', expected_payload,
mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_succeed_immediately(self):
timeutils.set_time_override()
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("ACTIVE"))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.get(mox.IsA(str)).AndReturn(mock_retention)
self._simple_prepare_worker_mock()
self.mox.StubOutWithMock(utils, "generate_notification")
utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str))
utils.generate_notification(None, "qonos.job.run.end", mox.IsA(dict), mox.IsA(str))
self.worker.get_qonos_client().AndReturn(self.qonos_client)
self.qonos_client.delete_schedule(mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_new_image_when_retrying_with_failed_image(self):
timeutils.set_time_override()
self.job['metadata']['image_id'] = IMAGE_ID
self.job['status'] = 'ERROR'
#make an error checking the previous image
self.nova_client.images.get(IMAGE_ID).AndRaise(
Exception())
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._simple_prepare_worker_mock()
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.retry', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
self.worker.get_qonos_client().AndReturn(self.qonos_client)
self.qonos_client.delete_schedule(mox.IsA(str))
self.worker.update_job(fakes.JOB_ID, 'DONE',
error_message=None, timeout=None).AndReturn(None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_update_status_only(self):
base_time = timeutils.utcnow()
time_seq = [
base_time,
base_time,
base_time + datetime.timedelta(seconds=305),
base_time + datetime.timedelta(seconds=605),
base_time + datetime.timedelta(seconds=905),
]
timeutils.set_time_override_seq(time_seq)
job = copy.deepcopy(self.job)
job["timeout"] = base_time + datetime.timedelta(minutes=60)
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("QUEUED"))
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING"))
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING"))
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("ACTIVE"))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.get(mox.IsA(str)).AndReturn(mock_retention)
self._simple_prepare_worker_mock(2)
self.mox.StubOutWithMock(utils, "generate_notification")
utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str))
utils.generate_notification(None, "qonos.job.run.end", mox.IsA(dict), mox.IsA(str))
self.worker.get_qonos_client().AndReturn(self.qonos_client)
self.qonos_client.delete_schedule(mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(job)
self.mox.VerifyAll()
def test_process_job_should_update_status_timestamp_no_retries(self):
base_time = timeutils.utcnow()
time_seq = [
base_time,
base_time,
base_time + datetime.timedelta(minutes=5, seconds=5),
base_time + datetime.timedelta(minutes=60, seconds=5),
base_time + datetime.timedelta(minutes=120, seconds=5),
base_time + datetime.timedelta(minutes=180, seconds=5),
base_time + datetime.timedelta(minutes=240, seconds=5),
]
timeutils.set_time_override_seq(time_seq)
job = copy.deepcopy(self.job)
job["timeout"] = base_time + datetime.timedelta(minutes=60)
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("QUEUED"))
self.nova_client.images.get(IMAGE_ID).MultipleTimes().AndReturn(MockImageStatus("SAVING"))
self._init_worker_mock()
self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=mox.IsA(datetime.datetime), error_message=None)
self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=mox.IsA(datetime.datetime), error_message=None)
self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=mox.IsA(datetime.datetime), error_message=None)
self.worker.update_job(fakes.JOB_ID, "TIMED_OUT", timeout=None, error_message=None)
self.mox.StubOutWithMock(utils, "generate_notification")
utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(job)
self.mox.VerifyAll()
def test_process_job_should_cancel_if_no_instance_id(self):
self._init_qonos_client(schedule=dict())
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
self.worker.update_job(fakes.JOB_ID, 'PROCESSING',
timeout=mox.IsA(datetime.datetime),
error_message=None).\
AndReturn({'status': 'PROCESSING', 'timeout': 'new_timeout'})
self.worker.update_job(fakes.JOB_ID, 'CANCELLED', timeout=None,
error_message=mox.IsA(str)).\
AndReturn({'status': 'CANCELLED', 'timeout': self.job['timeout']})
expected_processing_payload = \
{'job': {'status': 'PROCESSING',
'hard_timeout': self.job['hard_timeout'],
'created_at': self.job['created_at'],
'modified_at': self.job['modified_at'],
'retry_count': 1,
'schedule_id': '33333333-3333-3333-3333-33333333',
'worker_id': '11111111-1111-1111-1111-11111111',
'timeout': 'new_timeout',
'action': 'snapshot',
'id': '22222222-2222-2222-2222-22222222',
'tenant': '44444444-4444-4444-4444-44444444',
'metadata': {}}}
expected_cancelled_payload = \
{'job': {'status': 'CANCELLED',
'hard_timeout': self.job['hard_timeout'],
'created_at': self.job['created_at'],
'modified_at': self.job['modified_at'],
'retry_count': 1,
'schedule_id': '33333333-3333-3333-3333-33333333',
'worker_id': '11111111-1111-1111-1111-11111111',
'timeout': self.job['timeout'],
'action': 'snapshot',
'id': '22222222-2222-2222-2222-22222222',
'tenant': '44444444-4444-4444-4444-44444444',
'metadata': {}}}
utils.generate_notification(None, 'qonos.job.update',
expected_processing_payload,
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.failed',
expected_cancelled_payload,
mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
del self.job['metadata']['instance_id']
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_cancel_if_no_instance_id(self):
self._init_qonos_client(schedule=dict())
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
self.worker.update_job(fakes.JOB_ID, 'PROCESSING',
timeout=mox.IsA(datetime.datetime),
error_message=None).\
AndReturn({'status': 'PROCESSING', 'timeout': 'new_timeout'})
self.worker.update_job(fakes.JOB_ID, 'CANCELLED', timeout=None,
error_message=mox.IsA(str)).\
AndReturn({'status': 'CANCELLED', 'timeout': self.job['timeout']})
expected_processing_payload = \
{'job': {'status': 'PROCESSING',
'hard_timeout': self.job['hard_timeout'],
'created_at': self.job['created_at'],
'modified_at': self.job['modified_at'],
'retry_count': 1,
'schedule_id': '33333333-3333-3333-3333-33333333',
'worker_id': '11111111-1111-1111-1111-11111111',
'timeout': 'new_timeout',
'action': 'snapshot',
'id': '22222222-2222-2222-2222-22222222',
'tenant': '44444444-4444-4444-4444-44444444',
'metadata': {}}}
expected_cancelled_payload = \
{'job': {'status': 'CANCELLED',
'hard_timeout': self.job['hard_timeout'],
'created_at': self.job['created_at'],
'modified_at': self.job['modified_at'],
'retry_count': 1,
'schedule_id': '33333333-3333-3333-3333-33333333',
'worker_id': '11111111-1111-1111-1111-11111111',
'timeout': self.job['timeout'],
'action': 'snapshot',
'id': '22222222-2222-2222-2222-22222222',
'tenant': '44444444-4444-4444-4444-44444444',
'metadata': {}}}
utils.generate_notification(None, 'qonos.job.update',
expected_processing_payload,
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.failed',
expected_cancelled_payload,
mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
del self.job['metadata']['instance_id']
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_succeed_immediately(self):
timeutils.set_time_override()
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._init_worker_mock(False)
expected_processing_payload = \
{'job': {'status': 'PROCESSING',
'hard_timeout': self.job['hard_timeout'],
'created_at': self.job['created_at'],
'modified_at': self.job['modified_at'],
'retry_count': 1,
'schedule_id': '33333333-3333-3333-3333-33333333',
'worker_id': '11111111-1111-1111-1111-11111111',
'timeout': self.job['timeout'],
'action': 'snapshot',
'id': '22222222-2222-2222-2222-22222222',
'tenant': '44444444-4444-4444-4444-44444444',
'metadata': {'instance_id':
'55555555-5555-5555-5555-55555555'}}}
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.update',
expected_processing_payload,
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
self.worker.get_qonos_client().AndReturn(self.qonos_client)
self.qonos_client.delete_schedule(mox.IsA(str))
processor = TestableSnapshotProcessor(self.nova_client)
self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None,
error_message=None).\
AndReturn({'status': 'CANCELLED', 'timeout': self.job['timeout']})
self.mox.ReplayAll()
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_update_status_only(self):
base_time = timeutils.utcnow()
time_seq = [
base_time,
base_time,
base_time,
base_time + datetime.timedelta(seconds=305),
base_time + datetime.timedelta(seconds=605),
base_time + datetime.timedelta(seconds=905),
]
timeutils.set_time_override_seq(time_seq)
job = copy.deepcopy(self.job)
job['timeout'] = base_time + datetime.timedelta(minutes=60)
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(
mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('QUEUED'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._simple_prepare_worker_mock(2)
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict),
mox.IsA(str)).MultipleTimes()
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
self.worker.update_job(fakes.JOB_ID,
'DONE',
timeout=None,
error_message=None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(job)
self.mox.VerifyAll()
def test_process_job_should_succeed_immediately(self):
timeutils.set_time_override()
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(
mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._init_worker_mock(False)
expected_processing_payload = \
{'job': {'status': 'PROCESSING',
'hard_timeout': self.job['hard_timeout'],
'created_at': self.job['created_at'],
'modified_at': self.job['modified_at'],
'retry_count': 1,
'schedule_id': '33333333-3333-3333-3333-33333333',
'worker_id': '11111111-1111-1111-1111-11111111',
'timeout': self.job['timeout'],
'action': 'snapshot',
'id': '22222222-2222-2222-2222-22222222',
'tenant': '44444444-4444-4444-4444-44444444',
'metadata': {'instance_id':
'55555555-5555-5555-5555-55555555'}}}
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.update',
expected_processing_payload,
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
processor = TestableSnapshotProcessor(self.nova_client)
self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None,
error_message=None).\
AndReturn({'status': 'CANCELLED', 'timeout': self.job['timeout']})
self.mox.ReplayAll()
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_new_image_when_retrying_with_failed_image(self):
timeutils.set_time_override()
self.job['metadata']['image_id'] = IMAGE_ID
self.job['status'] = 'ERROR'
#make an error checking the previous image
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ERROR'))
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(
mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._simple_prepare_worker_mock()
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.retry', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict),
mox.IsA(str)).MultipleTimes()
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
self.worker.update_job(fakes.JOB_ID,
'DONE',
error_message=None,
timeout=None).AndReturn(None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def send_notification(self, event_type, payload, level='INFO'):
utils.generate_notification(None, event_type, payload, level)
def send_notification(self, event_type, payload, level='INFO'):
utils.generate_notification(None, event_type, payload, level)
def _do_test_process_job_should_update_image_error(self, error_status,
include_create=True,
include_queued=True,
is_retry=False,
job=None):
base_time = timeutils.utcnow()
time_seq = [
base_time,
base_time,
base_time,
base_time + datetime.timedelta(seconds=305),
base_time + datetime.timedelta(seconds=605),
base_time + datetime.timedelta(seconds=905),
base_time + datetime.timedelta(seconds=1205),
base_time + datetime.timedelta(seconds=1505),
]
timeutils.set_time_override_seq(time_seq)
if job is None:
job = copy.deepcopy(self.job)
job['timeout'] = base_time + datetime.timedelta(minutes=60)
if include_create:
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
if include_queued:
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('QUEUED'))
else:
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
error_status)
self._init_worker_mock(skip_metadata_update=(not include_create))
self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
error_message=None)
metadata = copy.copy(job['metadata'])
metadata['image_id'] = IMAGE_ID
backoff_factor = DEFAULT_BACKOFF_FACTOR ** job['retry_count']
timeout = time_seq[-1] + datetime.timedelta(minutes=
DEFAULT_TIMEOUT_INCR *
backoff_factor)
self.worker.update_job(fakes.JOB_ID, 'ERROR', timeout=timeout,
error_message=mox.IsA(unicode)).AndReturn(
{'status': 'ERROR', 'timeout': job['timeout']})
self.mox.StubOutWithMock(utils, 'generate_notification')
expected_job = copy.deepcopy(job)
expected_job['status'] = 'ERROR'
expected_job['metadata'] = metadata
if not is_retry:
utils.generate_notification(None, 'qonos.job.run.start',
mox.IsA(dict), mox.IsA(str))
else:
utils.generate_notification(None, 'qonos.job.retry',
mox.IsA(dict), mox.IsA(str))
utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict),
mox.IsA(str))
def assert_job_payload(job_payload):
self.assertTrue('error_message' in job_payload['job'])
del job_payload['job']['error_message']
self.assertEquals({'job': expected_job}, job_payload)
return True
utils.generate_notification(None, 'qonos.job.update',
mox.Func(assert_job_payload), 'ERROR')
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(job)
self.mox.VerifyAll()
def _do_test_process_job_should_update_image_error(
self, error_status, include_create=True, include_queued=True, is_retry=False, job=None
):
base_time = timeutils.utcnow()
time_seq = [
base_time,
base_time,
base_time + datetime.timedelta(seconds=305),
base_time + datetime.timedelta(seconds=605),
base_time + datetime.timedelta(seconds=905),
base_time + datetime.timedelta(seconds=1205),
base_time + datetime.timedelta(seconds=1505),
]
timeutils.set_time_override_seq(time_seq)
if job is None:
job = copy.deepcopy(self.job)
job["timeout"] = base_time + datetime.timedelta(minutes=60)
if include_create:
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
if include_queued:
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("QUEUED"))
else:
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING"))
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING"))
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING"))
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING"))
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING"))
self.nova_client.images.get(IMAGE_ID).AndReturn(error_status)
self._init_worker_mock(skip_metadata_update=(not include_create))
self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=None, error_message=None)
self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=None, error_message=None)
self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=None, error_message=None)
self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=None, error_message=None)
metadata = copy.copy(job["metadata"])
metadata["image_id"] = IMAGE_ID
backoff_factor = DEFAULT_BACKOFF_FACTOR ** job["retry_count"]
timeout = time_seq[-1] + datetime.timedelta(minutes=DEFAULT_TIMEOUT_INCR * backoff_factor)
self.worker.update_job(fakes.JOB_ID, "ERROR", timeout=timeout, error_message=mox.IsA(str))
self.mox.StubOutWithMock(utils, "generate_notification")
if not is_retry:
utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str))
else:
utils.generate_notification(None, "qonos.job.retry", mox.IsA(dict), mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(job)
self.mox.VerifyAll()
def create(self, request, body):
if (body is None or body.get('job') is None
or body['job'].get('schedule_id') is None):
raise webob.exc.HTTPBadRequest()
job = body['job']
try:
schedule = self.db_api.schedule_get_by_id(job['schedule_id'])
except exception.NotFound:
raise webob.exc.HTTPNotFound()
# Check integrity of schedule and update next run
expected_next_run = job.get('next_run')
if expected_next_run:
try:
expected_next_run = timeutils.parse_isotime(expected_next_run)
expected_next_run = expected_next_run.replace(tzinfo=None)
except ValueError as e:
msg = _('Invalid "next_run" value. Must be ISO 8601 format')
raise webob.exc.HTTPBadRequest(explanation=msg)
next_run = api_utils.schedule_to_next_run(schedule, timeutils.utcnow())
next_run = next_run.replace(tzinfo=None)
try:
self.db_api.schedule_test_and_set_next_run(schedule['id'],
expected_next_run,
next_run)
except exception.NotFound:
msg = _("Specified next run does not match the current next run"
" value. This could mean schedule has either changed"
"or has already been scheduled since you last expected.")
raise webob.exc.HTTPConflict(explanation=msg)
# Update schedule last_scheduled
values = {}
values['last_scheduled'] = timeutils.utcnow()
self.db_api.schedule_update(schedule['id'], values)
# Create job
values = {}
values.update(job)
values['tenant'] = schedule['tenant']
values['action'] = schedule['action']
values['status'] = 'QUEUED'
job_metadata = []
for metadata in schedule['schedule_metadata']:
job_metadata.append({
'key': metadata['key'],
'value': metadata['value']
})
values['job_metadata'] = job_metadata
job_action = values['action']
if not 'timeout' in values:
values['timeout'] = api_utils.get_new_timeout_by_action(job_action)
values['hard_timeout'] = \
api_utils.get_new_timeout_by_action(job_action)
job = self.db_api.job_create(values)
utils.serialize_datetimes(job)
api_utils.serialize_job_metadata(job)
job = {'job': job}
utils.generate_notification(None, 'qonos.job.create', job, 'INFO')
return job
def _do_test_process_job_should_update_image_error(self,
error_status,
include_create=True,
include_queued=True,
is_retry=False,
job=None):
base_time = timeutils.utcnow()
time_seq = [
base_time,
base_time,
base_time,
base_time + datetime.timedelta(seconds=305),
base_time + datetime.timedelta(seconds=605),
base_time + datetime.timedelta(seconds=905),
base_time + datetime.timedelta(seconds=1205),
base_time + datetime.timedelta(seconds=1505),
]
timeutils.set_time_override_seq(time_seq)
if job is None:
job = copy.deepcopy(self.job)
job['timeout'] = base_time + datetime.timedelta(minutes=60)
if include_create:
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(
mox.IsA(str), mox.IsA(str),
self.snapshot_meta).AndReturn(IMAGE_ID)
if include_queued:
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('QUEUED'))
else:
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(error_status)
self._init_worker_mock(skip_metadata_update=(not include_create))
self.worker.update_job(fakes.JOB_ID,
'PROCESSING',
timeout=None,
error_message=None)
self.worker.update_job(fakes.JOB_ID,
'PROCESSING',
timeout=None,
error_message=None)
self.worker.update_job(fakes.JOB_ID,
'PROCESSING',
timeout=None,
error_message=None)
self.worker.update_job(fakes.JOB_ID,
'PROCESSING',
timeout=None,
error_message=None)
metadata = copy.copy(job['metadata'])
metadata['image_id'] = IMAGE_ID
backoff_factor = DEFAULT_BACKOFF_FACTOR**job['retry_count']
timeout = time_seq[-1] + datetime.timedelta(
minutes=DEFAULT_TIMEOUT_INCR * backoff_factor)
self.worker.update_job(fakes.JOB_ID,
'ERROR',
timeout=timeout,
error_message=mox.IsA(unicode)).AndReturn({
'status':
'ERROR',
'timeout':
job['timeout']
})
self.mox.StubOutWithMock(utils, 'generate_notification')
expected_job = copy.deepcopy(job)
expected_job['status'] = 'ERROR'
expected_job['metadata'] = metadata
if not is_retry:
utils.generate_notification(None, 'qonos.job.run.start',
mox.IsA(dict), mox.IsA(str))
else:
utils.generate_notification(None, 'qonos.job.retry', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict),
mox.IsA(str))
def assert_job_payload(job_payload):
self.assertTrue('error_message' in job_payload['job'])
del job_payload['job']['error_message']
self.assertEquals({'job': expected_job}, job_payload)
return True
utils.generate_notification(None, 'qonos.job.update',
mox.Func(assert_job_payload), 'ERROR')
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(job)
self.mox.VerifyAll()
