def create(self, request, body): if (body is None or body.get('job') is None or body['job'].get('schedule_id') is None): raise webob.exc.HTTPBadRequest() job = body['job'] try: schedule = self.db_api.schedule_get_by_id(job['schedule_id']) except exception.NotFound: raise webob.exc.HTTPNotFound() # Check integrity of schedule and update next run expected_next_run = job.get('next_run') if expected_next_run: expected_next_run = timeutils.parse_isotime(job.get('next_run')) next_run = api_utils.schedule_to_next_run(schedule, timeutils.utcnow()) try: self.db_api.schedule_test_and_set_next_run(schedule['id'], expected_next_run, next_run) except exception.NotFound: msg = _("Specified next run does not match the current next run" " value. This could mean schedule has either changed" "or has already been scheduled since you last expected.") raise webob.exc.HTTPConflict(explanation=msg) # Update schedule last_scheduled values = {} values['last_scheduled'] = timeutils.utcnow() self.db_api.schedule_update(schedule['id'], values) # Create job values = {} values.update(job) values['tenant'] = schedule['tenant'] values['action'] = schedule['action'] values['status'] = 'QUEUED' job_metadata = [] for metadata in schedule['schedule_metadata']: job_metadata.append({ 'key': metadata['key'], 'value': metadata['value'] }) values['job_metadata'] = job_metadata job_action = values['action'] if not 'timeout' in values: values['timeout'] = api_utils.get_new_timeout_by_action(job_action) values['hard_timeout'] = \ api_utils.get_new_timeout_by_action(job_action) job = self.db_api.job_create(values) utils.serialize_datetimes(job) api_utils.serialize_job_metadata(job) job = {'job': job} utils.generate_notification(None, 'qonos.job.create', job, 'INFO') return job
def test_process_job_should_succeed_after_multiple_tries(self): timeutils.set_time_override() self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('QUEUED')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('ACTIVE')) mock_retention = MockRetention() self.nova_client.rax_scheduled_images_python_novaclient_ext.\ get(mox.IsA(str)).AndReturn(mock_retention) self._simple_prepare_worker_mock() self.mox.StubOutWithMock(utils, 'generate_notification') utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict), mox.IsA(str)) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(self.job) self.mox.VerifyAll()
def test_process_job_should_not_create_on_job_error_image_ok(self): timeutils.set_time_override() self.job['metadata']['image_id'] = IMAGE_ID self.job['status'] = 'ERROR' self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('ACTIVE')) mock_retention = MockRetention() self.nova_client.rax_scheduled_images_python_novaclient_ext.\ get(mox.IsA(str)).AndReturn(mock_retention) self._simple_prepare_worker_mock(skip_metadata_update=True) self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None, error_message=None) self.mox.StubOutWithMock(utils, 'generate_notification') utils.generate_notification(None, mox.IsA(str), mox.IsA(dict), mox.IsA(str)).MultipleTimes() self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(self.job) self.mox.VerifyAll()
def test_doesnt_delete_images_on_retention_error(self): timeutils.set_time_override() self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('ACTIVE')) self.nova_client.rax_scheduled_images_python_novaclient_ext.\ get(mox.IsA(str)).AndRaise(Exception()) mock_server = MockServer(retention=None) self._init_worker_mock() self.mox.StubOutWithMock(utils, 'generate_notification') utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict), mox.IsA(str)) self.worker.get_qonos_client().AndReturn(self.qonos_client) self.qonos_client.delete_schedule(mox.IsA(str)) self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None, error_message=None) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(self.job) self.mox.VerifyAll()
def test_deletes_images_more_than_retention(self): timeutils.set_time_override() instance_id = self.job["metadata"]["instance_id"] self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("ACTIVE")) mock_retention = MockRetention(3) self.nova_client.rax_scheduled_images_python_novaclient_ext.get(mox.IsA(str)).AndReturn(mock_retention) mock_server = MockServer(instance_id=instance_id) image_list = self._create_images_list(mock_server.id, 5) self.nova_client.images.list(detailed=True).AndReturn(image_list) # The image list happens to be in descending created order self.nova_client.images.delete(image_list[-2].id) self.nova_client.images.delete(image_list[-1].id) self._init_worker_mock() self.worker.update_job(fakes.JOB_ID, "DONE", timeout=None, error_message=None) self.mox.StubOutWithMock(utils, "generate_notification") utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str)) utils.generate_notification(None, "qonos.job.run.end", mox.IsA(dict), mox.IsA(str)) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(self.job) self.mox.VerifyAll()
def test_doesnt_delete_images_from_another_instance(self): timeutils.set_time_override() instance_id = self.job["metadata"]["instance_id"] self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("ACTIVE")) mock_retention = MockRetention(3) self.nova_client.rax_scheduled_images_python_novaclient_ext.get(mox.IsA(str)).AndReturn(mock_retention) mock_server = MockServer(instance_id=instance_id) image_list = self._create_images_list(mock_server.id, 5) to_delete = image_list[3:] image_list.extend(self._create_images_list(uuidutils.generate_uuid(), 3)) self.nova_client.images.list(detailed=True).AndReturn(image_list) # The image list happens to be in descending created order self.nova_client.images.delete(to_delete[0].id) self.nova_client.images.delete(to_delete[1].id) self._simple_prepare_worker_mock() self.mox.StubOutWithMock(utils, "generate_notification") utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str)) utils.generate_notification(None, "qonos.job.run.end", mox.IsA(dict), mox.IsA(str)) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(self.job) self.mox.VerifyAll()
def test_process_job_should_cancel_if_schedule_deleted(self): self._init_qonos_client() self.mox.StubOutWithMock(utils, 'generate_notification') utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) response = {'status': 'CANCELLED', 'timeout': self.job['timeout']} self.worker.update_job(fakes.JOB_ID, 'CANCELLED', timeout=None, error_message=mox.IsA(str)).AndReturn(response) expected_payload = {'job': {'status': 'CANCELLED', 'hard_timeout': self.job['hard_timeout'], 'created_at': self.job['created_at'], 'modified_at': self.job['modified_at'], 'retry_count': 1, 'schedule_id': '33333333-3333-3333-3333-33333333', 'worker_id': '11111111-1111-1111-1111-11111111', 'timeout': self.job['timeout'], 'action': 'snapshot', 'id': '22222222-2222-2222-2222-22222222', 'tenant': '44444444-4444-4444-4444-44444444', 'metadata': {'instance_id': '55555555-5555-5555-5555-55555555'}}} utils.generate_notification(None, 'qonos.job.failed', expected_payload, mox.IsA(str)) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(self.job) self.mox.VerifyAll()
def test_generate_notification(self): notification = {} def fake_publisher_id(service, host=None): if not host: host = 'localhost' return "%s.%s" % (service, host) def fake_notify(context, publisher_id, event_type, priority, payload): notification['context'] = context notification['publisher_id'] = publisher_id notification['event_type'] = event_type notification['priority'] = priority notification['payload'] = payload self.stubs.Set(notifier_api, 'publisher_id', fake_publisher_id) self.stubs.Set(notifier_api, 'notify', fake_notify) payload = {'id': 'fake-id'} self.assertEqual(notification, {}) self.config(host='localhost') utils.generate_notification(None, 'qonos.fake.event', payload) expected_notification = { 'context': None, 'publisher_id': 'qonos.localhost', 'event_type': 'qonos.fake.event', 'priority': 'INFO', 'payload': {'id': 'fake-id'} } self.assertEqual(notification, expected_notification)
def test_process_job_should_update_status_timestamp_no_retries(self): base_time = timeutils.utcnow() time_seq = [ base_time, base_time, base_time, base_time + datetime.timedelta(minutes=5, seconds=5), base_time + datetime.timedelta(minutes=60, seconds=5), base_time + datetime.timedelta(minutes=120, seconds=5), base_time + datetime.timedelta(minutes=180, seconds=5), base_time + datetime.timedelta(minutes=240, seconds=5), ] timeutils.set_time_override_seq(time_seq) job = copy.deepcopy(self.job) job['timeout'] = base_time + datetime.timedelta(minutes=60) self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image( mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('QUEUED')) self.nova_client.images.get(IMAGE_ID).MultipleTimes().AndReturn( MockImageStatus('SAVING')) self._init_worker_mock() self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=mox.IsA(datetime.datetime), error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=mox.IsA(datetime.datetime), error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=mox.IsA(datetime.datetime), error_message=None) self.worker.update_job(fakes.JOB_ID, 'TIMED_OUT', timeout=None, error_message=None) self.mox.StubOutWithMock(utils, 'generate_notification') utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict), mox.IsA(str)).MultipleTimes() self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(job) self.mox.VerifyAll()
def _do_test_process_job_should_update_image_error(self, error_status): base_time = timeutils.utcnow() time_seq = [ base_time, base_time, base_time + datetime.timedelta(seconds=305), base_time + datetime.timedelta(seconds=605), base_time + datetime.timedelta(seconds=905), base_time + datetime.timedelta(seconds=1205), base_time + datetime.timedelta(seconds=1505), ] timeutils.set_time_override_seq(time_seq) job = copy.deepcopy(self.job) job['timeout'] = base_time + datetime.timedelta(minutes=60) self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('QUEUED')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( error_status) self._init_worker_mock() self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, 'ERROR', timeout=None, error_message=mox.IsA(str)) self.mox.StubOutWithMock(utils, 'generate_notification') utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(job) self.mox.VerifyAll()
def test_process_job_should_cancel_if_schedule_deleted(self): self._init_qonos_client() self.mox.StubOutWithMock(utils, "generate_notification") utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str)) self.worker.update_job(fakes.JOB_ID, "CANCELLED", timeout=None, error_message=mox.IsA(str)) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(self.job) self.mox.VerifyAll()
def test_process_job_should_update_status_and_timestamp(self): base_time = timeutils.utcnow() time_seq = [ base_time, base_time, base_time + datetime.timedelta(seconds=305), base_time + datetime.timedelta(minutes=60, seconds=5), base_time + datetime.timedelta(minutes=60, seconds=305), ] timeutils.set_time_override_seq(time_seq) job = copy.deepcopy(self.job) job['timeout'] = base_time + datetime.timedelta(minutes=60) self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('QUEUED')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('ACTIVE')) mock_retention = MockRetention() self.nova_client.rax_scheduled_images_python_novaclient_ext.\ get(mox.IsA(str)).AndReturn(mock_retention) self._init_worker_mock() self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=mox.IsA(datetime.datetime), error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.mox.StubOutWithMock(utils, 'generate_notification') utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict), mox.IsA(str)) self.worker.get_qonos_client().AndReturn(self.qonos_client) self.qonos_client.delete_schedule(mox.IsA(str)) self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None, error_message=None) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(job) self.mox.VerifyAll()
def test_process_job_should_cancel_if_no_instance_id(self): self._init_qonos_client(schedule=MockSchedule()) self.mox.StubOutWithMock(utils, "generate_notification") utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str)) self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=mox.IsA(datetime.datetime), error_message=None) self.worker.update_job(fakes.JOB_ID, "CANCELLED", timeout=None, error_message=mox.IsA(str)) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) del self.job["metadata"]["instance_id"] processor.process_job(self.job) self.mox.VerifyAll()
def test_process_job_should_fail_if_it_reached_max_retry_count(self): mox.Reset(self.worker) self.mox.StubOutWithMock(utils, 'generate_notification') max_retry_count = 2 self.config(max_retry=max_retry_count, group='snapshot_worker') self.job['retry_count'] = max_retry_count + 1 utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) self.worker.update_job(fakes.JOB_ID, 'MAX_RETRIED', timeout=None, error_message=mox.IsA(str))\ .AndReturn({'status': 'MAX_RETRIED', 'timeout': self.job['timeout']}) expected_payload = { 'job': { 'status': 'MAX_RETRIED', 'hard_timeout': self.job['hard_timeout'], 'created_at': self.job['created_at'], 'modified_at': self.job['modified_at'], 'retry_count': self.job['retry_count'], 'schedule_id': '33333333-3333-3333-3333-33333333', 'worker_id': '11111111-1111-1111-1111-11111111', 'timeout': self.job['timeout'], 'action': 'snapshot', 'id': '22222222-2222-2222-2222-22222222', 'tenant': '44444444-4444-4444-4444-44444444', 'metadata': { 'instance_id': '55555555-5555-5555-5555-55555555' } } } utils.generate_notification(None, 'qonos.job.failed', expected_payload, mox.IsA(str)) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(self.job) self.mox.VerifyAll()
def test_process_job_should_cancel_if_instance_not_found(self): self._init_qonos_client(schedule=MockSchedule()) self.mox.StubOutWithMock(utils, "generate_notification") utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str)) self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndRaise( exceptions.NotFound("404") ) self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=mox.IsA(datetime.datetime), error_message=None) self.worker.update_job(fakes.JOB_ID, "CANCELLED", timeout=None, error_message=mox.IsA(str)) self.worker.get_qonos_client().AndReturn(self.qonos_client) self.qonos_client.delete_schedule(mox.IsA(str)) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(self.job) self.mox.VerifyAll()
def test_process_job_should_fail_if_hard_timed_out(self): mox.Reset(self.worker) self.mox.StubOutWithMock(utils, 'generate_notification') now = timeutils.utcnow() self.job['hard_timeout'] = timeutils.strtime(at=now) utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) self.worker.update_job(fakes.JOB_ID, 'HARD_TIMED_OUT', timeout=None, error_message=mox.IsA(str))\ .AndReturn({'status': 'HARD_TIMED_OUT', 'timeout': self.job['timeout']}) expected_payload = { 'job': { 'status': 'HARD_TIMED_OUT', 'hard_timeout': self.job['hard_timeout'], 'created_at': self.job['created_at'], 'modified_at': self.job['modified_at'], 'retry_count': 1, 'schedule_id': '33333333-3333-3333-3333-33333333', 'worker_id': '11111111-1111-1111-1111-11111111', 'timeout': self.job['timeout'], 'action': 'snapshot', 'id': '22222222-2222-2222-2222-22222222', 'tenant': '44444444-4444-4444-4444-44444444', 'metadata': { 'instance_id': '55555555-5555-5555-5555-55555555' } } } utils.generate_notification(None, 'qonos.job.failed', expected_payload, mox.IsA(str)) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(self.job) self.mox.VerifyAll()
def test_process_job_should_fail_if_it_reached_max_retry_count(self): mox.Reset(self.worker) self.mox.StubOutWithMock(utils, 'generate_notification') max_retry_count = 2 self.config(max_retry=max_retry_count, group='snapshot_worker') self.job['retry_count'] = max_retry_count + 1 utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) self.worker.update_job(fakes.JOB_ID, 'MAX_RETRIED', timeout=None, error_message=mox.IsA(str))\ .AndReturn({'status': 'MAX_RETRIED', 'timeout': self.job['timeout']}) expected_payload = {'job': {'status': 'MAX_RETRIED', 'hard_timeout': self.job['hard_timeout'], 'created_at': self.job['created_at'], 'modified_at': self.job['modified_at'], 'retry_count': self.job['retry_count'], 'schedule_id': '33333333-3333-3333-3333-33333333', 'worker_id': '11111111-1111-1111-1111-11111111', 'timeout': self.job['timeout'], 'action': 'snapshot', 'id': '22222222-2222-2222-2222-22222222', 'tenant': '44444444-4444-4444-4444-44444444', 'metadata': {'instance_id': '55555555-5555-5555-5555-55555555'}}} utils.generate_notification(None, 'qonos.job.failed', expected_payload, mox.IsA(str)) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(self.job) self.mox.VerifyAll()
def test_deletes_images_more_than_retention(self): timeutils.set_time_override() instance_id = self.job['metadata']['instance_id'] self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image( mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('ACTIVE')) mock_retention = MockRetention(3) self.nova_client.rax_scheduled_images_python_novaclient_ext.\ get(mox.IsA(str)).AndReturn(mock_retention) mock_server = MockServer(instance_id=instance_id) image_list = self._create_images_list(mock_server.id, 5) self.nova_client.images.list(detailed=True).AndReturn(image_list) # The image list happens to be in descending created order self.nova_client.images.delete(image_list[-2].id) self.nova_client.images.delete(image_list[-1].id) self._init_worker_mock() self.mox.StubOutWithMock(utils, 'generate_notification') utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict), mox.IsA(str)).MultipleTimes() self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None, error_message=None) utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict), mox.IsA(str)) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(self.job) self.mox.VerifyAll()
def test_doesnt_delete_images_on_retention_error(self): timeutils.set_time_override() self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image( mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('ACTIVE')) self.nova_client.rax_scheduled_images_python_novaclient_ext.\ get(mox.IsA(str)).AndRaise(Exception()) mock_server = MockServer(retention=None) self._init_worker_mock() self.mox.StubOutWithMock(utils, 'generate_notification') utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict), mox.IsA(str)).MultipleTimes() utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict), mox.IsA(str)) self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None, error_message=None) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(self.job) self.mox.VerifyAll()
def test_process_job_should_fail_if_hard_timed_out(self): mox.Reset(self.worker) self.mox.StubOutWithMock(utils, 'generate_notification') now = timeutils.utcnow() self.job['hard_timeout'] = timeutils.strtime(at=now) utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) self.worker.update_job(fakes.JOB_ID, 'HARD_TIMED_OUT', timeout=None, error_message=mox.IsA(str))\ .AndReturn({'status': 'HARD_TIMED_OUT', 'timeout': self.job['timeout']}) expected_payload = {'job': {'status': 'HARD_TIMED_OUT', 'hard_timeout': self.job['hard_timeout'], 'created_at': self.job['created_at'], 'modified_at': self.job['modified_at'], 'retry_count': 1, 'schedule_id': '33333333-3333-3333-3333-33333333', 'worker_id': '11111111-1111-1111-1111-11111111', 'timeout': self.job['timeout'], 'action': 'snapshot', 'id': '22222222-2222-2222-2222-22222222', 'tenant': '44444444-4444-4444-4444-44444444', 'metadata': {'instance_id': '55555555-5555-5555-5555-55555555'}}} utils.generate_notification(None, 'qonos.job.failed', expected_payload, mox.IsA(str)) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(self.job) self.mox.VerifyAll()
def test_process_job_should_succeed_immediately(self): timeutils.set_time_override() self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("ACTIVE")) mock_retention = MockRetention() self.nova_client.rax_scheduled_images_python_novaclient_ext.get(mox.IsA(str)).AndReturn(mock_retention) self._simple_prepare_worker_mock() self.mox.StubOutWithMock(utils, "generate_notification") utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str)) utils.generate_notification(None, "qonos.job.run.end", mox.IsA(dict), mox.IsA(str)) self.worker.get_qonos_client().AndReturn(self.qonos_client) self.qonos_client.delete_schedule(mox.IsA(str)) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(self.job) self.mox.VerifyAll()
def test_process_job_new_image_when_retrying_with_failed_image(self): timeutils.set_time_override() self.job['metadata']['image_id'] = IMAGE_ID self.job['status'] = 'ERROR' #make an error checking the previous image self.nova_client.images.get(IMAGE_ID).AndRaise( Exception()) self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('ACTIVE')) mock_retention = MockRetention() self.nova_client.rax_scheduled_images_python_novaclient_ext.\ get(mox.IsA(str)).AndReturn(mock_retention) self._simple_prepare_worker_mock() self.mox.StubOutWithMock(utils, 'generate_notification') utils.generate_notification(None, 'qonos.job.retry', mox.IsA(dict), mox.IsA(str)) utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict), mox.IsA(str)) self.worker.get_qonos_client().AndReturn(self.qonos_client) self.qonos_client.delete_schedule(mox.IsA(str)) self.worker.update_job(fakes.JOB_ID, 'DONE', error_message=None, timeout=None).AndReturn(None) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(self.job) self.mox.VerifyAll()
def test_process_job_should_update_status_only(self): base_time = timeutils.utcnow() time_seq = [ base_time, base_time, base_time + datetime.timedelta(seconds=305), base_time + datetime.timedelta(seconds=605), base_time + datetime.timedelta(seconds=905), ] timeutils.set_time_override_seq(time_seq) job = copy.deepcopy(self.job) job["timeout"] = base_time + datetime.timedelta(minutes=60) self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("QUEUED")) self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING")) self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING")) self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("ACTIVE")) mock_retention = MockRetention() self.nova_client.rax_scheduled_images_python_novaclient_ext.get(mox.IsA(str)).AndReturn(mock_retention) self._simple_prepare_worker_mock(2) self.mox.StubOutWithMock(utils, "generate_notification") utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str)) utils.generate_notification(None, "qonos.job.run.end", mox.IsA(dict), mox.IsA(str)) self.worker.get_qonos_client().AndReturn(self.qonos_client) self.qonos_client.delete_schedule(mox.IsA(str)) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(job) self.mox.VerifyAll()
def test_process_job_should_update_status_timestamp_no_retries(self): base_time = timeutils.utcnow() time_seq = [ base_time, base_time, base_time + datetime.timedelta(minutes=5, seconds=5), base_time + datetime.timedelta(minutes=60, seconds=5), base_time + datetime.timedelta(minutes=120, seconds=5), base_time + datetime.timedelta(minutes=180, seconds=5), base_time + datetime.timedelta(minutes=240, seconds=5), ] timeutils.set_time_override_seq(time_seq) job = copy.deepcopy(self.job) job["timeout"] = base_time + datetime.timedelta(minutes=60) self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("QUEUED")) self.nova_client.images.get(IMAGE_ID).MultipleTimes().AndReturn(MockImageStatus("SAVING")) self._init_worker_mock() self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=mox.IsA(datetime.datetime), error_message=None) self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=mox.IsA(datetime.datetime), error_message=None) self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=mox.IsA(datetime.datetime), error_message=None) self.worker.update_job(fakes.JOB_ID, "TIMED_OUT", timeout=None, error_message=None) self.mox.StubOutWithMock(utils, "generate_notification") utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str)) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(job) self.mox.VerifyAll()
def test_process_job_should_cancel_if_no_instance_id(self): self._init_qonos_client(schedule=dict()) self.mox.StubOutWithMock(utils, 'generate_notification') utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=mox.IsA(datetime.datetime), error_message=None).\ AndReturn({'status': 'PROCESSING', 'timeout': 'new_timeout'}) self.worker.update_job(fakes.JOB_ID, 'CANCELLED', timeout=None, error_message=mox.IsA(str)).\ AndReturn({'status': 'CANCELLED', 'timeout': self.job['timeout']}) expected_processing_payload = \ {'job': {'status': 'PROCESSING', 'hard_timeout': self.job['hard_timeout'], 'created_at': self.job['created_at'], 'modified_at': self.job['modified_at'], 'retry_count': 1, 'schedule_id': '33333333-3333-3333-3333-33333333', 'worker_id': '11111111-1111-1111-1111-11111111', 'timeout': 'new_timeout', 'action': 'snapshot', 'id': '22222222-2222-2222-2222-22222222', 'tenant': '44444444-4444-4444-4444-44444444', 'metadata': {}}} expected_cancelled_payload = \ {'job': {'status': 'CANCELLED', 'hard_timeout': self.job['hard_timeout'], 'created_at': self.job['created_at'], 'modified_at': self.job['modified_at'], 'retry_count': 1, 'schedule_id': '33333333-3333-3333-3333-33333333', 'worker_id': '11111111-1111-1111-1111-11111111', 'timeout': self.job['timeout'], 'action': 'snapshot', 'id': '22222222-2222-2222-2222-22222222', 'tenant': '44444444-4444-4444-4444-44444444', 'metadata': {}}} utils.generate_notification(None, 'qonos.job.update', expected_processing_payload, mox.IsA(str)) utils.generate_notification(None, 'qonos.job.failed', expected_cancelled_payload, mox.IsA(str)) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) del self.job['metadata']['instance_id'] processor.process_job(self.job) self.mox.VerifyAll()
def test_process_job_should_succeed_immediately(self): timeutils.set_time_override() self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('ACTIVE')) mock_retention = MockRetention() self.nova_client.rax_scheduled_images_python_novaclient_ext.\ get(mox.IsA(str)).AndReturn(mock_retention) self._init_worker_mock(False) expected_processing_payload = \ {'job': {'status': 'PROCESSING', 'hard_timeout': self.job['hard_timeout'], 'created_at': self.job['created_at'], 'modified_at': self.job['modified_at'], 'retry_count': 1, 'schedule_id': '33333333-3333-3333-3333-33333333', 'worker_id': '11111111-1111-1111-1111-11111111', 'timeout': self.job['timeout'], 'action': 'snapshot', 'id': '22222222-2222-2222-2222-22222222', 'tenant': '44444444-4444-4444-4444-44444444', 'metadata': {'instance_id': '55555555-5555-5555-5555-55555555'}}} self.mox.StubOutWithMock(utils, 'generate_notification') utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) utils.generate_notification(None, 'qonos.job.update', expected_processing_payload, mox.IsA(str)) utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict), mox.IsA(str)) self.worker.get_qonos_client().AndReturn(self.qonos_client) self.qonos_client.delete_schedule(mox.IsA(str)) processor = TestableSnapshotProcessor(self.nova_client) self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None, error_message=None).\ AndReturn({'status': 'CANCELLED', 'timeout': self.job['timeout']}) self.mox.ReplayAll() processor.init_processor(self.worker) processor.process_job(self.job) self.mox.VerifyAll()
def test_process_job_should_update_status_only(self): base_time = timeutils.utcnow() time_seq = [ base_time, base_time, base_time, base_time + datetime.timedelta(seconds=305), base_time + datetime.timedelta(seconds=605), base_time + datetime.timedelta(seconds=905), ] timeutils.set_time_override_seq(time_seq) job = copy.deepcopy(self.job) job['timeout'] = base_time + datetime.timedelta(minutes=60) self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image( mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('QUEUED')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('ACTIVE')) mock_retention = MockRetention() self.nova_client.rax_scheduled_images_python_novaclient_ext.\ get(mox.IsA(str)).AndReturn(mock_retention) self._simple_prepare_worker_mock(2) self.mox.StubOutWithMock(utils, 'generate_notification') utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict), mox.IsA(str)).MultipleTimes() utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict), mox.IsA(str)) self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None, error_message=None) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(job) self.mox.VerifyAll()
def test_process_job_should_succeed_immediately(self): timeutils.set_time_override() self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image( mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('ACTIVE')) mock_retention = MockRetention() self.nova_client.rax_scheduled_images_python_novaclient_ext.\ get(mox.IsA(str)).AndReturn(mock_retention) self._init_worker_mock(False) expected_processing_payload = \ {'job': {'status': 'PROCESSING', 'hard_timeout': self.job['hard_timeout'], 'created_at': self.job['created_at'], 'modified_at': self.job['modified_at'], 'retry_count': 1, 'schedule_id': '33333333-3333-3333-3333-33333333', 'worker_id': '11111111-1111-1111-1111-11111111', 'timeout': self.job['timeout'], 'action': 'snapshot', 'id': '22222222-2222-2222-2222-22222222', 'tenant': '44444444-4444-4444-4444-44444444', 'metadata': {'instance_id': '55555555-5555-5555-5555-55555555'}}} self.mox.StubOutWithMock(utils, 'generate_notification') utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) utils.generate_notification(None, 'qonos.job.update', expected_processing_payload, mox.IsA(str)) utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict), mox.IsA(str)) processor = TestableSnapshotProcessor(self.nova_client) self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None, error_message=None).\ AndReturn({'status': 'CANCELLED', 'timeout': self.job['timeout']}) self.mox.ReplayAll() processor.init_processor(self.worker) processor.process_job(self.job) self.mox.VerifyAll()
def test_process_job_new_image_when_retrying_with_failed_image(self): timeutils.set_time_override() self.job['metadata']['image_id'] = IMAGE_ID self.job['status'] = 'ERROR' #make an error checking the previous image self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('ERROR')) self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image( mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('ACTIVE')) mock_retention = MockRetention() self.nova_client.rax_scheduled_images_python_novaclient_ext.\ get(mox.IsA(str)).AndReturn(mock_retention) self._simple_prepare_worker_mock() self.mox.StubOutWithMock(utils, 'generate_notification') utils.generate_notification(None, 'qonos.job.retry', mox.IsA(dict), mox.IsA(str)) utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict), mox.IsA(str)).MultipleTimes() utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict), mox.IsA(str)) self.worker.update_job(fakes.JOB_ID, 'DONE', error_message=None, timeout=None).AndReturn(None) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(self.job) self.mox.VerifyAll()
def send_notification(self, event_type, payload, level='INFO'): utils.generate_notification(None, event_type, payload, level)
def _do_test_process_job_should_update_image_error(self, error_status, include_create=True, include_queued=True, is_retry=False, job=None): base_time = timeutils.utcnow() time_seq = [ base_time, base_time, base_time, base_time + datetime.timedelta(seconds=305), base_time + datetime.timedelta(seconds=605), base_time + datetime.timedelta(seconds=905), base_time + datetime.timedelta(seconds=1205), base_time + datetime.timedelta(seconds=1505), ] timeutils.set_time_override_seq(time_seq) if job is None: job = copy.deepcopy(self.job) job['timeout'] = base_time + datetime.timedelta(minutes=60) if include_create: self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) if include_queued: self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('QUEUED')) else: self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( error_status) self._init_worker_mock(skip_metadata_update=(not include_create)) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) metadata = copy.copy(job['metadata']) metadata['image_id'] = IMAGE_ID backoff_factor = DEFAULT_BACKOFF_FACTOR ** job['retry_count'] timeout = time_seq[-1] + datetime.timedelta(minutes= DEFAULT_TIMEOUT_INCR * backoff_factor) self.worker.update_job(fakes.JOB_ID, 'ERROR', timeout=timeout, error_message=mox.IsA(unicode)).AndReturn( {'status': 'ERROR', 'timeout': job['timeout']}) self.mox.StubOutWithMock(utils, 'generate_notification') expected_job = copy.deepcopy(job) expected_job['status'] = 'ERROR' expected_job['metadata'] = metadata if not is_retry: utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) else: utils.generate_notification(None, 'qonos.job.retry', mox.IsA(dict), mox.IsA(str)) utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict), mox.IsA(str)) def assert_job_payload(job_payload): self.assertTrue('error_message' in job_payload['job']) del job_payload['job']['error_message'] self.assertEquals({'job': expected_job}, job_payload) return True utils.generate_notification(None, 'qonos.job.update', mox.Func(assert_job_payload), 'ERROR') self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(job) self.mox.VerifyAll()
def _do_test_process_job_should_update_image_error( self, error_status, include_create=True, include_queued=True, is_retry=False, job=None ): base_time = timeutils.utcnow() time_seq = [ base_time, base_time, base_time + datetime.timedelta(seconds=305), base_time + datetime.timedelta(seconds=605), base_time + datetime.timedelta(seconds=905), base_time + datetime.timedelta(seconds=1205), base_time + datetime.timedelta(seconds=1505), ] timeutils.set_time_override_seq(time_seq) if job is None: job = copy.deepcopy(self.job) job["timeout"] = base_time + datetime.timedelta(minutes=60) if include_create: self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) if include_queued: self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("QUEUED")) else: self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING")) self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING")) self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING")) self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING")) self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING")) self.nova_client.images.get(IMAGE_ID).AndReturn(error_status) self._init_worker_mock(skip_metadata_update=(not include_create)) self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=None, error_message=None) metadata = copy.copy(job["metadata"]) metadata["image_id"] = IMAGE_ID backoff_factor = DEFAULT_BACKOFF_FACTOR ** job["retry_count"] timeout = time_seq[-1] + datetime.timedelta(minutes=DEFAULT_TIMEOUT_INCR * backoff_factor) self.worker.update_job(fakes.JOB_ID, "ERROR", timeout=timeout, error_message=mox.IsA(str)) self.mox.StubOutWithMock(utils, "generate_notification") if not is_retry: utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str)) else: utils.generate_notification(None, "qonos.job.retry", mox.IsA(dict), mox.IsA(str)) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(job) self.mox.VerifyAll()
def create(self, request, body): if (body is None or body.get('job') is None or body['job'].get('schedule_id') is None): raise webob.exc.HTTPBadRequest() job = body['job'] try: schedule = self.db_api.schedule_get_by_id(job['schedule_id']) except exception.NotFound: raise webob.exc.HTTPNotFound() # Check integrity of schedule and update next run expected_next_run = job.get('next_run') if expected_next_run: try: expected_next_run = timeutils.parse_isotime(expected_next_run) expected_next_run = expected_next_run.replace(tzinfo=None) except ValueError as e: msg = _('Invalid "next_run" value. Must be ISO 8601 format') raise webob.exc.HTTPBadRequest(explanation=msg) next_run = api_utils.schedule_to_next_run(schedule, timeutils.utcnow()) next_run = next_run.replace(tzinfo=None) try: self.db_api.schedule_test_and_set_next_run(schedule['id'], expected_next_run, next_run) except exception.NotFound: msg = _("Specified next run does not match the current next run" " value. This could mean schedule has either changed" "or has already been scheduled since you last expected.") raise webob.exc.HTTPConflict(explanation=msg) # Update schedule last_scheduled values = {} values['last_scheduled'] = timeutils.utcnow() self.db_api.schedule_update(schedule['id'], values) # Create job values = {} values.update(job) values['tenant'] = schedule['tenant'] values['action'] = schedule['action'] values['status'] = 'QUEUED' job_metadata = [] for metadata in schedule['schedule_metadata']: job_metadata.append({ 'key': metadata['key'], 'value': metadata['value'] }) values['job_metadata'] = job_metadata job_action = values['action'] if not 'timeout' in values: values['timeout'] = api_utils.get_new_timeout_by_action(job_action) values['hard_timeout'] = \ api_utils.get_new_timeout_by_action(job_action) job = self.db_api.job_create(values) utils.serialize_datetimes(job) api_utils.serialize_job_metadata(job) job = {'job': job} utils.generate_notification(None, 'qonos.job.create', job, 'INFO') return job
def _do_test_process_job_should_update_image_error(self, error_status, include_create=True, include_queued=True, is_retry=False, job=None): base_time = timeutils.utcnow() time_seq = [ base_time, base_time, base_time, base_time + datetime.timedelta(seconds=305), base_time + datetime.timedelta(seconds=605), base_time + datetime.timedelta(seconds=905), base_time + datetime.timedelta(seconds=1205), base_time + datetime.timedelta(seconds=1505), ] timeutils.set_time_override_seq(time_seq) if job is None: job = copy.deepcopy(self.job) job['timeout'] = base_time + datetime.timedelta(minutes=60) if include_create: self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image( mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) if include_queued: self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('QUEUED')) else: self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn(error_status) self._init_worker_mock(skip_metadata_update=(not include_create)) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) metadata = copy.copy(job['metadata']) metadata['image_id'] = IMAGE_ID backoff_factor = DEFAULT_BACKOFF_FACTOR**job['retry_count'] timeout = time_seq[-1] + datetime.timedelta( minutes=DEFAULT_TIMEOUT_INCR * backoff_factor) self.worker.update_job(fakes.JOB_ID, 'ERROR', timeout=timeout, error_message=mox.IsA(unicode)).AndReturn({ 'status': 'ERROR', 'timeout': job['timeout'] }) self.mox.StubOutWithMock(utils, 'generate_notification') expected_job = copy.deepcopy(job) expected_job['status'] = 'ERROR' expected_job['metadata'] = metadata if not is_retry: utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) else: utils.generate_notification(None, 'qonos.job.retry', mox.IsA(dict), mox.IsA(str)) utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict), mox.IsA(str)) def assert_job_payload(job_payload): self.assertTrue('error_message' in job_payload['job']) del job_payload['job']['error_message'] self.assertEquals({'job': expected_job}, job_payload) return True utils.generate_notification(None, 'qonos.job.update', mox.Func(assert_job_payload), 'ERROR') self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(job) self.mox.VerifyAll()