def test_process_job_should_update_status_and_timestamp(self): timeutils.set_time_override() base_time = timeutils.utcnow() time_seq = [ base_time, base_time, base_time + datetime.timedelta(seconds=305), base_time + datetime.timedelta(minutes=60, seconds=5), base_time + datetime.timedelta(minutes=60, seconds=305), ] timeutils.set_time_override_seq(time_seq) job = copy.deepcopy(self.job) job['timeout'] = base_time + datetime.timedelta(minutes=60) self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image( mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('QUEUED')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('ACTIVE')) mock_retention = MockRetention() self.nova_client.rax_scheduled_images_python_novaclient_ext.\ get(mox.IsA(str)).AndReturn(mock_retention) self._init_worker_mock() self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=mox.IsA(datetime.datetime), error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.mox.StubOutWithMock(utils, 'generate_notification') utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict), mox.IsA(str)).MultipleTimes() utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict), mox.IsA(str)) self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None, error_message=None) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(job) self.mox.VerifyAll()
def test_process_job_should_update_status_and_timestamp(self): base_time = timeutils.utcnow() time_seq = [ base_time, base_time, base_time + datetime.timedelta(seconds=305), base_time + datetime.timedelta(minutes=60, seconds=5), base_time + datetime.timedelta(minutes=60, seconds=305), ] timeutils.set_time_override_seq(time_seq) job = copy.deepcopy(self.job) job['timeout'] = base_time + datetime.timedelta(minutes=60) self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('QUEUED')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('ACTIVE')) mock_retention = MockRetention() self.nova_client.rax_scheduled_images_python_novaclient_ext.\ get(mox.IsA(str)).AndReturn(mock_retention) self._init_worker_mock() self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=mox.IsA(datetime.datetime), error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.mox.StubOutWithMock(utils, 'generate_notification') utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict), mox.IsA(str)).MultipleTimes() utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict), mox.IsA(str)) self.worker.get_qonos_client().AndReturn(self.qonos_client) self.qonos_client.delete_schedule(mox.IsA(str)) self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None, error_message=None) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(job) self.mox.VerifyAll()
def _do_test_process_job_should_update_image_error(self, error_status): base_time = timeutils.utcnow() time_seq = [ base_time, base_time, base_time + datetime.timedelta(seconds=305), base_time + datetime.timedelta(seconds=605), base_time + datetime.timedelta(seconds=905), base_time + datetime.timedelta(seconds=1205), base_time + datetime.timedelta(seconds=1505), ] timeutils.set_time_override_seq(time_seq) job = copy.deepcopy(self.job) job['timeout'] = base_time + datetime.timedelta(minutes=60) self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('QUEUED')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( error_status) self._init_worker_mock() self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, 'ERROR', timeout=None, error_message=mox.IsA(str)) self.mox.StubOutWithMock(utils, 'generate_notification') utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(job) self.mox.VerifyAll()
def test_process_job_should_update_status_timestamp_no_retries(self): base_time = timeutils.utcnow() time_seq = [ base_time, base_time, base_time + datetime.timedelta(minutes=5, seconds=5), base_time + datetime.timedelta(minutes=60, seconds=5), base_time + datetime.timedelta(minutes=120, seconds=5), base_time + datetime.timedelta(minutes=180, seconds=5), base_time + datetime.timedelta(minutes=240, seconds=5), ] print "Time_seq: %s" % str(time_seq) timeutils.set_time_override_seq(time_seq) job = copy.deepcopy(fakes.JOB['job']) job['timeout'] = base_time + datetime.timedelta(minutes=60) self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('QUEUED')) self.nova_client.images.get(IMAGE_ID).MultipleTimes().AndReturn( MockImageStatus('SAVING')) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=mox.IsA(datetime.datetime), error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=mox.IsA(datetime.datetime), error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=mox.IsA(datetime.datetime), error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=mox.IsA(datetime.datetime), error_message=None) self.worker.update_job(fakes.JOB_ID, 'TIMED_OUT', timeout=None, error_message=None) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(job) self.mox.VerifyAll()
def test_process_job_should_update_status_and_timestamp(self): base_time = timeutils.utcnow() time_seq = [ base_time, base_time, base_time + datetime.timedelta(seconds=305), base_time + datetime.timedelta(minutes=60, seconds=5), base_time + datetime.timedelta(minutes=60, seconds=305), ] timeutils.set_time_override_seq(time_seq) job = copy.deepcopy(fakes.JOB['job']) job['timeout'] = base_time + datetime.timedelta(minutes=60) self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('QUEUED')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('ACTIVE')) self.nova_client.servers.get(mox.IsA(str)).AndReturn( MockServer()) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=mox.IsA(datetime.datetime), error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=mox.IsA(datetime.datetime), error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None, error_message=None) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(job) self.mox.VerifyAll()
def test_process_job_should_update_status_only(self): base_time = timeutils.utcnow() time_seq = [ base_time, base_time, base_time + datetime.timedelta(seconds=305), base_time + datetime.timedelta(seconds=605), base_time + datetime.timedelta(seconds=905), ] timeutils.set_time_override_seq(time_seq) job = copy.deepcopy(self.job) job['timeout'] = base_time + datetime.timedelta(minutes=60) self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('QUEUED')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('ACTIVE')) mock_retention = MockRetention() self.nova_client.rax_scheduled_images_python_novaclient_ext.\ get(mox.IsA(str)).AndReturn(mock_retention) self._simple_prepare_worker_mock(2) self.mox.StubOutWithMock(utils, 'generate_notification') utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict), mox.IsA(str)) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(job) self.mox.VerifyAll()
def test_process_job_should_update_status_timestamp_no_retries(self): base_time = timeutils.utcnow() time_seq = [ base_time, base_time, base_time + datetime.timedelta(minutes=5, seconds=5), base_time + datetime.timedelta(minutes=60, seconds=5), base_time + datetime.timedelta(minutes=120, seconds=5), base_time + datetime.timedelta(minutes=180, seconds=5), base_time + datetime.timedelta(minutes=240, seconds=5), ] timeutils.set_time_override_seq(time_seq) job = copy.deepcopy(self.job) job["timeout"] = base_time + datetime.timedelta(minutes=60) self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("QUEUED")) self.nova_client.images.get(IMAGE_ID).MultipleTimes().AndReturn(MockImageStatus("SAVING")) self._init_worker_mock() self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=mox.IsA(datetime.datetime), error_message=None) self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=mox.IsA(datetime.datetime), error_message=None) self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=mox.IsA(datetime.datetime), error_message=None) self.worker.update_job(fakes.JOB_ID, "TIMED_OUT", timeout=None, error_message=None) self.mox.StubOutWithMock(utils, "generate_notification") utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str)) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(job) self.mox.VerifyAll()
def _do_test_process_job_should_update_image_error( self, error_status, include_create=True, include_queued=True, is_retry=False, job=None ): base_time = timeutils.utcnow() time_seq = [ base_time, base_time, base_time + datetime.timedelta(seconds=305), base_time + datetime.timedelta(seconds=605), base_time + datetime.timedelta(seconds=905), base_time + datetime.timedelta(seconds=1205), base_time + datetime.timedelta(seconds=1505), ] timeutils.set_time_override_seq(time_seq) if job is None: job = copy.deepcopy(self.job) job["timeout"] = base_time + datetime.timedelta(minutes=60) if include_create: self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) if include_queued: self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("QUEUED")) else: self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING")) self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING")) self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING")) self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING")) self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING")) self.nova_client.images.get(IMAGE_ID).AndReturn(error_status) self._init_worker_mock(skip_metadata_update=(not include_create)) self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=None, error_message=None) metadata = copy.copy(job["metadata"]) metadata["image_id"] = IMAGE_ID backoff_factor = DEFAULT_BACKOFF_FACTOR ** job["retry_count"] timeout = time_seq[-1] + datetime.timedelta(minutes=DEFAULT_TIMEOUT_INCR * backoff_factor) self.worker.update_job(fakes.JOB_ID, "ERROR", timeout=timeout, error_message=mox.IsA(str)) self.mox.StubOutWithMock(utils, "generate_notification") if not is_retry: utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str)) else: utils.generate_notification(None, "qonos.job.retry", mox.IsA(dict), mox.IsA(str)) self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(job) self.mox.VerifyAll()
def _do_test_process_job_should_update_image_error(self, error_status, include_create=True, include_queued=True, is_retry=False, job=None): base_time = timeutils.utcnow() time_seq = [ base_time, base_time, base_time, base_time + datetime.timedelta(seconds=305), base_time + datetime.timedelta(seconds=605), base_time + datetime.timedelta(seconds=905), base_time + datetime.timedelta(seconds=1205), base_time + datetime.timedelta(seconds=1505), ] timeutils.set_time_override_seq(time_seq) if job is None: job = copy.deepcopy(self.job) job['timeout'] = base_time + datetime.timedelta(minutes=60) if include_create: self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) if include_queued: self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('QUEUED')) else: self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( error_status) self._init_worker_mock(skip_metadata_update=(not include_create)) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) metadata = copy.copy(job['metadata']) metadata['image_id'] = IMAGE_ID backoff_factor = DEFAULT_BACKOFF_FACTOR ** job['retry_count'] timeout = time_seq[-1] + datetime.timedelta(minutes= DEFAULT_TIMEOUT_INCR * backoff_factor) self.worker.update_job(fakes.JOB_ID, 'ERROR', timeout=timeout, error_message=mox.IsA(unicode)).AndReturn( {'status': 'ERROR', 'timeout': job['timeout']}) self.mox.StubOutWithMock(utils, 'generate_notification') expected_job = copy.deepcopy(job) expected_job['status'] = 'ERROR' expected_job['metadata'] = metadata if not is_retry: utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) else: utils.generate_notification(None, 'qonos.job.retry', mox.IsA(dict), mox.IsA(str)) utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict), mox.IsA(str)) def assert_job_payload(job_payload): self.assertTrue('error_message' in job_payload['job']) del job_payload['job']['error_message'] self.assertEquals({'job': expected_job}, job_payload) return True utils.generate_notification(None, 'qonos.job.update', mox.Func(assert_job_payload), 'ERROR') self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(job) self.mox.VerifyAll()
def _do_test_process_job_should_update_image_error(self, error_status, include_create=True, include_queued=True, is_retry=False, job=None): base_time = timeutils.utcnow() time_seq = [ base_time, base_time, base_time, base_time + datetime.timedelta(seconds=305), base_time + datetime.timedelta(seconds=605), base_time + datetime.timedelta(seconds=905), base_time + datetime.timedelta(seconds=1205), base_time + datetime.timedelta(seconds=1505), ] timeutils.set_time_override_seq(time_seq) if job is None: job = copy.deepcopy(self.job) job['timeout'] = base_time + datetime.timedelta(minutes=60) if include_create: self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer()) self.nova_client.servers.create_image( mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID) if include_queued: self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('QUEUED')) else: self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn( MockImageStatus('SAVING')) self.nova_client.images.get(IMAGE_ID).AndReturn(error_status) self._init_worker_mock(skip_metadata_update=(not include_create)) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None, error_message=None) metadata = copy.copy(job['metadata']) metadata['image_id'] = IMAGE_ID backoff_factor = DEFAULT_BACKOFF_FACTOR**job['retry_count'] timeout = time_seq[-1] + datetime.timedelta( minutes=DEFAULT_TIMEOUT_INCR * backoff_factor) self.worker.update_job(fakes.JOB_ID, 'ERROR', timeout=timeout, error_message=mox.IsA(unicode)).AndReturn({ 'status': 'ERROR', 'timeout': job['timeout'] }) self.mox.StubOutWithMock(utils, 'generate_notification') expected_job = copy.deepcopy(job) expected_job['status'] = 'ERROR' expected_job['metadata'] = metadata if not is_retry: utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict), mox.IsA(str)) else: utils.generate_notification(None, 'qonos.job.retry', mox.IsA(dict), mox.IsA(str)) utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict), mox.IsA(str)) def assert_job_payload(job_payload): self.assertTrue('error_message' in job_payload['job']) del job_payload['job']['error_message'] self.assertEquals({'job': expected_job}, job_payload) return True utils.generate_notification(None, 'qonos.job.update', mox.Func(assert_job_payload), 'ERROR') self.mox.ReplayAll() processor = TestableSnapshotProcessor(self.nova_client) processor.init_processor(self.worker) processor.process_job(job) self.mox.VerifyAll()