def test_process_job_should_update_status_and_timestamp(self):
timeutils.set_time_override()
base_time = timeutils.utcnow()
time_seq = [
base_time,
base_time,
base_time + datetime.timedelta(seconds=305),
base_time + datetime.timedelta(minutes=60, seconds=5),
base_time + datetime.timedelta(minutes=60, seconds=305),
]
timeutils.set_time_override_seq(time_seq)
job = copy.deepcopy(self.job)
job['timeout'] = base_time + datetime.timedelta(minutes=60)
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(
mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('QUEUED'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._init_worker_mock()
self.worker.update_job(fakes.JOB_ID,
'PROCESSING',
timeout=mox.IsA(datetime.datetime),
error_message=None)
self.worker.update_job(fakes.JOB_ID,
'PROCESSING',
timeout=None,
error_message=None)
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict),
mox.IsA(str)).MultipleTimes()
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
self.worker.update_job(fakes.JOB_ID,
'DONE',
timeout=None,
error_message=None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(job)
self.mox.VerifyAll()
def test_process_job_should_update_status_and_timestamp(self):
base_time = timeutils.utcnow()
time_seq = [
base_time,
base_time,
base_time + datetime.timedelta(seconds=305),
base_time + datetime.timedelta(minutes=60, seconds=5),
base_time + datetime.timedelta(minutes=60, seconds=305),
]
timeutils.set_time_override_seq(time_seq)
job = copy.deepcopy(self.job)
job['timeout'] = base_time + datetime.timedelta(minutes=60)
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('QUEUED'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._init_worker_mock()
self.worker.update_job(fakes.JOB_ID, 'PROCESSING',
timeout=mox.IsA(datetime.datetime),
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
error_message=None)
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict),
mox.IsA(str)).MultipleTimes()
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
self.worker.get_qonos_client().AndReturn(self.qonos_client)
self.qonos_client.delete_schedule(mox.IsA(str))
self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None,
error_message=None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(job)
self.mox.VerifyAll()
def _do_test_process_job_should_update_image_error(self, error_status):
base_time = timeutils.utcnow()
time_seq = [
base_time,
base_time,
base_time + datetime.timedelta(seconds=305),
base_time + datetime.timedelta(seconds=605),
base_time + datetime.timedelta(seconds=905),
base_time + datetime.timedelta(seconds=1205),
base_time + datetime.timedelta(seconds=1505),
]
timeutils.set_time_override_seq(time_seq)
job = copy.deepcopy(self.job)
job['timeout'] = base_time + datetime.timedelta(minutes=60)
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('QUEUED'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
error_status)
self._init_worker_mock()
self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'ERROR', timeout=None,
error_message=mox.IsA(str))
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(job)
self.mox.VerifyAll()
def test_process_job_should_update_status_timestamp_no_retries(self):
base_time = timeutils.utcnow()
time_seq = [
base_time,
base_time,
base_time + datetime.timedelta(minutes=5, seconds=5),
base_time + datetime.timedelta(minutes=60, seconds=5),
base_time + datetime.timedelta(minutes=120, seconds=5),
base_time + datetime.timedelta(minutes=180, seconds=5),
base_time + datetime.timedelta(minutes=240, seconds=5),
]
print "Time_seq: %s" % str(time_seq)
timeutils.set_time_override_seq(time_seq)
job = copy.deepcopy(fakes.JOB['job'])
job['timeout'] = base_time + datetime.timedelta(minutes=60)
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('QUEUED'))
self.nova_client.images.get(IMAGE_ID).MultipleTimes().AndReturn(
MockImageStatus('SAVING'))
self.worker.update_job(fakes.JOB_ID, 'PROCESSING',
timeout=mox.IsA(datetime.datetime),
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'PROCESSING',
timeout=mox.IsA(datetime.datetime),
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'PROCESSING',
timeout=mox.IsA(datetime.datetime),
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'PROCESSING',
timeout=mox.IsA(datetime.datetime),
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'TIMED_OUT', timeout=None,
error_message=None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(job)
self.mox.VerifyAll()
def test_process_job_should_update_status_and_timestamp(self):
base_time = timeutils.utcnow()
time_seq = [
base_time,
base_time,
base_time + datetime.timedelta(seconds=305),
base_time + datetime.timedelta(minutes=60, seconds=5),
base_time + datetime.timedelta(minutes=60, seconds=305),
]
timeutils.set_time_override_seq(time_seq)
job = copy.deepcopy(fakes.JOB['job'])
job['timeout'] = base_time + datetime.timedelta(minutes=60)
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('QUEUED'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
self.nova_client.servers.get(mox.IsA(str)).AndReturn(
MockServer())
self.worker.update_job(fakes.JOB_ID, 'PROCESSING',
timeout=mox.IsA(datetime.datetime),
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'PROCESSING',
timeout=mox.IsA(datetime.datetime),
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None,
error_message=None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(job)
self.mox.VerifyAll()
def test_process_job_should_update_status_only(self):
base_time = timeutils.utcnow()
time_seq = [
base_time,
base_time,
base_time + datetime.timedelta(seconds=305),
base_time + datetime.timedelta(seconds=605),
base_time + datetime.timedelta(seconds=905),
]
timeutils.set_time_override_seq(time_seq)
job = copy.deepcopy(self.job)
job['timeout'] = base_time + datetime.timedelta(minutes=60)
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('QUEUED'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._simple_prepare_worker_mock(2)
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(job)
self.mox.VerifyAll()
def test_process_job_should_update_status_timestamp_no_retries(self):
base_time = timeutils.utcnow()
time_seq = [
base_time,
base_time,
base_time + datetime.timedelta(minutes=5, seconds=5),
base_time + datetime.timedelta(minutes=60, seconds=5),
base_time + datetime.timedelta(minutes=120, seconds=5),
base_time + datetime.timedelta(minutes=180, seconds=5),
base_time + datetime.timedelta(minutes=240, seconds=5),
]
timeutils.set_time_override_seq(time_seq)
job = copy.deepcopy(self.job)
job["timeout"] = base_time + datetime.timedelta(minutes=60)
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("QUEUED"))
self.nova_client.images.get(IMAGE_ID).MultipleTimes().AndReturn(MockImageStatus("SAVING"))
self._init_worker_mock()
self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=mox.IsA(datetime.datetime), error_message=None)
self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=mox.IsA(datetime.datetime), error_message=None)
self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=mox.IsA(datetime.datetime), error_message=None)
self.worker.update_job(fakes.JOB_ID, "TIMED_OUT", timeout=None, error_message=None)
self.mox.StubOutWithMock(utils, "generate_notification")
utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(job)
self.mox.VerifyAll()
def _do_test_process_job_should_update_image_error(
self, error_status, include_create=True, include_queued=True, is_retry=False, job=None
):
base_time = timeutils.utcnow()
time_seq = [
base_time,
base_time,
base_time + datetime.timedelta(seconds=305),
base_time + datetime.timedelta(seconds=605),
base_time + datetime.timedelta(seconds=905),
base_time + datetime.timedelta(seconds=1205),
base_time + datetime.timedelta(seconds=1505),
]
timeutils.set_time_override_seq(time_seq)
if job is None:
job = copy.deepcopy(self.job)
job["timeout"] = base_time + datetime.timedelta(minutes=60)
if include_create:
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
if include_queued:
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("QUEUED"))
else:
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING"))
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING"))
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING"))
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING"))
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING"))
self.nova_client.images.get(IMAGE_ID).AndReturn(error_status)
self._init_worker_mock(skip_metadata_update=(not include_create))
self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=None, error_message=None)
self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=None, error_message=None)
self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=None, error_message=None)
self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=None, error_message=None)
metadata = copy.copy(job["metadata"])
metadata["image_id"] = IMAGE_ID
backoff_factor = DEFAULT_BACKOFF_FACTOR ** job["retry_count"]
timeout = time_seq[-1] + datetime.timedelta(minutes=DEFAULT_TIMEOUT_INCR * backoff_factor)
self.worker.update_job(fakes.JOB_ID, "ERROR", timeout=timeout, error_message=mox.IsA(str))
self.mox.StubOutWithMock(utils, "generate_notification")
if not is_retry:
utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str))
else:
utils.generate_notification(None, "qonos.job.retry", mox.IsA(dict), mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(job)
self.mox.VerifyAll()
def _do_test_process_job_should_update_image_error(self, error_status,
include_create=True,
include_queued=True,
is_retry=False,
job=None):
base_time = timeutils.utcnow()
time_seq = [
base_time,
base_time,
base_time,
base_time + datetime.timedelta(seconds=305),
base_time + datetime.timedelta(seconds=605),
base_time + datetime.timedelta(seconds=905),
base_time + datetime.timedelta(seconds=1205),
base_time + datetime.timedelta(seconds=1505),
]
timeutils.set_time_override_seq(time_seq)
if job is None:
job = copy.deepcopy(self.job)
job['timeout'] = base_time + datetime.timedelta(minutes=60)
if include_create:
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
if include_queued:
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('QUEUED'))
else:
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
error_status)
self._init_worker_mock(skip_metadata_update=(not include_create))
self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
error_message=None)
metadata = copy.copy(job['metadata'])
metadata['image_id'] = IMAGE_ID
backoff_factor = DEFAULT_BACKOFF_FACTOR ** job['retry_count']
timeout = time_seq[-1] + datetime.timedelta(minutes=
DEFAULT_TIMEOUT_INCR *
backoff_factor)
self.worker.update_job(fakes.JOB_ID, 'ERROR', timeout=timeout,
error_message=mox.IsA(unicode)).AndReturn(
{'status': 'ERROR', 'timeout': job['timeout']})
self.mox.StubOutWithMock(utils, 'generate_notification')
expected_job = copy.deepcopy(job)
expected_job['status'] = 'ERROR'
expected_job['metadata'] = metadata
if not is_retry:
utils.generate_notification(None, 'qonos.job.run.start',
mox.IsA(dict), mox.IsA(str))
else:
utils.generate_notification(None, 'qonos.job.retry',
mox.IsA(dict), mox.IsA(str))
utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict),
mox.IsA(str))
def assert_job_payload(job_payload):
self.assertTrue('error_message' in job_payload['job'])
del job_payload['job']['error_message']
self.assertEquals({'job': expected_job}, job_payload)
return True
utils.generate_notification(None, 'qonos.job.update',
mox.Func(assert_job_payload), 'ERROR')
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(job)
self.mox.VerifyAll()
def _do_test_process_job_should_update_image_error(self,
error_status,
include_create=True,
include_queued=True,
is_retry=False,
job=None):
base_time = timeutils.utcnow()
time_seq = [
base_time,
base_time,
base_time,
base_time + datetime.timedelta(seconds=305),
base_time + datetime.timedelta(seconds=605),
base_time + datetime.timedelta(seconds=905),
base_time + datetime.timedelta(seconds=1205),
base_time + datetime.timedelta(seconds=1505),
]
timeutils.set_time_override_seq(time_seq)
if job is None:
job = copy.deepcopy(self.job)
job['timeout'] = base_time + datetime.timedelta(minutes=60)
if include_create:
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(
mox.IsA(str), mox.IsA(str),
self.snapshot_meta).AndReturn(IMAGE_ID)
if include_queued:
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('QUEUED'))
else:
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(error_status)
self._init_worker_mock(skip_metadata_update=(not include_create))
self.worker.update_job(fakes.JOB_ID,
'PROCESSING',
timeout=None,
error_message=None)
self.worker.update_job(fakes.JOB_ID,
'PROCESSING',
timeout=None,
error_message=None)
self.worker.update_job(fakes.JOB_ID,
'PROCESSING',
timeout=None,
error_message=None)
self.worker.update_job(fakes.JOB_ID,
'PROCESSING',
timeout=None,
error_message=None)
metadata = copy.copy(job['metadata'])
metadata['image_id'] = IMAGE_ID
backoff_factor = DEFAULT_BACKOFF_FACTOR**job['retry_count']
timeout = time_seq[-1] + datetime.timedelta(
minutes=DEFAULT_TIMEOUT_INCR * backoff_factor)
self.worker.update_job(fakes.JOB_ID,
'ERROR',
timeout=timeout,
error_message=mox.IsA(unicode)).AndReturn({
'status':
'ERROR',
'timeout':
job['timeout']
})
self.mox.StubOutWithMock(utils, 'generate_notification')
expected_job = copy.deepcopy(job)
expected_job['status'] = 'ERROR'
expected_job['metadata'] = metadata
if not is_retry:
utils.generate_notification(None, 'qonos.job.run.start',
mox.IsA(dict), mox.IsA(str))
else:
utils.generate_notification(None, 'qonos.job.retry', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict),
mox.IsA(str))
def assert_job_payload(job_payload):
self.assertTrue('error_message' in job_payload['job'])
del job_payload['job']['error_message']
self.assertEquals({'job': expected_job}, job_payload)
return True
utils.generate_notification(None, 'qonos.job.update',
mox.Func(assert_job_payload), 'ERROR')
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(job)
self.mox.VerifyAll()
