def test_process_job_should_continue_when_image_id_present(self):
timeutils.set_time_override()
self.job['metadata']['image_id'] = IMAGE_ID
self.job['status'] = 'PROCESSING'
# Note NO call to create_image is expected
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._simple_prepare_worker_mock(skip_metadata_update=True)
self.worker.get_qonos_client().AndReturn(self.qonos_client)
self.qonos_client.delete_schedule(mox.IsA(str))
self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None,
error_message=None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_succeed_immediately(self):
timeutils.set_time_override()
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._simple_prepare_worker_mock()
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
self.worker.get_qonos_client().AndReturn(self.qonos_client)
self.qonos_client.delete_schedule(mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_schedule_update_remove_metadata(self):
fixture = {
'id': str(uuid.uuid4()),
'tenant': str(uuid.uuid4()),
'action': 'snapshot',
'minute': 30,
'hour': 2,
'schedule_metadata': [
{
'key': 'instance_id',
'value': 'my_instance',
},
],
}
schedule = self.db_api.schedule_create(fixture)
fixture = {
'schedule_metadata': [],
}
timeutils.set_time_override()
timeutils.advance_time_seconds(2)
updated_schedule = self.db_api.schedule_update(schedule['id'], fixture)
timeutils.clear_time_override()
self.assertTrue(uuidutils.is_uuid_like(schedule['id']))
self.assertEqual(updated_schedule['tenant'], schedule['tenant'])
self.assertEqual(updated_schedule['action'], schedule['action'])
self.assertEqual(updated_schedule['minute'], schedule['minute'])
self.assertEqual(updated_schedule['hour'], schedule['hour'])
metadata = updated_schedule['schedule_metadata']
self.assertEqual(len(metadata), 0)
# updated child metadata collection doesn't update the parent schedule
self.assertEqual(updated_schedule['updated_at'],
schedule['updated_at'])
def test_process_job_should_succeed_after_multiple_tries(self):
timeutils.set_time_override()
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('QUEUED'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
self.nova_client.servers.get(mox.IsA(str)).AndReturn(
MockServer())
self.worker.update_job(fakes.JOB_ID, 'PROCESSING',
timeout=mox.IsA(datetime.datetime),
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None,
error_message=None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(fakes.JOB['job'])
self.mox.VerifyAll()
def test_doesnt_delete_images_on_retention_error(self):
timeutils.set_time_override()
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndRaise(Exception())
mock_server = MockServer(retention=None)
self._init_worker_mock()
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
self.worker.get_qonos_client().AndReturn(self.qonos_client)
self.qonos_client.delete_schedule(mox.IsA(str))
self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None,
error_message=None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_succeed_after_multiple_tries(self):
timeutils.set_time_override()
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('QUEUED'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._simple_prepare_worker_mock()
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict),
mox.IsA(str)).MultipleTimes()
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None,
error_message=None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_continue_when_image_id_present(self):
timeutils.set_time_override()
self.job['metadata']['image_id'] = IMAGE_ID
self.job['status'] = 'PROCESSING'
# Note NO call to create_image is expected
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._simple_prepare_worker_mock(skip_metadata_update=True)
self.worker.update_job(fakes.JOB_ID,
'DONE',
timeout=None,
error_message=None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_generate_image_name_weekly(self):
schedule = {'day_of_week': 1}
timeutils.set_time_override(datetime.datetime(2013, 3, 22, 22, 39, 27))
timestamp = '1363991967'
processor = TestableSnapshotProcessor(self.nova_client)
image_name = processor.generate_image_name(schedule, 'test')
self.assertEqual(image_name, 'Weekly-test-' + timestamp)
def test_doesnt_delete_images_from_another_instance(self):
timeutils.set_time_override()
instance_id = fakes.JOB['job']['metadata']['instance_id']
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_server = MockServer(instance_id=instance_id, retention=3)
self.nova_client.servers.get(mox.IsA(str)).AndReturn(mock_server)
image_list = self._create_images_list(mock_server.id, 5)
to_delete = image_list[3:]
image_list.extend(self._create_images_list(
uuidutils.generate_uuid(), 3))
self.nova_client.images.list(detailed=True).AndReturn(image_list)
# The image list happens to be in descending created order
self.nova_client.images.delete(to_delete[0].id)
self.nova_client.images.delete(to_delete[1].id)
self.worker.update_job(fakes.JOB_ID, 'PROCESSING',
timeout=mox.IsA(datetime.datetime),
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None,
error_message=None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(fakes.JOB['job'])
self.mox.VerifyAll()
def test_process_job_should_not_create_on_job_error_image_ok(self):
timeutils.set_time_override()
self.job['metadata']['image_id'] = IMAGE_ID
self.job['status'] = 'ERROR'
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._simple_prepare_worker_mock(skip_metadata_update=True)
self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None,
error_message=None)
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, mox.IsA(str), mox.IsA(dict),
mox.IsA(str)).MultipleTimes()
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_doesnt_delete_images_from_another_instance(self):
timeutils.set_time_override()
instance_id = self.job["metadata"]["instance_id"]
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("ACTIVE"))
mock_retention = MockRetention(3)
self.nova_client.rax_scheduled_images_python_novaclient_ext.get(mox.IsA(str)).AndReturn(mock_retention)
mock_server = MockServer(instance_id=instance_id)
image_list = self._create_images_list(mock_server.id, 5)
to_delete = image_list[3:]
image_list.extend(self._create_images_list(uuidutils.generate_uuid(), 3))
self.nova_client.images.list(detailed=True).AndReturn(image_list)
# The image list happens to be in descending created order
self.nova_client.images.delete(to_delete[0].id)
self.nova_client.images.delete(to_delete[1].id)
self._simple_prepare_worker_mock()
self.mox.StubOutWithMock(utils, "generate_notification")
utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str))
utils.generate_notification(None, "qonos.job.run.end", mox.IsA(dict), mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_doesnt_delete_images_on_retention_error(self):
timeutils.set_time_override()
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(
mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndRaise(Exception())
mock_server = MockServer(retention=None)
self._init_worker_mock()
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict),
mox.IsA(str)).MultipleTimes()
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
self.worker.update_job(fakes.JOB_ID,
'DONE',
timeout=None,
error_message=None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_not_create_on_job_error_image_ok(self):
timeutils.set_time_override()
self.job['metadata']['image_id'] = IMAGE_ID
self.job['status'] = 'ERROR'
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._simple_prepare_worker_mock(skip_metadata_update=True)
self.worker.get_qonos_client().AndReturn(self.qonos_client)
self.qonos_client.delete_schedule(mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_not_create_on_job_error_image_ok(self):
timeutils.set_time_override()
self.job['metadata']['image_id'] = IMAGE_ID
self.job['status'] = 'ERROR'
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._simple_prepare_worker_mock(skip_metadata_update=True)
self.worker.update_job(fakes.JOB_ID,
'DONE',
timeout=None,
error_message=None)
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, mox.IsA(str), mox.IsA(dict),
mox.IsA(str)).MultipleTimes()
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_deletes_images_more_than_retention(self):
timeutils.set_time_override()
instance_id = self.job['metadata']['instance_id']
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(
mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention(3)
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
mock_server = MockServer(instance_id=instance_id)
image_list = self._create_images_list(mock_server.id, 5)
self.nova_client.images.list(detailed=True).AndReturn(image_list)
# The image list happens to be in descending created order
self.nova_client.images.delete(image_list[-2].id)
self.nova_client.images.delete(image_list[-1].id)
self._init_worker_mock()
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict),
mox.IsA(str)).MultipleTimes()
self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None,
error_message=None)
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_create_on_job_error_image_failed(self):
timeutils.set_time_override()
self.job['metadata']['image_id'] = IMAGE_ID
self.job['status'] = 'ERROR'
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('KILLED'))
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(
mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._simple_prepare_worker_mock()
self.worker.update_job(fakes.JOB_ID,
'DONE',
timeout=None,
error_message=None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_create_on_job_error_image_failed(self):
timeutils.set_time_override()
self.job["metadata"]["image_id"] = IMAGE_ID
self.job["status"] = "ERROR"
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("KILLED"))
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("ACTIVE"))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.get(mox.IsA(str)).AndReturn(mock_retention)
self._simple_prepare_worker_mock()
self.worker.get_qonos_client().AndReturn(self.qonos_client)
self.qonos_client.delete_schedule(mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_deletes_images_more_than_retention(self):
timeutils.set_time_override()
instance_id = self.job["metadata"]["instance_id"]
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("ACTIVE"))
mock_retention = MockRetention(3)
self.nova_client.rax_scheduled_images_python_novaclient_ext.get(mox.IsA(str)).AndReturn(mock_retention)
mock_server = MockServer(instance_id=instance_id)
image_list = self._create_images_list(mock_server.id, 5)
self.nova_client.images.list(detailed=True).AndReturn(image_list)
# The image list happens to be in descending created order
self.nova_client.images.delete(image_list[-2].id)
self.nova_client.images.delete(image_list[-1].id)
self._init_worker_mock()
self.worker.update_job(fakes.JOB_ID, "DONE", timeout=None, error_message=None)
self.mox.StubOutWithMock(utils, "generate_notification")
utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str))
utils.generate_notification(None, "qonos.job.run.end", mox.IsA(dict), mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_not_continue_when_in_bad_status(self):
timeutils.set_time_override()
self.job['metadata']['image_id'] = IMAGE_ID
self.job['status'] = 'ERROR'
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._simple_prepare_worker_mock()
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_generate_image_name_weekly(self):
schedule = {'day_of_week': 1}
timeutils.set_time_override(datetime.datetime(2013, 3, 22, 22, 39, 27))
timestamp = '1363991967'
processor = TestableSnapshotProcessor(self.nova_client)
image_name = processor.generate_image_name(schedule, 'test')
self.assertEqual(image_name, 'Weekly-test-' + timestamp)
def test_process_job_should_create_on_job_error_image_failed(self):
timeutils.set_time_override()
self.job['metadata']['image_id'] = IMAGE_ID
self.job['status'] = 'ERROR'
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('KILLED'))
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._simple_prepare_worker_mock()
self.worker.get_qonos_client().AndReturn(self.qonos_client)
self.qonos_client.delete_schedule(mox.IsA(str))
self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None,
error_message=None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_generate_image_name(self):
job = self.job_fixture("INSTANCE_ID")
timeutils.set_time_override(datetime.datetime(2013, 3, 22, 22, 39, 27))
timestamp = '1363991967'
with TestableSnapshotProcessor(job, None, []) as processor:
image_name = processor.generate_image_name("test")
self.assertEqual(image_name, 'Daily-test-' + timestamp)
def test_jobs_cleanup_hard_timed_out(self):
jobs = self.db_api.job_get_all()
self.assertEqual(len(jobs), 2)
timeutils.set_time_override()
timeutils.advance_time_delta(datetime.timedelta(hours=4, minutes=1))
self.db_api._jobs_cleanup_hard_timed_out()
timeutils.clear_time_override()
jobs = self.db_api.job_get_all()
self.assertEqual(len(jobs), 0)
def test_generate_image_name_long_server_name(self):
timeutils.set_time_override(datetime.datetime(2013, 3, 22, 22, 39, 27))
timestamp = '1363991967'
processor = TestableSnapshotProcessor(self.nova_client)
fake_server_name = 'a' * 255
expected_server_name = 'a' * (255 - len(timestamp) - len('Daily--'))
image_name = processor.generate_image_name(fake_server_name)
expected_image_name = 'Daily-' + expected_server_name + '-' + timestamp
self.assertEqual(image_name, expected_image_name)
def test_process_job_should_update_status_and_timestamp(self):
timeutils.set_time_override()
base_time = timeutils.utcnow()
time_seq = [
base_time,
base_time,
base_time + datetime.timedelta(seconds=305),
base_time + datetime.timedelta(minutes=60, seconds=5),
base_time + datetime.timedelta(minutes=60, seconds=305),
]
timeutils.set_time_override_seq(time_seq)
job = copy.deepcopy(self.job)
job['timeout'] = base_time + datetime.timedelta(minutes=60)
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(
mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('QUEUED'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._init_worker_mock()
self.worker.update_job(fakes.JOB_ID,
'PROCESSING',
timeout=mox.IsA(datetime.datetime),
error_message=None)
self.worker.update_job(fakes.JOB_ID,
'PROCESSING',
timeout=None,
error_message=None)
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict),
mox.IsA(str)).MultipleTimes()
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
self.worker.update_job(fakes.JOB_ID,
'DONE',
timeout=None,
error_message=None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(job)
self.mox.VerifyAll()
def test_generate_image_name_long_server_name(self):
timeutils.set_time_override(datetime.datetime(2013, 3, 22, 22, 39, 27))
timestamp = "1363991967"
processor = TestableSnapshotProcessor(self.nova_client)
fake_server_name = "a" * 255
expected_server_name = "a" * (255 - len(timestamp) - len("Daily--"))
image_name = processor.generate_image_name(fake_server_name)
expected_image_name = "Daily-" + expected_server_name + "-" + timestamp
self.assertEqual(image_name, expected_image_name)
def test_get_schedules(self):
timeutils.set_time_override()
start_time = timeutils.isotime()
timeutils.advance_time_seconds(30)
end_time = timeutils.isotime()
filter_args = {'next_run_after': start_time,
'next_run_before': end_time}
self.client.list_schedules(filter_args=filter_args).AndReturn([])
self.mox.ReplayAll()
self.scheduler.get_schedules(start_time, end_time)
self.mox.VerifyAll()
def test_process_job_should_exponentially_increates_timeout(self):
status = MockImageStatus('ERROR')
job = copy.deepcopy(self.job)
self._do_test_process_job_should_update_image_error(status, job=job)
self._reset_mocks()
new_now = timeutils.utcnow() + datetime.timedelta(minutes=120)
timeutils.clear_time_override()
timeutils.set_time_override(new_now)
job['status'] = 'ERROR'
job['retry_count'] = 2
self._do_test_process_job_should_update_image_error(status,
include_create=False, include_queued=False, is_retry=True, job=job)
def test_process_job_should_update_status_and_timestamp(self):
timeutils.set_time_override()
base_time = timeutils.utcnow()
time_seq = [
base_time,
base_time,
base_time + datetime.timedelta(seconds=305),
base_time + datetime.timedelta(minutes=60, seconds=5),
base_time + datetime.timedelta(minutes=60, seconds=305),
]
timeutils.set_time_override_seq(time_seq)
job = copy.deepcopy(self.job)
job['timeout'] = base_time + datetime.timedelta(minutes=60)
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('QUEUED'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('SAVING'))
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._init_worker_mock()
self.worker.update_job(fakes.JOB_ID, 'PROCESSING',
timeout=mox.IsA(datetime.datetime),
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
error_message=None)
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict),
mox.IsA(str)).MultipleTimes()
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None,
error_message=None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(job)
self.mox.VerifyAll()
def test_generate_image_name_long_server_name(self):
job = self.job_fixture("INSTANCE_ID")
timeutils.set_time_override(datetime.datetime(2013, 3, 22, 22, 39, 27))
timestamp = '1363991967'
fake_server_name = 'a' * 255
expected_server_name = 'a' * (255 - len(timestamp) - len('Daily--'))
with TestableSnapshotProcessor(job, None, []) as processor:
image_name = processor.generate_image_name(fake_server_name)
expected_image_name = (
'Daily-' + expected_server_name + '-' + timestamp)
self.assertEqual(image_name, expected_image_name)
def test_generate_image_name_long_server_name(self):
schedule = {}
timeutils.set_time_override(datetime.datetime(2013, 3, 22, 22, 39, 27))
timestamp = '1363991967'
processor = TestableSnapshotProcessor(self.nova_client)
fake_server_name = 'a' * 255
prefix = 'Daily'
expected_server_name = 'a' * (255 - len(timestamp) -
len(prefix + '--'))
image_name = processor.generate_image_name(schedule, fake_server_name)
expected_image_name = (prefix + '-' + expected_server_name + '-' +
timestamp)
self.assertEqual(image_name, expected_image_name)
def test_polling_job_timeout_extension_with_max_retries(self):
timeout_extension = 3600
job_timeout_max_updates_count = 3
self.config(job_timeout_extension_sec=timeout_extension,
group='snapshot_worker')
self.config(job_timeout_max_updates=job_timeout_max_updates_count,
group='snapshot_worker')
self.config(job_timeout_initial_value_sec=10800,
group='snapshot_worker')
server = self.server_instance_fixture("INSTANCE_ID", "test")
job = self.job_fixture(server.id)
images = [self.image_fixture('IMAGE_ID', 'QUEUED', server.id),
self.image_fixture('IMAGE_ID', 'SAVING', server.id),
self.image_fixture('IMAGE_ID', 'SAVING', server.id),
self.image_fixture('IMAGE_ID', 'SAVING', server.id)]
now = timeutils.utcnow()
timeutils.set_time_override(now)
timeutils.advance_time_delta(
datetime.timedelta(seconds=timeout_extension))
try:
with TestableSnapshotProcessor(job, server, images) as p:
p.next_timeout = now + p.initial_timeout
p.next_update = now + p.update_interval
# NOTE(venkatesh): unfortunately had to use a protected method
# for testing. Else there seems to be no easier way to test
# this scenario. we need to fix this as part of refactoring
# SnapshotJobProcessor.
while True:
try:
p._update_job(job['id'], 'PROCESSING')
except exception.OutOfTimeException:
break
timeutils.advance_time_delta(
datetime.timedelta(seconds=timeout_extension))
total_timeout_duration = datetime.timedelta(
seconds=(timeout_extension * job_timeout_max_updates_count)
)
self.assertEqual(
now + (p.initial_timeout +
total_timeout_duration),
p.next_timeout
)
self.assertEqual(3, p.timeout_count)
finally:
timeutils.clear_time_override()
def test_polling_job_timeout_extension_with_max_retries(self):
timeout_extension = 3600
job_timeout_max_updates_count = 3
self.config(job_timeout_extension_sec=timeout_extension,
group='snapshot_worker')
self.config(job_timeout_max_updates=job_timeout_max_updates_count,
group='snapshot_worker')
self.config(job_timeout_initial_value_sec=10800,
group='snapshot_worker')
server = self.server_instance_fixture("INSTANCE_ID", "test")
job = self.job_fixture(server.id)
images = [
self.image_fixture('IMAGE_ID', 'QUEUED', server.id),
self.image_fixture('IMAGE_ID', 'SAVING', server.id),
self.image_fixture('IMAGE_ID', 'SAVING', server.id),
self.image_fixture('IMAGE_ID', 'SAVING', server.id)
]
now = timeutils.utcnow()
timeutils.set_time_override(now)
timeutils.advance_time_delta(
datetime.timedelta(seconds=timeout_extension))
try:
with TestableSnapshotProcessor(job, server, images) as p:
p.next_timeout = now + p.initial_timeout
p.next_update = now + p.update_interval
#NOTE(venkatesh): unfortunately had to use a protected method
# for testing. Else there seems to be no easier way to test
# this scenario. we need to fix this as part of refactoring
# SnapshotJobProcessor.
while True:
try:
p._update_job(job['id'], 'PROCESSING')
except exception.OutOfTimeException:
break
timeutils.advance_time_delta(
datetime.timedelta(seconds=timeout_extension))
total_timeout_duration = datetime.timedelta(
seconds=(timeout_extension *
job_timeout_max_updates_count))
self.assertEqual(
now + (p.initial_timeout + total_timeout_duration),
p.next_timeout)
self.assertEqual(3, p.timeout_count)
finally:
timeutils.clear_time_override()
def test_get_schedules(self):
timeutils.set_time_override()
start_time = timeutils.isotime()
timeutils.advance_time_seconds(30)
end_time = timeutils.isotime()
filter_args = {
'next_run_after': start_time,
'next_run_before': end_time
}
self.client.list_schedules(filter_args=filter_args).AndReturn([])
self.mox.ReplayAll()
self.scheduler.get_schedules(start_time, end_time)
self.mox.VerifyAll()
def test_process_job_should_succeed_immediately(self):
timeutils.set_time_override()
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._init_worker_mock(False)
expected_processing_payload = \
{'job': {'status': 'PROCESSING',
'hard_timeout': self.job['hard_timeout'],
'created_at': self.job['created_at'],
'modified_at': self.job['modified_at'],
'retry_count': 1,
'schedule_id': '33333333-3333-3333-3333-33333333',
'worker_id': '11111111-1111-1111-1111-11111111',
'timeout': self.job['timeout'],
'action': 'snapshot',
'id': '22222222-2222-2222-2222-22222222',
'tenant': '44444444-4444-4444-4444-44444444',
'metadata': {'instance_id':
'55555555-5555-5555-5555-55555555'}}}
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.update',
expected_processing_payload,
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
self.worker.get_qonos_client().AndReturn(self.qonos_client)
self.qonos_client.delete_schedule(mox.IsA(str))
processor = TestableSnapshotProcessor(self.nova_client)
self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None,
error_message=None).\
AndReturn({'status': 'CANCELLED', 'timeout': self.job['timeout']})
self.mox.ReplayAll()
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_should_exponentially_increase_timeout(self):
status = MockImageStatus('ERROR')
job = copy.deepcopy(self.job)
self._do_test_process_job_should_update_image_error(status, job=job)
self._reset_mocks()
new_now = timeutils.utcnow() + datetime.timedelta(minutes=120)
timeutils.clear_time_override()
timeutils.set_time_override(new_now)
job['status'] = 'ERROR'
job['retry_count'] = 2
job['hard_timeout'] = timeutils.strtime(
at=(new_now + datetime.timedelta(minutes=120)))
self._do_test_process_job_should_update_image_error(
status, include_create=False, include_queued=False, is_retry=True,
job=job)
def test_run_loop(self):
self.config(job_schedule_interval=5, group='scheduler')
timeutils.set_time_override()
current_time = timeutils.isotime()
called = {'enqueue_jobs': False}
def fake(end_time=None): # assert only end_time kwarg is passed
self.assertEqual(end_time, current_time)
called['enqueue_jobs'] = True
self.stubs.Set(self.scheduler, 'enqueue_jobs', fake)
fake_sleep = lambda x: None
self.stubs.Set(time, 'sleep', fake_sleep)
self.scheduler.run(run_once=True)
self.assertTrue(called['enqueue_jobs'])
def test_run_loop(self):
self.config(job_schedule_interval=5, group='scheduler')
timeutils.set_time_override()
current_time = timeutils.isotime()
called = {'enqueue_jobs': False}
def fake(end_time=None): # assert only end_time kwarg is passed
self.assertEqual(end_time, current_time)
called['enqueue_jobs'] = True
self.stubs.Set(self.scheduler, 'enqueue_jobs', fake)
fake_sleep = lambda x: None
self.stubs.Set(time, 'sleep', fake_sleep)
self.scheduler.run(run_once=True)
self.assertTrue(called['enqueue_jobs'])
def test_process_job_should_succeed_immediately(self):
timeutils.set_time_override()
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(
mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._init_worker_mock(False)
expected_processing_payload = \
{'job': {'status': 'PROCESSING',
'hard_timeout': self.job['hard_timeout'],
'created_at': self.job['created_at'],
'modified_at': self.job['modified_at'],
'retry_count': 1,
'schedule_id': '33333333-3333-3333-3333-33333333',
'worker_id': '11111111-1111-1111-1111-11111111',
'timeout': self.job['timeout'],
'action': 'snapshot',
'id': '22222222-2222-2222-2222-22222222',
'tenant': '44444444-4444-4444-4444-44444444',
'metadata': {'instance_id':
'55555555-5555-5555-5555-55555555'}}}
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.update',
expected_processing_payload,
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
processor = TestableSnapshotProcessor(self.nova_client)
self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None,
error_message=None).\
AndReturn({'status': 'CANCELLED', 'timeout': self.job['timeout']})
self.mox.ReplayAll()
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_schedule_to_next_run(self):
timeutils.set_time_override()
self.called = False
def fake_next_datetime(min, h, dom, m, dow, start_time):
self.called = True
self.assertEqual(min, '*')
self.assertEqual(h, '*')
self.assertEqual(dom, '*')
self.assertEqual(m, '*')
self.assertEqual(dow, '*')
self.assertEqual(timeutils.utcnow(), start_time)
self.stubs.Set(utils, 'cron_string_to_next_datetime',
fake_next_datetime)
api_utils.schedule_to_next_run({})
self.assertTrue(self.called)
timeutils.clear_time_override()
def test_schedule_to_next_run(self):
timeutils.set_time_override()
self.called = False
def fake_next_datetime(min, h, dom, m, dow, start_time):
self.called = True
self.assertEqual(min, '*')
self.assertEqual(h, '*')
self.assertEqual(dom, '*')
self.assertEqual(m, '*')
self.assertEqual(dow, '*')
self.assertEqual(timeutils.utcnow(), start_time)
self.stubs.Set(utils, 'cron_string_to_next_datetime',
fake_next_datetime)
api_utils.schedule_to_next_run({})
self.assertTrue(self.called)
timeutils.clear_time_override()
def test_process_job_should_continue_when_image_id_present(self):
timeutils.set_time_override()
self.job["metadata"]["image_id"] = IMAGE_ID
self.job["status"] = "PROCESSING"
# Note NO call to create_image is expected
self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("ACTIVE"))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.get(mox.IsA(str)).AndReturn(mock_retention)
self._simple_prepare_worker_mock(skip_metadata_update=True)
self.worker.get_qonos_client().AndReturn(self.qonos_client)
self.qonos_client.delete_schedule(mox.IsA(str))
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_new_image_when_retrying_with_failed_image(self):
timeutils.set_time_override()
self.job['metadata']['image_id'] = IMAGE_ID
self.job['status'] = 'ERROR'
#make an error checking the previous image
self.nova_client.images.get(IMAGE_ID).AndRaise(
Exception())
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._simple_prepare_worker_mock()
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.retry', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict),
mox.IsA(str)).MultipleTimes()
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
self.worker.get_qonos_client().AndReturn(self.qonos_client)
self.qonos_client.delete_schedule(mox.IsA(str))
self.worker.update_job(fakes.JOB_ID, 'DONE',
error_message=None, timeout=None).AndReturn(None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_process_job_new_image_when_retrying_with_failed_image(self):
timeutils.set_time_override()
self.job['metadata']['image_id'] = IMAGE_ID
self.job['status'] = 'ERROR'
#make an error checking the previous image
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ERROR'))
self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
self.nova_client.servers.create_image(
mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_retention = MockRetention()
self.nova_client.rax_scheduled_images_python_novaclient_ext.\
get(mox.IsA(str)).AndReturn(mock_retention)
self._simple_prepare_worker_mock()
self.mox.StubOutWithMock(utils, 'generate_notification')
utils.generate_notification(None, 'qonos.job.retry', mox.IsA(dict),
mox.IsA(str))
utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict),
mox.IsA(str)).MultipleTimes()
utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
mox.IsA(str))
self.worker.update_job(fakes.JOB_ID,
'DONE',
error_message=None,
timeout=None).AndReturn(None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(self.job)
self.mox.VerifyAll()
def test_doesnt_delete_images_less_than_retention(self):
timeutils.set_time_override()
self.nova_client.servers.create_image(mox.IsA(str),
mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
self.nova_client.images.get(IMAGE_ID).AndReturn(
MockImageStatus('ACTIVE'))
mock_server = MockServer(retention=3)
self.nova_client.servers.get(mox.IsA(str)).AndReturn(mock_server)
image_list = self._create_images_list(mock_server.id, 3)
self.nova_client.images.list(detailed=True).AndReturn(image_list)
self.worker.update_job(fakes.JOB_ID, 'PROCESSING',
timeout=mox.IsA(datetime.datetime),
error_message=None)
self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None,
error_message=None)
self.mox.ReplayAll()
processor = TestableSnapshotProcessor(self.nova_client)
processor.init_processor(self.worker)
processor.process_job(fakes.JOB['job'])
self.mox.VerifyAll()
def setUp(self):
super(TestJobsApi, self).setUp()
timeutils.set_time_override()
self.controller = jobs.JobsController(db_api=db_api)
self._create_jobs()
def test_generate_image_name(self):
timeutils.set_time_override(datetime.datetime(2013, 3, 22, 22, 39, 27))
timestamp = '1363991967'
processor = TestableSnapshotProcessor(self.nova_client)
image_name = processor.generate_image_name("test")
self.assertEqual(image_name, 'Daily-test-' + timestamp)
def setUp(self):
super(TestSchedulesApi, self).setUp()
timeutils.set_time_override(datetime.datetime(2013, 2, 17, 2, 0, 0, 0))
self.controller = schedules.SchedulesController(db_api=db_api)
self._create_schedules()