Exemple #1
0
 def test_process_job_should_exponentially_increase_timeout(self):
     status = MockImageStatus('ERROR')
     job = copy.deepcopy(self.job)
     self._do_test_process_job_should_update_image_error(status, job=job)
     self._reset_mocks()
     new_now = timeutils.utcnow() + datetime.timedelta(minutes=120)
     timeutils.clear_time_override()
     timeutils.set_time_override(new_now)
     job['status'] = 'ERROR'
     job['retry_count'] = 2
     job['hard_timeout'] = timeutils.strtime(
         at=(new_now + datetime.timedelta(minutes=120)))
     self._do_test_process_job_should_update_image_error(status,
         include_create=False, include_queued=False, is_retry=True, job=job)
Exemple #2
0
    def job_fixture(self, instance_id, **kwargs):
        now = timeutils.utcnow()
        timeout = timeutils.strtime(now + datetime.timedelta(hours=1))
        hard_timeout = timeutils.strtime(now + datetime.timedelta(hours=4))

        fixture = {
            'id': 'JOB_1',
            'schedule_id': 'SCH_1',
            'tenant': 'TENANT1',
            'worker_id': 'WORKER_1',
            'action': 'snapshot',
            'status': 'QUEUED',
            'timeout': timeout,
            'hard_timeout': hard_timeout,
            'retry_count': 0,
            'metadata': {
                'instance_id': instance_id,
                'value': 'my_instance',
            },
        }
        if kwargs:
            fixture.update(kwargs)
        return fixture
Exemple #3
0
    def job_fixture(self, instance_id, **kwargs):
        now = timeutils.utcnow()
        timeout = timeutils.strtime(now + datetime.timedelta(hours=1))
        hard_timeout = timeutils.strtime(now + datetime.timedelta(hours=4))

        fixture = {
            'id': 'JOB_1',
            'schedule_id': 'SCH_1',
            'tenant': 'TENANT1',
            'worker_id': 'WORKER_1',
            'action': 'snapshot',
            'status': 'QUEUED',
            'timeout': timeout,
            'hard_timeout': hard_timeout,
            'retry_count': 0,
            'metadata': {
                'instance_id': instance_id,
                'value': 'my_instance',
            },
        }
        if kwargs:
            fixture.update(kwargs)
        return fixture
Exemple #4
0
 def test_process_job_should_exponentially_increase_timeout(self):
     status = MockImageStatus('ERROR')
     job = copy.deepcopy(self.job)
     self._do_test_process_job_should_update_image_error(status, job=job)
     self._reset_mocks()
     new_now = timeutils.utcnow() + datetime.timedelta(minutes=120)
     timeutils.clear_time_override()
     timeutils.set_time_override(new_now)
     job['status'] = 'ERROR'
     job['retry_count'] = 2
     job['hard_timeout'] = timeutils.strtime(
         at=(new_now + datetime.timedelta(minutes=120)))
     self._do_test_process_job_should_update_image_error(
         status, include_create=False, include_queued=False, is_retry=True,
         job=job)
Exemple #5
0
    def test_notifications_for_cancelled_job_on_hard_timeout_reached(self):
        server = self.server_instance_fixture("INSTANCE_ID", "test")
        now = timeutils.utcnow()
        expired_hard_timeout = now - datetime.timedelta(hours=4)
        job = self.job_fixture(
            server.id, hard_timeout=timeutils.strtime(expired_hard_timeout))

        with TestableSnapshotProcessor(job, server, []) as processor:
            processor.process_job(job)

            self.assertEqual('HARD_TIMED_OUT', job['status'])
            expected_notifications = [
                ('qonos.job.run.start', 'INFO', 'QUEUED'),
                ('qonos.job.failed', 'ERROR', 'HARD_TIMED_OUT')
            ]
            self.assert_job_notification_events(processor,
                                                expected_notifications)
Exemple #6
0
    def test_notifications_for_cancelled_job_on_hard_timeout_reached(self):
        server = self.server_instance_fixture("INSTANCE_ID", "test")
        now = timeutils.utcnow()
        expired_hard_timeout = now - datetime.timedelta(hours=4)
        job = self.job_fixture(server.id,
                               hard_timeout=timeutils.strtime(
                                   expired_hard_timeout))

        with TestableSnapshotProcessor(job, server, []) as processor:
            processor.process_job(job)

            self.assertEqual('HARD_TIMED_OUT', job['status'])
            expected_notifications = [
                ('qonos.job.run.start', 'INFO', 'QUEUED'),
                ('qonos.job.failed', 'ERROR', 'HARD_TIMED_OUT')]
            self.assert_job_notification_events(processor,
                                                expected_notifications)
Exemple #7
0
    def test_process_job_should_fail_if_hard_timed_out(self):
        mox.Reset(self.worker)
        self.mox.StubOutWithMock(utils, 'generate_notification')
        now = timeutils.utcnow()
        self.job['hard_timeout'] = timeutils.strtime(at=now)

        utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
                                    mox.IsA(str))
        self.worker.update_job(fakes.JOB_ID,
                               'HARD_TIMED_OUT',
                               timeout=None,
                               error_message=mox.IsA(str))\
            .AndReturn({'status': 'HARD_TIMED_OUT',
                        'timeout': self.job['timeout']})
        expected_payload = {
            'job': {
                'status': 'HARD_TIMED_OUT',
                'hard_timeout': self.job['hard_timeout'],
                'created_at': self.job['created_at'],
                'modified_at': self.job['modified_at'],
                'retry_count': 1,
                'schedule_id': '33333333-3333-3333-3333-33333333',
                'worker_id': '11111111-1111-1111-1111-11111111',
                'timeout': self.job['timeout'],
                'action': 'snapshot',
                'id': '22222222-2222-2222-2222-22222222',
                'tenant': '44444444-4444-4444-4444-44444444',
                'metadata': {
                    'instance_id': '55555555-5555-5555-5555-55555555'
                }
            }
        }

        utils.generate_notification(None, 'qonos.job.failed', expected_payload,
                                    mox.IsA(str))
        self.mox.ReplayAll()

        processor = TestableSnapshotProcessor(self.nova_client)
        processor.init_processor(self.worker)

        processor.process_job(self.job)

        self.mox.VerifyAll()
Exemple #8
0
    def test_process_retention_for_hard_timedout_job(self):
        server = self.server_instance_fixture("INSTANCE_ID", "test",
                                              retention=2)

        now = timeutils.utcnow()
        expired_hard_timeout = now - datetime.timedelta(hours=4)
        expired_hard_timeout = timeutils.strtime(expired_hard_timeout)
        job = self.job_fixture(server.id, hard_timeout=expired_hard_timeout)

        existing_snapshot_images = [
            self.image_fixture('OLD_IMAGE_01', 'ACTIVE', server.id),
            self.image_fixture('OLD_IMAGE_02', 'ACTIVE', server.id),
            self.image_fixture('OLD_IMAGE_03', 'ACTIVE', server.id),
        ]

        current_image = self.image_fixture('IMAGE_ID', 'ACTIVE', server.id)
        all_instance_images = [current_image] + existing_snapshot_images

        with TestableSnapshotProcessor(job, server,
                                       [current_image]) as processor:
            processor.nova_client.images.list = mock.Mock(
                mock.ANY, return_value=all_instance_images)
            processor.process_job(job)

            self.assertEqual(2, processor.nova_client.images.delete.call_count)
            processor.nova_client.images.delete.has_calls(
                [mock.call('OLD_IMAGE_02'), mock.call('OLD_IMAGE_01')])

            self.assert_update_job_statuses(processor, ['HARD_TIMED_OUT'])
            self.assertEqual('HARD_TIMED_OUT', job['status'])

            error_msg = ('Job %(job_id)s has reached/exceeded its'
                         ' hard timeout: %(hard_timeout)s.' %
                         {'job_id': job['id'],
                          'hard_timeout': job['hard_timeout']})
            expected_status_values = {
                'status': 'HARD_TIMED_OUT',
                'error_message': error_msg
            }
            self.assert_job_status_values(processor, expected_status_values)
Exemple #9
0
    def test_process_job_should_fail_if_hard_timed_out(self):
        mox.Reset(self.worker)
        self.mox.StubOutWithMock(utils, 'generate_notification')
        now = timeutils.utcnow()
        self.job['hard_timeout'] = timeutils.strtime(at=now)

        utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
                                    mox.IsA(str))
        self.worker.update_job(fakes.JOB_ID,
                               'HARD_TIMED_OUT',
                               timeout=None,
                               error_message=mox.IsA(str))\
            .AndReturn({'status': 'HARD_TIMED_OUT',
                        'timeout': self.job['timeout']})
        expected_payload = {'job':
                            {'status': 'HARD_TIMED_OUT',
                             'hard_timeout': self.job['hard_timeout'],
                             'created_at': self.job['created_at'],
                             'modified_at': self.job['modified_at'],
                             'retry_count': 1,
                             'schedule_id': '33333333-3333-3333-3333-33333333',
                             'worker_id': '11111111-1111-1111-1111-11111111',
                             'timeout': self.job['timeout'],
                             'action': 'snapshot',
                             'id': '22222222-2222-2222-2222-22222222',
                             'tenant': '44444444-4444-4444-4444-44444444',
                             'metadata': {'instance_id':
                                          '55555555-5555-5555-5555-55555555'}}}

        utils.generate_notification(None, 'qonos.job.failed', expected_payload,
                                    mox.IsA(str))
        self.mox.ReplayAll()

        processor = TestableSnapshotProcessor(self.nova_client)
        processor.init_processor(self.worker)

        processor.process_job(self.job)

        self.mox.VerifyAll()
Exemple #10
0
    def test_process_job_should_cancel_if_job_hard_timed_out(self):
        server = self.server_instance_fixture("INSTANCE_ID", "test")

        now = timeutils.utcnow()
        expired_hard_timeout = now - datetime.timedelta(hours=4)
        job = self.job_fixture(
            server.id, hard_timeout=timeutils.strtime(expired_hard_timeout))

        with TestableSnapshotProcessor(job, server, []) as processor:
            processor.process_job(job)

            self.assert_update_job_statuses(processor, ['HARD_TIMED_OUT'])
            self.assertEqual('HARD_TIMED_OUT', job['status'])

            error_msg = ('Job %(job_id)s has reached/exceeded its'
                         ' hard timeout: %(hard_timeout)s.' % {
                             'job_id': job['id'],
                             'hard_timeout': job['hard_timeout']
                         })
            expected_status_values = {
                'status': 'HARD_TIMED_OUT',
                'error_message': error_msg
            }
            self.assert_job_status_values(processor, expected_status_values)
Exemple #11
0
    def test_process_job_should_cancel_if_job_hard_timed_out(self):
        server = self.server_instance_fixture("INSTANCE_ID", "test")

        now = timeutils.utcnow()
        expired_hard_timeout = now - datetime.timedelta(hours=4)
        job = self.job_fixture(server.id,
                               hard_timeout=timeutils.strtime(
                                   expired_hard_timeout))

        with TestableSnapshotProcessor(job, server, []) as processor:
            processor.process_job(job)

            self.assert_update_job_statuses(processor, ['HARD_TIMED_OUT'])
            self.assertEqual('HARD_TIMED_OUT', job['status'])

            error_msg = ('Job %(job_id)s has reached/exceeded its'
                         ' hard timeout: %(hard_timeout)s.' %
                         {'job_id': job['id'],
                          'hard_timeout': job['hard_timeout']})
            expected_status_values = {
                'status': 'HARD_TIMED_OUT',
                'error_message': error_msg
            }
            self.assert_job_status_values(processor, expected_status_values)
Exemple #12
0
"""
Fakes For Worker tests.
"""

import datetime

from qonos.common import timeutils

WORKER_ID = '11111111-1111-1111-1111-11111111'
JOB_ID = '22222222-2222-2222-2222-22222222'
SCHEDULE_ID = '33333333-3333-3333-3333-33333333'
TENANT = '44444444-4444-4444-4444-44444444'
INSTANCE_ID = '55555555-5555-5555-5555-55555555'

BASE_TIME_T = timeutils.utcnow()
BASE_TIME = timeutils.strtime(at=BASE_TIME_T)
TIMEOUT = timeutils.strtime(BASE_TIME_T + datetime.timedelta(hours=1))
HARD_TIMEOUT = timeutils.strtime(BASE_TIME_T + datetime.timedelta(hours=2))

JOB = {
    'job': {
        'id': JOB_ID,
        'created_at': BASE_TIME,
        'modified_at': BASE_TIME,
        'schedule_id': SCHEDULE_ID,
        'tenant': TENANT,
        'worker_id': WORKER_ID,
        'status': 'QUEUED',
        'action': 'snapshot',
        'retry_count': 1,
        'timeout': TIMEOUT,
Exemple #13
0
"""
Fakes For Worker tests.
"""

import datetime

from qonos.common import timeutils

WORKER_ID = '11111111-1111-1111-1111-11111111'
JOB_ID = '22222222-2222-2222-2222-22222222'
SCHEDULE_ID = '33333333-3333-3333-3333-33333333'
TENANT = '44444444-4444-4444-4444-44444444'
INSTANCE_ID = '55555555-5555-5555-5555-55555555'

BASE_TIME_T = timeutils.utcnow()
BASE_TIME = timeutils.strtime(at=BASE_TIME_T)
TIMEOUT = timeutils.strtime(BASE_TIME_T + datetime.timedelta(hours=1))
HARD_TIMEOUT = timeutils.strtime(BASE_TIME_T + datetime.timedelta(hours=2))

JOB = {
    'job': {
        'id': JOB_ID,
        'created_at': BASE_TIME,
        'modified_at': BASE_TIME,
        'schedule_id': SCHEDULE_ID,
        'tenant': TENANT,
        'worker_id': WORKER_ID,
        'status': 'QUEUED',
        'action': 'snapshot',
        'retry_count': 1,
        'timeout': TIMEOUT,