Ejemplo n.º 1
0
    def test_process_job_should_update_status_and_timestamp(self):
        timeutils.set_time_override()
        base_time = timeutils.utcnow()
        time_seq = [
            base_time,
            base_time,
            base_time + datetime.timedelta(seconds=305),
            base_time + datetime.timedelta(minutes=60, seconds=5),
            base_time + datetime.timedelta(minutes=60, seconds=305),
        ]
        timeutils.set_time_override_seq(time_seq)

        job = copy.deepcopy(self.job)
        job['timeout'] = base_time + datetime.timedelta(minutes=60)

        self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
        self.nova_client.servers.create_image(
            mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('QUEUED'))
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('SAVING'))
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('SAVING'))
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('ACTIVE'))
        mock_retention = MockRetention()
        self.nova_client.rax_scheduled_images_python_novaclient_ext.\
            get(mox.IsA(str)).AndReturn(mock_retention)
        self._init_worker_mock()
        self.worker.update_job(fakes.JOB_ID,
                               'PROCESSING',
                               timeout=mox.IsA(datetime.datetime),
                               error_message=None)
        self.worker.update_job(fakes.JOB_ID,
                               'PROCESSING',
                               timeout=None,
                               error_message=None)
        self.mox.StubOutWithMock(utils, 'generate_notification')
        utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
                                    mox.IsA(str))
        utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict),
                                    mox.IsA(str)).MultipleTimes()
        utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
                                    mox.IsA(str))
        self.worker.update_job(fakes.JOB_ID,
                               'DONE',
                               timeout=None,
                               error_message=None)
        self.mox.ReplayAll()

        processor = TestableSnapshotProcessor(self.nova_client)
        processor.init_processor(self.worker)

        processor.process_job(job)

        self.mox.VerifyAll()
Ejemplo n.º 2
0
    def test_process_job_should_update_status_and_timestamp(self):
        base_time = timeutils.utcnow()
        time_seq = [
            base_time,
            base_time,
            base_time + datetime.timedelta(seconds=305),
            base_time + datetime.timedelta(minutes=60, seconds=5),
            base_time + datetime.timedelta(minutes=60, seconds=305),
            ]
        timeutils.set_time_override_seq(time_seq)

        job = copy.deepcopy(self.job)
        job['timeout'] = base_time + datetime.timedelta(minutes=60)

        self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
        self.nova_client.servers.create_image(mox.IsA(str),
            mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('QUEUED'))
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('SAVING'))
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('SAVING'))
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('ACTIVE'))
        mock_retention = MockRetention()
        self.nova_client.rax_scheduled_images_python_novaclient_ext.\
            get(mox.IsA(str)).AndReturn(mock_retention)
        self._init_worker_mock()
        self.worker.update_job(fakes.JOB_ID, 'PROCESSING',
                               timeout=mox.IsA(datetime.datetime),
                               error_message=None)
        self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
                               error_message=None)
        self.mox.StubOutWithMock(utils, 'generate_notification')
        utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
                                    mox.IsA(str))
        utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict),
                                    mox.IsA(str)).MultipleTimes()
        utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
                                    mox.IsA(str))
        self.worker.get_qonos_client().AndReturn(self.qonos_client)
        self.qonos_client.delete_schedule(mox.IsA(str))
        self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None,
                               error_message=None)
        self.mox.ReplayAll()

        processor = TestableSnapshotProcessor(self.nova_client)
        processor.init_processor(self.worker)

        processor.process_job(job)

        self.mox.VerifyAll()
Ejemplo n.º 3
0
    def _do_test_process_job_should_update_image_error(self, error_status):
        base_time = timeutils.utcnow()
        time_seq = [
            base_time,
            base_time,
            base_time + datetime.timedelta(seconds=305),
            base_time + datetime.timedelta(seconds=605),
            base_time + datetime.timedelta(seconds=905),
            base_time + datetime.timedelta(seconds=1205),
            base_time + datetime.timedelta(seconds=1505),
            ]
        timeutils.set_time_override_seq(time_seq)

        job = copy.deepcopy(self.job)
        job['timeout'] = base_time + datetime.timedelta(minutes=60)

        self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
        self.nova_client.servers.create_image(mox.IsA(str),
            mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('QUEUED'))
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('SAVING'))
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('SAVING'))
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('SAVING'))
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            error_status)

        self._init_worker_mock()
        self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
                               error_message=None)
        self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
                               error_message=None)
        self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
                               error_message=None)
        self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
                               error_message=None)
        self.worker.update_job(fakes.JOB_ID, 'ERROR', timeout=None,
                               error_message=mox.IsA(str))

        self.mox.StubOutWithMock(utils, 'generate_notification')
        utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
                                    mox.IsA(str))
        self.mox.ReplayAll()

        processor = TestableSnapshotProcessor(self.nova_client)
        processor.init_processor(self.worker)

        processor.process_job(job)

        self.mox.VerifyAll()
Ejemplo n.º 4
0
    def test_process_job_should_update_status_timestamp_no_retries(self):
        base_time = timeutils.utcnow()
        time_seq = [
            base_time,
            base_time,
            base_time + datetime.timedelta(minutes=5, seconds=5),
            base_time + datetime.timedelta(minutes=60, seconds=5),
            base_time + datetime.timedelta(minutes=120, seconds=5),
            base_time + datetime.timedelta(minutes=180, seconds=5),
            base_time + datetime.timedelta(minutes=240, seconds=5),
            ]
        print "Time_seq: %s" % str(time_seq)
        timeutils.set_time_override_seq(time_seq)

        job = copy.deepcopy(fakes.JOB['job'])
        job['timeout'] = base_time + datetime.timedelta(minutes=60)

        self.nova_client.servers.create_image(mox.IsA(str),
            mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('QUEUED'))
        self.nova_client.images.get(IMAGE_ID).MultipleTimes().AndReturn(
            MockImageStatus('SAVING'))

        self.worker.update_job(fakes.JOB_ID, 'PROCESSING',
                               timeout=mox.IsA(datetime.datetime),
                               error_message=None)
        self.worker.update_job(fakes.JOB_ID, 'PROCESSING',
                               timeout=mox.IsA(datetime.datetime),
                               error_message=None)
        self.worker.update_job(fakes.JOB_ID, 'PROCESSING',
                               timeout=mox.IsA(datetime.datetime),
                               error_message=None)
        self.worker.update_job(fakes.JOB_ID, 'PROCESSING',
                               timeout=mox.IsA(datetime.datetime),
                               error_message=None)
        self.worker.update_job(fakes.JOB_ID, 'TIMED_OUT', timeout=None,
                               error_message=None)

        self.mox.ReplayAll()

        processor = TestableSnapshotProcessor(self.nova_client)
        processor.init_processor(self.worker)

        processor.process_job(job)

        self.mox.VerifyAll()
Ejemplo n.º 5
0
    def test_process_job_should_update_status_and_timestamp(self):
        base_time = timeutils.utcnow()
        time_seq = [
            base_time,
            base_time,
            base_time + datetime.timedelta(seconds=305),
            base_time + datetime.timedelta(minutes=60, seconds=5),
            base_time + datetime.timedelta(minutes=60, seconds=305),
            ]
        timeutils.set_time_override_seq(time_seq)

        job = copy.deepcopy(fakes.JOB['job'])
        job['timeout'] = base_time + datetime.timedelta(minutes=60)

        self.nova_client.servers.create_image(mox.IsA(str),
            mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('QUEUED'))
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('SAVING'))
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('SAVING'))
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('ACTIVE'))
        self.nova_client.servers.get(mox.IsA(str)).AndReturn(
            MockServer())
        self.worker.update_job(fakes.JOB_ID, 'PROCESSING',
                               timeout=mox.IsA(datetime.datetime),
                               error_message=None)
        self.worker.update_job(fakes.JOB_ID, 'PROCESSING',
                               timeout=mox.IsA(datetime.datetime),
                               error_message=None)
        self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
                               error_message=None)
        self.worker.update_job(fakes.JOB_ID, 'DONE', timeout=None,
                               error_message=None)
        self.mox.ReplayAll()

        processor = TestableSnapshotProcessor(self.nova_client)
        processor.init_processor(self.worker)

        processor.process_job(job)

        self.mox.VerifyAll()
Ejemplo n.º 6
0
    def test_process_job_should_update_status_only(self):
        base_time = timeutils.utcnow()
        time_seq = [
            base_time,
            base_time,
            base_time + datetime.timedelta(seconds=305),
            base_time + datetime.timedelta(seconds=605),
            base_time + datetime.timedelta(seconds=905),
            ]
        timeutils.set_time_override_seq(time_seq)

        job = copy.deepcopy(self.job)
        job['timeout'] = base_time + datetime.timedelta(minutes=60)

        self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
        self.nova_client.servers.create_image(mox.IsA(str),
            mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('QUEUED'))
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('SAVING'))
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('SAVING'))
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('ACTIVE'))
        mock_retention = MockRetention()
        self.nova_client.rax_scheduled_images_python_novaclient_ext.\
            get(mox.IsA(str)).AndReturn(mock_retention)
        self._simple_prepare_worker_mock(2)
        self.mox.StubOutWithMock(utils, 'generate_notification')
        utils.generate_notification(None, 'qonos.job.run.start', mox.IsA(dict),
                                    mox.IsA(str))
        utils.generate_notification(None, 'qonos.job.run.end', mox.IsA(dict),
                                    mox.IsA(str))
        self.mox.ReplayAll()

        processor = TestableSnapshotProcessor(self.nova_client)
        processor.init_processor(self.worker)

        processor.process_job(job)

        self.mox.VerifyAll()
Ejemplo n.º 7
0
    def test_process_job_should_update_status_timestamp_no_retries(self):
        base_time = timeutils.utcnow()
        time_seq = [
            base_time,
            base_time,
            base_time + datetime.timedelta(minutes=5, seconds=5),
            base_time + datetime.timedelta(minutes=60, seconds=5),
            base_time + datetime.timedelta(minutes=120, seconds=5),
            base_time + datetime.timedelta(minutes=180, seconds=5),
            base_time + datetime.timedelta(minutes=240, seconds=5),
        ]
        timeutils.set_time_override_seq(time_seq)

        job = copy.deepcopy(self.job)
        job["timeout"] = base_time + datetime.timedelta(minutes=60)

        self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
        self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)
        self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("QUEUED"))
        self.nova_client.images.get(IMAGE_ID).MultipleTimes().AndReturn(MockImageStatus("SAVING"))

        self._init_worker_mock()
        self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=mox.IsA(datetime.datetime), error_message=None)
        self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=mox.IsA(datetime.datetime), error_message=None)
        self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=mox.IsA(datetime.datetime), error_message=None)
        self.worker.update_job(fakes.JOB_ID, "TIMED_OUT", timeout=None, error_message=None)
        self.mox.StubOutWithMock(utils, "generate_notification")
        utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str))
        self.mox.ReplayAll()

        processor = TestableSnapshotProcessor(self.nova_client)
        processor.init_processor(self.worker)

        processor.process_job(job)

        self.mox.VerifyAll()
Ejemplo n.º 8
0
    def _do_test_process_job_should_update_image_error(
        self, error_status, include_create=True, include_queued=True, is_retry=False, job=None
    ):
        base_time = timeutils.utcnow()
        time_seq = [
            base_time,
            base_time,
            base_time + datetime.timedelta(seconds=305),
            base_time + datetime.timedelta(seconds=605),
            base_time + datetime.timedelta(seconds=905),
            base_time + datetime.timedelta(seconds=1205),
            base_time + datetime.timedelta(seconds=1505),
        ]
        timeutils.set_time_override_seq(time_seq)

        if job is None:
            job = copy.deepcopy(self.job)

        job["timeout"] = base_time + datetime.timedelta(minutes=60)

        if include_create:
            self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
            self.nova_client.servers.create_image(mox.IsA(str), mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)

        if include_queued:
            self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("QUEUED"))
        else:
            self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING"))
            self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING"))

        self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING"))
        self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING"))
        self.nova_client.images.get(IMAGE_ID).AndReturn(MockImageStatus("SAVING"))
        self.nova_client.images.get(IMAGE_ID).AndReturn(error_status)

        self._init_worker_mock(skip_metadata_update=(not include_create))

        self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=None, error_message=None)
        self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=None, error_message=None)
        self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=None, error_message=None)
        self.worker.update_job(fakes.JOB_ID, "PROCESSING", timeout=None, error_message=None)

        metadata = copy.copy(job["metadata"])
        metadata["image_id"] = IMAGE_ID

        backoff_factor = DEFAULT_BACKOFF_FACTOR ** job["retry_count"]
        timeout = time_seq[-1] + datetime.timedelta(minutes=DEFAULT_TIMEOUT_INCR * backoff_factor)
        self.worker.update_job(fakes.JOB_ID, "ERROR", timeout=timeout, error_message=mox.IsA(str))

        self.mox.StubOutWithMock(utils, "generate_notification")

        if not is_retry:
            utils.generate_notification(None, "qonos.job.run.start", mox.IsA(dict), mox.IsA(str))
        else:
            utils.generate_notification(None, "qonos.job.retry", mox.IsA(dict), mox.IsA(str))

        self.mox.ReplayAll()

        processor = TestableSnapshotProcessor(self.nova_client)
        processor.init_processor(self.worker)

        processor.process_job(job)

        self.mox.VerifyAll()
Ejemplo n.º 9
0
    def _do_test_process_job_should_update_image_error(self, error_status,
                                                       include_create=True,
                                                       include_queued=True,
                                                       is_retry=False,
                                                       job=None):
        base_time = timeutils.utcnow()
        time_seq = [
            base_time,
            base_time,
            base_time,
            base_time + datetime.timedelta(seconds=305),
            base_time + datetime.timedelta(seconds=605),
            base_time + datetime.timedelta(seconds=905),
            base_time + datetime.timedelta(seconds=1205),
            base_time + datetime.timedelta(seconds=1505),
            ]
        timeutils.set_time_override_seq(time_seq)

        if job is None:
            job = copy.deepcopy(self.job)

        job['timeout'] = base_time + datetime.timedelta(minutes=60)

        if include_create:
            self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
            self.nova_client.servers.create_image(mox.IsA(str),
                mox.IsA(str), self.snapshot_meta).AndReturn(IMAGE_ID)

        if include_queued:
            self.nova_client.images.get(IMAGE_ID).AndReturn(
                MockImageStatus('QUEUED'))
        else:
            self.nova_client.images.get(IMAGE_ID).AndReturn(
                MockImageStatus('SAVING'))
            self.nova_client.images.get(IMAGE_ID).AndReturn(
                MockImageStatus('SAVING'))

        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('SAVING'))
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('SAVING'))
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('SAVING'))
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            error_status)

        self._init_worker_mock(skip_metadata_update=(not include_create))

        self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
                               error_message=None)
        self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
                               error_message=None)
        self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
                               error_message=None)
        self.worker.update_job(fakes.JOB_ID, 'PROCESSING', timeout=None,
                               error_message=None)

        metadata = copy.copy(job['metadata'])
        metadata['image_id'] = IMAGE_ID

        backoff_factor = DEFAULT_BACKOFF_FACTOR ** job['retry_count']
        timeout = time_seq[-1] + datetime.timedelta(minutes=
                                                    DEFAULT_TIMEOUT_INCR *
                                                    backoff_factor)
        self.worker.update_job(fakes.JOB_ID, 'ERROR', timeout=timeout,
                               error_message=mox.IsA(unicode)).AndReturn(
                                {'status': 'ERROR', 'timeout': job['timeout']})

        self.mox.StubOutWithMock(utils, 'generate_notification')

        expected_job = copy.deepcopy(job)
        expected_job['status'] = 'ERROR'
        expected_job['metadata'] = metadata

        if not is_retry:
            utils.generate_notification(None, 'qonos.job.run.start',
                                        mox.IsA(dict), mox.IsA(str))
        else:
            utils.generate_notification(None, 'qonos.job.retry',
                                        mox.IsA(dict), mox.IsA(str))

        utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict),
                                    mox.IsA(str))

        def assert_job_payload(job_payload):
            self.assertTrue('error_message' in job_payload['job'])
            del job_payload['job']['error_message']
            self.assertEquals({'job': expected_job}, job_payload)
            return True

        utils.generate_notification(None, 'qonos.job.update',
                                    mox.Func(assert_job_payload), 'ERROR')

        self.mox.ReplayAll()

        processor = TestableSnapshotProcessor(self.nova_client)
        processor.init_processor(self.worker)

        processor.process_job(job)

        self.mox.VerifyAll()
Ejemplo n.º 10
0
    def _do_test_process_job_should_update_image_error(self,
                                                       error_status,
                                                       include_create=True,
                                                       include_queued=True,
                                                       is_retry=False,
                                                       job=None):
        base_time = timeutils.utcnow()
        time_seq = [
            base_time,
            base_time,
            base_time,
            base_time + datetime.timedelta(seconds=305),
            base_time + datetime.timedelta(seconds=605),
            base_time + datetime.timedelta(seconds=905),
            base_time + datetime.timedelta(seconds=1205),
            base_time + datetime.timedelta(seconds=1505),
        ]
        timeutils.set_time_override_seq(time_seq)

        if job is None:
            job = copy.deepcopy(self.job)

        job['timeout'] = base_time + datetime.timedelta(minutes=60)

        if include_create:
            self.nova_client.servers.get(mox.IsA(str)).AndReturn(MockServer())
            self.nova_client.servers.create_image(
                mox.IsA(str), mox.IsA(str),
                self.snapshot_meta).AndReturn(IMAGE_ID)

        if include_queued:
            self.nova_client.images.get(IMAGE_ID).AndReturn(
                MockImageStatus('QUEUED'))
        else:
            self.nova_client.images.get(IMAGE_ID).AndReturn(
                MockImageStatus('SAVING'))
            self.nova_client.images.get(IMAGE_ID).AndReturn(
                MockImageStatus('SAVING'))

        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('SAVING'))
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('SAVING'))
        self.nova_client.images.get(IMAGE_ID).AndReturn(
            MockImageStatus('SAVING'))
        self.nova_client.images.get(IMAGE_ID).AndReturn(error_status)

        self._init_worker_mock(skip_metadata_update=(not include_create))

        self.worker.update_job(fakes.JOB_ID,
                               'PROCESSING',
                               timeout=None,
                               error_message=None)
        self.worker.update_job(fakes.JOB_ID,
                               'PROCESSING',
                               timeout=None,
                               error_message=None)
        self.worker.update_job(fakes.JOB_ID,
                               'PROCESSING',
                               timeout=None,
                               error_message=None)
        self.worker.update_job(fakes.JOB_ID,
                               'PROCESSING',
                               timeout=None,
                               error_message=None)

        metadata = copy.copy(job['metadata'])
        metadata['image_id'] = IMAGE_ID

        backoff_factor = DEFAULT_BACKOFF_FACTOR**job['retry_count']
        timeout = time_seq[-1] + datetime.timedelta(
            minutes=DEFAULT_TIMEOUT_INCR * backoff_factor)
        self.worker.update_job(fakes.JOB_ID,
                               'ERROR',
                               timeout=timeout,
                               error_message=mox.IsA(unicode)).AndReturn({
                                   'status':
                                   'ERROR',
                                   'timeout':
                                   job['timeout']
                               })

        self.mox.StubOutWithMock(utils, 'generate_notification')

        expected_job = copy.deepcopy(job)
        expected_job['status'] = 'ERROR'
        expected_job['metadata'] = metadata

        if not is_retry:
            utils.generate_notification(None, 'qonos.job.run.start',
                                        mox.IsA(dict), mox.IsA(str))
        else:
            utils.generate_notification(None, 'qonos.job.retry', mox.IsA(dict),
                                        mox.IsA(str))

        utils.generate_notification(None, 'qonos.job.update', mox.IsA(dict),
                                    mox.IsA(str))

        def assert_job_payload(job_payload):
            self.assertTrue('error_message' in job_payload['job'])
            del job_payload['job']['error_message']
            self.assertEquals({'job': expected_job}, job_payload)
            return True

        utils.generate_notification(None, 'qonos.job.update',
                                    mox.Func(assert_job_payload), 'ERROR')

        self.mox.ReplayAll()

        processor = TestableSnapshotProcessor(self.nova_client)
        processor.init_processor(self.worker)

        processor.process_job(job)

        self.mox.VerifyAll()