Пример #1
0
 def test_retry_delay(self):
     # The job is retried every minute, unless it just made one of its
     # first four attempts to poll the status endpoint, in which case the
     # delays are 15/15/30/30 seconds.
     self.useFixture(FakeLogger())
     snapbuild = self.makeSnapBuild()
     job = SnapStoreUploadJob.create(snapbuild)
     client = FakeSnapStoreClient()
     client.upload.failure = UploadFailedResponse("Proxy error",
                                                  can_retry=True)
     self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient))
     with dbuser(config.ISnapStoreUploadJobSource.dbuser):
         JobRunner([job]).runAll()
     self.assertNotIn("status_url", job.metadata)
     self.assertEqual(timedelta(seconds=60), job.retry_delay)
     job.scheduled_start = None
     client.upload.failure = None
     client.upload.result = self.status_url
     client.checkStatus.failure = UploadNotScannedYetResponse()
     for expected_delay in (15, 15, 30, 30, 60):
         with dbuser(config.ISnapStoreUploadJobSource.dbuser):
             JobRunner([job]).runAll()
         self.assertIn("status_url", job.snapbuild.store_upload_metadata)
         self.assertIsNone(job.store_url)
         self.assertEqual(timedelta(seconds=expected_delay),
                          job.retry_delay)
         job.scheduled_start = None
     client.checkStatus.failure = None
     client.checkStatus.result = (self.store_url, 1)
     with dbuser(config.ISnapStoreUploadJobSource.dbuser):
         JobRunner([job]).runAll()
     self.assertEqual(self.store_url, job.store_url)
     self.assertIsNone(job.error_message)
     self.assertEqual([], pop_notifications())
     self.assertEqual(JobStatus.COMPLETED, job.job.status)
Пример #2
0
 def test_runJob_exceeding_max_retries(self):
     """If a job exceeds maximum retries, it should raise normally."""
     job = RaisingRetryJob('completion')
     JobRunner([job]).runJob(job, None)
     self.assertEqual(JobStatus.WAITING, job.status)
     runner = JobRunner([job])
     with ExpectedException(RetryError, ''):
         runner.runJob(job, None)
     self.assertEqual(JobStatus.FAILED, job.status)
     self.assertNotIn(job, runner.completed_jobs)
     self.assertIn(job, runner.incomplete_jobs)
Пример #3
0
    def test_runAll_requires_IRunnable(self):
        """Supplied classes must implement IRunnableJob.

        If they don't, we get a TypeError.  If they do, then we get an
        AttributeError, because we don't actually implement the interface.
        """
        runner = JobRunner([object()])
        self.assertRaises(TypeError, runner.runAll)

        class Runnable:
            implements(IRunnableJob)

        runner = JobRunner([Runnable()])
        self.assertRaises(AttributeError, runner.runAll)
Пример #4
0
    def test_runAll_mails_user_errors(self):
        """User errors should be mailed out without oopsing.

        User errors are identified by the RunnableJob.user_error_types
        attribute.  They do not cause an oops to be recorded, and their
        error messages are mailed to interested parties verbatim.
        """
        job_1, job_2 = self.makeTwoJobs()

        class ExampleError(Exception):
            pass

        def raiseError():
            raise ExampleError('Fake exception.  Foobar, I say!')

        job_1.run = raiseError
        job_1.user_error_types = (ExampleError, )
        job_1.error_recipients = ['*****@*****.**']
        runner = JobRunner([job_1, job_2])
        runner.runAll()
        self.assertEqual([], self.oopses)
        notifications = pop_notifications()
        self.assertEqual(1, len(notifications))
        body = notifications[0].get_payload(decode=True)
        self.assertEqual(
            'Launchpad encountered an error during the following operation:'
            ' appending a string to a list.  Fake exception.  Foobar, I say!',
            body)
        self.assertEqual('Launchpad error while appending a string to a list',
                         notifications[0]['subject'])
Пример #5
0
 def test_runJob(self):
     """Ensure status is set to completed when a job runs to completion."""
     job_1, job_2 = self.makeTwoJobs()
     runner = JobRunner(job_1)
     runner.runJob(job_1, None)
     self.assertEqual(JobStatus.COMPLETED, job_1.job.status)
     self.assertEqual([job_1], runner.completed_jobs)
Пример #6
0
 def test_run_branches_empty(self):
     """If the branches are empty, we tell the user."""
     # If the job has been waiting for a significant period of time (15
     # minutes for now), we run the job anyway.  The checkReady method
     # then raises and this is caught as a user error by the job system,
     # and as such sends an email to the error recipients, which for this
     # job is the merge proposal registrant.
     eric = self.factory.makePerson(name='eric', email='*****@*****.**')
     bmp = self.factory.makeBranchMergeProposal(registrant=eric)
     job = UpdatePreviewDiffJob.create(bmp)
     pop_notifications()
     JobRunner([job]).runAll()
     [email] = pop_notifications()
     self.assertEqual('Eric <*****@*****.**>', email['to'])
     self.assertEqual(
         'Launchpad error while generating the diff for a merge proposal',
         email['subject'])
     branch = bmp.source_branch
     self.assertEqual(
         'Launchpad encountered an error during the following operation: '
         'generating the diff for a merge proposal.  '
         'The source branch of http://code.launchpad.dev/~%s/%s/%s/'
         '+merge/%d has no revisions.' %
         (branch.owner.name, branch.target.name, branch.name, bmp.id),
         email.get_payload(decode=True))
Пример #7
0
 def test_triggers_webhooks(self):
     # Jobs trigger any relevant webhooks when they're enabled.
     self.useFixture(FeatureFixture({'code.git.webhooks.enabled': 'on'}))
     repository = self.factory.makeGitRepository()
     self.factory.makeGitRefs(
         repository, paths=['refs/heads/master', 'refs/tags/1.0'])
     hook = self.factory.makeWebhook(
         target=repository, event_types=['git:push:0.1'])
     job = GitRefScanJob.create(repository)
     paths = ('refs/heads/master', 'refs/tags/2.0')
     self.useFixture(GitHostingFixture(refs=self.makeFakeRefs(paths)))
     with dbuser('branchscanner'):
         JobRunner([job]).runAll()
     delivery = hook.deliveries.one()
     sha1 = lambda s: hashlib.sha1(s).hexdigest()
     self.assertThat(
         delivery,
         MatchesStructure(
             event_type=Equals('git:push:0.1'),
             payload=MatchesDict({
                 'git_repository': Equals('/' + repository.unique_name),
                 'git_repository_path': Equals(repository.unique_name),
                 'ref_changes': Equals({
                     'refs/tags/1.0': {
                         'old': {'commit_sha1': sha1('refs/tags/1.0')},
                         'new': None},
                     'refs/tags/2.0': {
                         'old': None,
                         'new': {'commit_sha1': sha1('refs/tags/2.0')}},
                 })})))
     with dbuser(config.IWebhookDeliveryJobSource.dbuser):
         self.assertEqual(
             "<WebhookDeliveryJob for webhook %d on %r>" % (
                 hook.id, hook.target),
             repr(delivery))
Пример #8
0
    def test_works_in_job(self):
        # `BranchHostingClient` is usable from a running job.
        blob = b"".join(chr(i) for i in range(256))

        @implementer(IRunnableJob)
        class GetBlobJob(BaseRunnableJob):
            def __init__(self, testcase):
                super(GetBlobJob, self).__init__()
                self.job = Job()
                self.testcase = testcase

            def run(self):
                with self.testcase.mockRequests("GET",
                                                body=blob,
                                                set_default_timeout=False):
                    self.blob = self.testcase.client.getBlob(123, "file-id")
                # We must make this assertion inside the job, since the job
                # runner creates a separate timeline.
                self.testcase.assertRequest(
                    "+branch-id/123/download/head%3A/file-id")

        job = GetBlobJob(self)
        JobRunner([job]).runAll()
        self.assertEqual(JobStatus.COMPLETED, job.job.status)
        self.assertEqual(blob, job.blob)
Пример #9
0
 def test_run_all(self):
     """The job can be run under the JobRunner successfully."""
     job = make_runnable_incremental_diff_job(self)
     with dbuser("merge-proposal-jobs"):
         runner = JobRunner([job])
         runner.runAll()
     self.assertEqual([job], runner.completed_jobs)
Пример #10
0
 def test_run_sends_email(self):
     """MergeProposalCreationJob.run sends an email."""
     bmp = self.createProposalWithEmptyBranches()
     job = MergeProposalNeedsReviewEmailJob.create(bmp)
     self.assertEqual([], pop_notifications())
     with dbuser("merge-proposal-jobs"):
         JobRunner([job]).runAll()
     self.assertEqual(2, len(pop_notifications()))
Пример #11
0
 def test_run(self):
     bmp = self.createExampleBzrMerge()[0]
     job = UpdatePreviewDiffJob.create(bmp)
     self.factory.makeRevisionsForBranch(bmp.source_branch, count=1)
     bmp.source_branch.next_mirror_time = None
     with dbuser("merge-proposal-jobs"):
         JobRunner([job]).runAll()
     self.checkExampleBzrMerge(bmp.preview_diff.text)
Пример #12
0
 def test_runJob_raising_retry_error(self):
     """If a job raises a retry_error, it should be re-queued."""
     job = RaisingRetryJob('completion')
     runner = JobRunner([job])
     with self.expectedLog('Scheduling retry due to RetryError'):
         runner.runJob(job, None)
     self.assertEqual(JobStatus.WAITING, job.status)
     self.assertNotIn(job, runner.completed_jobs)
     self.assertIn(job, runner.incomplete_jobs)
Пример #13
0
 def test_runJob_records_failure(self):
     """When a job fails, the failure needs to be recorded."""
     job = RaisingJob('boom')
     runner = JobRunner([job])
     self.assertRaises(RaisingJobException, runner.runJob, job, None)
     # Abort the transaction to confirm that the update of the job status
     # has been committed.
     transaction.abort()
     self.assertEqual(JobStatus.FAILED, job.job.status)
Пример #14
0
 def test_runJobHandleErrors_oops_timeline_detail_filter(self):
     """A job can choose to filter oops timeline details."""
     job = RaisingJobTimelineMessage('boom')
     job.timeline_detail_filter = lambda _, detail: '<redacted>'
     flush_database_updates()
     runner = JobRunner([job])
     runner.runJobHandleError(job)
     self.assertEqual(1, len(self.oopses))
     actions = [action[2:4] for action in self.oopses[0]['timeline']]
     self.assertIn(('job', '<redacted>'), actions)
Пример #15
0
    def test_runJobHandleErrors_oops_generated_user_notify_fails(self):
        """A second oops is logged if the notification of the oops fails.

        In this test case the error is a user expected error, so the
        notifyUserError is called, and in this case the notify raises too.
        """
        job = RaisingJobRaisingNotifyUserError('boom')
        runner = JobRunner([job])
        runner.runJobHandleError(job)
        self.assertEqual(1, len(self.oopses))
Пример #16
0
 def test_run(self):
     # A proper test run closes bugs.
     spr = self.factory.makeSourcePackageRelease(
         distroseries=self.distroseries, changelog_entry="changelog")
     bug = self.factory.makeBug()
     bugtask = self.factory.makeBugTask(target=spr.sourcepackage, bug=bug)
     self.assertEqual(BugTaskStatus.NEW, bugtask.status)
     job = self.makeJob(spr=spr, bug_ids=[bug.id])
     JobRunner([job]).runAll()
     self.assertEqual(BugTaskStatus.FIXRELEASED, bugtask.status)
Пример #17
0
    def test_runJob_with_SuspendJobException(self):
        # A job that raises SuspendJobError should end up suspended.
        job = NullJob('suspended')
        job.run = FakeMethod(failure=SuspendJobException())
        runner = JobRunner([job])
        runner.runJob(job, None)

        self.assertEqual(JobStatus.SUSPENDED, job.status)
        self.assertNotIn(job, runner.completed_jobs)
        self.assertIn(job, runner.incomplete_jobs)
Пример #18
0
 def test_run_scan_pending_retries(self):
     # A run that finds that the store has not yet finished scanning the
     # package schedules itself to be retried.
     self.useFixture(FakeLogger())
     snapbuild = self.makeSnapBuild()
     self.assertContentEqual([], snapbuild.store_upload_jobs)
     job = SnapStoreUploadJob.create(snapbuild)
     client = FakeSnapStoreClient()
     client.upload.result = self.status_url
     client.checkStatus.failure = UploadNotScannedYetResponse()
     self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient))
     with dbuser(config.ISnapStoreUploadJobSource.dbuser):
         JobRunner([job]).runAll()
     self.assertEqual([((snapbuild, ), {})], client.upload.calls)
     self.assertEqual([((self.status_url, ), {})], client.checkStatus.calls)
     self.assertEqual([], client.release.calls)
     self.assertContentEqual([job], snapbuild.store_upload_jobs)
     self.assertIsNone(job.store_url)
     self.assertIsNone(job.store_revision)
     self.assertIsNone(job.error_message)
     self.assertEqual([], pop_notifications())
     self.assertEqual(JobStatus.WAITING, job.job.status)
     self.assertWebhookDeliveries(snapbuild, ["Pending"])
     # Try again.  The upload part of the job is not retried, and this
     # time the scan completes.
     job.scheduled_start = None
     client.upload.calls = []
     client.checkStatus.calls = []
     client.checkStatus.failure = None
     client.checkStatus.result = (self.store_url, 1)
     with dbuser(config.ISnapStoreUploadJobSource.dbuser):
         JobRunner([job]).runAll()
     self.assertEqual([], client.upload.calls)
     self.assertEqual([((self.status_url, ), {})], client.checkStatus.calls)
     self.assertEqual([], client.release.calls)
     self.assertContentEqual([job], snapbuild.store_upload_jobs)
     self.assertEqual(self.store_url, job.store_url)
     self.assertEqual(1, job.store_revision)
     self.assertIsNone(job.error_message)
     self.assertEqual([], pop_notifications())
     self.assertEqual(JobStatus.COMPLETED, job.job.status)
     self.assertWebhookDeliveries(snapbuild, ["Pending", "Uploaded"])
Пример #19
0
 def test_run_502_retries(self):
     # A run that gets a 502 error from the store schedules itself to be
     # retried.
     self.useFixture(FakeLogger())
     snapbuild = self.makeSnapBuild()
     self.assertContentEqual([], snapbuild.store_upload_jobs)
     job = SnapStoreUploadJob.create(snapbuild)
     client = FakeSnapStoreClient()
     client.upload.failure = UploadFailedResponse("Proxy error",
                                                  can_retry=True)
     self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient))
     with dbuser(config.ISnapStoreUploadJobSource.dbuser):
         JobRunner([job]).runAll()
     self.assertEqual([((snapbuild, ), {})], client.upload.calls)
     self.assertEqual([], client.checkStatus.calls)
     self.assertEqual([], client.release.calls)
     self.assertContentEqual([job], snapbuild.store_upload_jobs)
     self.assertIsNone(job.store_url)
     self.assertIsNone(job.store_revision)
     self.assertIsNone(job.error_message)
     self.assertEqual([], pop_notifications())
     self.assertEqual(JobStatus.WAITING, job.job.status)
     self.assertWebhookDeliveries(snapbuild, ["Pending"])
     # Try again.  The upload part of the job is retried, and this time
     # it succeeds.
     job.scheduled_start = None
     client.upload.calls = []
     client.upload.failure = None
     client.upload.result = self.status_url
     client.checkStatus.result = (self.store_url, 1)
     with dbuser(config.ISnapStoreUploadJobSource.dbuser):
         JobRunner([job]).runAll()
     self.assertEqual([((snapbuild, ), {})], client.upload.calls)
     self.assertEqual([((self.status_url, ), {})], client.checkStatus.calls)
     self.assertEqual([], client.release.calls)
     self.assertContentEqual([job], snapbuild.store_upload_jobs)
     self.assertEqual(self.store_url, job.store_url)
     self.assertEqual(1, job.store_revision)
     self.assertIsNone(job.error_message)
     self.assertEqual([], pop_notifications())
     self.assertEqual(JobStatus.COMPLETED, job.job.status)
     self.assertWebhookDeliveries(snapbuild, ["Pending", "Uploaded"])
Пример #20
0
 def test_logs_bad_ref_info(self):
     repository = self.factory.makeGitRepository()
     job = GitRefScanJob.create(repository)
     self.useFixture(GitHostingFixture(refs={"refs/heads/master": {}}))
     expected_message = (
         'Unconvertible ref refs/heads/master {}: '
         'ref info does not contain "object" key')
     with self.expectedLog(expected_message):
         with dbuser("branchscanner"):
             JobRunner([job]).runAll()
     self.assertEqual([], list(repository.refs))
Пример #21
0
 def test_triggers_webhooks_git(self):
     self.useFixture(
         FeatureFixture({BRANCH_MERGE_PROPOSAL_WEBHOOKS_FEATURE_FLAG:
                         "on"}))
     bmp = self.createExampleGitMerge()[0]
     hook = self.factory.makeWebhook(target=bmp.target_git_repository,
                                     event_types=["merge-proposal:0.1"])
     job = UpdatePreviewDiffJob.create(bmp)
     with dbuser("merge-proposal-jobs"):
         JobRunner([job]).runAll()
     self.assertCorrectPreviewDiffDelivery(bmp, hook.deliveries.one())
Пример #22
0
 def test_runAll_skips_lease_failures(self):
     """Ensure runAll skips jobs whose leases can't be acquired."""
     job_1, job_2 = self.makeTwoJobs()
     job_2.job.acquireLease()
     runner = JobRunner([job_1, job_2])
     runner.runAll()
     self.assertEqual(JobStatus.COMPLETED, job_1.job.status)
     self.assertEqual(JobStatus.WAITING, job_2.job.status)
     self.assertEqual([job_1], runner.completed_jobs)
     self.assertEqual([job_2], runner.incomplete_jobs)
     self.assertEqual([], self.oopses)
Пример #23
0
 def test_runJobHandleErrors_oops_timeline(self):
     """The oops timeline only covers the job itself."""
     timeline = get_request_timeline(get_current_browser_request())
     timeline.start('test', 'sentinel').finish()
     job = RaisingJobTimelineMessage('boom')
     flush_database_updates()
     runner = JobRunner([job])
     runner.runJobHandleError(job)
     self.assertEqual(1, len(self.oopses))
     actions = [action[2:4] for action in self.oopses[0]['timeline']]
     self.assertIn(('job', 'boom'), actions)
     self.assertNotIn(('test', 'sentinel'), actions)
Пример #24
0
 def test_run(self):
     # Running a job to reclaim space sends a request to the hosting
     # service.
     hosting_fixture = self.useFixture(GitHostingFixture())
     name = "/~owner/+git/gone"
     path = "1"
     job = ReclaimGitRepositorySpaceJob.create(name, path)
     self.makeJobReady(job)
     [job] = list(ReclaimGitRepositorySpaceJob.iterReady())
     with dbuser("branchscanner"):
         JobRunner([job]).runAll()
     self.assertEqual([(path,)], hosting_fixture.delete.extract_args())
Пример #25
0
 def test_runAll(self):
     """Ensure runAll works in the normal case."""
     job_1, job_2 = self.makeTwoJobs()
     runner = JobRunner([job_1, job_2])
     runner.runAll()
     self.assertEqual(JobStatus.COMPLETED, job_1.job.status)
     self.assertEqual(JobStatus.COMPLETED, job_2.job.status)
     msg1 = NullJob.JOB_COMPLETIONS.pop()
     msg2 = NullJob.JOB_COMPLETIONS.pop()
     self.assertEqual(msg1, "job 2")
     self.assertEqual(msg2, "job 1")
     self.assertEqual([job_1, job_2], runner.completed_jobs)
Пример #26
0
 def runJob(self, job):
     with dbuser("webhookrunner"):
         runner = JobRunner([job])
         runner.runAll()
     job.lease_expires = None
     if len(runner.completed_jobs) == 1 and not runner.incomplete_jobs:
         return True
     if len(runner.incomplete_jobs) == 1 and not runner.completed_jobs:
         return False
     if not runner.incomplete_jobs and not runner.completed_jobs:
         return None
     raise Exception("Unexpected jobs.")
Пример #27
0
 def test_triggers_webhooks_bzr(self):
     self.useFixture(
         FeatureFixture({BRANCH_MERGE_PROPOSAL_WEBHOOKS_FEATURE_FLAG:
                         "on"}))
     bmp = self.createExampleBzrMerge()[0]
     hook = self.factory.makeWebhook(target=bmp.target_branch,
                                     event_types=["merge-proposal:0.1"])
     job = UpdatePreviewDiffJob.create(bmp)
     self.factory.makeRevisionsForBranch(bmp.source_branch, count=1)
     bmp.source_branch.next_mirror_time = None
     with dbuser("merge-proposal-jobs"):
         JobRunner([job]).runAll()
     self.assertCorrectPreviewDiffDelivery(bmp, hook.deliveries.one())
Пример #28
0
 def test_run_upload_failure_notifies(self):
     # A run that gets some other upload failure from the store sends
     # mail.
     requester = self.factory.makePerson(name="requester")
     requester_team = self.factory.makeTeam(owner=requester,
                                            name="requester-team",
                                            members=[requester])
     snapbuild = self.makeSnapBuild(requester=requester_team,
                                    name="test-snap",
                                    owner=requester_team)
     self.assertContentEqual([], snapbuild.store_upload_jobs)
     job = SnapStoreUploadJob.create(snapbuild)
     client = FakeSnapStoreClient()
     client.upload.failure = UploadFailedResponse(
         "Failed to upload", detail="The proxy exploded.\n")
     self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient))
     with dbuser(config.ISnapStoreUploadJobSource.dbuser):
         JobRunner([job]).runAll()
     self.assertEqual([((snapbuild, ), {})], client.upload.calls)
     self.assertEqual([], client.checkStatus.calls)
     self.assertEqual([], client.release.calls)
     self.assertContentEqual([job], snapbuild.store_upload_jobs)
     self.assertIsNone(job.store_url)
     self.assertIsNone(job.store_revision)
     self.assertEqual("Failed to upload", job.error_message)
     [notification] = pop_notifications()
     self.assertEqual(config.canonical.noreply_from_address,
                      notification["From"])
     self.assertEqual("Requester <%s>" % requester.preferredemail.email,
                      notification["To"])
     subject = notification["Subject"].replace("\n ", " ")
     self.assertEqual("Store upload failed for test-snap", subject)
     self.assertEqual("Requester @requester-team",
                      notification["X-Launchpad-Message-Rationale"])
     self.assertEqual(requester_team.name,
                      notification["X-Launchpad-Message-For"])
     self.assertEqual("snap-build-upload-failed",
                      notification["X-Launchpad-Notification-Type"])
     body, footer = notification.get_payload(decode=True).split("\n-- \n")
     self.assertIn("Failed to upload", body)
     build_url = (
         "http://launchpad.dev/~requester-team/+snap/test-snap/+build/%d" %
         snapbuild.id)
     self.assertIn(build_url, body)
     self.assertEqual(
         "%s\nYour team Requester Team is the requester of the build.\n" %
         build_url, footer)
     self.assertWebhookDeliveries(snapbuild,
                                  ["Pending", "Failed to upload"])
     self.assertIn(("error_detail", "The proxy exploded.\n"),
                   job.getOopsVars())
Пример #29
0
 def test_date_first_sent(self):
     job, reqs = self.makeAndRunJob(response_status=404)
     self.assertEqual(job.date_first_sent, job.date_sent)
     orig_first_sent = job.date_first_sent
     self.assertEqual(JobStatus.WAITING, job.status)
     self.assertEqual(1, job.attempt_count)
     job.lease_expires = None
     job.scheduled_start = None
     with dbuser("webhookrunner"):
         JobRunner([job]).runAll()
     self.assertEqual(JobStatus.WAITING, job.status)
     self.assertEqual(2, job.attempt_count)
     self.assertNotEqual(job.date_first_sent, job.date_sent)
     self.assertEqual(orig_first_sent, job.date_first_sent)
Пример #30
0
 def test_run_release_manual_review_notifies(self):
     # A run configured to automatically release the package to certain
     # channels but that encounters the manual review state on upload
     # sends mail.
     requester = self.factory.makePerson(name="requester")
     requester_team = self.factory.makeTeam(owner=requester,
                                            name="requester-team",
                                            members=[requester])
     snapbuild = self.makeSnapBuild(requester=requester_team,
                                    name="test-snap",
                                    owner=requester_team,
                                    store_channels=["stable", "edge"])
     self.assertContentEqual([], snapbuild.store_upload_jobs)
     job = SnapStoreUploadJob.create(snapbuild)
     client = FakeSnapStoreClient()
     client.upload.result = self.status_url
     client.checkStatus.result = (self.store_url, None)
     self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient))
     with dbuser(config.ISnapStoreUploadJobSource.dbuser):
         JobRunner([job]).runAll()
     self.assertEqual([((snapbuild, ), {})], client.upload.calls)
     self.assertEqual([((self.status_url, ), {})], client.checkStatus.calls)
     self.assertEqual([], client.release.calls)
     self.assertContentEqual([job], snapbuild.store_upload_jobs)
     self.assertEqual(self.store_url, job.store_url)
     self.assertIsNone(job.store_revision)
     self.assertEqual(
         "Package held for manual review on the store; "
         "cannot release it automatically.", job.error_message)
     [notification] = pop_notifications()
     self.assertEqual(config.canonical.noreply_from_address,
                      notification["From"])
     self.assertEqual("Requester <%s>" % requester.preferredemail.email,
                      notification["To"])
     subject = notification["Subject"].replace("\n ", " ")
     self.assertEqual("test-snap held for manual review", subject)
     self.assertEqual("Requester @requester-team",
                      notification["X-Launchpad-Message-Rationale"])
     self.assertEqual(requester_team.name,
                      notification["X-Launchpad-Message-For"])
     self.assertEqual("snap-build-release-manual-review",
                      notification["X-Launchpad-Notification-Type"])
     body, footer = notification.get_payload(decode=True).split("\n-- \n")
     self.assertIn(self.store_url, body)
     self.assertEqual(
         "http://launchpad.dev/~requester-team/+snap/test-snap/+build/%d\n"
         "Your team Requester Team is the requester of the build.\n" %
         snapbuild.id, footer)
     self.assertWebhookDeliveries(
         snapbuild, ["Pending", "Failed to release to channels"])