def test_retry_delay(self): # The job is retried every minute, unless it just made one of its # first four attempts to poll the status endpoint, in which case the # delays are 15/15/30/30 seconds. self.useFixture(FakeLogger()) snapbuild = self.makeSnapBuild() job = SnapStoreUploadJob.create(snapbuild) client = FakeSnapStoreClient() client.upload.failure = UploadFailedResponse("Proxy error", can_retry=True) self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient)) with dbuser(config.ISnapStoreUploadJobSource.dbuser): JobRunner([job]).runAll() self.assertNotIn("status_url", job.metadata) self.assertEqual(timedelta(seconds=60), job.retry_delay) job.scheduled_start = None client.upload.failure = None client.upload.result = self.status_url client.checkStatus.failure = UploadNotScannedYetResponse() for expected_delay in (15, 15, 30, 30, 60): with dbuser(config.ISnapStoreUploadJobSource.dbuser): JobRunner([job]).runAll() self.assertIn("status_url", job.snapbuild.store_upload_metadata) self.assertIsNone(job.store_url) self.assertEqual(timedelta(seconds=expected_delay), job.retry_delay) job.scheduled_start = None client.checkStatus.failure = None client.checkStatus.result = (self.store_url, 1) with dbuser(config.ISnapStoreUploadJobSource.dbuser): JobRunner([job]).runAll() self.assertEqual(self.store_url, job.store_url) self.assertIsNone(job.error_message) self.assertEqual([], pop_notifications()) self.assertEqual(JobStatus.COMPLETED, job.job.status)
def test_runJob_exceeding_max_retries(self): """If a job exceeds maximum retries, it should raise normally.""" job = RaisingRetryJob('completion') JobRunner([job]).runJob(job, None) self.assertEqual(JobStatus.WAITING, job.status) runner = JobRunner([job]) with ExpectedException(RetryError, ''): runner.runJob(job, None) self.assertEqual(JobStatus.FAILED, job.status) self.assertNotIn(job, runner.completed_jobs) self.assertIn(job, runner.incomplete_jobs)
def test_runAll_requires_IRunnable(self): """Supplied classes must implement IRunnableJob. If they don't, we get a TypeError. If they do, then we get an AttributeError, because we don't actually implement the interface. """ runner = JobRunner([object()]) self.assertRaises(TypeError, runner.runAll) class Runnable: implements(IRunnableJob) runner = JobRunner([Runnable()]) self.assertRaises(AttributeError, runner.runAll)
def test_runAll_mails_user_errors(self): """User errors should be mailed out without oopsing. User errors are identified by the RunnableJob.user_error_types attribute. They do not cause an oops to be recorded, and their error messages are mailed to interested parties verbatim. """ job_1, job_2 = self.makeTwoJobs() class ExampleError(Exception): pass def raiseError(): raise ExampleError('Fake exception. Foobar, I say!') job_1.run = raiseError job_1.user_error_types = (ExampleError, ) job_1.error_recipients = ['*****@*****.**'] runner = JobRunner([job_1, job_2]) runner.runAll() self.assertEqual([], self.oopses) notifications = pop_notifications() self.assertEqual(1, len(notifications)) body = notifications[0].get_payload(decode=True) self.assertEqual( 'Launchpad encountered an error during the following operation:' ' appending a string to a list. Fake exception. Foobar, I say!', body) self.assertEqual('Launchpad error while appending a string to a list', notifications[0]['subject'])
def test_runJob(self): """Ensure status is set to completed when a job runs to completion.""" job_1, job_2 = self.makeTwoJobs() runner = JobRunner(job_1) runner.runJob(job_1, None) self.assertEqual(JobStatus.COMPLETED, job_1.job.status) self.assertEqual([job_1], runner.completed_jobs)
def test_run_branches_empty(self): """If the branches are empty, we tell the user.""" # If the job has been waiting for a significant period of time (15 # minutes for now), we run the job anyway. The checkReady method # then raises and this is caught as a user error by the job system, # and as such sends an email to the error recipients, which for this # job is the merge proposal registrant. eric = self.factory.makePerson(name='eric', email='*****@*****.**') bmp = self.factory.makeBranchMergeProposal(registrant=eric) job = UpdatePreviewDiffJob.create(bmp) pop_notifications() JobRunner([job]).runAll() [email] = pop_notifications() self.assertEqual('Eric <*****@*****.**>', email['to']) self.assertEqual( 'Launchpad error while generating the diff for a merge proposal', email['subject']) branch = bmp.source_branch self.assertEqual( 'Launchpad encountered an error during the following operation: ' 'generating the diff for a merge proposal. ' 'The source branch of http://code.launchpad.dev/~%s/%s/%s/' '+merge/%d has no revisions.' % (branch.owner.name, branch.target.name, branch.name, bmp.id), email.get_payload(decode=True))
def test_triggers_webhooks(self): # Jobs trigger any relevant webhooks when they're enabled. self.useFixture(FeatureFixture({'code.git.webhooks.enabled': 'on'})) repository = self.factory.makeGitRepository() self.factory.makeGitRefs( repository, paths=['refs/heads/master', 'refs/tags/1.0']) hook = self.factory.makeWebhook( target=repository, event_types=['git:push:0.1']) job = GitRefScanJob.create(repository) paths = ('refs/heads/master', 'refs/tags/2.0') self.useFixture(GitHostingFixture(refs=self.makeFakeRefs(paths))) with dbuser('branchscanner'): JobRunner([job]).runAll() delivery = hook.deliveries.one() sha1 = lambda s: hashlib.sha1(s).hexdigest() self.assertThat( delivery, MatchesStructure( event_type=Equals('git:push:0.1'), payload=MatchesDict({ 'git_repository': Equals('/' + repository.unique_name), 'git_repository_path': Equals(repository.unique_name), 'ref_changes': Equals({ 'refs/tags/1.0': { 'old': {'commit_sha1': sha1('refs/tags/1.0')}, 'new': None}, 'refs/tags/2.0': { 'old': None, 'new': {'commit_sha1': sha1('refs/tags/2.0')}}, })}))) with dbuser(config.IWebhookDeliveryJobSource.dbuser): self.assertEqual( "<WebhookDeliveryJob for webhook %d on %r>" % ( hook.id, hook.target), repr(delivery))
def test_works_in_job(self): # `BranchHostingClient` is usable from a running job. blob = b"".join(chr(i) for i in range(256)) @implementer(IRunnableJob) class GetBlobJob(BaseRunnableJob): def __init__(self, testcase): super(GetBlobJob, self).__init__() self.job = Job() self.testcase = testcase def run(self): with self.testcase.mockRequests("GET", body=blob, set_default_timeout=False): self.blob = self.testcase.client.getBlob(123, "file-id") # We must make this assertion inside the job, since the job # runner creates a separate timeline. self.testcase.assertRequest( "+branch-id/123/download/head%3A/file-id") job = GetBlobJob(self) JobRunner([job]).runAll() self.assertEqual(JobStatus.COMPLETED, job.job.status) self.assertEqual(blob, job.blob)
def test_run_all(self): """The job can be run under the JobRunner successfully.""" job = make_runnable_incremental_diff_job(self) with dbuser("merge-proposal-jobs"): runner = JobRunner([job]) runner.runAll() self.assertEqual([job], runner.completed_jobs)
def test_run_sends_email(self): """MergeProposalCreationJob.run sends an email.""" bmp = self.createProposalWithEmptyBranches() job = MergeProposalNeedsReviewEmailJob.create(bmp) self.assertEqual([], pop_notifications()) with dbuser("merge-proposal-jobs"): JobRunner([job]).runAll() self.assertEqual(2, len(pop_notifications()))
def test_run(self): bmp = self.createExampleBzrMerge()[0] job = UpdatePreviewDiffJob.create(bmp) self.factory.makeRevisionsForBranch(bmp.source_branch, count=1) bmp.source_branch.next_mirror_time = None with dbuser("merge-proposal-jobs"): JobRunner([job]).runAll() self.checkExampleBzrMerge(bmp.preview_diff.text)
def test_runJob_raising_retry_error(self): """If a job raises a retry_error, it should be re-queued.""" job = RaisingRetryJob('completion') runner = JobRunner([job]) with self.expectedLog('Scheduling retry due to RetryError'): runner.runJob(job, None) self.assertEqual(JobStatus.WAITING, job.status) self.assertNotIn(job, runner.completed_jobs) self.assertIn(job, runner.incomplete_jobs)
def test_runJob_records_failure(self): """When a job fails, the failure needs to be recorded.""" job = RaisingJob('boom') runner = JobRunner([job]) self.assertRaises(RaisingJobException, runner.runJob, job, None) # Abort the transaction to confirm that the update of the job status # has been committed. transaction.abort() self.assertEqual(JobStatus.FAILED, job.job.status)
def test_runJobHandleErrors_oops_timeline_detail_filter(self): """A job can choose to filter oops timeline details.""" job = RaisingJobTimelineMessage('boom') job.timeline_detail_filter = lambda _, detail: '<redacted>' flush_database_updates() runner = JobRunner([job]) runner.runJobHandleError(job) self.assertEqual(1, len(self.oopses)) actions = [action[2:4] for action in self.oopses[0]['timeline']] self.assertIn(('job', '<redacted>'), actions)
def test_runJobHandleErrors_oops_generated_user_notify_fails(self): """A second oops is logged if the notification of the oops fails. In this test case the error is a user expected error, so the notifyUserError is called, and in this case the notify raises too. """ job = RaisingJobRaisingNotifyUserError('boom') runner = JobRunner([job]) runner.runJobHandleError(job) self.assertEqual(1, len(self.oopses))
def test_run(self): # A proper test run closes bugs. spr = self.factory.makeSourcePackageRelease( distroseries=self.distroseries, changelog_entry="changelog") bug = self.factory.makeBug() bugtask = self.factory.makeBugTask(target=spr.sourcepackage, bug=bug) self.assertEqual(BugTaskStatus.NEW, bugtask.status) job = self.makeJob(spr=spr, bug_ids=[bug.id]) JobRunner([job]).runAll() self.assertEqual(BugTaskStatus.FIXRELEASED, bugtask.status)
def test_runJob_with_SuspendJobException(self): # A job that raises SuspendJobError should end up suspended. job = NullJob('suspended') job.run = FakeMethod(failure=SuspendJobException()) runner = JobRunner([job]) runner.runJob(job, None) self.assertEqual(JobStatus.SUSPENDED, job.status) self.assertNotIn(job, runner.completed_jobs) self.assertIn(job, runner.incomplete_jobs)
def test_run_scan_pending_retries(self): # A run that finds that the store has not yet finished scanning the # package schedules itself to be retried. self.useFixture(FakeLogger()) snapbuild = self.makeSnapBuild() self.assertContentEqual([], snapbuild.store_upload_jobs) job = SnapStoreUploadJob.create(snapbuild) client = FakeSnapStoreClient() client.upload.result = self.status_url client.checkStatus.failure = UploadNotScannedYetResponse() self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient)) with dbuser(config.ISnapStoreUploadJobSource.dbuser): JobRunner([job]).runAll() self.assertEqual([((snapbuild, ), {})], client.upload.calls) self.assertEqual([((self.status_url, ), {})], client.checkStatus.calls) self.assertEqual([], client.release.calls) self.assertContentEqual([job], snapbuild.store_upload_jobs) self.assertIsNone(job.store_url) self.assertIsNone(job.store_revision) self.assertIsNone(job.error_message) self.assertEqual([], pop_notifications()) self.assertEqual(JobStatus.WAITING, job.job.status) self.assertWebhookDeliveries(snapbuild, ["Pending"]) # Try again. The upload part of the job is not retried, and this # time the scan completes. job.scheduled_start = None client.upload.calls = [] client.checkStatus.calls = [] client.checkStatus.failure = None client.checkStatus.result = (self.store_url, 1) with dbuser(config.ISnapStoreUploadJobSource.dbuser): JobRunner([job]).runAll() self.assertEqual([], client.upload.calls) self.assertEqual([((self.status_url, ), {})], client.checkStatus.calls) self.assertEqual([], client.release.calls) self.assertContentEqual([job], snapbuild.store_upload_jobs) self.assertEqual(self.store_url, job.store_url) self.assertEqual(1, job.store_revision) self.assertIsNone(job.error_message) self.assertEqual([], pop_notifications()) self.assertEqual(JobStatus.COMPLETED, job.job.status) self.assertWebhookDeliveries(snapbuild, ["Pending", "Uploaded"])
def test_run_502_retries(self): # A run that gets a 502 error from the store schedules itself to be # retried. self.useFixture(FakeLogger()) snapbuild = self.makeSnapBuild() self.assertContentEqual([], snapbuild.store_upload_jobs) job = SnapStoreUploadJob.create(snapbuild) client = FakeSnapStoreClient() client.upload.failure = UploadFailedResponse("Proxy error", can_retry=True) self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient)) with dbuser(config.ISnapStoreUploadJobSource.dbuser): JobRunner([job]).runAll() self.assertEqual([((snapbuild, ), {})], client.upload.calls) self.assertEqual([], client.checkStatus.calls) self.assertEqual([], client.release.calls) self.assertContentEqual([job], snapbuild.store_upload_jobs) self.assertIsNone(job.store_url) self.assertIsNone(job.store_revision) self.assertIsNone(job.error_message) self.assertEqual([], pop_notifications()) self.assertEqual(JobStatus.WAITING, job.job.status) self.assertWebhookDeliveries(snapbuild, ["Pending"]) # Try again. The upload part of the job is retried, and this time # it succeeds. job.scheduled_start = None client.upload.calls = [] client.upload.failure = None client.upload.result = self.status_url client.checkStatus.result = (self.store_url, 1) with dbuser(config.ISnapStoreUploadJobSource.dbuser): JobRunner([job]).runAll() self.assertEqual([((snapbuild, ), {})], client.upload.calls) self.assertEqual([((self.status_url, ), {})], client.checkStatus.calls) self.assertEqual([], client.release.calls) self.assertContentEqual([job], snapbuild.store_upload_jobs) self.assertEqual(self.store_url, job.store_url) self.assertEqual(1, job.store_revision) self.assertIsNone(job.error_message) self.assertEqual([], pop_notifications()) self.assertEqual(JobStatus.COMPLETED, job.job.status) self.assertWebhookDeliveries(snapbuild, ["Pending", "Uploaded"])
def test_logs_bad_ref_info(self): repository = self.factory.makeGitRepository() job = GitRefScanJob.create(repository) self.useFixture(GitHostingFixture(refs={"refs/heads/master": {}})) expected_message = ( 'Unconvertible ref refs/heads/master {}: ' 'ref info does not contain "object" key') with self.expectedLog(expected_message): with dbuser("branchscanner"): JobRunner([job]).runAll() self.assertEqual([], list(repository.refs))
def test_triggers_webhooks_git(self): self.useFixture( FeatureFixture({BRANCH_MERGE_PROPOSAL_WEBHOOKS_FEATURE_FLAG: "on"})) bmp = self.createExampleGitMerge()[0] hook = self.factory.makeWebhook(target=bmp.target_git_repository, event_types=["merge-proposal:0.1"]) job = UpdatePreviewDiffJob.create(bmp) with dbuser("merge-proposal-jobs"): JobRunner([job]).runAll() self.assertCorrectPreviewDiffDelivery(bmp, hook.deliveries.one())
def test_runAll_skips_lease_failures(self): """Ensure runAll skips jobs whose leases can't be acquired.""" job_1, job_2 = self.makeTwoJobs() job_2.job.acquireLease() runner = JobRunner([job_1, job_2]) runner.runAll() self.assertEqual(JobStatus.COMPLETED, job_1.job.status) self.assertEqual(JobStatus.WAITING, job_2.job.status) self.assertEqual([job_1], runner.completed_jobs) self.assertEqual([job_2], runner.incomplete_jobs) self.assertEqual([], self.oopses)
def test_runJobHandleErrors_oops_timeline(self): """The oops timeline only covers the job itself.""" timeline = get_request_timeline(get_current_browser_request()) timeline.start('test', 'sentinel').finish() job = RaisingJobTimelineMessage('boom') flush_database_updates() runner = JobRunner([job]) runner.runJobHandleError(job) self.assertEqual(1, len(self.oopses)) actions = [action[2:4] for action in self.oopses[0]['timeline']] self.assertIn(('job', 'boom'), actions) self.assertNotIn(('test', 'sentinel'), actions)
def test_run(self): # Running a job to reclaim space sends a request to the hosting # service. hosting_fixture = self.useFixture(GitHostingFixture()) name = "/~owner/+git/gone" path = "1" job = ReclaimGitRepositorySpaceJob.create(name, path) self.makeJobReady(job) [job] = list(ReclaimGitRepositorySpaceJob.iterReady()) with dbuser("branchscanner"): JobRunner([job]).runAll() self.assertEqual([(path,)], hosting_fixture.delete.extract_args())
def test_runAll(self): """Ensure runAll works in the normal case.""" job_1, job_2 = self.makeTwoJobs() runner = JobRunner([job_1, job_2]) runner.runAll() self.assertEqual(JobStatus.COMPLETED, job_1.job.status) self.assertEqual(JobStatus.COMPLETED, job_2.job.status) msg1 = NullJob.JOB_COMPLETIONS.pop() msg2 = NullJob.JOB_COMPLETIONS.pop() self.assertEqual(msg1, "job 2") self.assertEqual(msg2, "job 1") self.assertEqual([job_1, job_2], runner.completed_jobs)
def runJob(self, job): with dbuser("webhookrunner"): runner = JobRunner([job]) runner.runAll() job.lease_expires = None if len(runner.completed_jobs) == 1 and not runner.incomplete_jobs: return True if len(runner.incomplete_jobs) == 1 and not runner.completed_jobs: return False if not runner.incomplete_jobs and not runner.completed_jobs: return None raise Exception("Unexpected jobs.")
def test_triggers_webhooks_bzr(self): self.useFixture( FeatureFixture({BRANCH_MERGE_PROPOSAL_WEBHOOKS_FEATURE_FLAG: "on"})) bmp = self.createExampleBzrMerge()[0] hook = self.factory.makeWebhook(target=bmp.target_branch, event_types=["merge-proposal:0.1"]) job = UpdatePreviewDiffJob.create(bmp) self.factory.makeRevisionsForBranch(bmp.source_branch, count=1) bmp.source_branch.next_mirror_time = None with dbuser("merge-proposal-jobs"): JobRunner([job]).runAll() self.assertCorrectPreviewDiffDelivery(bmp, hook.deliveries.one())
def test_run_upload_failure_notifies(self): # A run that gets some other upload failure from the store sends # mail. requester = self.factory.makePerson(name="requester") requester_team = self.factory.makeTeam(owner=requester, name="requester-team", members=[requester]) snapbuild = self.makeSnapBuild(requester=requester_team, name="test-snap", owner=requester_team) self.assertContentEqual([], snapbuild.store_upload_jobs) job = SnapStoreUploadJob.create(snapbuild) client = FakeSnapStoreClient() client.upload.failure = UploadFailedResponse( "Failed to upload", detail="The proxy exploded.\n") self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient)) with dbuser(config.ISnapStoreUploadJobSource.dbuser): JobRunner([job]).runAll() self.assertEqual([((snapbuild, ), {})], client.upload.calls) self.assertEqual([], client.checkStatus.calls) self.assertEqual([], client.release.calls) self.assertContentEqual([job], snapbuild.store_upload_jobs) self.assertIsNone(job.store_url) self.assertIsNone(job.store_revision) self.assertEqual("Failed to upload", job.error_message) [notification] = pop_notifications() self.assertEqual(config.canonical.noreply_from_address, notification["From"]) self.assertEqual("Requester <%s>" % requester.preferredemail.email, notification["To"]) subject = notification["Subject"].replace("\n ", " ") self.assertEqual("Store upload failed for test-snap", subject) self.assertEqual("Requester @requester-team", notification["X-Launchpad-Message-Rationale"]) self.assertEqual(requester_team.name, notification["X-Launchpad-Message-For"]) self.assertEqual("snap-build-upload-failed", notification["X-Launchpad-Notification-Type"]) body, footer = notification.get_payload(decode=True).split("\n-- \n") self.assertIn("Failed to upload", body) build_url = ( "http://launchpad.dev/~requester-team/+snap/test-snap/+build/%d" % snapbuild.id) self.assertIn(build_url, body) self.assertEqual( "%s\nYour team Requester Team is the requester of the build.\n" % build_url, footer) self.assertWebhookDeliveries(snapbuild, ["Pending", "Failed to upload"]) self.assertIn(("error_detail", "The proxy exploded.\n"), job.getOopsVars())
def test_date_first_sent(self): job, reqs = self.makeAndRunJob(response_status=404) self.assertEqual(job.date_first_sent, job.date_sent) orig_first_sent = job.date_first_sent self.assertEqual(JobStatus.WAITING, job.status) self.assertEqual(1, job.attempt_count) job.lease_expires = None job.scheduled_start = None with dbuser("webhookrunner"): JobRunner([job]).runAll() self.assertEqual(JobStatus.WAITING, job.status) self.assertEqual(2, job.attempt_count) self.assertNotEqual(job.date_first_sent, job.date_sent) self.assertEqual(orig_first_sent, job.date_first_sent)
def test_run_release_manual_review_notifies(self): # A run configured to automatically release the package to certain # channels but that encounters the manual review state on upload # sends mail. requester = self.factory.makePerson(name="requester") requester_team = self.factory.makeTeam(owner=requester, name="requester-team", members=[requester]) snapbuild = self.makeSnapBuild(requester=requester_team, name="test-snap", owner=requester_team, store_channels=["stable", "edge"]) self.assertContentEqual([], snapbuild.store_upload_jobs) job = SnapStoreUploadJob.create(snapbuild) client = FakeSnapStoreClient() client.upload.result = self.status_url client.checkStatus.result = (self.store_url, None) self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient)) with dbuser(config.ISnapStoreUploadJobSource.dbuser): JobRunner([job]).runAll() self.assertEqual([((snapbuild, ), {})], client.upload.calls) self.assertEqual([((self.status_url, ), {})], client.checkStatus.calls) self.assertEqual([], client.release.calls) self.assertContentEqual([job], snapbuild.store_upload_jobs) self.assertEqual(self.store_url, job.store_url) self.assertIsNone(job.store_revision) self.assertEqual( "Package held for manual review on the store; " "cannot release it automatically.", job.error_message) [notification] = pop_notifications() self.assertEqual(config.canonical.noreply_from_address, notification["From"]) self.assertEqual("Requester <%s>" % requester.preferredemail.email, notification["To"]) subject = notification["Subject"].replace("\n ", " ") self.assertEqual("test-snap held for manual review", subject) self.assertEqual("Requester @requester-team", notification["X-Launchpad-Message-Rationale"]) self.assertEqual(requester_team.name, notification["X-Launchpad-Message-For"]) self.assertEqual("snap-build-release-manual-review", notification["X-Launchpad-Notification-Type"]) body, footer = notification.get_payload(decode=True).split("\n-- \n") self.assertIn(self.store_url, body) self.assertEqual( "http://launchpad.dev/~requester-team/+snap/test-snap/+build/%d\n" "Your team Requester Team is the requester of the build.\n" % snapbuild.id, footer) self.assertWebhookDeliveries( snapbuild, ["Pending", "Failed to release to channels"])