def test_retry_delay(self):
# The job is retried every minute, unless it just made one of its
# first four attempts to poll the status endpoint, in which case the
# delays are 15/15/30/30 seconds.
self.useFixture(FakeLogger())
snapbuild = self.makeSnapBuild()
job = SnapStoreUploadJob.create(snapbuild)
client = FakeSnapStoreClient()
client.upload.failure = UploadFailedResponse("Proxy error",
can_retry=True)
self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient))
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertNotIn("status_url", job.metadata)
self.assertEqual(timedelta(seconds=60), job.retry_delay)
job.scheduled_start = None
client.upload.failure = None
client.upload.result = self.status_url
client.checkStatus.failure = UploadNotScannedYetResponse()
for expected_delay in (15, 15, 30, 30, 60):
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertIn("status_url", job.snapbuild.store_upload_metadata)
self.assertIsNone(job.store_url)
self.assertEqual(timedelta(seconds=expected_delay),
job.retry_delay)
job.scheduled_start = None
client.checkStatus.failure = None
client.checkStatus.result = (self.store_url, 1)
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertEqual(self.store_url, job.store_url)
self.assertIsNone(job.error_message)
self.assertEqual([], pop_notifications())
self.assertEqual(JobStatus.COMPLETED, job.job.status)
def test_runJob_exceeding_max_retries(self):
"""If a job exceeds maximum retries, it should raise normally."""
job = RaisingRetryJob('completion')
JobRunner([job]).runJob(job, None)
self.assertEqual(JobStatus.WAITING, job.status)
runner = JobRunner([job])
with ExpectedException(RetryError, ''):
runner.runJob(job, None)
self.assertEqual(JobStatus.FAILED, job.status)
self.assertNotIn(job, runner.completed_jobs)
self.assertIn(job, runner.incomplete_jobs)
def test_runAll_requires_IRunnable(self):
"""Supplied classes must implement IRunnableJob.
If they don't, we get a TypeError. If they do, then we get an
AttributeError, because we don't actually implement the interface.
"""
runner = JobRunner([object()])
self.assertRaises(TypeError, runner.runAll)
class Runnable:
implements(IRunnableJob)
runner = JobRunner([Runnable()])
self.assertRaises(AttributeError, runner.runAll)
def test_runAll_mails_user_errors(self):
"""User errors should be mailed out without oopsing.
User errors are identified by the RunnableJob.user_error_types
attribute. They do not cause an oops to be recorded, and their
error messages are mailed to interested parties verbatim.
"""
job_1, job_2 = self.makeTwoJobs()
class ExampleError(Exception):
pass
def raiseError():
raise ExampleError('Fake exception. Foobar, I say!')
job_1.run = raiseError
job_1.user_error_types = (ExampleError, )
job_1.error_recipients = ['*****@*****.**']
runner = JobRunner([job_1, job_2])
runner.runAll()
self.assertEqual([], self.oopses)
notifications = pop_notifications()
self.assertEqual(1, len(notifications))
body = notifications[0].get_payload(decode=True)
self.assertEqual(
'Launchpad encountered an error during the following operation:'
' appending a string to a list. Fake exception. Foobar, I say!',
body)
self.assertEqual('Launchpad error while appending a string to a list',
notifications[0]['subject'])
def test_runJob(self):
"""Ensure status is set to completed when a job runs to completion."""
job_1, job_2 = self.makeTwoJobs()
runner = JobRunner(job_1)
runner.runJob(job_1, None)
self.assertEqual(JobStatus.COMPLETED, job_1.job.status)
self.assertEqual([job_1], runner.completed_jobs)
def test_run_branches_empty(self):
"""If the branches are empty, we tell the user."""
# If the job has been waiting for a significant period of time (15
# minutes for now), we run the job anyway. The checkReady method
# then raises and this is caught as a user error by the job system,
# and as such sends an email to the error recipients, which for this
# job is the merge proposal registrant.
eric = self.factory.makePerson(name='eric', email='*****@*****.**')
bmp = self.factory.makeBranchMergeProposal(registrant=eric)
job = UpdatePreviewDiffJob.create(bmp)
pop_notifications()
JobRunner([job]).runAll()
[email] = pop_notifications()
self.assertEqual('Eric <*****@*****.**>', email['to'])
self.assertEqual(
'Launchpad error while generating the diff for a merge proposal',
email['subject'])
branch = bmp.source_branch
self.assertEqual(
'Launchpad encountered an error during the following operation: '
'generating the diff for a merge proposal. '
'The source branch of http://code.launchpad.dev/~%s/%s/%s/'
'+merge/%d has no revisions.' %
(branch.owner.name, branch.target.name, branch.name, bmp.id),
email.get_payload(decode=True))
def test_triggers_webhooks(self):
# Jobs trigger any relevant webhooks when they're enabled.
self.useFixture(FeatureFixture({'code.git.webhooks.enabled': 'on'}))
repository = self.factory.makeGitRepository()
self.factory.makeGitRefs(
repository, paths=['refs/heads/master', 'refs/tags/1.0'])
hook = self.factory.makeWebhook(
target=repository, event_types=['git:push:0.1'])
job = GitRefScanJob.create(repository)
paths = ('refs/heads/master', 'refs/tags/2.0')
self.useFixture(GitHostingFixture(refs=self.makeFakeRefs(paths)))
with dbuser('branchscanner'):
JobRunner([job]).runAll()
delivery = hook.deliveries.one()
sha1 = lambda s: hashlib.sha1(s).hexdigest()
self.assertThat(
delivery,
MatchesStructure(
event_type=Equals('git:push:0.1'),
payload=MatchesDict({
'git_repository': Equals('/' + repository.unique_name),
'git_repository_path': Equals(repository.unique_name),
'ref_changes': Equals({
'refs/tags/1.0': {
'old': {'commit_sha1': sha1('refs/tags/1.0')},
'new': None},
'refs/tags/2.0': {
'old': None,
'new': {'commit_sha1': sha1('refs/tags/2.0')}},
})})))
with dbuser(config.IWebhookDeliveryJobSource.dbuser):
self.assertEqual(
"<WebhookDeliveryJob for webhook %d on %r>" % (
hook.id, hook.target),
repr(delivery))
def test_works_in_job(self):
# `BranchHostingClient` is usable from a running job.
blob = b"".join(chr(i) for i in range(256))
@implementer(IRunnableJob)
class GetBlobJob(BaseRunnableJob):
def __init__(self, testcase):
super(GetBlobJob, self).__init__()
self.job = Job()
self.testcase = testcase
def run(self):
with self.testcase.mockRequests("GET",
body=blob,
set_default_timeout=False):
self.blob = self.testcase.client.getBlob(123, "file-id")
# We must make this assertion inside the job, since the job
# runner creates a separate timeline.
self.testcase.assertRequest(
"+branch-id/123/download/head%3A/file-id")
job = GetBlobJob(self)
JobRunner([job]).runAll()
self.assertEqual(JobStatus.COMPLETED, job.job.status)
self.assertEqual(blob, job.blob)
def test_run_all(self):
"""The job can be run under the JobRunner successfully."""
job = make_runnable_incremental_diff_job(self)
with dbuser("merge-proposal-jobs"):
runner = JobRunner([job])
runner.runAll()
self.assertEqual([job], runner.completed_jobs)
def test_run_sends_email(self):
"""MergeProposalCreationJob.run sends an email."""
bmp = self.createProposalWithEmptyBranches()
job = MergeProposalNeedsReviewEmailJob.create(bmp)
self.assertEqual([], pop_notifications())
with dbuser("merge-proposal-jobs"):
JobRunner([job]).runAll()
self.assertEqual(2, len(pop_notifications()))
def test_run(self):
bmp = self.createExampleBzrMerge()[0]
job = UpdatePreviewDiffJob.create(bmp)
self.factory.makeRevisionsForBranch(bmp.source_branch, count=1)
bmp.source_branch.next_mirror_time = None
with dbuser("merge-proposal-jobs"):
JobRunner([job]).runAll()
self.checkExampleBzrMerge(bmp.preview_diff.text)
def test_runJob_raising_retry_error(self):
"""If a job raises a retry_error, it should be re-queued."""
job = RaisingRetryJob('completion')
runner = JobRunner([job])
with self.expectedLog('Scheduling retry due to RetryError'):
runner.runJob(job, None)
self.assertEqual(JobStatus.WAITING, job.status)
self.assertNotIn(job, runner.completed_jobs)
self.assertIn(job, runner.incomplete_jobs)
def test_runJob_records_failure(self):
"""When a job fails, the failure needs to be recorded."""
job = RaisingJob('boom')
runner = JobRunner([job])
self.assertRaises(RaisingJobException, runner.runJob, job, None)
# Abort the transaction to confirm that the update of the job status
# has been committed.
transaction.abort()
self.assertEqual(JobStatus.FAILED, job.job.status)
def test_runJobHandleErrors_oops_timeline_detail_filter(self):
"""A job can choose to filter oops timeline details."""
job = RaisingJobTimelineMessage('boom')
job.timeline_detail_filter = lambda _, detail: '<redacted>'
flush_database_updates()
runner = JobRunner([job])
runner.runJobHandleError(job)
self.assertEqual(1, len(self.oopses))
actions = [action[2:4] for action in self.oopses[0]['timeline']]
self.assertIn(('job', '<redacted>'), actions)
def test_runJobHandleErrors_oops_generated_user_notify_fails(self):
"""A second oops is logged if the notification of the oops fails.
In this test case the error is a user expected error, so the
notifyUserError is called, and in this case the notify raises too.
"""
job = RaisingJobRaisingNotifyUserError('boom')
runner = JobRunner([job])
runner.runJobHandleError(job)
self.assertEqual(1, len(self.oopses))
def test_run(self):
# A proper test run closes bugs.
spr = self.factory.makeSourcePackageRelease(
distroseries=self.distroseries, changelog_entry="changelog")
bug = self.factory.makeBug()
bugtask = self.factory.makeBugTask(target=spr.sourcepackage, bug=bug)
self.assertEqual(BugTaskStatus.NEW, bugtask.status)
job = self.makeJob(spr=spr, bug_ids=[bug.id])
JobRunner([job]).runAll()
self.assertEqual(BugTaskStatus.FIXRELEASED, bugtask.status)
def test_runJob_with_SuspendJobException(self):
# A job that raises SuspendJobError should end up suspended.
job = NullJob('suspended')
job.run = FakeMethod(failure=SuspendJobException())
runner = JobRunner([job])
runner.runJob(job, None)
self.assertEqual(JobStatus.SUSPENDED, job.status)
self.assertNotIn(job, runner.completed_jobs)
self.assertIn(job, runner.incomplete_jobs)
def test_run_scan_pending_retries(self):
# A run that finds that the store has not yet finished scanning the
# package schedules itself to be retried.
self.useFixture(FakeLogger())
snapbuild = self.makeSnapBuild()
self.assertContentEqual([], snapbuild.store_upload_jobs)
job = SnapStoreUploadJob.create(snapbuild)
client = FakeSnapStoreClient()
client.upload.result = self.status_url
client.checkStatus.failure = UploadNotScannedYetResponse()
self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient))
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertEqual([((snapbuild, ), {})], client.upload.calls)
self.assertEqual([((self.status_url, ), {})], client.checkStatus.calls)
self.assertEqual([], client.release.calls)
self.assertContentEqual([job], snapbuild.store_upload_jobs)
self.assertIsNone(job.store_url)
self.assertIsNone(job.store_revision)
self.assertIsNone(job.error_message)
self.assertEqual([], pop_notifications())
self.assertEqual(JobStatus.WAITING, job.job.status)
self.assertWebhookDeliveries(snapbuild, ["Pending"])
# Try again. The upload part of the job is not retried, and this
# time the scan completes.
job.scheduled_start = None
client.upload.calls = []
client.checkStatus.calls = []
client.checkStatus.failure = None
client.checkStatus.result = (self.store_url, 1)
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertEqual([], client.upload.calls)
self.assertEqual([((self.status_url, ), {})], client.checkStatus.calls)
self.assertEqual([], client.release.calls)
self.assertContentEqual([job], snapbuild.store_upload_jobs)
self.assertEqual(self.store_url, job.store_url)
self.assertEqual(1, job.store_revision)
self.assertIsNone(job.error_message)
self.assertEqual([], pop_notifications())
self.assertEqual(JobStatus.COMPLETED, job.job.status)
self.assertWebhookDeliveries(snapbuild, ["Pending", "Uploaded"])
def test_run_502_retries(self):
# A run that gets a 502 error from the store schedules itself to be
# retried.
self.useFixture(FakeLogger())
snapbuild = self.makeSnapBuild()
self.assertContentEqual([], snapbuild.store_upload_jobs)
job = SnapStoreUploadJob.create(snapbuild)
client = FakeSnapStoreClient()
client.upload.failure = UploadFailedResponse("Proxy error",
can_retry=True)
self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient))
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertEqual([((snapbuild, ), {})], client.upload.calls)
self.assertEqual([], client.checkStatus.calls)
self.assertEqual([], client.release.calls)
self.assertContentEqual([job], snapbuild.store_upload_jobs)
self.assertIsNone(job.store_url)
self.assertIsNone(job.store_revision)
self.assertIsNone(job.error_message)
self.assertEqual([], pop_notifications())
self.assertEqual(JobStatus.WAITING, job.job.status)
self.assertWebhookDeliveries(snapbuild, ["Pending"])
# Try again. The upload part of the job is retried, and this time
# it succeeds.
job.scheduled_start = None
client.upload.calls = []
client.upload.failure = None
client.upload.result = self.status_url
client.checkStatus.result = (self.store_url, 1)
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertEqual([((snapbuild, ), {})], client.upload.calls)
self.assertEqual([((self.status_url, ), {})], client.checkStatus.calls)
self.assertEqual([], client.release.calls)
self.assertContentEqual([job], snapbuild.store_upload_jobs)
self.assertEqual(self.store_url, job.store_url)
self.assertEqual(1, job.store_revision)
self.assertIsNone(job.error_message)
self.assertEqual([], pop_notifications())
self.assertEqual(JobStatus.COMPLETED, job.job.status)
self.assertWebhookDeliveries(snapbuild, ["Pending", "Uploaded"])
def test_logs_bad_ref_info(self):
repository = self.factory.makeGitRepository()
job = GitRefScanJob.create(repository)
self.useFixture(GitHostingFixture(refs={"refs/heads/master": {}}))
expected_message = (
'Unconvertible ref refs/heads/master {}: '
'ref info does not contain "object" key')
with self.expectedLog(expected_message):
with dbuser("branchscanner"):
JobRunner([job]).runAll()
self.assertEqual([], list(repository.refs))
def test_triggers_webhooks_git(self):
self.useFixture(
FeatureFixture({BRANCH_MERGE_PROPOSAL_WEBHOOKS_FEATURE_FLAG:
"on"}))
bmp = self.createExampleGitMerge()[0]
hook = self.factory.makeWebhook(target=bmp.target_git_repository,
event_types=["merge-proposal:0.1"])
job = UpdatePreviewDiffJob.create(bmp)
with dbuser("merge-proposal-jobs"):
JobRunner([job]).runAll()
self.assertCorrectPreviewDiffDelivery(bmp, hook.deliveries.one())
def test_runAll_skips_lease_failures(self):
"""Ensure runAll skips jobs whose leases can't be acquired."""
job_1, job_2 = self.makeTwoJobs()
job_2.job.acquireLease()
runner = JobRunner([job_1, job_2])
runner.runAll()
self.assertEqual(JobStatus.COMPLETED, job_1.job.status)
self.assertEqual(JobStatus.WAITING, job_2.job.status)
self.assertEqual([job_1], runner.completed_jobs)
self.assertEqual([job_2], runner.incomplete_jobs)
self.assertEqual([], self.oopses)
def test_runJobHandleErrors_oops_timeline(self):
"""The oops timeline only covers the job itself."""
timeline = get_request_timeline(get_current_browser_request())
timeline.start('test', 'sentinel').finish()
job = RaisingJobTimelineMessage('boom')
flush_database_updates()
runner = JobRunner([job])
runner.runJobHandleError(job)
self.assertEqual(1, len(self.oopses))
actions = [action[2:4] for action in self.oopses[0]['timeline']]
self.assertIn(('job', 'boom'), actions)
self.assertNotIn(('test', 'sentinel'), actions)
def test_run(self):
# Running a job to reclaim space sends a request to the hosting
# service.
hosting_fixture = self.useFixture(GitHostingFixture())
name = "/~owner/+git/gone"
path = "1"
job = ReclaimGitRepositorySpaceJob.create(name, path)
self.makeJobReady(job)
[job] = list(ReclaimGitRepositorySpaceJob.iterReady())
with dbuser("branchscanner"):
JobRunner([job]).runAll()
self.assertEqual([(path,)], hosting_fixture.delete.extract_args())
def test_runAll(self):
"""Ensure runAll works in the normal case."""
job_1, job_2 = self.makeTwoJobs()
runner = JobRunner([job_1, job_2])
runner.runAll()
self.assertEqual(JobStatus.COMPLETED, job_1.job.status)
self.assertEqual(JobStatus.COMPLETED, job_2.job.status)
msg1 = NullJob.JOB_COMPLETIONS.pop()
msg2 = NullJob.JOB_COMPLETIONS.pop()
self.assertEqual(msg1, "job 2")
self.assertEqual(msg2, "job 1")
self.assertEqual([job_1, job_2], runner.completed_jobs)
def runJob(self, job):
with dbuser("webhookrunner"):
runner = JobRunner([job])
runner.runAll()
job.lease_expires = None
if len(runner.completed_jobs) == 1 and not runner.incomplete_jobs:
return True
if len(runner.incomplete_jobs) == 1 and not runner.completed_jobs:
return False
if not runner.incomplete_jobs and not runner.completed_jobs:
return None
raise Exception("Unexpected jobs.")
def test_triggers_webhooks_bzr(self):
self.useFixture(
FeatureFixture({BRANCH_MERGE_PROPOSAL_WEBHOOKS_FEATURE_FLAG:
"on"}))
bmp = self.createExampleBzrMerge()[0]
hook = self.factory.makeWebhook(target=bmp.target_branch,
event_types=["merge-proposal:0.1"])
job = UpdatePreviewDiffJob.create(bmp)
self.factory.makeRevisionsForBranch(bmp.source_branch, count=1)
bmp.source_branch.next_mirror_time = None
with dbuser("merge-proposal-jobs"):
JobRunner([job]).runAll()
self.assertCorrectPreviewDiffDelivery(bmp, hook.deliveries.one())
def test_run_upload_failure_notifies(self):
# A run that gets some other upload failure from the store sends
# mail.
requester = self.factory.makePerson(name="requester")
requester_team = self.factory.makeTeam(owner=requester,
name="requester-team",
members=[requester])
snapbuild = self.makeSnapBuild(requester=requester_team,
name="test-snap",
owner=requester_team)
self.assertContentEqual([], snapbuild.store_upload_jobs)
job = SnapStoreUploadJob.create(snapbuild)
client = FakeSnapStoreClient()
client.upload.failure = UploadFailedResponse(
"Failed to upload", detail="The proxy exploded.\n")
self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient))
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertEqual([((snapbuild, ), {})], client.upload.calls)
self.assertEqual([], client.checkStatus.calls)
self.assertEqual([], client.release.calls)
self.assertContentEqual([job], snapbuild.store_upload_jobs)
self.assertIsNone(job.store_url)
self.assertIsNone(job.store_revision)
self.assertEqual("Failed to upload", job.error_message)
[notification] = pop_notifications()
self.assertEqual(config.canonical.noreply_from_address,
notification["From"])
self.assertEqual("Requester <%s>" % requester.preferredemail.email,
notification["To"])
subject = notification["Subject"].replace("\n ", " ")
self.assertEqual("Store upload failed for test-snap", subject)
self.assertEqual("Requester @requester-team",
notification["X-Launchpad-Message-Rationale"])
self.assertEqual(requester_team.name,
notification["X-Launchpad-Message-For"])
self.assertEqual("snap-build-upload-failed",
notification["X-Launchpad-Notification-Type"])
body, footer = notification.get_payload(decode=True).split("\n-- \n")
self.assertIn("Failed to upload", body)
build_url = (
"http://launchpad.dev/~requester-team/+snap/test-snap/+build/%d" %
snapbuild.id)
self.assertIn(build_url, body)
self.assertEqual(
"%s\nYour team Requester Team is the requester of the build.\n" %
build_url, footer)
self.assertWebhookDeliveries(snapbuild,
["Pending", "Failed to upload"])
self.assertIn(("error_detail", "The proxy exploded.\n"),
job.getOopsVars())
def test_date_first_sent(self):
job, reqs = self.makeAndRunJob(response_status=404)
self.assertEqual(job.date_first_sent, job.date_sent)
orig_first_sent = job.date_first_sent
self.assertEqual(JobStatus.WAITING, job.status)
self.assertEqual(1, job.attempt_count)
job.lease_expires = None
job.scheduled_start = None
with dbuser("webhookrunner"):
JobRunner([job]).runAll()
self.assertEqual(JobStatus.WAITING, job.status)
self.assertEqual(2, job.attempt_count)
self.assertNotEqual(job.date_first_sent, job.date_sent)
self.assertEqual(orig_first_sent, job.date_first_sent)
def test_run_release_manual_review_notifies(self):
# A run configured to automatically release the package to certain
# channels but that encounters the manual review state on upload
# sends mail.
requester = self.factory.makePerson(name="requester")
requester_team = self.factory.makeTeam(owner=requester,
name="requester-team",
members=[requester])
snapbuild = self.makeSnapBuild(requester=requester_team,
name="test-snap",
owner=requester_team,
store_channels=["stable", "edge"])
self.assertContentEqual([], snapbuild.store_upload_jobs)
job = SnapStoreUploadJob.create(snapbuild)
client = FakeSnapStoreClient()
client.upload.result = self.status_url
client.checkStatus.result = (self.store_url, None)
self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient))
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertEqual([((snapbuild, ), {})], client.upload.calls)
self.assertEqual([((self.status_url, ), {})], client.checkStatus.calls)
self.assertEqual([], client.release.calls)
self.assertContentEqual([job], snapbuild.store_upload_jobs)
self.assertEqual(self.store_url, job.store_url)
self.assertIsNone(job.store_revision)
self.assertEqual(
"Package held for manual review on the store; "
"cannot release it automatically.", job.error_message)
[notification] = pop_notifications()
self.assertEqual(config.canonical.noreply_from_address,
notification["From"])
self.assertEqual("Requester <%s>" % requester.preferredemail.email,
notification["To"])
subject = notification["Subject"].replace("\n ", " ")
self.assertEqual("test-snap held for manual review", subject)
self.assertEqual("Requester @requester-team",
notification["X-Launchpad-Message-Rationale"])
self.assertEqual(requester_team.name,
notification["X-Launchpad-Message-For"])
self.assertEqual("snap-build-release-manual-review",
notification["X-Launchpad-Notification-Type"])
body, footer = notification.get_payload(decode=True).split("\n-- \n")
self.assertIn(self.store_url, body)
self.assertEqual(
"http://launchpad.dev/~requester-team/+snap/test-snap/+build/%d\n"
"Your team Requester Team is the requester of the build.\n" %
snapbuild.id, footer)
self.assertWebhookDeliveries(
snapbuild, ["Pending", "Failed to release to channels"])
