def test_retry_delay(self):
# The job is retried every minute, unless it just made one of its
# first four attempts to poll the status endpoint, in which case the
# delays are 15/15/30/30 seconds.
self.useFixture(FakeLogger())
snapbuild = self.makeSnapBuild()
job = SnapStoreUploadJob.create(snapbuild)
client = FakeSnapStoreClient()
client.upload.failure = UploadFailedResponse("Proxy error",
can_retry=True)
self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient))
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertNotIn("status_url", job.metadata)
self.assertEqual(timedelta(seconds=60), job.retry_delay)
job.scheduled_start = None
client.upload.failure = None
client.upload.result = self.status_url
client.checkStatus.failure = UploadNotScannedYetResponse()
for expected_delay in (15, 15, 30, 30, 60):
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertIn("status_url", job.snapbuild.store_upload_metadata)
self.assertIsNone(job.store_url)
self.assertEqual(timedelta(seconds=expected_delay),
job.retry_delay)
job.scheduled_start = None
client.checkStatus.failure = None
client.checkStatus.result = (self.store_url, 1)
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertEqual(self.store_url, job.store_url)
self.assertIsNone(job.error_message)
self.assertEqual([], pop_notifications())
self.assertEqual(JobStatus.COMPLETED, job.job.status)
def test_runAll_mails_user_errors(self):
"""User errors should be mailed out without oopsing.
User errors are identified by the RunnableJob.user_error_types
attribute. They do not cause an oops to be recorded, and their
error messages are mailed to interested parties verbatim.
"""
job_1, job_2 = self.makeTwoJobs()
class ExampleError(Exception):
pass
def raiseError():
raise ExampleError('Fake exception. Foobar, I say!')
job_1.run = raiseError
job_1.user_error_types = (ExampleError, )
job_1.error_recipients = ['*****@*****.**']
runner = JobRunner([job_1, job_2])
runner.runAll()
self.assertEqual([], self.oopses)
notifications = pop_notifications()
self.assertEqual(1, len(notifications))
body = notifications[0].get_payload(decode=True)
self.assertEqual(
'Launchpad encountered an error during the following operation:'
' appending a string to a list. Fake exception. Foobar, I say!',
body)
self.assertEqual('Launchpad error while appending a string to a list',
notifications[0]['subject'])
def test_runAll_mails_user_errors(self):
"""User errors should be mailed out without oopsing.
User errors are identified by the RunnableJob.user_error_types
attribute. They do not cause an oops to be recorded, and their
error messages are mailed to interested parties verbatim.
"""
job_1, job_2 = self.makeTwoJobs()
class ExampleError(Exception):
pass
def raiseError():
raise ExampleError('Fake exception. Foobar, I say!')
job_1.run = raiseError
job_1.user_error_types = (ExampleError,)
job_1.error_recipients = ['*****@*****.**']
runner = JobRunner([job_1, job_2])
runner.runAll()
self.assertEqual([], self.oopses)
notifications = pop_notifications()
self.assertEqual(1, len(notifications))
body = notifications[0].get_payload(decode=True)
self.assertEqual(
'Launchpad encountered an error during the following operation:'
' appending a string to a list. Fake exception. Foobar, I say!',
body)
self.assertEqual(
'Launchpad error while appending a string to a list',
notifications[0]['subject'])
def test_runJob(self):
"""Ensure status is set to completed when a job runs to completion."""
job_1, job_2 = self.makeTwoJobs()
runner = JobRunner(job_1)
runner.runJob(job_1, None)
self.assertEqual(JobStatus.COMPLETED, job_1.job.status)
self.assertEqual([job_1], runner.completed_jobs)
def test_runJob(self):
"""Ensure status is set to completed when a job runs to completion."""
job_1, job_2 = self.makeTwoJobs()
runner = JobRunner(job_1)
runner.runJob(job_1, None)
self.assertEqual(JobStatus.COMPLETED, job_1.job.status)
self.assertEqual([job_1], runner.completed_jobs)
def test_run_all(self):
"""The job can be run under the JobRunner successfully."""
job = make_runnable_incremental_diff_job(self)
with dbuser("merge-proposal-jobs"):
runner = JobRunner([job])
runner.runAll()
self.assertEqual([job], runner.completed_jobs)
def test_runJob_raising_retry_error(self):
"""If a job raises a retry_error, it should be re-queued."""
job = RaisingRetryJob('completion')
runner = JobRunner([job])
with self.expectedLog('Scheduling retry due to RetryError'):
runner.runJob(job, None)
self.assertEqual(JobStatus.WAITING, job.status)
self.assertNotIn(job, runner.completed_jobs)
self.assertIn(job, runner.incomplete_jobs)
def test_runJob_raising_retry_error(self):
"""If a job raises a retry_error, it should be re-queued."""
job = RaisingRetryJob('completion')
runner = JobRunner([job])
with self.expectedLog('Scheduling retry due to RetryError'):
runner.runJob(job, None)
self.assertEqual(JobStatus.WAITING, job.status)
self.assertNotIn(job, runner.completed_jobs)
self.assertIn(job, runner.incomplete_jobs)
def test_runJobHandleErrors_oops_generated_user_notify_fails(self):
"""A second oops is logged if the notification of the oops fails.
In this test case the error is a user expected error, so the
notifyUserError is called, and in this case the notify raises too.
"""
job = RaisingJobRaisingNotifyUserError('boom')
runner = JobRunner([job])
runner.runJobHandleError(job)
self.assertEqual(1, len(self.oopses))
def test_runJobHandleErrors_oops_timeline_detail_filter(self):
"""A job can choose to filter oops timeline details."""
job = RaisingJobTimelineMessage('boom')
job.timeline_detail_filter = lambda _, detail: '<redacted>'
flush_database_updates()
runner = JobRunner([job])
runner.runJobHandleError(job)
self.assertEqual(1, len(self.oopses))
actions = [action[2:4] for action in self.oopses[0]['timeline']]
self.assertIn(('job', '<redacted>'), actions)
def test_runJob_with_SuspendJobException(self):
# A job that raises SuspendJobError should end up suspended.
job = NullJob('suspended')
job.run = FakeMethod(failure=SuspendJobException())
runner = JobRunner([job])
runner.runJob(job, None)
self.assertEqual(JobStatus.SUSPENDED, job.status)
self.assertNotIn(job, runner.completed_jobs)
self.assertIn(job, runner.incomplete_jobs)
def test_runJobHandleErrors_oops_generated_user_notify_fails(self):
"""A second oops is logged if the notification of the oops fails.
In this test case the error is a user expected error, so the
notifyUserError is called, and in this case the notify raises too.
"""
job = RaisingJobRaisingNotifyUserError('boom')
runner = JobRunner([job])
runner.runJobHandleError(job)
self.assertEqual(1, len(self.oopses))
def test_runJob_with_SuspendJobException(self):
# A job that raises SuspendJobError should end up suspended.
job = NullJob('suspended')
job.run = FakeMethod(failure=SuspendJobException())
runner = JobRunner([job])
runner.runJob(job, None)
self.assertEqual(JobStatus.SUSPENDED, job.status)
self.assertNotIn(job, runner.completed_jobs)
self.assertIn(job, runner.incomplete_jobs)
def test_runAll_skips_lease_failures(self):
"""Ensure runAll skips jobs whose leases can't be acquired."""
job_1, job_2 = self.makeTwoJobs()
job_2.job.acquireLease()
runner = JobRunner([job_1, job_2])
runner.runAll()
self.assertEqual(JobStatus.COMPLETED, job_1.job.status)
self.assertEqual(JobStatus.WAITING, job_2.job.status)
self.assertEqual([job_1], runner.completed_jobs)
self.assertEqual([job_2], runner.incomplete_jobs)
self.assertEqual([], self.oopses)
def test_runJob_exceeding_max_retries(self):
"""If a job exceeds maximum retries, it should raise normally."""
job = RaisingRetryJob('completion')
JobRunner([job]).runJob(job, None)
self.assertEqual(JobStatus.WAITING, job.status)
runner = JobRunner([job])
with ExpectedException(RetryError, ''):
runner.runJob(job, None)
self.assertEqual(JobStatus.FAILED, job.status)
self.assertNotIn(job, runner.completed_jobs)
self.assertIn(job, runner.incomplete_jobs)
def test_runJob_exceeding_max_retries(self):
"""If a job exceeds maximum retries, it should raise normally."""
job = RaisingRetryJob('completion')
JobRunner([job]).runJob(job, None)
self.assertEqual(JobStatus.WAITING, job.status)
runner = JobRunner([job])
with ExpectedException(RetryError, ''):
runner.runJob(job, None)
self.assertEqual(JobStatus.FAILED, job.status)
self.assertNotIn(job, runner.completed_jobs)
self.assertIn(job, runner.incomplete_jobs)
def test_runAll_skips_lease_failures(self):
"""Ensure runAll skips jobs whose leases can't be acquired."""
job_1, job_2 = self.makeTwoJobs()
job_2.job.acquireLease()
runner = JobRunner([job_1, job_2])
runner.runAll()
self.assertEqual(JobStatus.COMPLETED, job_1.job.status)
self.assertEqual(JobStatus.WAITING, job_2.job.status)
self.assertEqual([job_1], runner.completed_jobs)
self.assertEqual([job_2], runner.incomplete_jobs)
self.assertEqual([], self.oopses)
def test_empty_branch(self):
self.makeBzrSync(self.db_branch).syncBranchAndClose()
JobRunner.fromReady(getUtility(IRevisionMailJobSource)).runAll()
self.assertEqual(len(stub.test_emails), 1)
[initial_email] = stub.test_emails
expected = 'First scan of the branch detected 0 revisions'
message = email.message_from_string(initial_email[2])
email_body = message.get_payload()
self.assertTextIn(expected, email_body)
self.assertEmailHeadersEqual(
'[Branch %s] 0 revisions' % self.db_branch.unique_name,
message['Subject'])
def test_runAll(self):
"""Ensure runAll works in the normal case."""
job_1, job_2 = self.makeTwoJobs()
runner = JobRunner([job_1, job_2])
runner.runAll()
self.assertEqual(JobStatus.COMPLETED, job_1.job.status)
self.assertEqual(JobStatus.COMPLETED, job_2.job.status)
msg1 = NullJob.JOB_COMPLETIONS.pop()
msg2 = NullJob.JOB_COMPLETIONS.pop()
self.assertEqual(msg1, "job 2")
self.assertEqual(msg2, "job 1")
self.assertEqual([job_1, job_2], runner.completed_jobs)
def test_runAll(self):
"""Ensure runAll works in the normal case."""
job_1, job_2 = self.makeTwoJobs()
runner = JobRunner([job_1, job_2])
runner.runAll()
self.assertEqual(JobStatus.COMPLETED, job_1.job.status)
self.assertEqual(JobStatus.COMPLETED, job_2.job.status)
msg1 = NullJob.JOB_COMPLETIONS.pop()
msg2 = NullJob.JOB_COMPLETIONS.pop()
self.assertEqual(msg1, "job 2")
self.assertEqual(msg2, "job 1")
self.assertEqual([job_1, job_2], runner.completed_jobs)
def test_runJobHandleErrors_oops_timeline(self):
"""The oops timeline only covers the job itself."""
timeline = get_request_timeline(get_current_browser_request())
timeline.start('test', 'sentinel').finish()
job = RaisingJobTimelineMessage('boom')
flush_database_updates()
runner = JobRunner([job])
runner.runJobHandleError(job)
self.assertEqual(1, len(self.oopses))
actions = [action[2:4] for action in self.oopses[0]['timeline']]
self.assertIn(('job', 'boom'), actions)
self.assertNotIn(('test', 'sentinel'), actions)
def test_empty_branch(self):
self.makeBzrSync(self.db_branch).syncBranchAndClose()
JobRunner.fromReady(getUtility(IRevisionMailJobSource)).runAll()
self.assertEqual(len(stub.test_emails), 1)
[initial_email] = stub.test_emails
expected = 'First scan of the branch detected 0 revisions'
message = email.message_from_string(initial_email[2])
email_body = message.get_payload()
self.assertIn(expected, email_body)
self.assertEmailHeadersEqual(
'[Branch %s] 0 revisions' % self.db_branch.unique_name,
message['Subject'])
def runJob(self, job):
with dbuser("webhookrunner"):
runner = JobRunner([job])
runner.runAll()
job.lease_expires = None
if len(runner.completed_jobs) == 1 and not runner.incomplete_jobs:
return True
if len(runner.incomplete_jobs) == 1 and not runner.completed_jobs:
return False
if not runner.incomplete_jobs and not runner.completed_jobs:
return None
raise Exception("Unexpected jobs.")
def test_runAll_requires_IRunnable(self):
"""Supplied classes must implement IRunnableJob.
If they don't, we get a TypeError. If they do, then we get an
AttributeError, because we don't actually implement the interface.
"""
runner = JobRunner([object()])
self.assertRaises(TypeError, runner.runAll)
class Runnable:
implements(IRunnableJob)
runner = JobRunner([Runnable()])
self.assertRaises(AttributeError, runner.runAll)
def test_import_revision(self):
self.commitRevision()
self.makeBzrSync(self.db_branch).syncBranchAndClose()
JobRunner.fromReady(getUtility(IRevisionMailJobSource)).runAll()
self.assertEqual(len(stub.test_emails), 1)
[initial_email] = stub.test_emails
expected = ('First scan of the branch detected 1 revision'
' in the revision history of the=\n branch.')
message = email.message_from_string(initial_email[2])
email_body = message.get_payload()
self.assertTextIn(expected, email_body)
self.assertEmailHeadersEqual(
'[Branch %s] 1 revision' % self.db_branch.unique_name,
message['Subject'])
def test_oops_messages_used_when_handling(self):
"""Oops messages should appear even when exceptions are handled."""
job_1, job_2 = self.makeTwoJobs()
def handleError():
reporter = errorlog.globalErrorUtility
try:
raise ValueError('Fake exception. Foobar, I say!')
except ValueError:
reporter.raising(sys.exc_info())
job_1.run = handleError
runner = JobRunner([job_1, job_2])
runner.runAll()
oops = self.oopses[-1]
self.assertEqual(["{'foo': 'bar'}"], oops['req_vars'].values())
def test_runAll_aborts_transaction_on_error(self):
"""runAll should abort the transaction on oops."""
class DBAlterJob(NullJob):
def __init__(self):
super(DBAlterJob, self).__init__('')
def run(self):
self.job.log = 'hello'
raise ValueError
job = DBAlterJob()
runner = JobRunner([job])
runner.runAll()
# If the transaction was committed, job.log == 'hello'. If it was
# aborted, it is None.
self.assertIs(None, job.job.log)
def test_oops_messages_used_when_handling(self):
"""Oops messages should appear even when exceptions are handled."""
job_1, job_2 = self.makeTwoJobs()
def handleError():
reporter = errorlog.globalErrorUtility
try:
raise ValueError('Fake exception. Foobar, I say!')
except ValueError:
reporter.raising(sys.exc_info())
job_1.run = handleError
runner = JobRunner([job_1, job_2])
runner.runAll()
oops = self.oopses[-1]
self.assertEqual(["{'foo': 'bar'}"], oops['req_vars'].values())
def test_runJob_raising_retry_error(self):
"""If a job raises a retry_error, it should be re-queued."""
job = RaisingRetryJob('completion')
runner = JobRunner([job])
self.assertIs(None, job.scheduled_start)
with self.expectedLog('Scheduling retry due to RetryError'):
runner.runJob(job, None)
self.assertEqual(JobStatus.WAITING, job.status)
expected_delay = datetime.now(UTC) + timedelta(minutes=10)
self.assertThat(
job.scheduled_start,
MatchesAll(GreaterThan(expected_delay - timedelta(minutes=1)),
LessThan(expected_delay + timedelta(minutes=1))))
self.assertIsNone(job.lease_expires)
self.assertNotIn(job, runner.completed_jobs)
self.assertIn(job, runner.incomplete_jobs)
def test_triggers_webhooks(self):
# Jobs trigger any relevant webhooks when they're enabled.
self.useFixture(FeatureFixture({'code.git.webhooks.enabled': 'on'}))
repository = self.factory.makeGitRepository()
self.factory.makeGitRefs(
repository, paths=['refs/heads/master', 'refs/tags/1.0'])
hook = self.factory.makeWebhook(
target=repository, event_types=['git:push:0.1'])
job = GitRefScanJob.create(repository)
paths = ('refs/heads/master', 'refs/tags/2.0')
self.useFixture(GitHostingFixture(refs=self.makeFakeRefs(paths)))
with dbuser('branchscanner'):
JobRunner([job]).runAll()
delivery = hook.deliveries.one()
sha1 = lambda s: hashlib.sha1(s).hexdigest()
self.assertThat(
delivery,
MatchesStructure(
event_type=Equals('git:push:0.1'),
payload=MatchesDict({
'git_repository': Equals('/' + repository.unique_name),
'git_repository_path': Equals(repository.unique_name),
'ref_changes': Equals({
'refs/tags/1.0': {
'old': {'commit_sha1': sha1('refs/tags/1.0')},
'new': None},
'refs/tags/2.0': {
'old': None,
'new': {'commit_sha1': sha1('refs/tags/2.0')}},
})})))
with dbuser(config.IWebhookDeliveryJobSource.dbuser):
self.assertEqual(
"<WebhookDeliveryJob for webhook %d on %r>" % (
hook.id, hook.target),
repr(delivery))
def run_mail_jobs():
"""Process job queues that send out emails.
If a new job type is added that sends emails, this function can be
extended to run those jobs, so that testing emails doesn't require a
bunch of different function calls to process different queues.
"""
# Circular import.
from lp.testing.pages import permissive_security_policy
# Commit the transaction to make sure that the JobRunner can find
# the queued jobs.
transaction.commit()
for interface in (
IExpiringMembershipNotificationJobSource,
IMembershipNotificationJobSource,
ISelfRenewalNotificationJobSource,
ITeamInvitationNotificationJobSource,
ITeamJoinNotificationJobSource,
):
job_source = getUtility(interface)
logger = DevNullLogger()
dbuser_name = getattr(config, interface.__name__).dbuser
with permissive_security_policy(dbuser_name):
runner = JobRunner.fromReady(job_source, logger)
runner.runAll()
def test_works_in_job(self):
# `BranchHostingClient` is usable from a running job.
blob = b"".join(chr(i) for i in range(256))
@implementer(IRunnableJob)
class GetBlobJob(BaseRunnableJob):
def __init__(self, testcase):
super(GetBlobJob, self).__init__()
self.job = Job()
self.testcase = testcase
def run(self):
with self.testcase.mockRequests("GET",
body=blob,
set_default_timeout=False):
self.blob = self.testcase.client.getBlob(123, "file-id")
# We must make this assertion inside the job, since the job
# runner creates a separate timeline.
self.testcase.assertRequest(
"+branch-id/123/download/head%3A/file-id")
job = GetBlobJob(self)
JobRunner([job]).runAll()
self.assertEqual(JobStatus.COMPLETED, job.job.status)
self.assertEqual(blob, job.blob)
def test_run_branches_empty(self):
"""If the branches are empty, we tell the user."""
# If the job has been waiting for a significant period of time (15
# minutes for now), we run the job anyway. The checkReady method
# then raises and this is caught as a user error by the job system,
# and as such sends an email to the error recipients, which for this
# job is the merge proposal registrant.
eric = self.factory.makePerson(name='eric', email='*****@*****.**')
bmp = self.factory.makeBranchMergeProposal(registrant=eric)
job = UpdatePreviewDiffJob.create(bmp)
pop_notifications()
JobRunner([job]).runAll()
[email] = pop_notifications()
self.assertEqual('Eric <*****@*****.**>', email['to'])
self.assertEqual(
'Launchpad error while generating the diff for a merge proposal',
email['subject'])
branch = bmp.source_branch
self.assertEqual(
'Launchpad encountered an error during the following operation: '
'generating the diff for a merge proposal. '
'The source branch of http://code.launchpad.dev/~%s/%s/%s/'
'+merge/%d has no revisions.' %
(branch.owner.name, branch.target.name, branch.name, bmp.id),
email.get_payload(decode=True))
def test_import_uncommit(self):
self.commitRevision()
self.makeBzrSync(self.db_branch).syncBranchAndClose()
JobRunner.fromReady(getUtility(IRevisionMailJobSource)).runAll()
stub.test_emails = []
self.uncommitRevision()
self.makeBzrSync(self.db_branch).syncBranchAndClose()
JobRunner.fromReady(getUtility(IRevisionMailJobSource)).runAll()
self.assertEqual(len(stub.test_emails), 1)
[uncommit_email] = stub.test_emails
expected = '1 revision was removed from the branch.'
message = email.message_from_string(uncommit_email[2])
email_body = message.get_payload()
self.assertTextIn(expected, email_body)
self.assertEmailHeadersEqual(
'[Branch %s] 1 revision removed' % self.db_branch.unique_name,
message['Subject'])
def test_import_uncommit(self):
self.commitRevision()
self.makeBzrSync(self.db_branch).syncBranchAndClose()
JobRunner.fromReady(getUtility(IRevisionMailJobSource)).runAll()
stub.test_emails = []
self.uncommitRevision()
self.makeBzrSync(self.db_branch).syncBranchAndClose()
JobRunner.fromReady(getUtility(IRevisionMailJobSource)).runAll()
self.assertEqual(len(stub.test_emails), 1)
[uncommit_email] = stub.test_emails
expected = '1 revision was removed from the branch.'
message = email.message_from_string(uncommit_email[2])
email_body = message.get_payload()
self.assertIn(expected, email_body)
self.assertEmailHeadersEqual(
'[Branch %s] 1 revision removed' % self.db_branch.unique_name,
message['Subject'])
def test_run(self):
bmp = self.createExampleBzrMerge()[0]
job = UpdatePreviewDiffJob.create(bmp)
self.factory.makeRevisionsForBranch(bmp.source_branch, count=1)
bmp.source_branch.next_mirror_time = None
with dbuser("merge-proposal-jobs"):
JobRunner([job]).runAll()
self.checkExampleBzrMerge(bmp.preview_diff.text)
def test_run_sends_email(self):
"""MergeProposalCreationJob.run sends an email."""
bmp = self.createProposalWithEmptyBranches()
job = MergeProposalNeedsReviewEmailJob.create(bmp)
self.assertEqual([], pop_notifications())
with dbuser("merge-proposal-jobs"):
JobRunner([job]).runAll()
self.assertEqual(2, len(pop_notifications()))
def test_runAll_aborts_transaction_on_error(self):
"""runAll should abort the transaction on oops."""
class DBAlterJob(NullJob):
def __init__(self):
super(DBAlterJob, self).__init__('')
def run(self):
self.job.log = 'hello'
raise ValueError
job = DBAlterJob()
runner = JobRunner([job])
runner.runAll()
# If the transaction was committed, job.log == 'hello'. If it was
# aborted, it is None.
self.assertIs(None, job.job.log)
def test_runJob_records_failure(self):
"""When a job fails, the failure needs to be recorded."""
job = RaisingJob('boom')
runner = JobRunner([job])
self.assertRaises(RaisingJobException, runner.runJob, job, None)
# Abort the transaction to confirm that the update of the job status
# has been committed.
transaction.abort()
self.assertEqual(JobStatus.FAILED, job.job.status)
def test_runAll_reports_oopses(self):
"""When an error is encountered, report an oops and continue."""
job_1, job_2 = self.makeTwoJobs()
def raiseError():
# Ensure that jobs which call transaction.abort work, too.
transaction.abort()
raise Exception('Fake exception. Foobar, I say!')
job_1.run = raiseError
runner = JobRunner([job_1, job_2])
runner.runAll()
self.assertEqual([], pop_notifications())
self.assertEqual([job_2], runner.completed_jobs)
self.assertEqual([job_1], runner.incomplete_jobs)
self.assertEqual(JobStatus.FAILED, job_1.job.status)
self.assertEqual(JobStatus.COMPLETED, job_2.job.status)
oops = self.oopses[-1]
self.assertIn('Fake exception. Foobar, I say!', oops['tb_text'])
self.assertEqual(["{'foo': 'bar'}"], oops['req_vars'].values())
def test_run(self):
# A proper test run closes bugs.
spr = self.factory.makeSourcePackageRelease(
distroseries=self.distroseries, changelog_entry="changelog")
bug = self.factory.makeBug()
bugtask = self.factory.makeBugTask(target=spr.sourcepackage, bug=bug)
self.assertEqual(BugTaskStatus.NEW, bugtask.status)
job = self.makeJob(spr=spr, bug_ids=[bug.id])
JobRunner([job]).runAll()
self.assertEqual(BugTaskStatus.FIXRELEASED, bugtask.status)
def test_run_scan_pending_retries(self):
# A run that finds that the store has not yet finished scanning the
# package schedules itself to be retried.
self.useFixture(FakeLogger())
snapbuild = self.makeSnapBuild()
self.assertContentEqual([], snapbuild.store_upload_jobs)
job = SnapStoreUploadJob.create(snapbuild)
client = FakeSnapStoreClient()
client.upload.result = self.status_url
client.checkStatus.failure = UploadNotScannedYetResponse()
self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient))
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertEqual([((snapbuild, ), {})], client.upload.calls)
self.assertEqual([((self.status_url, ), {})], client.checkStatus.calls)
self.assertEqual([], client.release.calls)
self.assertContentEqual([job], snapbuild.store_upload_jobs)
self.assertIsNone(job.store_url)
self.assertIsNone(job.store_revision)
self.assertIsNone(job.error_message)
self.assertEqual([], pop_notifications())
self.assertEqual(JobStatus.WAITING, job.job.status)
self.assertWebhookDeliveries(snapbuild, ["Pending"])
# Try again. The upload part of the job is not retried, and this
# time the scan completes.
job.scheduled_start = None
client.upload.calls = []
client.checkStatus.calls = []
client.checkStatus.failure = None
client.checkStatus.result = (self.store_url, 1)
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertEqual([], client.upload.calls)
self.assertEqual([((self.status_url, ), {})], client.checkStatus.calls)
self.assertEqual([], client.release.calls)
self.assertContentEqual([job], snapbuild.store_upload_jobs)
self.assertEqual(self.store_url, job.store_url)
self.assertEqual(1, job.store_revision)
self.assertIsNone(job.error_message)
self.assertEqual([], pop_notifications())
self.assertEqual(JobStatus.COMPLETED, job.job.status)
self.assertWebhookDeliveries(snapbuild, ["Pending", "Uploaded"])
def test_run_502_retries(self):
# A run that gets a 502 error from the store schedules itself to be
# retried.
self.useFixture(FakeLogger())
snapbuild = self.makeSnapBuild()
self.assertContentEqual([], snapbuild.store_upload_jobs)
job = SnapStoreUploadJob.create(snapbuild)
client = FakeSnapStoreClient()
client.upload.failure = UploadFailedResponse("Proxy error",
can_retry=True)
self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient))
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertEqual([((snapbuild, ), {})], client.upload.calls)
self.assertEqual([], client.checkStatus.calls)
self.assertEqual([], client.release.calls)
self.assertContentEqual([job], snapbuild.store_upload_jobs)
self.assertIsNone(job.store_url)
self.assertIsNone(job.store_revision)
self.assertIsNone(job.error_message)
self.assertEqual([], pop_notifications())
self.assertEqual(JobStatus.WAITING, job.job.status)
self.assertWebhookDeliveries(snapbuild, ["Pending"])
# Try again. The upload part of the job is retried, and this time
# it succeeds.
job.scheduled_start = None
client.upload.calls = []
client.upload.failure = None
client.upload.result = self.status_url
client.checkStatus.result = (self.store_url, 1)
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertEqual([((snapbuild, ), {})], client.upload.calls)
self.assertEqual([((self.status_url, ), {})], client.checkStatus.calls)
self.assertEqual([], client.release.calls)
self.assertContentEqual([job], snapbuild.store_upload_jobs)
self.assertEqual(self.store_url, job.store_url)
self.assertEqual(1, job.store_revision)
self.assertIsNone(job.error_message)
self.assertEqual([], pop_notifications())
self.assertEqual(JobStatus.COMPLETED, job.job.status)
self.assertWebhookDeliveries(snapbuild, ["Pending", "Uploaded"])
def test_runAll_mails_oopses(self):
"""Email interested parties about OOPses."""
job_1, job_2 = self.makeTwoJobs()
def raiseError():
# Ensure that jobs which call transaction.abort work, too.
transaction.abort()
raise Exception('Fake exception. Foobar, I say!')
job_1.run = raiseError
job_1.oops_recipients = ['*****@*****.**']
runner = JobRunner([job_1, job_2])
runner.runAll()
(notification,) = pop_notifications()
oops = self.oopses[-1]
self.assertIn(
'Launchpad encountered an internal error during the following'
' operation: appending a string to a list. It was logged with id'
' %s. Sorry for the inconvenience.' % oops['id'],
notification.get_payload(decode=True))
self.assertNotIn('Fake exception. Foobar, I say!',
notification.get_payload(decode=True))
self.assertEqual('Launchpad internal error', notification['subject'])
def test_import_recommit(self):
# When scanning the uncommit and new commit there should be an email
# generated saying that 1 (in this case) revision has been removed,
# and another email with the diff and log message.
self.commitRevision('first')
self.makeBzrSync(self.db_branch).syncBranchAndClose()
JobRunner.fromReady(getUtility(IRevisionMailJobSource)).runAll()
stub.test_emails = []
self.uncommitRevision()
self.writeToFile(filename="hello.txt",
contents="Hello World\n")
author = self.factory.getUniqueString()
self.commitRevision('second', committer=author)
self.makeBzrSync(self.db_branch).syncBranchAndClose()
JobRunner.fromReady(getUtility(IRevisionsAddedJobSource)).runAll()
JobRunner.fromReady(getUtility(IRevisionMailJobSource)).runAll()
self.assertEqual(len(stub.test_emails), 2)
[recommit_email, uncommit_email] = stub.test_emails
uncommit_email_body = uncommit_email[2]
expected = '1 revision was removed from the branch.'
self.assertTextIn(expected, uncommit_email_body)
subject = (
'Subject: [Branch %s] Test branch' % self.db_branch.unique_name)
self.assertTextIn(expected, uncommit_email_body)
recommit_email_msg = email.message_from_string(recommit_email[2])
recommit_email_body = recommit_email_msg.get_payload()[0].get_payload(
decode=True)
subject = '[Branch %s] Rev 1: second' % self.db_branch.unique_name
self.assertEmailHeadersEqual(subject, recommit_email_msg['Subject'])
body_bits = [
'revno: 1',
'committer: %s' % author,
'branch nick: %s' % self.bzr_branch.nick,
'message:\n second',
'added:\n hello.txt',
]
for bit in body_bits:
self.assertTextIn(bit, recommit_email_body)
def test_runJobHandleErrors_oops_generated_notify_fails(self):
"""A second oops is logged if the notification of the oops fails."""
job = RaisingJobRaisingNotifyOops('boom')
runner = JobRunner([job])
runner.runJobHandleError(job)
self.assertEqual(2, len(self.oopses))
def test_runJobHandleErrors_oops_generated(self):
"""The handle errors method records an oops for raised errors."""
job = RaisingJob('boom')
runner = JobRunner([job])
runner.runJobHandleError(job)
self.assertEqual(1, len(self.oopses))
def test_runJobHandleErrors_user_error_no_oops(self):
"""If the job raises a user error, there is no oops."""
job = RaisingJobUserError('boom')
runner = JobRunner([job])
runner.runJobHandleError(job)
self.assertEqual(0, len(self.oopses))
