def test_automatic_retries(self):
hook = self.factory.makeWebhook()
job = WebhookDeliveryJob.create(hook, 'test', payload={'foo': 'bar'})
client = MockWebhookClient(response_status=503)
self.useFixture(ZopeUtilityFixture(client, IWebhookClient))
# The first attempt fails but schedules a retry five minutes later.
self.assertEqual(False, self.runJob(job))
self.assertEqual(JobStatus.WAITING, job.status)
self.assertEqual(False, job.successful)
self.assertTrue(job.pending)
self.assertIsNot(None, job.date_sent)
last_date_sent = job.date_sent
# Pretend we're five minutes in the future and try again. The
# job will be retried again.
job.json_data['date_first_sent'] = (
job.date_first_sent - timedelta(minutes=5)).isoformat()
job.scheduled_start -= timedelta(minutes=5)
self.assertEqual(False, self.runJob(job))
self.assertEqual(JobStatus.WAITING, job.status)
self.assertEqual(False, job.successful)
self.assertTrue(job.pending)
self.assertThat(job.date_sent, GreaterThan(last_date_sent))
# If the job was first tried a day ago, the next attempt gives up.
job.json_data['date_first_sent'] = (
job.date_first_sent - timedelta(hours=24)).isoformat()
job.scheduled_start -= timedelta(hours=24)
self.assertEqual(False, self.runJob(job))
self.assertEqual(JobStatus.FAILED, job.status)
self.assertEqual(False, job.successful)
self.assertFalse(job.pending)
def test_snap__repr__(self):
# `WebhookDeliveryJob` objects for snaps have an informative __repr__.
snap = self.factory.makeSnap()
hook = self.factory.makeWebhook(target=snap)
job = WebhookDeliveryJob.create(hook, 'test', payload={'foo': 'bar'})
self.assertEqual(
"<WebhookDeliveryJob for webhook %d on %r>" % (hook.id, snap),
repr(job))
def test_iterReady_orders_by_job_id(self):
# Older jobs are run first.
hook = self.factory.makeWebhook()
jobs = [
WebhookJob(hook, WebhookJobType.DELIVERY, {}) for _ in range(3)]
IStore(WebhookJob).flush()
self.assertEqual(
[job.job_id for job in jobs],
[job.job_id for job in WebhookDeliveryJob.iterReady()])
def test_branch__repr__(self):
# `WebhookDeliveryJob` objects for Bazaar branches have an
# informative __repr__.
branch = self.factory.makeAnyBranch()
hook = self.factory.makeWebhook(target=branch)
job = WebhookDeliveryJob.create(hook, 'test', payload={'foo': 'bar'})
self.assertEqual(
"<WebhookDeliveryJob for webhook %d on %r>" % (hook.id, branch),
repr(job))
def test_create(self):
# `WebhookDeliveryJob` objects provide `IWebhookDeliveryJob`.
hook = self.factory.makeWebhook()
job = WebhookDeliveryJob.create(hook, 'test', payload={'foo': 'bar'})
self.assertProvides(job, IWebhookDeliveryJob)
self.assertThat(
job,
MatchesStructure.byEquality(
webhook=hook, event_type='test', payload={'foo': 'bar'}))
def test_gitrepository__repr__(self):
# `WebhookDeliveryJob` objects for Git repositories have an
# informative __repr__.
repository = self.factory.makeGitRepository()
hook = self.factory.makeWebhook(target=repository)
job = WebhookDeliveryJob.create(hook, 'test', payload={'foo': 'bar'})
self.assertEqual(
"<WebhookDeliveryJob for webhook %d on %r>" % (
hook.id, repository),
repr(job))
def makeAndRunJob(self, response_status=200, raises=None, mock=True,
secret=None, active=True):
hook = self.factory.makeWebhook(
delivery_url=u'http://example.com/ep', secret=secret,
active=active)
job = WebhookDeliveryJob.create(hook, 'test', payload={'foo': 'bar'})
client = MockWebhookClient(
response_status=response_status, raises=raises)
if mock:
self.useFixture(ZopeUtilityFixture(client, IWebhookClient))
with dbuser("webhookrunner"):
JobRunner([job]).runAll()
return job, client.requests
def test_WebhookDeliveryJob(self):
"""WebhookDeliveryJob runs under Celery."""
hook = self.factory.makeWebhook(delivery_url=u'http://example.com/ep')
self.useFixture(FeatureFixture(
{'jobs.celery.enabled_classes': 'WebhookDeliveryJob'}))
with block_on_job():
job = WebhookDeliveryJob.create(
hook, 'test', payload={'foo': 'bar'})
transaction.commit()
self.assertEqual(JobStatus.WAITING, job.status)
self.assertIn(
'Cannot connect to proxy',
job.json_data['result']['connection_error'])
def test_manual_retries(self):
hook = self.factory.makeWebhook()
job = WebhookDeliveryJob.create(hook, 'test', payload={'foo': 'bar'})
client = MockWebhookClient(response_status=503)
self.useFixture(ZopeUtilityFixture(client, IWebhookClient))
# Simulate a first attempt failure.
self.assertEqual(False, self.runJob(job))
self.assertEqual(JobStatus.WAITING, job.status)
self.assertIsNot(None, job.scheduled_start)
# A manual retry brings the scheduled start forward.
job.retry()
self.assertEqual(JobStatus.WAITING, job.status)
self.assertIs(None, job.scheduled_start)
# Force the next attempt to fail hard by pretending it was more
# than 24 hours later.
job.json_data['date_first_sent'] = (
job.date_first_sent - timedelta(hours=24)).isoformat()
self.assertEqual(False, self.runJob(job))
self.assertEqual(JobStatus.FAILED, job.status)
# A manual retry brings the job out of FAILED and schedules it
# to run as soon as possible. If it fails again, it fails hard;
# the initial attempt more than 24 hours ago is remembered.
job.retry()
self.assertEqual(JobStatus.WAITING, job.status)
self.assertIs(None, job.scheduled_start)
self.assertEqual(False, self.runJob(job))
self.assertEqual(JobStatus.FAILED, job.status)
# A completed job can be retried just like a failed one. The
# endpoint may have erroneously returned a 200 without recording
# the event.
client.response_status = 200
job.retry()
self.assertEqual(JobStatus.WAITING, job.status)
self.assertEqual(True, self.runJob(job))
self.assertEqual(JobStatus.COMPLETED, job.status)
job.retry()
self.assertEqual(JobStatus.WAITING, job.status)
self.assertEqual(True, self.runJob(job))
self.assertEqual(JobStatus.COMPLETED, job.status)
def test_run_from_cronscript(self):
hook = self.factory.makeWebhook(delivery_url=u'http://example.com/ep')
job = WebhookDeliveryJob.create(hook, 'test', payload={'foo': 'bar'})
self.assertEqual(JobStatus.WAITING, job.status)
transaction.commit()
retcode, stdout, stderr = run_script(
'cronscripts/process-job-source.py', ['IWebhookDeliveryJobSource'],
expect_returncode=0)
self.assertEqual('', stdout)
self.assertIn(
'WARNING Scheduling retry due to WebhookDeliveryRetry', stderr)
self.assertIn(
'INFO 1 WebhookDeliveryJob jobs did not complete.\n', stderr)
self.assertEqual(JobStatus.WAITING, job.status)
self.assertIn(
'Cannot connect to proxy',
job.json_data['result']['connection_error'])
def test_manual_retry_with_reset(self):
# retry(reset=True) unsets date_first_sent so the automatic
# retries can be resumed. This can be useful for recovering from
# systemic errors that erroneously failed many deliveries.
hook = self.factory.makeWebhook()
job = WebhookDeliveryJob.create(hook, 'test', payload={'foo': 'bar'})
client = MockWebhookClient(response_status=503)
self.useFixture(ZopeUtilityFixture(client, IWebhookClient))
# Simulate a first attempt failure.
self.assertEqual(False, self.runJob(job))
self.assertEqual(JobStatus.WAITING, job.status)
self.assertIsNot(None, job.date_first_sent)
# A manual retry brings the scheduled start forward.
job.retry()
self.assertEqual(JobStatus.WAITING, job.status)
self.assertIsNot(None, job.date_first_sent)
# When reset=True, date_first_sent is unset to restart the 24
# hour auto-retry window.
job.retry(reset=True)
self.assertEqual(JobStatus.WAITING, job.status)
self.assertIs(None, job.date_first_sent)