def test_automatic_retries(self): hook = self.factory.makeWebhook() job = WebhookDeliveryJob.create(hook, 'test', payload={'foo': 'bar'}) client = MockWebhookClient(response_status=503) self.useFixture(ZopeUtilityFixture(client, IWebhookClient)) # The first attempt fails but schedules a retry five minutes later. self.assertEqual(False, self.runJob(job)) self.assertEqual(JobStatus.WAITING, job.status) self.assertEqual(False, job.successful) self.assertTrue(job.pending) self.assertIsNot(None, job.date_sent) last_date_sent = job.date_sent # Pretend we're five minutes in the future and try again. The # job will be retried again. job.json_data['date_first_sent'] = ( job.date_first_sent - timedelta(minutes=5)).isoformat() job.scheduled_start -= timedelta(minutes=5) self.assertEqual(False, self.runJob(job)) self.assertEqual(JobStatus.WAITING, job.status) self.assertEqual(False, job.successful) self.assertTrue(job.pending) self.assertThat(job.date_sent, GreaterThan(last_date_sent)) # If the job was first tried a day ago, the next attempt gives up. job.json_data['date_first_sent'] = ( job.date_first_sent - timedelta(hours=24)).isoformat() job.scheduled_start -= timedelta(hours=24) self.assertEqual(False, self.runJob(job)) self.assertEqual(JobStatus.FAILED, job.status) self.assertEqual(False, job.successful) self.assertFalse(job.pending)
def test_snap__repr__(self): # `WebhookDeliveryJob` objects for snaps have an informative __repr__. snap = self.factory.makeSnap() hook = self.factory.makeWebhook(target=snap) job = WebhookDeliveryJob.create(hook, 'test', payload={'foo': 'bar'}) self.assertEqual( "<WebhookDeliveryJob for webhook %d on %r>" % (hook.id, snap), repr(job))
def test_iterReady_orders_by_job_id(self): # Older jobs are run first. hook = self.factory.makeWebhook() jobs = [ WebhookJob(hook, WebhookJobType.DELIVERY, {}) for _ in range(3)] IStore(WebhookJob).flush() self.assertEqual( [job.job_id for job in jobs], [job.job_id for job in WebhookDeliveryJob.iterReady()])
def test_branch__repr__(self): # `WebhookDeliveryJob` objects for Bazaar branches have an # informative __repr__. branch = self.factory.makeAnyBranch() hook = self.factory.makeWebhook(target=branch) job = WebhookDeliveryJob.create(hook, 'test', payload={'foo': 'bar'}) self.assertEqual( "<WebhookDeliveryJob for webhook %d on %r>" % (hook.id, branch), repr(job))
def test_create(self): # `WebhookDeliveryJob` objects provide `IWebhookDeliveryJob`. hook = self.factory.makeWebhook() job = WebhookDeliveryJob.create(hook, 'test', payload={'foo': 'bar'}) self.assertProvides(job, IWebhookDeliveryJob) self.assertThat( job, MatchesStructure.byEquality( webhook=hook, event_type='test', payload={'foo': 'bar'}))
def test_gitrepository__repr__(self): # `WebhookDeliveryJob` objects for Git repositories have an # informative __repr__. repository = self.factory.makeGitRepository() hook = self.factory.makeWebhook(target=repository) job = WebhookDeliveryJob.create(hook, 'test', payload={'foo': 'bar'}) self.assertEqual( "<WebhookDeliveryJob for webhook %d on %r>" % ( hook.id, repository), repr(job))
def makeAndRunJob(self, response_status=200, raises=None, mock=True, secret=None, active=True): hook = self.factory.makeWebhook( delivery_url=u'http://example.com/ep', secret=secret, active=active) job = WebhookDeliveryJob.create(hook, 'test', payload={'foo': 'bar'}) client = MockWebhookClient( response_status=response_status, raises=raises) if mock: self.useFixture(ZopeUtilityFixture(client, IWebhookClient)) with dbuser("webhookrunner"): JobRunner([job]).runAll() return job, client.requests
def test_WebhookDeliveryJob(self): """WebhookDeliveryJob runs under Celery.""" hook = self.factory.makeWebhook(delivery_url=u'http://example.com/ep') self.useFixture(FeatureFixture( {'jobs.celery.enabled_classes': 'WebhookDeliveryJob'})) with block_on_job(): job = WebhookDeliveryJob.create( hook, 'test', payload={'foo': 'bar'}) transaction.commit() self.assertEqual(JobStatus.WAITING, job.status) self.assertIn( 'Cannot connect to proxy', job.json_data['result']['connection_error'])
def test_manual_retries(self): hook = self.factory.makeWebhook() job = WebhookDeliveryJob.create(hook, 'test', payload={'foo': 'bar'}) client = MockWebhookClient(response_status=503) self.useFixture(ZopeUtilityFixture(client, IWebhookClient)) # Simulate a first attempt failure. self.assertEqual(False, self.runJob(job)) self.assertEqual(JobStatus.WAITING, job.status) self.assertIsNot(None, job.scheduled_start) # A manual retry brings the scheduled start forward. job.retry() self.assertEqual(JobStatus.WAITING, job.status) self.assertIs(None, job.scheduled_start) # Force the next attempt to fail hard by pretending it was more # than 24 hours later. job.json_data['date_first_sent'] = ( job.date_first_sent - timedelta(hours=24)).isoformat() self.assertEqual(False, self.runJob(job)) self.assertEqual(JobStatus.FAILED, job.status) # A manual retry brings the job out of FAILED and schedules it # to run as soon as possible. If it fails again, it fails hard; # the initial attempt more than 24 hours ago is remembered. job.retry() self.assertEqual(JobStatus.WAITING, job.status) self.assertIs(None, job.scheduled_start) self.assertEqual(False, self.runJob(job)) self.assertEqual(JobStatus.FAILED, job.status) # A completed job can be retried just like a failed one. The # endpoint may have erroneously returned a 200 without recording # the event. client.response_status = 200 job.retry() self.assertEqual(JobStatus.WAITING, job.status) self.assertEqual(True, self.runJob(job)) self.assertEqual(JobStatus.COMPLETED, job.status) job.retry() self.assertEqual(JobStatus.WAITING, job.status) self.assertEqual(True, self.runJob(job)) self.assertEqual(JobStatus.COMPLETED, job.status)
def test_run_from_cronscript(self): hook = self.factory.makeWebhook(delivery_url=u'http://example.com/ep') job = WebhookDeliveryJob.create(hook, 'test', payload={'foo': 'bar'}) self.assertEqual(JobStatus.WAITING, job.status) transaction.commit() retcode, stdout, stderr = run_script( 'cronscripts/process-job-source.py', ['IWebhookDeliveryJobSource'], expect_returncode=0) self.assertEqual('', stdout) self.assertIn( 'WARNING Scheduling retry due to WebhookDeliveryRetry', stderr) self.assertIn( 'INFO 1 WebhookDeliveryJob jobs did not complete.\n', stderr) self.assertEqual(JobStatus.WAITING, job.status) self.assertIn( 'Cannot connect to proxy', job.json_data['result']['connection_error'])
def test_manual_retry_with_reset(self): # retry(reset=True) unsets date_first_sent so the automatic # retries can be resumed. This can be useful for recovering from # systemic errors that erroneously failed many deliveries. hook = self.factory.makeWebhook() job = WebhookDeliveryJob.create(hook, 'test', payload={'foo': 'bar'}) client = MockWebhookClient(response_status=503) self.useFixture(ZopeUtilityFixture(client, IWebhookClient)) # Simulate a first attempt failure. self.assertEqual(False, self.runJob(job)) self.assertEqual(JobStatus.WAITING, job.status) self.assertIsNot(None, job.date_first_sent) # A manual retry brings the scheduled start forward. job.retry() self.assertEqual(JobStatus.WAITING, job.status) self.assertIsNot(None, job.date_first_sent) # When reset=True, date_first_sent is unset to restart the 24 # hour auto-retry window. job.retry(reset=True) self.assertEqual(JobStatus.WAITING, job.status) self.assertIs(None, job.date_first_sent)