def test_retry_delay(self):
# The job is retried every minute, unless it just made one of its
# first four attempts to poll the status endpoint, in which case the
# delays are 15/15/30/30 seconds.
self.useFixture(FakeLogger())
snapbuild = self.makeSnapBuild()
job = SnapStoreUploadJob.create(snapbuild)
client = FakeSnapStoreClient()
client.upload.failure = UploadFailedResponse("Proxy error",
can_retry=True)
self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient))
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertNotIn("status_url", job.metadata)
self.assertEqual(timedelta(seconds=60), job.retry_delay)
job.scheduled_start = None
client.upload.failure = None
client.upload.result = self.status_url
client.checkStatus.failure = UploadNotScannedYetResponse()
for expected_delay in (15, 15, 30, 30, 60):
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertIn("status_url", job.snapbuild.store_upload_metadata)
self.assertIsNone(job.store_url)
self.assertEqual(timedelta(seconds=expected_delay),
job.retry_delay)
job.scheduled_start = None
client.checkStatus.failure = None
client.checkStatus.result = (self.store_url, 1)
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertEqual(self.store_url, job.store_url)
self.assertIsNone(job.error_message)
self.assertEqual([], pop_notifications())
self.assertEqual(JobStatus.COMPLETED, job.job.status)
def test___repr__(self):
# `SnapStoreUploadJob` objects have an informative __repr__.
snapbuild = self.factory.makeSnapBuild()
job = SnapStoreUploadJob.create(snapbuild)
self.assertEqual(
"<SnapStoreUploadJob for ~%s/+snap/%s/+build/%d>" %
(snapbuild.snap.owner.name, snapbuild.snap.name, snapbuild.id),
repr(job))
def test_with_snapbuild_metadata_as_none(self):
db_build = self.factory.makeSnapBuild()
unsecure_db_build = removeSecurityProxy(db_build)
unsecure_db_build.store_upload_metadata = None
store = IStore(SnapBuild)
store.flush()
loaded_build = store.find(SnapBuild, id=unsecure_db_build.id).one()
job = SnapStoreUploadJob.create(loaded_build)
self.assertEqual({}, job.store_metadata)
def test_run_upload_failure_notifies(self):
# A run that gets some other upload failure from the store sends
# mail.
requester = self.factory.makePerson(name="requester")
requester_team = self.factory.makeTeam(owner=requester,
name="requester-team",
members=[requester])
snapbuild = self.makeSnapBuild(requester=requester_team,
name="test-snap",
owner=requester_team)
self.assertContentEqual([], snapbuild.store_upload_jobs)
job = SnapStoreUploadJob.create(snapbuild)
client = FakeSnapStoreClient()
client.upload.failure = UploadFailedResponse(
"Failed to upload", detail="The proxy exploded.\n")
self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient))
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertEqual([((snapbuild, ), {})], client.upload.calls)
self.assertEqual([], client.checkStatus.calls)
self.assertEqual([], client.release.calls)
self.assertContentEqual([job], snapbuild.store_upload_jobs)
self.assertIsNone(job.store_url)
self.assertIsNone(job.store_revision)
self.assertEqual("Failed to upload", job.error_message)
[notification] = pop_notifications()
self.assertEqual(config.canonical.noreply_from_address,
notification["From"])
self.assertEqual("Requester <%s>" % requester.preferredemail.email,
notification["To"])
subject = notification["Subject"].replace("\n ", " ")
self.assertEqual("Store upload failed for test-snap", subject)
self.assertEqual("Requester @requester-team",
notification["X-Launchpad-Message-Rationale"])
self.assertEqual(requester_team.name,
notification["X-Launchpad-Message-For"])
self.assertEqual("snap-build-upload-failed",
notification["X-Launchpad-Notification-Type"])
body, footer = notification.get_payload(decode=True).split("\n-- \n")
self.assertIn("Failed to upload", body)
build_url = (
"http://launchpad.dev/~requester-team/+snap/test-snap/+build/%d" %
snapbuild.id)
self.assertIn(build_url, body)
self.assertEqual(
"%s\nYour team Requester Team is the requester of the build.\n" %
build_url, footer)
self.assertWebhookDeliveries(snapbuild,
["Pending", "Failed to upload"])
self.assertIn(("error_detail", "The proxy exploded.\n"),
job.getOopsVars())
def test_run_release_manual_review_notifies(self):
# A run configured to automatically release the package to certain
# channels but that encounters the manual review state on upload
# sends mail.
requester = self.factory.makePerson(name="requester")
requester_team = self.factory.makeTeam(owner=requester,
name="requester-team",
members=[requester])
snapbuild = self.makeSnapBuild(requester=requester_team,
name="test-snap",
owner=requester_team,
store_channels=["stable", "edge"])
self.assertContentEqual([], snapbuild.store_upload_jobs)
job = SnapStoreUploadJob.create(snapbuild)
client = FakeSnapStoreClient()
client.upload.result = self.status_url
client.checkStatus.result = (self.store_url, None)
self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient))
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertEqual([((snapbuild, ), {})], client.upload.calls)
self.assertEqual([((self.status_url, ), {})], client.checkStatus.calls)
self.assertEqual([], client.release.calls)
self.assertContentEqual([job], snapbuild.store_upload_jobs)
self.assertEqual(self.store_url, job.store_url)
self.assertIsNone(job.store_revision)
self.assertEqual(
"Package held for manual review on the store; "
"cannot release it automatically.", job.error_message)
[notification] = pop_notifications()
self.assertEqual(config.canonical.noreply_from_address,
notification["From"])
self.assertEqual("Requester <%s>" % requester.preferredemail.email,
notification["To"])
subject = notification["Subject"].replace("\n ", " ")
self.assertEqual("test-snap held for manual review", subject)
self.assertEqual("Requester @requester-team",
notification["X-Launchpad-Message-Rationale"])
self.assertEqual(requester_team.name,
notification["X-Launchpad-Message-For"])
self.assertEqual("snap-build-release-manual-review",
notification["X-Launchpad-Notification-Type"])
body, footer = notification.get_payload(decode=True).split("\n-- \n")
self.assertIn(self.store_url, body)
self.assertEqual(
"http://launchpad.dev/~requester-team/+snap/test-snap/+build/%d\n"
"Your team Requester Team is the requester of the build.\n" %
snapbuild.id, footer)
self.assertWebhookDeliveries(
snapbuild, ["Pending", "Failed to release to channels"])
def test_with_snapbuild_metadata_as_none_set_status(self):
db_build = self.factory.makeSnapBuild()
unsecure_db_build = removeSecurityProxy(db_build)
unsecure_db_build.store_upload_metadata = None
store = IStore(SnapBuild)
store.flush()
loaded_build = store.find(SnapBuild, id=unsecure_db_build.id).one()
job = SnapStoreUploadJob.create(loaded_build)
job.status_url = 'http://example.org'
store.flush()
loaded_build = store.find(SnapBuild, id=unsecure_db_build.id).one()
self.assertEqual('http://example.org',
loaded_build.store_upload_metadata['status_url'])
def test_run_scan_pending_retries(self):
# A run that finds that the store has not yet finished scanning the
# package schedules itself to be retried.
self.useFixture(FakeLogger())
snapbuild = self.makeSnapBuild()
self.assertContentEqual([], snapbuild.store_upload_jobs)
job = SnapStoreUploadJob.create(snapbuild)
client = FakeSnapStoreClient()
client.upload.result = self.status_url
client.checkStatus.failure = UploadNotScannedYetResponse()
self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient))
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertEqual([((snapbuild, ), {})], client.upload.calls)
self.assertEqual([((self.status_url, ), {})], client.checkStatus.calls)
self.assertEqual([], client.release.calls)
self.assertContentEqual([job], snapbuild.store_upload_jobs)
self.assertIsNone(job.store_url)
self.assertIsNone(job.store_revision)
self.assertIsNone(job.error_message)
self.assertEqual([], pop_notifications())
self.assertEqual(JobStatus.WAITING, job.job.status)
self.assertWebhookDeliveries(snapbuild, ["Pending"])
# Try again. The upload part of the job is not retried, and this
# time the scan completes.
job.scheduled_start = None
client.upload.calls = []
client.checkStatus.calls = []
client.checkStatus.failure = None
client.checkStatus.result = (self.store_url, 1)
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertEqual([], client.upload.calls)
self.assertEqual([((self.status_url, ), {})], client.checkStatus.calls)
self.assertEqual([], client.release.calls)
self.assertContentEqual([job], snapbuild.store_upload_jobs)
self.assertEqual(self.store_url, job.store_url)
self.assertEqual(1, job.store_revision)
self.assertIsNone(job.error_message)
self.assertEqual([], pop_notifications())
self.assertEqual(JobStatus.COMPLETED, job.job.status)
self.assertWebhookDeliveries(snapbuild, ["Pending", "Uploaded"])
def test_run_502_retries(self):
# A run that gets a 502 error from the store schedules itself to be
# retried.
self.useFixture(FakeLogger())
snapbuild = self.makeSnapBuild()
self.assertContentEqual([], snapbuild.store_upload_jobs)
job = SnapStoreUploadJob.create(snapbuild)
client = FakeSnapStoreClient()
client.upload.failure = UploadFailedResponse("Proxy error",
can_retry=True)
self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient))
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertEqual([((snapbuild, ), {})], client.upload.calls)
self.assertEqual([], client.checkStatus.calls)
self.assertEqual([], client.release.calls)
self.assertContentEqual([job], snapbuild.store_upload_jobs)
self.assertIsNone(job.store_url)
self.assertIsNone(job.store_revision)
self.assertIsNone(job.error_message)
self.assertEqual([], pop_notifications())
self.assertEqual(JobStatus.WAITING, job.job.status)
self.assertWebhookDeliveries(snapbuild, ["Pending"])
# Try again. The upload part of the job is retried, and this time
# it succeeds.
job.scheduled_start = None
client.upload.calls = []
client.upload.failure = None
client.upload.result = self.status_url
client.checkStatus.result = (self.store_url, 1)
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertEqual([((snapbuild, ), {})], client.upload.calls)
self.assertEqual([((self.status_url, ), {})], client.checkStatus.calls)
self.assertEqual([], client.release.calls)
self.assertContentEqual([job], snapbuild.store_upload_jobs)
self.assertEqual(self.store_url, job.store_url)
self.assertEqual(1, job.store_revision)
self.assertIsNone(job.error_message)
self.assertEqual([], pop_notifications())
self.assertEqual(JobStatus.COMPLETED, job.job.status)
self.assertWebhookDeliveries(snapbuild, ["Pending", "Uploaded"])
def test_run_failed(self):
# A failed run sets the store upload status to FAILED.
snapbuild = self.makeSnapBuild()
self.assertContentEqual([], snapbuild.store_upload_jobs)
job = SnapStoreUploadJob.create(snapbuild)
client = FakeSnapStoreClient()
client.upload.failure = ValueError("An upload failure")
self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient))
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertEqual([((snapbuild, ), {})], client.upload.calls)
self.assertEqual([], client.checkStatus.calls)
self.assertEqual([], client.release.calls)
self.assertContentEqual([job], snapbuild.store_upload_jobs)
self.assertIsNone(job.store_url)
self.assertIsNone(job.store_revision)
self.assertEqual("An upload failure", job.error_message)
self.assertEqual([], pop_notifications())
self.assertWebhookDeliveries(snapbuild,
["Pending", "Failed to upload"])
def test_run_release(self):
# A run configured to automatically release the package to certain
# channels does so.
snapbuild = self.makeSnapBuild(store_channels=["stable", "edge"])
self.assertContentEqual([], snapbuild.store_upload_jobs)
job = SnapStoreUploadJob.create(snapbuild)
client = FakeSnapStoreClient()
client.upload.result = self.status_url
client.checkStatus.result = (self.store_url, 1)
self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient))
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertEqual([((snapbuild, ), {})], client.upload.calls)
self.assertEqual([((self.status_url, ), {})], client.checkStatus.calls)
self.assertEqual([((snapbuild, 1), {})], client.release.calls)
self.assertContentEqual([job], snapbuild.store_upload_jobs)
self.assertEqual(self.store_url, job.store_url)
self.assertEqual(1, job.store_revision)
self.assertIsNone(job.error_message)
self.assertEqual([], pop_notifications())
self.assertWebhookDeliveries(snapbuild, ["Pending", "Uploaded"])
def test_run(self):
# The job uploads the build to the store and records the store URL
# and revision.
snapbuild = self.makeSnapBuild()
self.assertContentEqual([], snapbuild.store_upload_jobs)
job = SnapStoreUploadJob.create(snapbuild)
client = FakeSnapStoreClient()
client.upload.result = self.status_url
client.checkStatus.result = (self.store_url, 1)
self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient))
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
self.assertEqual([((snapbuild, ), {})], client.upload.calls)
self.assertEqual([((self.status_url, ), {})], client.checkStatus.calls)
self.assertEqual([], client.release.calls)
self.assertContentEqual([job], snapbuild.store_upload_jobs)
self.assertEqual(self.store_url, job.store_url)
self.assertEqual(1, job.store_revision)
self.assertIsNone(job.error_message)
self.assertEqual([], pop_notifications())
self.assertWebhookDeliveries(snapbuild, ["Pending", "Uploaded"])
def test_retry_after_upload_does_not_upload(self):
# If the job has uploaded, but failed to release, it should
# not attempt to upload again on the next run.
self.useFixture(FakeLogger())
snapbuild = self.makeSnapBuild(store_channels=["stable", "edge"])
self.assertContentEqual([], snapbuild.store_upload_jobs)
job = SnapStoreUploadJob.create(snapbuild)
client = FakeSnapStoreClient()
client.upload.result = self.status_url
client.checkStatus.result = (self.store_url, 1)
client.release.failure = UploadFailedResponse("Proxy error",
can_retry=True)
self.useFixture(ZopeUtilityFixture(client, ISnapStoreClient))
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
previous_upload = client.upload.calls
previous_checkStatus = client.checkStatus.calls
len_previous_release = len(client.release.calls)
# Check we uploaded as expected
self.assertEqual(self.store_url, job.store_url)
self.assertEqual(1, job.store_revision)
self.assertEqual(timedelta(seconds=60), job.retry_delay)
self.assertEqual(1, len(client.upload.calls))
self.assertIsNone(job.error_message)
# Run the job again
with dbuser(config.ISnapStoreUploadJobSource.dbuser):
JobRunner([job]).runAll()
# We should not have called `upload`, but moved straight to `release`
self.assertEqual(previous_upload, client.upload.calls)
self.assertEqual(previous_checkStatus, client.checkStatus.calls)
self.assertEqual(len_previous_release + 1, len(client.release.calls))
self.assertIsNone(job.error_message)
def test_provides_interface(self):
# `SnapStoreUploadJob` objects provide `ISnapStoreUploadJob`.
snapbuild = self.factory.makeSnapBuild()
job = SnapStoreUploadJob.create(snapbuild)
self.assertProvides(job, ISnapStoreUploadJob)