def test_reap(self):
archive = self.factory.makeArchive()
archive_files = [
self.factory.makeArchiveFile(archive=archive, container="foo")
for _ in range(3)
]
archive_files.append(self.factory.makeArchiveFile(archive=archive))
other_archive = self.factory.makeArchive()
archive_files.append(
self.factory.makeArchiveFile(archive=other_archive))
now = get_transaction_timestamp(Store.of(archive_files[0]))
removeSecurityProxy(
archive_files[0]).scheduled_deletion_date = (now -
timedelta(days=1))
removeSecurityProxy(
archive_files[1]).scheduled_deletion_date = (now +
timedelta(days=1))
removeSecurityProxy(
archive_files[3]).scheduled_deletion_date = (now -
timedelta(days=1))
removeSecurityProxy(
archive_files[4]).scheduled_deletion_date = (now -
timedelta(days=1))
archive_file_set = getUtility(IArchiveFileSet)
expected_rows = [
("foo", archive_files[0].path,
archive_files[0].library_file.content.sha256),
]
rows = archive_file_set.reap(archive, container="foo")
self.assertContentEqual(expected_rows, rows)
self.assertContentEqual(archive_files[1:4],
archive_file_set.getByArchive(archive))
def test_setGrants_matching_rule(self):
repository = self.factory.makeGitRepository()
[ref] = self.factory.makeGitRefs(repository=repository)
rule = self.factory.makeGitRule(repository=repository,
ref_pattern=ref.path)
date_created = get_transaction_timestamp(Store.of(rule))
transaction.commit()
with person_logged_in(repository.owner):
ref.setGrants([
IGitNascentRuleGrant({
"grantee_type": GitGranteeType.REPOSITORY_OWNER,
"can_force_push": True,
}),
], repository.owner)
self.assertThat(
list(repository.rules),
MatchesListwise([
MatchesStructure(repository=Equals(repository),
ref_pattern=Equals(ref.path),
date_created=Equals(date_created),
grants=MatchesSetwise(
MatchesStructure(
grantee_type=Equals(
GitGranteeType.REPOSITORY_OWNER),
grantee=Is(None),
can_create=Is(False),
can_push=Is(False),
can_force_push=Is(True)))),
]))
def test_properties_owner(self):
owner = self.factory.makeTeam()
member = self.factory.makePerson(member_of=[owner])
rule = self.factory.makeGitRule(owner=owner)
grant = self.factory.makeGitRuleGrant(
rule=rule,
grantee=GitGranteeType.REPOSITORY_OWNER,
grantor=member,
can_create=True,
can_force_push=True)
now = get_transaction_timestamp(Store.of(grant))
self.assertThat(
grant,
MatchesStructure(
repository=Equals(rule.repository),
rule=Equals(rule),
grantee_type=Equals(GitGranteeType.REPOSITORY_OWNER),
grantee=Is(None),
combined_grantee=Equals(GitGranteeType.REPOSITORY_OWNER),
can_create=Is(True),
can_push=Is(False),
can_force_push=Is(True),
grantor=Equals(member),
date_created=Equals(now),
date_last_modified=Equals(now)))
def test_sql_passed_through(self):
# create() passes SQL() expressions through untouched.
bug = self.factory.makeBug()
person = self.factory.makePerson()
[sub] = bulk.create(
(BugSubscription.bug, BugSubscription.person,
BugSubscription.subscribed_by, BugSubscription.date_created,
BugSubscription.bug_notification_level),
[(bug, person, person, SQL("CURRENT_TIMESTAMP AT TIME ZONE 'UTC'"),
BugNotificationLevel.LIFECYCLE)],
get_objects=True)
self.assertEqual(get_transaction_timestamp(), sub.date_created)
def test_sql_passed_through(self):
# create() passes SQL() expressions through untouched.
bug = self.factory.makeBug()
person = self.factory.makePerson()
[sub] = bulk.create(
(BugSubscription.bug, BugSubscription.person,
BugSubscription.subscribed_by, BugSubscription.date_created,
BugSubscription.bug_notification_level),
[(bug, person, person,
SQL("CURRENT_TIMESTAMP AT TIME ZONE 'UTC'"),
BugNotificationLevel.LIFECYCLE)], get_objects=True)
self.assertEqual(get_transaction_timestamp(), sub.date_created)
def test_scheduleDeletion(self):
archive_files = [self.factory.makeArchiveFile() for _ in range(3)]
expected_rows = [(archive_file.container, archive_file.path,
archive_file.library_file.content.sha256)
for archive_file in archive_files[:2]]
rows = getUtility(IArchiveFileSet).scheduleDeletion(
archive_files[:2], timedelta(days=1))
self.assertContentEqual(expected_rows, rows)
flush_database_caches()
tomorrow = (get_transaction_timestamp(Store.of(archive_files[0])) +
timedelta(days=1))
self.assertEqual(tomorrow, archive_files[0].scheduled_deletion_date)
self.assertEqual(tomorrow, archive_files[1].scheduled_deletion_date)
self.assertIsNone(archive_files[2].scheduled_deletion_date)
def test_run(self):
# The job requests builds and records the result.
distroseries, processors = self.makeSeriesAndProcessors(
["avr2001", "sparc64", "x32"])
[git_ref] = self.factory.makeGitRefs()
snap = self.factory.makeSnap(
git_ref=git_ref, distroseries=distroseries, processors=processors)
expected_date_created = get_transaction_timestamp(IStore(snap))
job = SnapRequestBuildsJob.create(
snap, snap.registrant, distroseries.main_archive,
PackagePublishingPocket.RELEASE, {"core": "stable"})
snapcraft_yaml = dedent("""\
architectures:
- build-on: avr2001
- build-on: x32
""")
self.useFixture(GitHostingFixture(blob=snapcraft_yaml))
with dbuser(config.ISnapRequestBuildsJobSource.dbuser):
JobRunner([job]).runAll()
now = get_transaction_timestamp(IStore(snap))
self.assertEmailQueueLength(0)
self.assertThat(job, MatchesStructure(
job=MatchesStructure.byEquality(status=JobStatus.COMPLETED),
date_created=Equals(expected_date_created),
date_finished=MatchesAll(
GreaterThan(expected_date_created), LessThan(now)),
error_message=Is(None),
builds=AfterPreprocessing(set, MatchesSetwise(*[
MatchesStructure(
build_request=MatchesStructure.byEquality(id=job.job.id),
requester=Equals(snap.registrant),
snap=Equals(snap),
archive=Equals(distroseries.main_archive),
distro_arch_series=Equals(distroseries[arch]),
pocket=Equals(PackagePublishingPocket.RELEASE),
channels=Equals({"core": "stable"}))
for arch in ("avr2001", "x32")]))))
def test_properties(self):
owner = self.factory.makeTeam()
member = self.factory.makePerson(member_of=[owner])
repository = self.factory.makeGitRepository(owner=owner)
rule = self.factory.makeGitRule(repository=repository,
ref_pattern="refs/heads/stable/*",
creator=member)
now = get_transaction_timestamp(Store.of(rule))
self.assertThat(
rule,
MatchesStructure.byEquality(repository=repository,
ref_pattern="refs/heads/stable/*",
creator=member,
date_created=now,
date_last_modified=now))
def test_run_failed(self):
# A failed run sets the job status to FAILED and records the error
# message.
# The job requests builds and records the result.
distroseries, processors = self.makeSeriesAndProcessors(
["avr2001", "sparc64", "x32"])
[git_ref] = self.factory.makeGitRefs()
snap = self.factory.makeSnap(
git_ref=git_ref, distroseries=distroseries, processors=processors)
expected_date_created = get_transaction_timestamp(IStore(snap))
job = SnapRequestBuildsJob.create(
snap, snap.registrant, distroseries.main_archive,
PackagePublishingPocket.RELEASE, {"core": "stable"})
self.useFixture(GitHostingFixture()).getBlob.failure = (
CannotParseSnapcraftYaml("Nonsense on stilts"))
with dbuser(config.ISnapRequestBuildsJobSource.dbuser):
JobRunner([job]).runAll()
now = get_transaction_timestamp(IStore(snap))
[notification] = self.assertEmailQueueLength(1)
self.assertThat(dict(notification), ContainsDict({
"From": Equals(config.canonical.noreply_from_address),
"To": Equals(format_address_for_person(snap.registrant)),
"Subject": Equals(
"Launchpad error while requesting builds of %s" % snap.name),
}))
self.assertEqual(
"Launchpad encountered an error during the following operation: "
"requesting builds of %s. Nonsense on stilts" % snap.name,
notification.get_payload(decode=True))
self.assertThat(job, MatchesStructure(
job=MatchesStructure.byEquality(status=JobStatus.FAILED),
date_created=Equals(expected_date_created),
date_finished=MatchesAll(
GreaterThan(expected_date_created), LessThan(now)),
error_message=Equals("Nonsense on stilts"),
builds=AfterPreprocessing(set, MatchesSetwise())))
def test_properties(self):
repository = self.factory.makeGitRepository()
changee = self.factory.makePerson()
activity = GitActivity(
repository, repository.owner, GitActivityType.RULE_ADDED,
changee=changee, old_value={"old": None}, new_value={"new": None})
now = get_transaction_timestamp(Store.of(activity))
self.assertThat(activity, MatchesStructure.byEquality(
repository=repository,
date_changed=now,
changer=repository.owner,
changee=changee,
what_changed=GitActivityType.RULE_ADDED,
old_value={"old": None},
new_value={"new": None}))
def test_unscheduleDeletion(self):
archive_files = [self.factory.makeArchiveFile() for _ in range(3)]
now = get_transaction_timestamp(Store.of(archive_files[0]))
for archive_file in archive_files:
removeSecurityProxy(archive_file).scheduled_deletion_date = now
expected_rows = [(archive_file.container, archive_file.path,
archive_file.library_file.content.sha256)
for archive_file in archive_files[:2]]
rows = getUtility(IArchiveFileSet).unscheduleDeletion(
archive_files[:2])
self.assertContentEqual(expected_rows, rows)
flush_database_caches()
self.assertIsNone(archive_files[0].scheduled_deletion_date)
self.assertIsNone(archive_files[1].scheduled_deletion_date)
self.assertEqual(now, archive_files[2].scheduled_deletion_date)
def confirm_no_clock_skew(store):
"""Raise an exception if there is significant clock skew between the
database and this machine.
It is theoretically possible to lose data if there is more than several
hours of skew.
"""
db_now = get_transaction_timestamp(store)
local_now = _utcnow()
five_minutes = timedelta(minutes=5)
if -five_minutes < local_now - db_now < five_minutes:
return
else:
raise Exception("%s clock skew between librarian and database" %
(local_now - db_now, ))
def test_getByArchive(self):
archives = [self.factory.makeArchive(), self.factory.makeArchive()]
archive_files = []
now = get_transaction_timestamp(Store.of(archives[0]))
for archive in archives:
archive_files.append(
self.factory.makeArchiveFile(archive=archive,
scheduled_deletion_date=now))
archive_files.append(
self.factory.makeArchiveFile(archive=archive, container="foo"))
archive_file_set = getUtility(IArchiveFileSet)
self.assertContentEqual(archive_files[:2],
archive_file_set.getByArchive(archives[0]))
self.assertContentEqual([archive_files[1]],
archive_file_set.getByArchive(archives[0],
container="foo"))
self.assertContentEqual([],
archive_file_set.getByArchive(archives[0],
container="bar"))
self.assertContentEqual([archive_files[1]],
archive_file_set.getByArchive(
archives[0], path=archive_files[1].path))
self.assertContentEqual([],
archive_file_set.getByArchive(archives[0],
path="other"))
self.assertContentEqual([archive_files[0]],
archive_file_set.getByArchive(
archives[0], only_condemned=True))
self.assertContentEqual(archive_files[2:],
archive_file_set.getByArchive(archives[1]))
self.assertContentEqual([archive_files[3]],
archive_file_set.getByArchive(archives[1],
container="foo"))
self.assertContentEqual([],
archive_file_set.getByArchive(archives[1],
container="bar"))
self.assertContentEqual([archive_files[3]],
archive_file_set.getByArchive(
archives[1], path=archive_files[3].path))
self.assertContentEqual([],
archive_file_set.getByArchive(archives[1],
path="other"))
self.assertContentEqual([archive_files[2]],
archive_file_set.getByArchive(
archives[1], only_condemned=True))
def test_handles_archive_subscriptions_and_tokens(self):
person = self.factory.makePerson()
ppa = self.factory.makeArchive(private=True)
subscription = ppa.newSubscription(person, ppa.owner)
other_subscription = ppa.newSubscription(self.factory.makePerson(),
ppa.owner)
ppa.newAuthToken(person)
self.assertEqual(ArchiveSubscriberStatus.CURRENT, subscription.status)
self.assertIsNotNone(ppa.getAuthToken(person))
person_id = person.id
account_id = person.account.id
script = self.makeScript([six.ensure_str(person.name)])
with dbuser('launchpad'):
now = get_transaction_timestamp(Store.of(person))
self.runScript(script)
self.assertRemoved(account_id, person_id)
self.assertEqual(ArchiveSubscriberStatus.CANCELLED,
subscription.status)
self.assertEqual(now, subscription.date_cancelled)
self.assertEqual(ArchiveSubscriberStatus.CURRENT,
other_subscription.status)
self.assertIsNotNone(ppa.getAuthToken(person))
def test_dateDueRecentPreviousResult(self):
# If there is a CodeImportResult for the CodeImport that is more
# recent than the effective_update_interval, then the new
# CodeImportJob has date_due set in the future.
code_import = self.getCodeImportForDateDueTest()
# A code import job is automatically started when a reviewed code import
# is created. Remove it, so a "clean" one can be created later.
removeSecurityProxy(code_import).import_job.destroySelf()
# Create a CodeImportResult that started a long time ago. This one
# must be superseded by the more recent one created below.
machine = self.factory.makeCodeImportMachine()
FAILURE = CodeImportResultStatus.FAILURE
CodeImportResult(
code_import=code_import, machine=machine, status=FAILURE,
date_job_started=datetime(2000, 1, 1, 12, 0, 0, tzinfo=UTC),
date_created=datetime(2000, 1, 1, 12, 5, 0, tzinfo=UTC))
# Create a CodeImportResult that started a shorter time ago than the
# effective update interval of the code import. This is the most
# recent one and must supersede the older one.
# XXX 2008-06-09 jamesh:
# psycopg2 isn't correctly substituting intervals into the
# expression (it doesn't include the "INTERVAL" keyword).
# This causes problems for the "UTC_NOW - interval / 2"
# expression below.
interval = code_import.effective_update_interval
from lp.services.database.sqlbase import get_transaction_timestamp
recent_result = CodeImportResult(
code_import=code_import, machine=machine, status=FAILURE,
date_job_started=get_transaction_timestamp() - interval / 2)
# When we create the job, its date_due should be set to the date_due
# of the job that was deleted when the CodeImport review status
# changed from REVIEWED. That is the date_job_started of the most
# recent CodeImportResult plus the effective update interval.
getUtility(ICodeImportJobWorkflow).newJob(code_import)
self.assertSqlAttributeEqualsDate(
code_import.import_job, 'date_due',
recent_result.date_job_started + interval)
def test_getContainersToReap(self):
archive = self.factory.makeArchive()
archive_files = []
for container in ("release:foo", "other:bar", "baz"):
for _ in range(2):
archive_files.append(
self.factory.makeArchiveFile(archive=archive,
container=container))
other_archive = self.factory.makeArchive()
archive_files.append(
self.factory.makeArchiveFile(archive=other_archive,
container="baz"))
now = get_transaction_timestamp(Store.of(archive_files[0]))
removeSecurityProxy(
archive_files[0]).scheduled_deletion_date = (now -
timedelta(days=1))
removeSecurityProxy(
archive_files[1]).scheduled_deletion_date = (now -
timedelta(days=1))
removeSecurityProxy(
archive_files[2]).scheduled_deletion_date = (now +
timedelta(days=1))
removeSecurityProxy(
archive_files[6]).scheduled_deletion_date = (now -
timedelta(days=1))
archive_file_set = getUtility(IArchiveFileSet)
self.assertContentEqual(["release:foo"],
archive_file_set.getContainersToReap(archive))
self.assertContentEqual(
["baz"], archive_file_set.getContainersToReap(other_archive))
removeSecurityProxy(
archive_files[3]).scheduled_deletion_date = (now -
timedelta(days=1))
self.assertContentEqual(["release:foo", "other:bar"],
archive_file_set.getContainersToReap(archive))
self.assertContentEqual(["release:foo"],
archive_file_set.getContainersToReap(
archive, container_prefix="release:"))
def test_setGrants_remove(self):
owner = self.factory.makeTeam()
members = [
self.factory.makePerson(member_of=[owner]) for _ in range(2)
]
rule = self.factory.makeGitRule(owner=owner)
grantees = [self.factory.makePerson() for _ in range(2)]
self.factory.makeGitRuleGrant(rule=rule,
grantee=GitGranteeType.REPOSITORY_OWNER,
grantor=members[0],
can_create=True)
self.factory.makeGitRuleGrant(rule=rule,
grantee=grantees[0],
grantor=members[0],
can_push=True)
self.factory.makeGitRuleGrant(rule=rule,
grantee=grantees[1],
grantor=members[0],
can_force_push=True)
date_created = get_transaction_timestamp(Store.of(rule))
transaction.commit()
removeSecurityProxy(rule.repository.getActivity()).remove()
with person_logged_in(members[1]):
rule.setGrants([
IGitNascentRuleGrant({
"grantee_type": GitGranteeType.PERSON,
"grantee": grantees[0],
"can_push": True,
}),
], members[1])
self.assertThat(
rule.grants,
MatchesSetwise(
MatchesStructure(rule=Equals(rule),
grantor=Equals(members[0]),
grantee_type=Equals(GitGranteeType.PERSON),
grantee=Equals(grantees[0]),
can_create=Is(False),
can_push=Is(True),
can_force_push=Is(False),
date_created=Equals(date_created),
date_last_modified=Equals(date_created))))
self.assertThat(
list(rule.repository.getActivity()),
MatchesSetwise(
MatchesStructure(repository=Equals(rule.repository),
changer=Equals(members[1]),
changee=Is(None),
what_changed=Equals(
GitActivityType.GRANT_REMOVED),
old_value=MatchesDict({
"changee_type":
Equals("Repository owner"),
"ref_pattern":
Equals(rule.ref_pattern),
"can_create":
Is(True),
"can_push":
Is(False),
"can_force_push":
Is(False),
}),
new_value=Is(None)),
MatchesStructure(repository=Equals(rule.repository),
changer=Equals(members[1]),
changee=Equals(grantees[1]),
what_changed=Equals(
GitActivityType.GRANT_REMOVED),
old_value=MatchesDict({
"changee_type":
Equals("Person"),
"ref_pattern":
Equals(rule.ref_pattern),
"can_create":
Is(False),
"can_push":
Is(False),
"can_force_push":
Is(True),
}),
new_value=Is(None)),
))