def make_all_result_types(code_import, factory, machine, start, count):
"""Make a code import result of each possible type for the code import."""
start_dates = time_counter(
datetime(2007,12,1,12, tzinfo=UTC), timedelta(days=1))
end_dates = time_counter(
datetime(2007,12,1,13, tzinfo=UTC), timedelta(days=1, hours=1))
for result_status in sorted(CodeImportResultStatus.items)[start:start+count]:
factory.makeCodeImportResult(
code_import, result_status, start_dates.next(), end_dates.next(),
machine=machine)
def make_all_result_types(code_import, factory, machine, start, count):
"""Make a code import result of each possible type for the code import."""
start_dates = time_counter(
datetime(2007, 12, 1, 12, tzinfo=UTC), timedelta(days=1))
end_dates = time_counter(
datetime(2007, 12, 1, 13, tzinfo=UTC), timedelta(days=1, hours=1))
for result_status in sorted(CodeImportResultStatus.items)[
start:start + count]:
factory.makeCodeImportResult(
code_import, result_status, start_dates.next(), end_dates.next(),
machine=machine)
def makeBranches(self, branch_count, official_count=0):
"""Make some package branches.
Make `branch_count` branches, and make `official_count` of those
official branches.
"""
distroseries = self.distro.series[0]
# Make the branches created in the past in order.
time_gen = time_counter(delta=timedelta(days=-1))
branches = [
self.factory.makePackageBranch(
distroseries=distroseries,
sourcepackagename=self.sourcepackagename,
date_created=time_gen.next()) for i in range(branch_count)
]
official = []
# Sort the pocket items so RELEASE is last, and thus first popped.
pockets = sorted(PackagePublishingPocket.items, reverse=True)
for i in range(official_count):
branch = branches.pop()
pocket = pockets.pop()
SeriesSourcePackageBranchSet.new(distroseries, pocket,
self.sourcepackagename, branch,
branch.owner)
official.append(branch)
return distroseries, branches, official
def makeBranch(self, product=None, last_commit_date=None, private=False,
revision_count=None):
"""Make a product branch with a particular last commit date"""
if revision_count is None:
revision_count = 5
delta = timedelta(days=1)
if last_commit_date is None:
# By default we create revisions that are within the last 30 days.
date_generator = time_counter(
datetime.now(pytz.UTC) - timedelta(days=25), delta)
else:
start_date = last_commit_date - delta * (revision_count - 1)
date_generator = time_counter(start_date, delta)
branch = make_project_branch_with_revisions(
self.factory, date_generator, product, private, revision_count)
return branch
def make_package_branches(factory, series, sourcepackagename, branch_count,
official_count=0, owner=None, registrant=None):
"""Make some package branches.
Make `branch_count` branches, and make `official_count` of those
official branches.
"""
if zisinstance(sourcepackagename, basestring):
sourcepackagename = factory.getOrMakeSourcePackageName(
sourcepackagename)
# Make the branches created in the past in order.
time_gen = time_counter(delta=timedelta(days=-1))
branch_names = consistent_branch_names()
branches = [
factory.makePackageBranch(
distroseries=series,
sourcepackagename=sourcepackagename,
date_created=time_gen.next(),
name=branch_names.next(), owner=owner, registrant=registrant)
for i in range(branch_count)]
official = []
# Sort the pocket items so RELEASE is last, and thus first popped.
pockets = sorted(PackagePublishingPocket.items, reverse=True)
# Since there can be only one link per pocket, max out the number of
# official branches at the pocket count.
for i in range(min(official_count, len(pockets))):
branch = branches.pop()
pocket = pockets.pop()
SeriesSourcePackageBranchSet.new(
series, pocket, sourcepackagename, branch, branch.owner)
official.append(branch)
return series, branches, official
def test_committers_count_private_branch_non_subscriber(self):
# Test that calling committer_count will return the proper value
# for a private branch.
fsm = self.factory.makePerson(email='*****@*****.**')
product, branch = self.makeProductAndDevelopmentFocusBranch(
owner=fsm, information_type=InformationType.USERDATA)
date_generator = time_counter(
datetime.now(pytz.UTC) - timedelta(days=30), timedelta(days=1))
login_person(fsm)
self.factory.makeRevisionsForBranch(branch,
author='*****@*****.**',
date_generator=date_generator)
getUtility(IRevisionSet).updateRevisionCacheForBranch(branch)
observer = self.factory.makePerson()
login_person(observer)
view = create_initialized_view(product,
'+branch-summary',
rootsite='code',
principal=observer)
self.assertEqual(view.branch_count, 0)
self.assertEqual(view.committer_count, 1)
view = create_initialized_view(product,
'+portlet-product-branchstatistics',
rootsite='code',
principal=observer)
commit_section = find_tag_by_id(view.render(), 'commits')
self.assertIs(None, commit_section)
def makeBranches(self, branch_count, official_count=0):
"""Make some package branches.
Make `branch_count` branches, and make `official_count` of those
official branches.
"""
distroseries = self.distro.series[0]
# Make the branches created in the past in order.
time_gen = time_counter(delta=timedelta(days=-1))
branches = [
self.factory.makePackageBranch(
distroseries=distroseries,
sourcepackagename=self.sourcepackagename,
date_created=time_gen.next())
for i in range(branch_count)]
official = []
# Sort the pocket items so RELEASE is last, and thus first popped.
pockets = sorted(PackagePublishingPocket.items, reverse=True)
for i in range(official_count):
branch = branches.pop()
pocket = pockets.pop()
SeriesSourcePackageBranchSet.new(
distroseries, pocket, self.sourcepackagename,
branch, branch.owner)
official.append(branch)
return distroseries, branches, official
def setUp(self):
# Use an administrator to set branch privacy easily.
TestCaseWithFactory.setUp(self, "*****@*****.**")
# Since the tests order by date, but also limit to the last 30
# days, we want a time counter that starts 10 days ago.
self.date_generator = time_counter(datetime.now(pytz.UTC) -
timedelta(days=10),
delta=timedelta(days=1))
def setUp(self):
# Use an administrator to set branch privacy easily.
TestCaseWithFactory.setUp(self, "*****@*****.**")
# Since the tests order by date, but also limit to the last 30
# days, we want a time counter that starts 10 days ago.
self.date_generator = time_counter(
datetime.now(pytz.UTC) - timedelta(days=10),
delta=timedelta(days=1))
def test_revisions_bound_by_date(self):
# Only revisions in the last 30 days are returned, even if the
# revision cache table hasn't been trimmed lately.
tc = time_counter(origin=datetime.now(pytz.UTC) - timedelta(days=27),
delta=timedelta(days=-2))
# Make four cached revisions spanning 33, 31, 29, and 27 days ago.
for i in range(4):
self.makeCachedRevision(revision=self.factory.makeRevision(
revision_date=tc.next()))
cache = getUtility(IRevisionCache)
self.assertEqual(2, cache.count())
def makeBranch(self,
product=None,
last_commit_date=None,
private=False,
revision_count=None):
"""Make a product branch with a particular last commit date"""
if revision_count is None:
revision_count = 5
delta = timedelta(days=1)
if last_commit_date is None:
# By default we create revisions that are within the last 30 days.
date_generator = time_counter(
datetime.now(pytz.UTC) - timedelta(days=25), delta)
else:
start_date = last_commit_date - delta * (revision_count - 1)
date_generator = time_counter(start_date, delta)
branch = make_project_branch_with_revisions(self.factory,
date_generator, product,
private, revision_count)
return branch
def test_revisions_bound_by_date(self):
# Only revisions in the last 30 days are returned, even if the
# revision cache table hasn't been trimmed lately.
tc = time_counter(
origin=datetime.now(pytz.UTC) - timedelta(days=27),
delta=timedelta(days=-2))
# Make four cached revisions spanning 33, 31, 29, and 27 days ago.
for i in range(4):
self.makeCachedRevision(
revision=self.factory.makeRevision(revision_date=tc.next()))
cache = getUtility(IRevisionCache)
self.assertEqual(2, cache.count())
def test_old_revisions_removed(self):
# Revisions older than 30 days are removed.
date_generator = time_counter(datetime.now(pytz.UTC) -
timedelta(days=33),
delta=timedelta(days=2))
for i in range(4):
revision = self.factory.makeRevision(
revision_date=date_generator.next())
cache = RevisionCache(revision)
self.store.add(cache)
RevisionSet.pruneRevisionCache(5)
self.assertEqual(2, len(self._getRevisionCache()))
def test_pruning_limit(self):
# The prune will only remove at most the parameter rows.
date_generator = time_counter(
datetime.now(pytz.UTC) - timedelta(days=33),
delta=timedelta(days=2))
for i in range(4):
revision = self.factory.makeRevision(
revision_date=date_generator.next())
cache = RevisionCache(revision)
self.store.add(cache)
RevisionSet.pruneRevisionCache(1)
self.assertEqual(3, len(self._getRevisionCache()))
def test_old_revisions_removed(self):
# Revisions older than 30 days are removed.
date_generator = time_counter(
datetime.now(pytz.UTC) - timedelta(days=33),
delta=timedelta(days=2))
for i in range(4):
revision = self.factory.makeRevision(
revision_date=date_generator.next())
cache = RevisionCache(revision)
self.store.add(cache)
RevisionSet.pruneRevisionCache(5)
self.assertEqual(2, len(self._getRevisionCache()))
def test_pruning_limit(self):
# The prune will only remove at most the parameter rows.
date_generator = time_counter(datetime.now(pytz.UTC) -
timedelta(days=33),
delta=timedelta(days=2))
for i in range(4):
revision = self.factory.makeRevision(
revision_date=date_generator.next())
cache = RevisionCache(revision)
self.store.add(cache)
RevisionSet.pruneRevisionCache(1)
self.assertEqual(3, len(self._getRevisionCache()))
def test_revision_ordering(self):
# Revisions are returned most recent first.
tc = time_counter(origin=datetime.now(pytz.UTC) - timedelta(days=15),
delta=timedelta(days=1))
# Make four cached revisions spanning 15, 14, 13 and 12 days ago.
# Create from oldest to newest, then check that the ordering from the
# query is the reverse order.
revisions = [
self.makeCachedRevision(revision=self.factory.makeRevision(
revision_date=tc.next())) for i in range(4)
]
revisions.reverse()
cache = getUtility(IRevisionCache)
self.assertEqual(revisions, list(cache.getRevisions()))
def test_committer_count_with_revision_authors(self):
# Test that the code pathing for calling committer_count with
# valid revision authors is truly tested.
self.factory.makePerson(email='*****@*****.**')
product, branch = self.makeProductAndDevelopmentFocusBranch()
date_generator = time_counter(
datetime.now(pytz.UTC) - timedelta(days=30), timedelta(days=1))
self.factory.makeRevisionsForBranch(branch,
author='*****@*****.**',
date_generator=date_generator)
getUtility(IRevisionSet).updateRevisionCacheForBranch(branch)
view = create_initialized_view(product, '+code-index', rootsite='code')
self.assertEqual(view.committer_count, 1)
def test_revision_ordering(self):
# Revisions are returned most recent first.
tc = time_counter(
origin=datetime.now(pytz.UTC) - timedelta(days=15),
delta=timedelta(days=1))
# Make four cached revisions spanning 15, 14, 13 and 12 days ago.
# Create from oldest to newest, then check that the ordering from the
# query is the reverse order.
revisions = [
self.makeCachedRevision(
revision=self.factory.makeRevision(revision_date=tc.next()))
for i in range(4)]
revisions.reverse()
cache = getUtility(IRevisionCache)
self.assertEqual(revisions, list(cache.getRevisions()))
def test_run(self):
# Ensure the job scans the repository.
repository = self.factory.makeGitRepository()
job = GitRefScanJob.create(repository)
paths = ("refs/heads/master", "refs/tags/1.0")
author = repository.owner
author_date_start = datetime(2015, 1, 1, tzinfo=pytz.UTC)
author_date_gen = time_counter(author_date_start, timedelta(days=1))
self.useFixture(GitHostingFixture(
refs=self.makeFakeRefs(paths),
commits=self.makeFakeCommits(author, author_date_gen, paths)))
with dbuser("branchscanner"):
JobRunner([job]).runAll()
self.assertRefsMatch(repository.refs, repository, paths)
self.assertEqual("refs/heads/master", repository.default_branch)
def test_only_recent_revisions_counted(self):
# If the revision cache has revisions for the project, but they are
# over 30 days old, we don't count them.
product = self.factory.makeProduct()
date_generator = time_counter(datetime.now(pytz.UTC) -
timedelta(days=33),
delta=timedelta(days=2))
store = Store.of(product)
for i in range(4):
revision = self.factory.makeRevision(
revision_date=date_generator.next())
cache = RevisionCache(revision)
cache.product = product
store.add(cache)
self.assertEqual([(product.name, 2, 2, revision.revision_date)],
self.getProductsWithInfo())
def test_committer_count_with_revision_authors(self):
# Test that the code pathing for calling committer_count with
# valid revision authors is truly tested.
self.factory.makePerson(email='*****@*****.**')
product, branch = self.makeProductAndDevelopmentFocusBranch()
date_generator = time_counter(
datetime.now(pytz.UTC) - timedelta(days=30),
timedelta(days=1))
self.factory.makeRevisionsForBranch(
branch, author='*****@*****.**',
date_generator=date_generator)
getUtility(IRevisionSet).updateRevisionCacheForBranch(branch)
view = create_initialized_view(product, '+code-index',
rootsite='code')
self.assertEqual(view.committer_count, 1)
def make_project_cloud_data(factory, details):
"""Make test data to populate the project cloud.
Details is a list of tuples containing:
(project-name, num_commits, num_authors, last_commit)
"""
delta = timedelta(seconds=1)
for project_name, num_commits, num_authors, last_commit in details:
project = factory.makeProduct(name=project_name)
start_date = last_commit - delta * (num_commits - 1)
gen = time_counter(start_date, delta)
commits_each = num_commits / num_authors
for committer in range(num_authors - 1):
make_project_branch_with_revisions(factory, gen, project, commits_each)
num_commits -= commits_each
make_project_branch_with_revisions(factory, gen, project, revision_count=num_commits)
transaction.commit()
def test_only_recent_revisions_counted(self):
# If the revision cache has revisions for the project, but they are
# over 30 days old, we don't count them.
product = self.factory.makeProduct()
date_generator = time_counter(
datetime.now(pytz.UTC) - timedelta(days=33),
delta=timedelta(days=2))
store = Store.of(product)
for i in range(4):
revision = self.factory.makeRevision(
revision_date=date_generator.next())
cache = RevisionCache(revision)
cache.product = product
store.add(cache)
self.assertEqual(
[(product.name, 2, 2, revision.revision_date)],
self.getProductsWithInfo())
def test_new_revisions_added(self):
# If there are already revisions in the cache for the branch, updating
# the branch again will only add the new revisions.
date_generator = time_counter(
datetime.now(pytz.UTC) - timedelta(days=29),
delta=timedelta(days=1))
# Initially add in 4 revisions.
branch = self.factory.makeAnyBranch()
self.factory.makeRevisionsForBranch(
branch, count=4, date_generator=date_generator)
RevisionSet.updateRevisionCacheForBranch(branch)
# Now add two more revisions.
self.factory.makeRevisionsForBranch(
branch, count=2, date_generator=date_generator)
RevisionSet.updateRevisionCacheForBranch(branch)
# There will be only six revisions cached.
cached = self._getRevisionCache()
self.assertEqual(6, len(cached))
def setUp(self):
TestCaseWithFactory.setUp(self)
self.user = self.factory.makePerson()
login_person(self.user)
self.barney = self.factory.makePerson(name='barney')
self.bambam = self.factory.makeProduct(name='bambam')
time_gen = time_counter(delta=timedelta(days=-1))
self.branches = [
self.factory.makeProductBranch(
product=self.bambam, owner=self.barney,
date_created=time_gen.next())
for i in range(10)]
self.bug = self.factory.makeBug()
self.bug.linkBranch(self.branches[0], self.barney)
self.spec = self.factory.makeSpecification()
self.spec.linkBranch(self.branches[1], self.barney)
def test_result_ordering(self):
# The results query will order the results by job started time, with
# the most recent import first.
when = time_counter(
origin=datetime(2007, 9, 9, 12, tzinfo=pytz.UTC),
delta=timedelta(days=1))
first = self.factory.makeCodeImportResult(
self.code_import, date_started=when.next())
second = self.factory.makeCodeImportResult(
self.code_import, date_started=when.next())
third = self.factory.makeCodeImportResult(
self.code_import, date_started=when.next())
self.assertTrue(first.date_job_started < second.date_job_started)
self.assertTrue(second.date_job_started < third.date_job_started)
results = list(self.code_import.results)
self.assertEqual(third, results[0])
self.assertEqual(second, results[1])
self.assertEqual(first, results[2])
def test_searchQuestions(self):
date_gen = time_counter(datetime(2015, 1, 1, tzinfo=pytz.UTC),
timedelta(days=1))
created = [
self.factory.makeQuestion(title="foo", datecreated=next(date_gen))
for i in range(10)
]
webservice = webservice_for_person(self.factory.makePerson())
collection = webservice.named_get('/questions',
'searchQuestions',
search_text='foo',
sort='oldest first',
api_version='devel').jsonBody()
# The first few matching questions are returned.
self.assertEqual([q.id for q in created[:5]], [
int(q['self_link'].rsplit('/', 1)[-1])
for q in collection['entries']
])
def test_old_revisions_not_added(self):
# Revisions older than the 30 day epoch are not added to the cache.
# Start 33 days ago.
epoch = datetime.now(pytz.UTC) - timedelta(days=30)
date_generator = time_counter(
epoch - timedelta(days=3), delta=timedelta(days=2))
# And add 4 revisions at 33, 31, 29, and 27 days ago
branch = self.factory.makeAnyBranch()
self.factory.makeRevisionsForBranch(
branch, count=4, date_generator=date_generator)
RevisionSet.updateRevisionCacheForBranch(branch)
cached = self._getRevisionCache()
# Only two revisions are within the 30 day cutoff.
self.assertEqual(2, len(cached))
# And both the revisions stored are a date create after the epoch.
for rev in cached:
self.assertTrue(rev.revision_date > epoch)
def make_project_cloud_data(factory, details):
"""Make test data to populate the project cloud.
Details is a list of tuples containing:
(project-name, num_commits, num_authors, last_commit)
"""
delta = timedelta(seconds=1)
for project_name, num_commits, num_authors, last_commit in details:
project = factory.makeProduct(name=project_name)
start_date = last_commit - delta * (num_commits - 1)
gen = time_counter(start_date, delta)
commits_each = num_commits / num_authors
for committer in range(num_authors - 1):
make_project_branch_with_revisions(
factory, gen, project, commits_each)
num_commits -= commits_each
make_project_branch_with_revisions(
factory, gen, project, revision_count=num_commits)
transaction.commit()
def test_result_ordering_paranoia(self):
# Similar to test_result_ordering, but with results created in reverse
# order (this wouldn't really happen) but it shows that the id of the
# import result isn't used to sort by.
when = time_counter(
origin=datetime(2007, 9, 11, 12, tzinfo=pytz.UTC),
delta=timedelta(days=-1))
first = self.factory.makeCodeImportResult(
self.code_import, date_started=when.next())
second = self.factory.makeCodeImportResult(
self.code_import, date_started=when.next())
third = self.factory.makeCodeImportResult(
self.code_import, date_started=when.next())
self.assertTrue(first.date_job_started > second.date_job_started)
self.assertTrue(second.date_job_started > third.date_job_started)
results = list(self.code_import.results)
self.assertEqual(first, results[0])
self.assertEqual(second, results[1])
self.assertEqual(third, results[2])
def setUp(self):
TestCaseWithFactory.setUp(self)
self.user = self.factory.makePerson()
login_person(self.user)
self.barney = self.factory.makePerson(name='barney')
self.bambam = self.factory.makeProduct(name='bambam')
time_gen = time_counter(delta=timedelta(days=-1))
self.branches = [
self.factory.makeProductBranch(product=self.bambam,
owner=self.barney,
date_created=time_gen.next())
for i in range(10)
]
self.bug = self.factory.makeBug()
self.bug.linkBranch(self.branches[0], self.barney)
self.spec = self.factory.makeSpecification()
self.spec.linkBranch(self.branches[1], self.barney)
def test_old_revisions_not_added(self):
# Revisions older than the 30 day epoch are not added to the cache.
# Start 33 days ago.
epoch = datetime.now(pytz.UTC) - timedelta(days=30)
date_generator = time_counter(epoch - timedelta(days=3),
delta=timedelta(days=2))
# And add 4 revisions at 33, 31, 29, and 27 days ago
branch = self.factory.makeAnyBranch()
self.factory.makeRevisionsForBranch(branch,
count=4,
date_generator=date_generator)
RevisionSet.updateRevisionCacheForBranch(branch)
cached = self._getRevisionCache()
# Only two revisions are within the 30 day cutoff.
self.assertEqual(2, len(cached))
# And both the revisions stored are a date create after the epoch.
for rev in cached:
self.assertTrue(rev.revision_date > epoch)
def test_new_revisions_added(self):
# If there are already revisions in the cache for the branch, updating
# the branch again will only add the new revisions.
date_generator = time_counter(datetime.now(pytz.UTC) -
timedelta(days=29),
delta=timedelta(days=1))
# Initially add in 4 revisions.
branch = self.factory.makeAnyBranch()
self.factory.makeRevisionsForBranch(branch,
count=4,
date_generator=date_generator)
RevisionSet.updateRevisionCacheForBranch(branch)
# Now add two more revisions.
self.factory.makeRevisionsForBranch(branch,
count=2,
date_generator=date_generator)
RevisionSet.updateRevisionCacheForBranch(branch)
# There will be only six revisions cached.
cached = self._getRevisionCache()
self.assertEqual(6, len(cached))
def test_builds(self):
# LiveFSView.builds produces reasonable results.
livefs = self.makeLiveFS()
# Create oldest builds first so that they sort properly by id.
date_gen = time_counter(datetime(2000, 1, 1, tzinfo=pytz.UTC),
timedelta(days=1))
builds = [
self.makeBuild(livefs=livefs, date_created=next(date_gen))
for i in range(11)
]
view = LiveFSView(livefs, None)
self.assertEqual(list(reversed(builds)), view.builds)
self.setStatus(builds[10], BuildStatus.FULLYBUILT)
self.setStatus(builds[9], BuildStatus.FAILEDTOBUILD)
# When there are >= 9 pending builds, only the most recent of any
# completed builds is returned.
self.assertEqual(
list(reversed(builds[:9])) + [builds[10]], view.builds)
for build in builds[:9]:
self.setStatus(build, BuildStatus.FULLYBUILT)
self.assertEqual(list(reversed(builds[1:])), view.builds)
def test_committers_count_private_branch(self):
# Test that calling committer_count will return the proper value
# for a private branch.
fsm = self.factory.makePerson(email='*****@*****.**')
product, branch = self.makeProductAndDevelopmentFocusBranch(
owner=fsm, information_type=InformationType.USERDATA)
date_generator = time_counter(
datetime.now(pytz.UTC) - timedelta(days=30),
timedelta(days=1))
login_person(fsm)
self.factory.makeRevisionsForBranch(
branch, author='*****@*****.**',
date_generator=date_generator)
getUtility(IRevisionSet).updateRevisionCacheForBranch(branch)
view = create_initialized_view(product, '+code-index',
rootsite='code', principal=fsm)
self.assertEqual(view.committer_count, 1)
commit_section = find_tag_by_id(view.render(), 'commits')
self.assertIsNot(None, commit_section)
def make_package_branches(
factory, series, sourcepackagename, branch_count, official_count=0, owner=None, registrant=None
):
"""Make some package branches.
Make `branch_count` branches, and make `official_count` of those
official branches.
"""
if zisinstance(sourcepackagename, basestring):
sourcepackagename = factory.getOrMakeSourcePackageName(sourcepackagename)
# Make the branches created in the past in order.
time_gen = time_counter(delta=timedelta(days=-1))
branch_names = consistent_branch_names()
branches = [
factory.makePackageBranch(
distroseries=series,
sourcepackagename=sourcepackagename,
date_created=time_gen.next(),
name=branch_names.next(),
owner=owner,
registrant=registrant,
)
for i in range(branch_count)
]
official = []
# Sort the pocket items so RELEASE is last, and thus first popped.
pockets = sorted(PackagePublishingPocket.items, reverse=True)
# Since there can be only one link per pocket, max out the number of
# official branches at the pocket count.
for i in range(min(official_count, len(pockets))):
branch = branches.pop()
pocket = pockets.pop()
SeriesSourcePackageBranchSet.new(series, pocket, sourcepackagename, branch, branch.owner)
official.append(branch)
return series, branches, official
