def _scopedGetMergeProposals(self, statuses, eager_load=False):
expressions = (self._filter_expressions +
self._getRepositoryVisibilityExpression())
with_expr = With(
"candidate_repositories",
Select(GitRepository.id,
tables=[GitRepository] + self._tables.values(),
where=And(*expressions) if expressions else True))
expressions = [
SQL("""
source_git_repository IN
(SELECT id FROM candidate_repositories) AND
target_git_repository IN
(SELECT id FROM candidate_repositories)""")
]
tables = [BranchMergeProposal]
if self._asymmetric_filter_expressions:
# Need to filter on GitRepository beyond the with constraints.
expressions += self._asymmetric_filter_expressions
expressions.append(BranchMergeProposal.source_git_repositoryID ==
GitRepository.id)
tables.append(GitRepository)
tables.extend(self._asymmetric_tables.values())
if statuses is not None:
expressions.append(
BranchMergeProposal.queue_status.is_in(statuses))
resultset = self.store.with_(with_expr).using(*tables).find(
BranchMergeProposal, *expressions)
if not eager_load:
return resultset
else:
loader = partial(BranchMergeProposal.preloadDataForBMPs,
user=self._user)
return DecoratedResultSet(resultset, pre_iter_hook=loader)
def get_bugsummary_filter_for_user(user):
"""Build a Storm expression to filter BugSummary by visibility.
:param user: The user for which visible rows should be calculated.
:return: (with_clauses, where_clauses)
"""
# Admins get to see every bug, everyone else only sees bugs
# viewable by them-or-their-teams.
# Note that because admins can see every bug regardless of
# subscription they will see rather inflated counts. Admins get to
# deal.
public_filter = And(BugSummary.viewed_by_id == None,
BugSummary.access_policy_id == None)
if user is None:
return [], [public_filter]
elif IPersonRoles(user).in_admin:
return [], []
else:
with_clauses = [
With(
'teams',
Select(TeamParticipation.teamID,
tables=[TeamParticipation],
where=(TeamParticipation.personID == user.id))),
With(
'policies',
Select(AccessPolicyGrant.policy_id,
tables=[AccessPolicyGrant],
where=(AccessPolicyGrant.grantee_id.is_in(
SQL("SELECT team FROM teams"))))),
]
where_clauses = [
Or(
public_filter,
BugSummary.viewed_by_id.is_in(SQL("SELECT team FROM teams")),
BugSummary.access_policy_id.is_in(
SQL("SELECT policy FROM policies")))
]
return with_clauses, where_clauses
def _getSharedPillars(self, person, user, pillar_class, extra_filter=None):
"""Helper method for getSharedProjects and getSharedDistributions.
pillar_class is either Product or Distribution. Products define the
owner foreign key attribute as _owner so we need to account for that,
but otherwise the logic is the same for both pillar types.
"""
if user is None:
return []
store = IStore(AccessPolicyGrantFlat)
roles = IPersonRoles(user)
if roles.in_admin:
filter = True
else:
with_statement = With("teams",
Select(TeamParticipation.teamID,
tables=TeamParticipation,
where=TeamParticipation.person == user.id))
teams_sql = SQL("SELECT team from teams")
store = store.with_(with_statement)
if IProduct.implementedBy(pillar_class):
ownerID = pillar_class._ownerID
else:
ownerID = pillar_class.ownerID
filter = Or(
extra_filter or False,
ownerID.is_in(teams_sql),
pillar_class.driverID.is_in(teams_sql))
tables = [
AccessPolicyGrantFlat,
Join(
AccessPolicy,
AccessPolicyGrantFlat.policy_id == AccessPolicy.id)]
if IProduct.implementedBy(pillar_class):
access_policy_column = AccessPolicy.product_id
else:
access_policy_column = AccessPolicy.distribution_id
result_set = store.find(
pillar_class,
pillar_class.id.is_in(
Select(
columns=access_policy_column, tables=tables,
where=(AccessPolicyGrantFlat.grantee_id == person.id))
), filter)
return result_set
def load_teams_and_permissions(grantees):
# We now have the grantees we want in the result so load any
# associated team memberships and permissions and cache them.
if permissions_cache:
return
store = IStore(cls)
for grantee in grantees:
grantees_by_id[grantee[0].id] = grantee[0]
# Find any teams associated with the grantees. If grantees is a
# sliced list (for batching), it may contain indirect grantees but
# not the team they belong to so that needs to be fixed below.
with_expr = With(
"grantees",
store.find(cls.grantee_id,
cls.policy_id.is_in(policies_by_id.keys())).config(
distinct=True)._get_select())
result_set = store.with_(with_expr).find(
(TeamParticipation.teamID, TeamParticipation.personID),
TeamParticipation.personID.is_in(grantees_by_id.keys()),
TeamParticipation.teamID.is_in(
Select((SQL("grantees.grantee"), ),
tables="grantees",
distinct=True)))
team_ids = set()
direct_grantee_ids = set()
for team_id, team_member_id in result_set:
if team_member_id == team_id:
direct_grantee_ids.add(team_member_id)
else:
via_teams_cache[team_member_id].append(team_id)
team_ids.add(team_id)
# Remove from the via_teams cache all the direct grantees.
for direct_grantee_id in direct_grantee_ids:
if direct_grantee_id in via_teams_cache:
del via_teams_cache[direct_grantee_id]
# Load and cache the additional required teams.
persons = store.find(Person, Person.id.is_in(team_ids))
for person in persons:
grantees_by_id[person.id] = person
cls._populatePermissionsCache(permissions_cache,
shared_artifact_info_types,
grantees_by_id.keys(),
policies_by_id, grantees_by_id)
def preloadBuildsData(self, builds):
# Circular import.
from lp.snappy.model.snap import Snap
load_related(Person, builds, ["requester_id"])
lfas = load_related(LibraryFileAlias, builds, ["log_id"])
load_related(LibraryFileContent, lfas, ["contentID"])
archives = load_related(Archive, builds, ["archive_id"])
load_related(Person, archives, ["ownerID"])
distroarchseries = load_related(
DistroArchSeries, builds, ['distro_arch_series_id'])
distroseries = load_related(
DistroSeries, distroarchseries, ['distroseriesID'])
load_related(Distribution, distroseries, ['distributionID'])
snaps = load_related(Snap, builds, ["snap_id"])
getUtility(ISnapSet).preloadDataForSnaps(snaps)
snapbuild_ids = set(map(attrgetter("id"), builds))
latest_jobs_cte = With("LatestJobs", Select(
(SnapBuildJob.job_id,
SQL(
"rank() OVER "
"(PARTITION BY snapbuild ORDER BY job DESC) AS rank")),
tables=SnapBuildJob,
where=And(
SnapBuildJob.snapbuild_id.is_in(snapbuild_ids),
SnapBuildJob.job_type == SnapBuildJobType.STORE_UPLOAD)))
LatestJobs = Table("LatestJobs")
sbjs = list(IStore(SnapBuildJob).with_(latest_jobs_cte).using(
SnapBuildJob, LatestJobs).find(
SnapBuildJob,
SnapBuildJob.job_id == Column("job", LatestJobs),
Column("rank", LatestJobs) == 1))
sbj_map = {}
for sbj in sbjs:
sbj_map[sbj.snapbuild] = sbj.makeDerived()
for build in builds:
get_property_cache(build).last_store_upload_job = (
sbj_map.get(build))
load_related(Job, sbjs, ["job_id"])
def calculate_bugsummary_rows(target):
"""Calculate BugSummary row fragments for the given `IBugTarget`.
The data is re-aggregated from BugTaskFlat, BugTag and BugSubscription.
"""
# Use a CTE to prepare a subset of BugTaskFlat, filtered to the
# relevant target and to exclude duplicates, and with has_patch
# calculated.
relevant_tasks = With(
'relevant_task',
Select((BugTaskFlat.bug_id, BugTaskFlat.information_type,
BugTaskFlat.status, BugTaskFlat.milestone_id,
BugTaskFlat.importance,
Alias(BugTaskFlat.latest_patch_uploaded != None, 'has_patch'),
BugTaskFlat.access_grants, BugTaskFlat.access_policies),
tables=[BugTaskFlat],
where=And(BugTaskFlat.duplicateof_id == None,
*get_bugtaskflat_constraint(target))))
# Storm class to reference the CTE.
class RelevantTask(BugTaskFlat):
__storm_table__ = 'relevant_task'
has_patch = Bool()
# Storm class to reference the union.
class BugSummaryPrototype(RawBugSummary):
__storm_table__ = 'bugsummary_prototype'
# Prepare a union for all combination of privacy and taggedness.
# It'll return a full set of
# (status, milestone, importance, has_patch, tag, viewed_by, access_policy)
# rows.
common_cols = (RelevantTask.status, RelevantTask.milestone_id,
RelevantTask.importance, RelevantTask.has_patch)
null_tag = Alias(Cast(None, 'text'), 'tag')
null_viewed_by = Alias(Cast(None, 'integer'), 'viewed_by')
null_policy = Alias(Cast(None, 'integer'), 'access_policy')
tag_join = Join(BugTag, BugTag.bugID == RelevantTask.bug_id)
public_constraint = RelevantTask.information_type.is_in(
PUBLIC_INFORMATION_TYPES)
private_constraint = RelevantTask.information_type.is_in(
PRIVATE_INFORMATION_TYPES)
unions = Union(
# Public, tagless
Select(common_cols + (null_tag, null_viewed_by, null_policy),
tables=[RelevantTask],
where=public_constraint),
# Public, tagged
Select(common_cols + (BugTag.tag, null_viewed_by, null_policy),
tables=[RelevantTask, tag_join],
where=public_constraint),
# Private, access grant, tagless
Select(common_cols +
(null_tag, Unnest(RelevantTask.access_grants), null_policy),
tables=[RelevantTask],
where=private_constraint),
# Private, access grant, tagged
Select(common_cols +
(BugTag.tag, Unnest(RelevantTask.access_grants), null_policy),
tables=[RelevantTask, tag_join],
where=private_constraint),
# Private, access policy, tagless
Select(
common_cols +
(null_tag, null_viewed_by, Unnest(RelevantTask.access_policies)),
tables=[RelevantTask],
where=private_constraint),
# Private, access policy, tagged
Select(
common_cols +
(BugTag.tag, null_viewed_by, Unnest(RelevantTask.access_policies)),
tables=[RelevantTask, tag_join],
where=private_constraint),
all=True)
# Select the relevant bits of the prototype rows and aggregate them.
proto_key_cols = (BugSummaryPrototype.status,
BugSummaryPrototype.milestone_id,
BugSummaryPrototype.importance,
BugSummaryPrototype.has_patch, BugSummaryPrototype.tag,
BugSummaryPrototype.viewed_by_id,
BugSummaryPrototype.access_policy_id)
origin = IStore(BugTaskFlat).with_(relevant_tasks).using(
Alias(unions, 'bugsummary_prototype'))
results = origin.find(proto_key_cols + (Count(), ))
results = results.group_by(*proto_key_cols).order_by(*proto_key_cols)
return results
def _composePOFileReviewerCTEs(self, no_older_than):
"""Compose Storm CTEs for common `POFile` queries.
Returns a list of Storm CTEs, much the same as
_composePOFileReviewerJoins."""
clause = [
POFileTranslator.personID == self.person.id,
POFile.language != getUtility(ILaunchpadCelebrities).english]
if no_older_than:
clause.append(POFileTranslator.date_last_touched >= no_older_than)
RecentPOFiles = With("recent_pofiles",
Select(
(POFile.id,),
tables=[
POFileTranslator,
Join(POFile, POFileTranslator.pofile == POFile.id)],
where=And(*clause)))
ReviewableGroups = With("reviewable_groups",
Select(
(TranslationGroup.id, Translator.languageID),
tables=[
TranslationGroup,
Join(
Translator,
Translator.translationgroupID == TranslationGroup.id),
Join(
TeamParticipation,
And(
TeamParticipation.teamID ==
Translator.translatorID,
TeamParticipation.personID == self.person.id))]))
TranslatableDistroSeries = With("translatable_distroseries",
Select(
(DistroSeries.id, SQL('reviewable_groups.language')),
tables=[
DistroSeries,
Join(
Distribution,
And(
Distribution.id == DistroSeries.distributionID,
Distribution.translations_usage ==
ServiceUsage.LAUNCHPAD,
Distribution.translation_focusID ==
DistroSeries.id)),
Join(
SQL('reviewable_groups'),
SQL('reviewable_groups.id') ==
Distribution.translationgroupID)]))
TranslatableProductSeries = With("translatable_productseries",
Select(
(ProductSeries.id, SQL('reviewable_groups.language')),
tables=[
ProductSeries,
Join(
Product, And(
Product.id == ProductSeries.productID,
Product.translations_usage ==
ServiceUsage.LAUNCHPAD,
Product.active == True)),
LeftJoin(
ProjectGroup, ProjectGroup.id == Product.projectID),
Join(
SQL('reviewable_groups'),
SQL('reviewable_groups.id') ==
Product.translationgroupID)]))
return [
RecentPOFiles, ReviewableGroups, TranslatableDistroSeries,
TranslatableProductSeries]
def search_specifications(context,
base_clauses,
user,
sort=None,
quantity=None,
spec_filter=None,
tables=[],
default_acceptance=False,
need_people=True,
need_branches=True,
need_workitems=False):
store = IStore(Specification)
if not default_acceptance:
default = SpecificationFilter.INCOMPLETE
options = set(
[SpecificationFilter.COMPLETE, SpecificationFilter.INCOMPLETE])
else:
default = SpecificationFilter.ACCEPTED
options = set([
SpecificationFilter.ACCEPTED, SpecificationFilter.DECLINED,
SpecificationFilter.PROPOSED
])
if not spec_filter:
spec_filter = [default]
if not set(spec_filter) & options:
spec_filter.append(default)
if not tables:
tables = [Specification]
clauses = base_clauses
product_tables, product_clauses = get_specification_active_product_filter(
context)
tables.extend(product_tables)
clauses.extend(product_clauses)
# If there are any base or product clauses, they typically have good
# selectivity, so use a CTE to force PostgreSQL to calculate them
# up-front rather than doing a sequential scan for visible
# specifications.
if clauses:
RelevantSpecification = Table('RelevantSpecification')
relevant_specification_cte = With(
RelevantSpecification.name,
Select(Specification.id, And(clauses), tables=tables))
store = store.with_(relevant_specification_cte)
tables = [Specification]
clauses = [
Specification.id.is_in(Select(Column('id',
RelevantSpecification))),
]
clauses.extend(get_specification_privacy_filter(user))
clauses.extend(get_specification_filters(spec_filter))
# Sort by priority descending, by default.
if sort is None or sort == SpecificationSort.PRIORITY:
order = [
Desc(Specification.priority), Specification.definition_status,
Specification.name
]
elif sort == SpecificationSort.DATE:
if SpecificationFilter.COMPLETE in spec_filter:
# If we are showing completed, we care about date completed.
order = [Desc(Specification.date_completed), Specification.id]
else:
# If not specially looking for complete, we care about date
# registered.
order = []
show_proposed = set(
[SpecificationFilter.ALL, SpecificationFilter.PROPOSED])
if default_acceptance and not (set(spec_filter) & show_proposed):
order.append(Desc(Specification.date_goal_decided))
order.extend([Desc(Specification.datecreated), Specification.id])
else:
order = [sort]
# Set the _known_viewers property for each specification, as well as
# preloading the objects involved, if asked.
def preload_hook(rows):
person_ids = set()
work_items_by_spec = defaultdict(list)
for spec in rows:
if need_people:
person_ids |= set(
[spec._assigneeID, spec._approverID, spec._drafterID])
if need_branches:
get_property_cache(spec).linked_branches = []
if need_workitems:
work_items = load_referencing(
SpecificationWorkItem,
rows, ['specification_id'],
extra_conditions=[SpecificationWorkItem.deleted == False])
for workitem in work_items:
person_ids.add(workitem.assignee_id)
work_items_by_spec[workitem.specification_id].append(workitem)
person_ids -= set([None])
if need_people:
list(
getUtility(IPersonSet).getPrecachedPersonsFromIDs(
person_ids, need_validity=True))
if need_workitems:
for spec in rows:
get_property_cache(spec).work_items = sorted(
work_items_by_spec[spec.id], key=lambda wi: wi.sequence)
if need_branches:
spec_branches = load_referencing(SpecificationBranch, rows,
['specificationID'])
for sbranch in spec_branches:
spec_cache = get_property_cache(sbranch.specification)
spec_cache.linked_branches.append(sbranch)
decorators = []
if user is not None and not IPersonRoles(user).in_admin:
decorators.append(_make_cache_user_can_view_spec(user))
results = store.using(*tables).find(
Specification, *clauses).order_by(*order).config(limit=quantity)
return DecoratedResultSet(
results,
lambda row: reduce(lambda task, dec: dec(task), decorators, row),
pre_iter_hook=preload_hook)