def _MakePromises(self):
config_dict = self.services.config.GetProjectConfigs(
self.mr.cnxn, self.query_project_ids)
self.harmonized_config = tracker_bizobj.HarmonizeConfigs(
list(config_dict.values()))
self.canned_query = savedqueries_helpers.SavedQueryIDToCond(
self.mr.cnxn, self.services.features, self.mr.can)
self.canned_query, warnings = searchpipeline.ReplaceKeywordsWithUserIDs(
self.me_user_ids, self.canned_query)
self.mr.warnings.extend(warnings)
self.user_query, warnings = searchpipeline.ReplaceKeywordsWithUserIDs(
self.me_user_ids, self.mr.query)
self.mr.warnings.extend(warnings)
logging.debug('Searching query: %s %s', self.canned_query,
self.user_query)
slice_term = ('Issue.shard = %s', [self.mr.shard_id])
sd = sorting.ComputeSortDirectives(self.harmonized_config,
self.mr.group_by_spec,
self.mr.sort_spec)
self.result_iids_promise = framework_helpers.Promise(
_GetQueryResultIIDs, self.mr.cnxn, self.services,
self.canned_query, self.user_query, self.query_project_ids,
self.harmonized_config, sd, slice_term, self.mr.shard_id,
self.mr.invalidation_timestep)
def SetUpPromises(self, exp_query):
self.mox.StubOutWithMock(framework_helpers, 'Promise')
framework_helpers.Promise(
backendsearchpipeline._GetQueryResultIIDs, self.mr.cnxn,
self.services, 'is:open', exp_query, [789],
mox.IsA(tracker_pb2.ProjectIssueConfig), ['project', 'id'],
('Issue.shard = %s', [2]), 2, self.mr.invalidation_timestep
).AndReturn('fake promise 1')
def GetNonviewableIIDs(
self, cnxn, user, effective_ids, project, perms, shard_id):
"""Return a list of IIDs that the user cannot view in the project shard."""
# Project owners and site admins can see all issues.
if not perms.consider_restrictions:
return []
# There are two main parts to the computation that we do in parallel:
# getting at-risk IIDs and getting OK-iids.
cnxn_2 = sql.MonorailConnection()
at_risk_iids_promise = framework_helpers.Promise(
self.GetAtRiskIIDs, cnxn_2, user, effective_ids, project, perms, shard_id)
ok_iids = self.GetViewableIIDs(
cnxn, effective_ids, project.project_id, shard_id)
at_risk_iids = at_risk_iids_promise.WaitAndGetValue()
# The set of non-viewable issues is the at-risk ones minus the ones where
# the user is the reporter, owner, CC'd, or granted "View" permission.
nonviewable_iids = set(at_risk_iids).difference(ok_iids)
return list(nonviewable_iids)
def _CreateIssueSearchDocuments(issues, comments_dict, users_by_id,
config_dict):
"""Make the GAE search index documents for the given issue batch.
Args:
issues: list of issues to index.
comments_dict: prefetched dictionary of comments on those issues.
users_by_id: dictionary {user_id: UserView} so that the email
addresses of users who left comments can be found via search.
config_dict: dict {project_id: config} for all the projects that
the given issues are in.
"""
documents_by_shard = collections.defaultdict(list)
for issue in issues:
summary = issue.summary
# TODO(jrobbins): allow search specifically on explicit vs derived
# fields.
owner_id = tracker_bizobj.GetOwnerId(issue)
owner_email = users_by_id[owner_id].email
config = config_dict[issue.project_id]
component_paths = []
for component_id in issue.component_ids:
cd = tracker_bizobj.FindComponentDefByID(component_id, config)
if cd:
component_paths.append(cd.path)
field_values = [
tracker_bizobj.GetFieldValue(fv, users_by_id)
for fv in issue.field_values
]
# Convert to string only the values that are not strings already.
# This is done because the default encoding in appengine seems to be 'ascii'
# and string values might contain unicode characters, so str will fail to
# encode them.
field_values = [
value if isinstance(value, string_types) else str(value)
for value in field_values
]
metadata = '%s %s %s %s %s %s' % (
tracker_bizobj.GetStatus(issue), owner_email, [
users_by_id[cc_id].email
for cc_id in tracker_bizobj.GetCcIds(issue)
], ' '.join(component_paths), ' '.join(field_values), ' '.join(
tracker_bizobj.GetLabels(issue)))
custom_fields = _BuildCustomFTSFields(issue)
comments = comments_dict.get(issue.issue_id, [])
room_for_comments = (framework_constants.MAX_FTS_FIELD_SIZE -
len(summary) - len(metadata) -
sum(len(cf.value) for cf in custom_fields))
comments = _IndexableComments(comments,
users_by_id,
remaining_chars=room_for_comments)
logging.info('len(comments) is %r', len(comments))
if comments:
description = _ExtractCommentText(comments[0], users_by_id)
description = description[:framework_constants.MAX_FTS_FIELD_SIZE]
all_comments = ' '.join(
_ExtractCommentText(c, users_by_id) for c in comments[1:])
all_comments = all_comments[:framework_constants.
MAX_FTS_FIELD_SIZE]
else:
description = ''
all_comments = ''
logging.info('Issue %s:%r has zero indexable comments',
issue.project_name, issue.local_id)
logging.info('Building document for %s:%d', issue.project_name,
issue.local_id)
logging.info('len(summary) = %d', len(summary))
logging.info('len(metadata) = %d', len(metadata))
logging.info('len(description) = %d', len(description))
logging.info('len(comment) = %d', len(all_comments))
for cf in custom_fields:
logging.info('len(%s) = %d', cf.name, len(cf.value))
doc = search.Document(
doc_id=str(issue.issue_id),
fields=[
search.NumberField(name='project_id', value=issue.project_id),
search.TextField(name='summary', value=summary),
search.TextField(name='metadata', value=metadata),
search.TextField(name='description', value=description),
search.TextField(name='comment', value=all_comments),
] + custom_fields)
shard_id = issue.issue_id % settings.num_logical_shards
documents_by_shard[shard_id].append(doc)
start_time = time.time()
promises = []
for shard_id, documents in documents_by_shard.items():
if documents:
promises.append(
framework_helpers.Promise(_IndexDocsInShard, shard_id,
documents))
for promise in promises:
promise.WaitAndGetValue()
logging.info('Finished %d indexing in shards in %d ms',
len(documents_by_shard), int(
(time.time() - start_time) * 1000))
def GatherPageData(self, mr):
"""Build up a dictionary of data values to use when rendering the page.
Args:
mr: commonly used info parsed from the request.
Returns:
Dict of values used by EZT for rendering the page.
"""
if mr.local_id is None:
self.abort(404, 'no issue specified')
with work_env.WorkEnv(mr, self.services) as we:
# Signed in users could edit the issue, so it must be fresh.
use_cache = not mr.auth.user_id
issue = we.GetIssueByLocalID(
mr.project_id, mr.local_id, use_cache=use_cache)
# We give no explanation of missing issues on the peek page.
if issue.deleted:
self.abort(404, 'issue not found')
star_cnxn = sql.MonorailConnection()
star_promise = framework_helpers.Promise(
we.IsIssueStarred, issue, cnxn=star_cnxn)
config = we.GetProjectConfig(mr.project_id)
comments = we.ListIssueComments(issue)
descriptions, visible_comments, cmnt_pagination = PaginateComments(
mr, issue, comments, config, self.services)
with mr.profiler.Phase('making user proxies'):
involved_user_ids = tracker_bizobj.UsersInvolvedInIssues([issue])
group_ids = self.services.usergroup.DetermineWhichUserIDsAreGroups(
mr.cnxn, involved_user_ids)
comment_user_ids = tracker_bizobj.UsersInvolvedInCommentList(
descriptions + visible_comments)
users_by_id = framework_views.MakeAllUserViews(
mr.cnxn, self.services.user, involved_user_ids,
comment_user_ids, group_ids=group_ids)
framework_views.RevealAllEmailsToMembers(mr.auth, mr.project, users_by_id)
(issue_view, description_views,
comment_views) = self._MakeIssueAndCommentViews(
mr, issue, users_by_id, descriptions, visible_comments, config,
issue_reporters=[], comment_reporters=[])
with mr.profiler.Phase('getting starring info'):
starred = star_promise.WaitAndGetValue()
star_cnxn.Close()
permit_edit = permissions.CanEditIssue(
mr.auth.effective_ids, mr.perms, mr.project, issue)
mr.ComputeColSpec(config)
restrict_to_known = config.restrict_to_known
page_perms = self.MakePagePerms(
mr, issue,
permissions.CREATE_ISSUE,
permissions.SET_STAR,
permissions.EDIT_ISSUE,
permissions.EDIT_ISSUE_SUMMARY,
permissions.EDIT_ISSUE_STATUS,
permissions.EDIT_ISSUE_OWNER,
permissions.EDIT_ISSUE_CC,
permissions.DELETE_ISSUE,
permissions.ADD_ISSUE_COMMENT,
permissions.DELETE_OWN,
permissions.DELETE_ANY,
permissions.VIEW_INBOUND_MESSAGES)
page_perms.EditIssue = ezt.boolean(permit_edit)
prevent_restriction_removal = (
mr.project.only_owners_remove_restrictions and
not framework_bizobj.UserOwnsProject(
mr.project, mr.auth.effective_ids))
cmd_slots, default_slot_num = self.services.features.GetRecentCommands(
mr.cnxn, mr.auth.user_id, mr.project_id)
cmd_slot_views = [
template_helpers.EZTItem(
slot_num=slot_num, command=command, comment=comment)
for slot_num, command, comment in cmd_slots]
previous_locations = self.GetPreviousLocations(mr, issue)
return {
'issue_tab_mode': 'issueDetail',
'issue': issue_view,
'description': description_views,
'comments': comment_views,
'labels': issue.labels,
'num_detail_rows': len(comment_views) + 4,
'noisy': ezt.boolean(tracker_helpers.IsNoisy(
len(comment_views), issue.star_count)),
'cmnt_pagination': cmnt_pagination,
'colspec': mr.col_spec,
'searchtip': 'You can jump to any issue by number',
'starred': ezt.boolean(starred),
'pagegen': str(int(time.time() * 1000000)),
'restrict_to_known': ezt.boolean(restrict_to_known),
'prevent_restriction_removal': ezt.boolean(
prevent_restriction_removal),
'statuses_offer_merge': config.statuses_offer_merge,
'page_perms': page_perms,
'cmd_slots': cmd_slot_views,
'default_slot_num': default_slot_num,
'quick_edit_submit_url': tracker_helpers.FormatRelativeIssueURL(
issue.project_name, urls.ISSUE_PEEK + '.do', id=issue.local_id),
'previous_locations': previous_locations,
# for template issue-meta-part shared by issuedetail servlet
'user_remaining_hotlists': [],
'user_issue_hotlists': [],
'involved_users_issue_hotlists': [],
'remaining_issue_hotlists': [],
}
def _CreateIssueSearchDocuments(issues, comments_dict, users_by_id,
config_dict):
"""Make the GAE search index documents for the given issue batch.
Args:
issues: list of issues to index.
comments_dict: prefetched dictionary of comments on those issues.
users_by_id: dictionary {user_id: UserView} so that the email
addresses of users who left comments can be found via search.
config_dict: dict {project_id: config} for all the projects that
the given issues are in.
"""
documents_by_shard = collections.defaultdict(list)
for issue in issues:
comments = comments_dict.get(issue.issue_id, [])
comments = _IndexableComments(comments, users_by_id)
summary = issue.summary
# TODO(jrobbins): allow search specifically on explicit vs derived
# fields.
owner_id = tracker_bizobj.GetOwnerId(issue)
owner_email = users_by_id[owner_id].email
config = config_dict[issue.project_id]
component_paths = []
for component_id in issue.component_ids:
cd = tracker_bizobj.FindComponentDefByID(component_id, config)
if cd:
component_paths.append(cd.path)
field_values = [
str(tracker_bizobj.GetFieldValue(fv, users_by_id))
for fv in issue.field_values
]
metadata = '%s %s %s %s %s %s' % (
tracker_bizobj.GetStatus(issue), owner_email, [
users_by_id[cc_id].email
for cc_id in tracker_bizobj.GetCcIds(issue)
], ' '.join(component_paths), ' '.join(field_values), ' '.join(
tracker_bizobj.GetLabels(issue)))
assert comments, 'issues should always have at least the description'
description = _ExtractCommentText(comments[0], users_by_id)
description = description[:framework_constants.MAX_FTS_FIELD_SIZE]
all_comments = ' '.join(
_ExtractCommentText(c, users_by_id) for c in comments[1:])
all_comments = all_comments[:framework_constants.MAX_FTS_FIELD_SIZE]
custom_fields = _BuildCustomFTSFields(issue)
doc = search.Document(
doc_id=str(issue.issue_id),
fields=[
search.NumberField(name='project_id', value=issue.project_id),
search.TextField(name='summary', value=summary),
search.TextField(name='metadata', value=metadata),
search.TextField(name='description', value=description),
search.TextField(name='comment', value=all_comments),
] + custom_fields)
shard_id = issue.issue_id % settings.num_logical_shards
documents_by_shard[shard_id].append(doc)
start_time = time.time()
promises = []
for shard_id, documents in documents_by_shard.iteritems():
if documents:
promises.append(
framework_helpers.Promise(_IndexDocsInShard, shard_id,
documents))
for promise in promises:
promise.WaitAndGetValue()
logging.info('Finished %d indexing in shards in %d ms',
len(documents_by_shard), int(
(time.time() - start_time) * 1000))
def QueryIssueSnapshots(self,
cnxn,
services,
unixtime,
effective_ids,
project,
perms,
group_by=None,
label_prefix=None,
query=None,
canned_query=None):
"""Queries historical issue counts grouped by label or component.
Args:
cnxn: A MonorailConnection instance.
services: A Services instance.
unixtime: An integer representing the Unix time in seconds.
effective_ids: The effective User IDs associated with the current user.
project: A project object representing the current project.
perms: A permissions object associated with the current user.
group_by (str, optional): Which dimension to group by. Values can
be 'label', 'component', or None, in which case no grouping will
be applied.
label_prefix: Required when group_by is 'label.' Will limit the query to
only labels with the specified prefix (for example 'Pri').
query (str, optional): A query string from the request to apply to
the snapshot query.
canned_query (str, optional): Parsed canned query applied to the query
scope.
Returns:
1. A dict of {'2nd dimension or "total"': number of occurences}.
2. A list of any unsupported query conditions in query.
3. A boolean that is true if any results were capped.
"""
project_config = services.config.GetProjectConfig(
cnxn, project.project_id)
try:
query_left_joins, query_where, unsupported_conds = self._QueryToWhere(
cnxn, services, project_config, query, canned_query, project)
except ast2select.NoPossibleResults:
return {}, ['Invalid query.'], False
restricted_label_ids = search_helpers.GetPersonalAtRiskLabelIDs(
cnxn, None, self.config_service, effective_ids, project, perms)
left_joins = [
('Issue ON IssueSnapshot.issue_id = Issue.id', []),
]
if restricted_label_ids:
left_joins.append((('Issue2Label AS Forbidden_label'
' ON Issue.id = Forbidden_label.issue_id'
' AND Forbidden_label.label_id IN (%s)' %
(sql.PlaceHolders(restricted_label_ids))),
restricted_label_ids))
if effective_ids:
left_joins.append(
('Issue2Cc AS I2cc'
' ON Issue.id = I2cc.issue_id'
' AND I2cc.cc_id IN (%s)' % sql.PlaceHolders(effective_ids),
effective_ids))
# TODO(jeffcarp): Handle case where there are issues with no labels.
where = [
('IssueSnapshot.period_start <= %s', [unixtime]),
('IssueSnapshot.period_end > %s', [unixtime]),
('IssueSnapshot.project_id = %s', [project.project_id]),
('Issue.is_spam = %s', [False]),
('Issue.deleted = %s', [False]),
]
forbidden_label_clause = 'Forbidden_label.label_id IS NULL'
if effective_ids:
if restricted_label_ids:
forbidden_label_clause = ' OR %s' % forbidden_label_clause
else:
forbidden_label_clause = ''
where.append(
(('(Issue.reporter_id IN (%s)'
' OR Issue.owner_id IN (%s)'
' OR I2cc.cc_id IS NOT NULL'
'%s)') %
(sql.PlaceHolders(effective_ids),
sql.PlaceHolders(effective_ids), forbidden_label_clause),
list(effective_ids) + list(effective_ids)))
else:
where.append((forbidden_label_clause, []))
if group_by == 'component':
cols = ['Comp.path', 'IssueSnapshot.issue_id']
left_joins.extend([
(('IssueSnapshot2Component AS Is2c ON'
' Is2c.issuesnapshot_id = IssueSnapshot.id'), []),
('ComponentDef AS Comp ON Comp.id = Is2c.component_id', []),
])
group_by = ['Comp.path']
elif group_by == 'label':
cols = ['Lab.label', 'IssueSnapshot.issue_id']
left_joins.extend([
(('IssueSnapshot2Label AS Is2l'
' ON Is2l.issuesnapshot_id = IssueSnapshot.id'), []),
('LabelDef AS Lab ON Lab.id = Is2l.label_id', []),
])
if not label_prefix:
raise ValueError(
'`label_prefix` required when grouping by label.')
# TODO(jeffcarp): If LookupIDsOfLabelsMatching() is called on output,
# ensure regex is case-insensitive.
where.append(
('LOWER(Lab.label) LIKE %s', [label_prefix.lower() + '-%']))
group_by = ['Lab.label']
elif group_by == 'open':
cols = ['IssueSnapshot.is_open', 'IssueSnapshot.issue_id']
group_by = ['IssueSnapshot.is_open']
elif group_by == 'status':
left_joins.append(('StatusDef AS Stats ON ' \
'Stats.id = IssueSnapshot.status_id', []))
cols = ['Stats.status', 'IssueSnapshot.issue_id']
group_by = ['Stats.status']
elif group_by == 'owner':
cols = ['IssueSnapshot.owner_id', 'IssueSnapshot.issue_id']
group_by = ['IssueSnapshot.owner_id']
elif not group_by:
cols = ['IssueSnapshot.issue_id']
else:
raise ValueError('`group_by` must be label, component, ' \
'open, status, owner or None.')
if query_left_joins:
left_joins.extend(query_left_joins)
if query_where:
where.extend(query_where)
promises = []
for shard_id in range(settings.num_logical_shards):
count_stmt, stmt_args = self._BuildSnapshotQuery(cols=cols,
where=where,
joins=left_joins,
group_by=group_by,
shard_id=shard_id)
promises.append(
framework_helpers.Promise(cnxn.Execute,
count_stmt,
stmt_args,
shard_id=shard_id))
shard_values_dict = {}
search_limit_reached = False
for promise in promises:
# Wait for each query to complete and add it to the dict.
shard_values = list(promise.WaitAndGetValue())
if not shard_values:
continue
if group_by:
for name, count in shard_values:
if count >= settings.chart_query_max_rows:
search_limit_reached = True
shard_values_dict.setdefault(name, 0)
shard_values_dict[name] += count
else:
if shard_values[0][0] >= settings.chart_query_max_rows:
search_limit_reached = True
shard_values_dict.setdefault('total', 0)
shard_values_dict['total'] += shard_values[0][0]
unsupported_field_names = list(
set([
field.field_name for cond in unsupported_conds
for field in cond.field_defs
]))
return shard_values_dict, unsupported_field_names, search_limit_reached