def testGetLabels(self):
issue = tracker_pb2.Issue()
self.assertEquals(tracker_bizobj.GetLabels(issue), [])
issue.derived_labels.extend(['a', 'b', 'c'])
self.assertEquals(tracker_bizobj.GetLabels(issue), ['a', 'b', 'c'])
issue.labels.extend(['d', 'e', 'f'])
self.assertEquals(tracker_bizobj.GetLabels(issue),
['d', 'e', 'f', 'a', 'b', 'c'])
def EvaluateSubscriptions(
cnxn, issue, users_to_queries, services, config):
"""Determine subscribers who have subs that match the given issue."""
# Note: unlike filter rule, subscriptions see explicit & derived values.
lower_labels = [lab.lower() for lab in tracker_bizobj.GetLabels(issue)]
label_set = set(lower_labels)
subscribers_to_notify = []
for uid, saved_queries in users_to_queries.items():
for sq in saved_queries:
if sq.subscription_mode != 'immediate':
continue
if issue.project_id not in sq.executes_in_project_ids:
continue
cond = savedqueries_helpers.SavedQueryToCond(sq)
# TODO(jrobbins): Support linked accounts me_user_ids.
cond, _warnings = searchpipeline.ReplaceKeywordsWithUserIDs([uid], cond)
cond_ast = query2ast.ParseUserQuery(
cond, '', query2ast.BUILTIN_ISSUE_FIELDS, config)
if filterrules_helpers.EvalPredicate(
cnxn, services, cond_ast, issue, label_set, config,
tracker_bizobj.GetOwnerId(issue), tracker_bizobj.GetCcIds(issue),
tracker_bizobj.GetStatus(issue)):
subscribers_to_notify.append(uid)
break # Don't bother looking at the user's other saved quereies.
return subscribers_to_notify
def GetRestrictions(issue, perm=''):
"""Return a list of restriction labels on the given issue."""
if not issue:
return []
return [lab.lower() for lab in tracker_bizobj.GetLabels(issue)
if IsRestrictLabel(lab, perm=perm)]
def _SortableLabelValues(art, col_name, well_known_value_indexes):
"""Return a list of ints and strings for labels relevant to one UI column."""
col_name_dash = col_name + '-'
sortable_value_list = []
for label in tracker_bizobj.GetLabels(art):
idx_or_lex = well_known_value_indexes.get(label)
if idx_or_lex == IGNORABLE_INDICATOR:
continue # Label is known to not have the desired prefix.
if idx_or_lex is None:
if '-' not in label:
# Skip an irrelevant OneWord label and remember to ignore it later.
well_known_value_indexes[label] = IGNORABLE_INDICATOR
continue
label_lower = label.lower()
if label_lower.startswith(col_name_dash):
# Label is a key-value label with an odd-ball value, remember it
value = label_lower[len(col_name_dash):]
idx_or_lex = value
well_known_value_indexes[label] = value
else:
# Label was a key-value label that is not relevant to this column.
# Remember to ignore it later.
well_known_value_indexes[label] = IGNORABLE_INDICATOR
continue
sortable_value_list.append(idx_or_lex)
return sortable_value_list
def Accessor(art):
"""Return a list of label values on the given artifact."""
result = [
label.lower() for label in tracker_bizobj.GetLabels(art)
if label.lower().startswith(prefix)
]
return result
def MakeLabelValuesDict(art):
"""Return a dict of label values and a list of one-word labels.
Args:
art: artifact object, e.g., an issue PB.
Returns:
A dict {prefix: [suffix,...], ...} for each key-value label.
"""
label_values = collections.defaultdict(list)
for label_name in tracker_bizobj.GetLabels(art):
if '-' in label_name:
key, value = label_name.split('-', 1)
label_values[key.lower()].append(value)
return label_values
def __init__(self, issue):
# List of restrictions that don't map to a known action kind.
self.other = []
restrictions_by_action = collections.defaultdict(list)
# We can't use GetRestrictions here, as we prefer to preserve
# the case of the label when showing restrictions in the UI.
for label in tracker_bizobj.GetLabels(issue):
if permissions.IsRestrictLabel(label):
_kw, action_kind, needed_perm = label.split('-', 2)
action_kind = action_kind.lower()
if action_kind in self._KNOWN_ACTION_KINDS:
restrictions_by_action[action_kind].append(needed_perm)
else:
self.other.append(label)
self.view = ' and '.join(restrictions_by_action[self._VIEW])
self.add_comment = ' and '.join(restrictions_by_action[self._ADD_COMMENT])
self.edit = ' and '.join(restrictions_by_action[self._EDIT])
self.has_restrictions = ezt.boolean(
self.view or self.add_comment or self.edit or self.other)
def ComputeUnshownColumns(results, shown_columns, config, built_in_cols):
"""Return a list of unshown columns that the user could add.
Args:
results: list of search result PBs. Each must have labels.
shown_columns: list of column names to be used in results table.
config: harmonized config for the issue search, including all
well known labels and custom fields.
built_in_cols: list of other column names that are built into the tool.
E.g., star count, or creation date.
Returns:
List of column names to append to the "..." menu.
"""
unshown_set = set() # lowercases column names
unshown_list = [] # original-case column names
shown_set = {col.lower() for col in shown_columns}
labels_already_seen = set() # whole labels, original case
def _MaybeAddLabel(label_name):
"""Add the key part of the given label if needed."""
if label_name.lower() in labels_already_seen:
return
labels_already_seen.add(label_name.lower())
if '-' in label_name:
col, _value = label_name.split('-', 1)
_MaybeAddCol(col)
def _MaybeAddCol(col):
if col.lower() not in shown_set and col.lower() not in unshown_set:
unshown_list.append(col)
unshown_set.add(col.lower())
# The user can always add any of the default columns.
for col in config.default_col_spec.split():
_MaybeAddCol(col)
# The user can always add any of the built-in columns.
for col in built_in_cols:
_MaybeAddCol(col)
# The user can add a column for any well-known labels
for wkl in config.well_known_labels:
_MaybeAddLabel(wkl.label)
phase_names = set(
itertools.chain.from_iterable((phase.name.lower()
for phase in result.phases)
for result in results))
# The user can add a column for any custom field
field_ids_alread_seen = set()
for fd in config.field_defs:
field_lower = fd.field_name.lower()
field_ids_alread_seen.add(fd.field_id)
if fd.is_phase_field:
for name in phase_names:
phase_field_col = name + '.' + field_lower
if (phase_field_col not in shown_set
and phase_field_col not in unshown_set):
unshown_list.append(phase_field_col)
unshown_set.add(phase_field_col)
elif field_lower not in shown_set and field_lower not in unshown_set:
unshown_list.append(fd.field_name)
unshown_set.add(field_lower)
if fd.field_type == tracker_pb2.FieldTypes.APPROVAL_TYPE:
approval_lower_approver = (field_lower +
tracker_constants.APPROVER_COL_SUFFIX)
if (approval_lower_approver not in shown_set
and approval_lower_approver not in unshown_set):
unshown_list.append(fd.field_name +
tracker_constants.APPROVER_COL_SUFFIX)
unshown_set.add(approval_lower_approver)
# The user can add a column for any key-value label or field in the results.
for r in results:
for label_name in tracker_bizobj.GetLabels(r):
_MaybeAddLabel(label_name)
for field_value in r.field_values:
if field_value.field_id not in field_ids_alread_seen:
field_ids_alread_seen.add(field_value.field_id)
fd = tracker_bizobj.FindFieldDefByID(field_value.field_id,
config)
if fd: # could be None for a foreign field, which we don't display.
field_lower = fd.field_name.lower()
if field_lower not in shown_set and field_lower not in unshown_set:
unshown_list.append(fd.field_name)
unshown_set.add(field_lower)
return sorted(unshown_list)
def ExtractUniqueValues(columns,
artifact_list,
users_by_id,
config,
related_issues,
hotlist_context_dict=None):
"""Build a nested list of unique values so the user can auto-filter.
Args:
columns: a list of lowercase column name strings, which may contain
combined columns like "priority/pri".
artifact_list: a list of artifacts in the complete set of search results.
users_by_id: dict mapping user_ids to UserViews.
config: ProjectIssueConfig PB for the current project.
related_issues: dict {issue_id: issue} of pre-fetched related issues.
hotlist_context_dict: dict for building a hotlist grid table
Returns:
[EZTItem(col1, colname1, [val11, val12,...]), ...]
A list of EZTItems, each of which has a col_index, column_name,
and a list of unique values that appear in that column.
"""
column_values = {col_name: {} for col_name in columns}
# For each combined column "a/b/c", add entries that point from "a" back
# to "a/b/c", from "b" back to "a/b/c", and from "c" back to "a/b/c".
combined_column_parts = collections.defaultdict(list)
for col in columns:
if '/' in col:
for col_part in col.split('/'):
combined_column_parts[col_part].append(col)
unique_labels = set()
for art in artifact_list:
unique_labels.update(tracker_bizobj.GetLabels(art))
for label in unique_labels:
if '-' in label:
col, val = label.split('-', 1)
col = col.lower()
if col in column_values:
column_values[col][val.lower()] = val
if col in combined_column_parts:
for combined_column in combined_column_parts[col]:
column_values[combined_column][val.lower()] = val
else:
if 'summary' in column_values:
column_values['summary'][label.lower()] = label
# TODO(jrobbins): Consider refacting some of this to tracker_bizobj
# or a new builtins.py to reduce duplication.
if 'reporter' in column_values:
for art in artifact_list:
reporter_id = art.reporter_id
if reporter_id and reporter_id in users_by_id:
reporter_username = users_by_id[reporter_id].display_name
column_values['reporter'][
reporter_username] = reporter_username
if 'owner' in column_values:
for art in artifact_list:
owner_id = tracker_bizobj.GetOwnerId(art)
if owner_id and owner_id in users_by_id:
owner_username = users_by_id[owner_id].display_name
column_values['owner'][owner_username] = owner_username
if 'cc' in column_values:
for art in artifact_list:
cc_ids = tracker_bizobj.GetCcIds(art)
for cc_id in cc_ids:
if cc_id and cc_id in users_by_id:
cc_username = users_by_id[cc_id].display_name
column_values['cc'][cc_username] = cc_username
if 'component' in column_values:
for art in artifact_list:
all_comp_ids = list(art.component_ids) + list(
art.derived_component_ids)
for component_id in all_comp_ids:
cd = tracker_bizobj.FindComponentDefByID(component_id, config)
if cd:
column_values['component'][cd.path] = cd.path
if 'stars' in column_values:
for art in artifact_list:
star_count = art.star_count
column_values['stars'][star_count] = star_count
if 'status' in column_values:
for art in artifact_list:
status = tracker_bizobj.GetStatus(art)
if status:
column_values['status'][status.lower()] = status
if 'project' in column_values:
for art in artifact_list:
project_name = art.project_name
column_values['project'][project_name] = project_name
if 'mergedinto' in column_values:
for art in artifact_list:
if art.merged_into and art.merged_into != 0:
merged_issue = related_issues[art.merged_into]
merged_issue_ref = tracker_bizobj.FormatIssueRef(
(merged_issue.project_name, merged_issue.local_id))
column_values['mergedinto'][
merged_issue_ref] = merged_issue_ref
if 'blocked' in column_values:
for art in artifact_list:
if art.blocked_on_iids:
column_values['blocked']['is_blocked'] = 'Yes'
else:
column_values['blocked']['is_not_blocked'] = 'No'
if 'blockedon' in column_values:
for art in artifact_list:
if art.blocked_on_iids:
for blocked_on_iid in art.blocked_on_iids:
blocked_on_issue = related_issues[blocked_on_iid]
blocked_on_ref = tracker_bizobj.FormatIssueRef(
(blocked_on_issue.project_name,
blocked_on_issue.local_id))
column_values['blockedon'][blocked_on_ref] = blocked_on_ref
if 'blocking' in column_values:
for art in artifact_list:
if art.blocking_iids:
for blocking_iid in art.blocking_iids:
blocking_issue = related_issues[blocking_iid]
blocking_ref = tracker_bizobj.FormatIssueRef(
(blocking_issue.project_name, blocking_issue.local_id))
column_values['blocking'][blocking_ref] = blocking_ref
if 'added' in column_values:
for art in artifact_list:
if hotlist_context_dict and hotlist_context_dict[art.issue_id]:
issue_dict = hotlist_context_dict[art.issue_id]
date_added = issue_dict['date_added']
column_values['added'][date_added] = date_added
if 'adder' in column_values:
for art in artifact_list:
if hotlist_context_dict and hotlist_context_dict[art.issue_id]:
issue_dict = hotlist_context_dict[art.issue_id]
adder_id = issue_dict['adder_id']
adder = users_by_id[adder_id].display_name
column_values['adder'][adder] = adder
if 'note' in column_values:
for art in artifact_list:
if hotlist_context_dict and hotlist_context_dict[art.issue_id]:
issue_dict = hotlist_context_dict[art.issue_id]
note = issue_dict['note']
if issue_dict['note']:
column_values['note'][note] = note
if 'attachments' in column_values:
for art in artifact_list:
attachment_count = art.attachment_count
column_values['attachments'][attachment_count] = attachment_count
# Add all custom field values if the custom field name is a shown column.
field_id_to_col = {}
for art in artifact_list:
for fv in art.field_values:
field_col, field_type = field_id_to_col.get(
fv.field_id, (None, None))
if field_col == 'NOT_SHOWN':
continue
if field_col is None:
fd = tracker_bizobj.FindFieldDefByID(fv.field_id, config)
if not fd:
field_id_to_col[fv.field_id] = 'NOT_SHOWN', None
continue
field_col = fd.field_name.lower()
field_type = fd.field_type
if field_col not in column_values:
field_id_to_col[fv.field_id] = 'NOT_SHOWN', None
continue
field_id_to_col[fv.field_id] = field_col, field_type
if field_type == tracker_pb2.FieldTypes.ENUM_TYPE:
continue # Already handled by label parsing
elif field_type == tracker_pb2.FieldTypes.INT_TYPE:
val = fv.int_value
elif field_type == tracker_pb2.FieldTypes.STR_TYPE:
val = fv.str_value
elif field_type == tracker_pb2.FieldTypes.USER_TYPE:
user = users_by_id.get(fv.user_id)
val = user.email if user else framework_constants.NO_USER_NAME
elif field_type == tracker_pb2.FieldTypes.DATE_TYPE:
val = fv.int_value # TODO(jrobbins): convert to date
elif field_type == tracker_pb2.FieldTypes.BOOL_TYPE:
val = 'Yes' if fv.int_value else 'No'
column_values[field_col][val] = val
# TODO(jrobbins): make the capitalization of well-known unique label and
# status values match the way it is written in the issue config.
# Return EZTItems for each column in left-to-right display order.
result = []
for i, col_name in enumerate(columns):
# TODO(jrobbins): sort each set of column values top-to-bottom, by the
# order specified in the project artifact config. For now, just sort
# lexicographically to make expected output defined.
sorted_col_values = sorted(column_values[col_name].values())
result.append(
template_helpers.EZTItem(col_index=i,
column_name=col_name,
filter_values=sorted_col_values))
return result
def _CreateIssueSearchDocuments(issues, comments_dict, users_by_id,
config_dict):
"""Make the GAE search index documents for the given issue batch.
Args:
issues: list of issues to index.
comments_dict: prefetched dictionary of comments on those issues.
users_by_id: dictionary {user_id: UserView} so that the email
addresses of users who left comments can be found via search.
config_dict: dict {project_id: config} for all the projects that
the given issues are in.
"""
documents_by_shard = collections.defaultdict(list)
for issue in issues:
summary = issue.summary
# TODO(jrobbins): allow search specifically on explicit vs derived
# fields.
owner_id = tracker_bizobj.GetOwnerId(issue)
owner_email = users_by_id[owner_id].email
config = config_dict[issue.project_id]
component_paths = []
for component_id in issue.component_ids:
cd = tracker_bizobj.FindComponentDefByID(component_id, config)
if cd:
component_paths.append(cd.path)
field_values = [
tracker_bizobj.GetFieldValue(fv, users_by_id)
for fv in issue.field_values
]
# Convert to string only the values that are not strings already.
# This is done because the default encoding in appengine seems to be 'ascii'
# and string values might contain unicode characters, so str will fail to
# encode them.
field_values = [
value if isinstance(value, string_types) else str(value)
for value in field_values
]
metadata = '%s %s %s %s %s %s' % (
tracker_bizobj.GetStatus(issue), owner_email, [
users_by_id[cc_id].email
for cc_id in tracker_bizobj.GetCcIds(issue)
], ' '.join(component_paths), ' '.join(field_values), ' '.join(
tracker_bizobj.GetLabels(issue)))
custom_fields = _BuildCustomFTSFields(issue)
comments = comments_dict.get(issue.issue_id, [])
room_for_comments = (framework_constants.MAX_FTS_FIELD_SIZE -
len(summary) - len(metadata) -
sum(len(cf.value) for cf in custom_fields))
comments = _IndexableComments(comments,
users_by_id,
remaining_chars=room_for_comments)
logging.info('len(comments) is %r', len(comments))
if comments:
description = _ExtractCommentText(comments[0], users_by_id)
description = description[:framework_constants.MAX_FTS_FIELD_SIZE]
all_comments = ' '.join(
_ExtractCommentText(c, users_by_id) for c in comments[1:])
all_comments = all_comments[:framework_constants.
MAX_FTS_FIELD_SIZE]
else:
description = ''
all_comments = ''
logging.info('Issue %s:%r has zero indexable comments',
issue.project_name, issue.local_id)
logging.info('Building document for %s:%d', issue.project_name,
issue.local_id)
logging.info('len(summary) = %d', len(summary))
logging.info('len(metadata) = %d', len(metadata))
logging.info('len(description) = %d', len(description))
logging.info('len(comment) = %d', len(all_comments))
for cf in custom_fields:
logging.info('len(%s) = %d', cf.name, len(cf.value))
doc = search.Document(
doc_id=str(issue.issue_id),
fields=[
search.NumberField(name='project_id', value=issue.project_id),
search.TextField(name='summary', value=summary),
search.TextField(name='metadata', value=metadata),
search.TextField(name='description', value=description),
search.TextField(name='comment', value=all_comments),
] + custom_fields)
shard_id = issue.issue_id % settings.num_logical_shards
documents_by_shard[shard_id].append(doc)
start_time = time.time()
promises = []
for shard_id, documents in documents_by_shard.items():
if documents:
promises.append(
framework_helpers.Promise(_IndexDocsInShard, shard_id,
documents))
for promise in promises:
promise.WaitAndGetValue()
logging.info('Finished %d indexing in shards in %d ms',
len(documents_by_shard), int(
(time.time() - start_time) * 1000))
def _CreateIssueSearchDocuments(issues, comments_dict, users_by_id,
config_dict):
"""Make the GAE search index documents for the given issue batch.
Args:
issues: list of issues to index.
comments_dict: prefetched dictionary of comments on those issues.
users_by_id: dictionary {user_id: UserView} so that the email
addresses of users who left comments can be found via search.
config_dict: dict {project_id: config} for all the projects that
the given issues are in.
"""
documents_by_shard = collections.defaultdict(list)
for issue in issues:
comments = comments_dict.get(issue.issue_id, [])
comments = _IndexableComments(comments, users_by_id)
summary = issue.summary
# TODO(jrobbins): allow search specifically on explicit vs derived
# fields.
owner_id = tracker_bizobj.GetOwnerId(issue)
owner_email = users_by_id[owner_id].email
config = config_dict[issue.project_id]
component_paths = []
for component_id in issue.component_ids:
cd = tracker_bizobj.FindComponentDefByID(component_id, config)
if cd:
component_paths.append(cd.path)
field_values = [
str(tracker_bizobj.GetFieldValue(fv, users_by_id))
for fv in issue.field_values
]
metadata = '%s %s %s %s %s %s' % (
tracker_bizobj.GetStatus(issue), owner_email, [
users_by_id[cc_id].email
for cc_id in tracker_bizobj.GetCcIds(issue)
], ' '.join(component_paths), ' '.join(field_values), ' '.join(
tracker_bizobj.GetLabels(issue)))
assert comments, 'issues should always have at least the description'
description = _ExtractCommentText(comments[0], users_by_id)
description = description[:framework_constants.MAX_FTS_FIELD_SIZE]
all_comments = ' '.join(
_ExtractCommentText(c, users_by_id) for c in comments[1:])
all_comments = all_comments[:framework_constants.MAX_FTS_FIELD_SIZE]
custom_fields = _BuildCustomFTSFields(issue)
doc = search.Document(
doc_id=str(issue.issue_id),
fields=[
search.NumberField(name='project_id', value=issue.project_id),
search.TextField(name='summary', value=summary),
search.TextField(name='metadata', value=metadata),
search.TextField(name='description', value=description),
search.TextField(name='comment', value=all_comments),
] + custom_fields)
shard_id = issue.issue_id % settings.num_logical_shards
documents_by_shard[shard_id].append(doc)
start_time = time.time()
promises = []
for shard_id, documents in documents_by_shard.iteritems():
if documents:
promises.append(
framework_helpers.Promise(_IndexDocsInShard, shard_id,
documents))
for promise in promises:
promise.WaitAndGetValue()
logging.info('Finished %d indexing in shards in %d ms',
len(documents_by_shard), int(
(time.time() - start_time) * 1000))
def StoreIssueSnapshots(self, cnxn, issues, commit=True):
"""Adds an IssueSnapshot and updates the previous one for each issue."""
for issue in issues:
right_now = self._currentTime()
# Look for an existing (latest) IssueSnapshot with this issue_id.
previous_snapshots = self.issuesnapshot_tbl.Select(
cnxn,
cols=ISSUESNAPSHOT_COLS,
issue_id=issue.issue_id,
limit=1,
order_by=[('period_start DESC', [])])
if len(previous_snapshots) > 0:
previous_snapshot_id = previous_snapshots[0][0]
logging.info('Found previous IssueSnapshot with id: %s',
previous_snapshot_id)
# Update previous snapshot's end time to right now.
delta = {'period_end': right_now}
where = [('IssueSnapshot.id = %s', [previous_snapshot_id])]
self.issuesnapshot_tbl.Update(cnxn,
delta,
commit=commit,
where=where)
config = self.config_service.GetProjectConfig(
cnxn, issue.project_id)
period_end = settings.maximum_snapshot_period_end
is_open = tracker_helpers.MeansOpenInProject(
tracker_bizobj.GetStatus(issue), config)
shard = issue.issue_id % settings.num_logical_shards
status = tracker_bizobj.GetStatus(issue)
status_id = self.config_service.LookupStatusID(
cnxn, issue.project_id, status) or None
owner_id = tracker_bizobj.GetOwnerId(issue) or None
issuesnapshot_rows = [(issue.issue_id, shard, issue.project_id,
issue.local_id, issue.reporter_id, owner_id,
status_id, right_now, period_end, is_open)]
ids = self.issuesnapshot_tbl.InsertRows(cnxn,
ISSUESNAPSHOT_COLS[1:],
issuesnapshot_rows,
replace=True,
commit=commit,
return_generated_ids=True)
issuesnapshot_id = ids[0]
# Add all labels to IssueSnapshot2Label.
label_rows = [
(issuesnapshot_id,
self.config_service.LookupLabelID(cnxn, issue.project_id,
label))
for label in tracker_bizobj.GetLabels(issue)
]
self.issuesnapshot2label_tbl.InsertRows(cnxn,
ISSUESNAPSHOT2LABEL_COLS,
label_rows,
replace=True,
commit=commit)
# Add all CCs to IssueSnapshot2Cc.
cc_rows = [(issuesnapshot_id, cc_id)
for cc_id in tracker_bizobj.GetCcIds(issue)]
self.issuesnapshot2cc_tbl.InsertRows(cnxn,
ISSUESNAPSHOT2CC_COLS,
cc_rows,
replace=True,
commit=commit)
# Add all components to IssueSnapshot2Component.
component_rows = [(issuesnapshot_id, component_id)
for component_id in issue.component_ids]
self.issuesnapshot2component_tbl.InsertRows(
cnxn,
ISSUESNAPSHOT2COMPONENT_COLS,
component_rows,
replace=True,
commit=commit)
# Add all components to IssueSnapshot2Hotlist.
# This is raw SQL to obviate passing FeaturesService down through
# the call stack wherever this function is called.
# TODO(jrobbins): sort out dependencies between service classes.
cnxn.Execute(
'''
INSERT INTO IssueSnapshot2Hotlist (issuesnapshot_id, hotlist_id)
SELECT %s, hotlist_id FROM Hotlist2Issue WHERE issue_id = %s
''', [issuesnapshot_id, issue.issue_id])
