def testGetLabels(self): issue = tracker_pb2.Issue() self.assertEquals(tracker_bizobj.GetLabels(issue), []) issue.derived_labels.extend(['a', 'b', 'c']) self.assertEquals(tracker_bizobj.GetLabels(issue), ['a', 'b', 'c']) issue.labels.extend(['d', 'e', 'f']) self.assertEquals(tracker_bizobj.GetLabels(issue), ['d', 'e', 'f', 'a', 'b', 'c'])
def EvaluateSubscriptions( cnxn, issue, users_to_queries, services, config): """Determine subscribers who have subs that match the given issue.""" # Note: unlike filter rule, subscriptions see explicit & derived values. lower_labels = [lab.lower() for lab in tracker_bizobj.GetLabels(issue)] label_set = set(lower_labels) subscribers_to_notify = [] for uid, saved_queries in users_to_queries.items(): for sq in saved_queries: if sq.subscription_mode != 'immediate': continue if issue.project_id not in sq.executes_in_project_ids: continue cond = savedqueries_helpers.SavedQueryToCond(sq) # TODO(jrobbins): Support linked accounts me_user_ids. cond, _warnings = searchpipeline.ReplaceKeywordsWithUserIDs([uid], cond) cond_ast = query2ast.ParseUserQuery( cond, '', query2ast.BUILTIN_ISSUE_FIELDS, config) if filterrules_helpers.EvalPredicate( cnxn, services, cond_ast, issue, label_set, config, tracker_bizobj.GetOwnerId(issue), tracker_bizobj.GetCcIds(issue), tracker_bizobj.GetStatus(issue)): subscribers_to_notify.append(uid) break # Don't bother looking at the user's other saved quereies. return subscribers_to_notify
def GetRestrictions(issue, perm=''): """Return a list of restriction labels on the given issue.""" if not issue: return [] return [lab.lower() for lab in tracker_bizobj.GetLabels(issue) if IsRestrictLabel(lab, perm=perm)]
def _SortableLabelValues(art, col_name, well_known_value_indexes): """Return a list of ints and strings for labels relevant to one UI column.""" col_name_dash = col_name + '-' sortable_value_list = [] for label in tracker_bizobj.GetLabels(art): idx_or_lex = well_known_value_indexes.get(label) if idx_or_lex == IGNORABLE_INDICATOR: continue # Label is known to not have the desired prefix. if idx_or_lex is None: if '-' not in label: # Skip an irrelevant OneWord label and remember to ignore it later. well_known_value_indexes[label] = IGNORABLE_INDICATOR continue label_lower = label.lower() if label_lower.startswith(col_name_dash): # Label is a key-value label with an odd-ball value, remember it value = label_lower[len(col_name_dash):] idx_or_lex = value well_known_value_indexes[label] = value else: # Label was a key-value label that is not relevant to this column. # Remember to ignore it later. well_known_value_indexes[label] = IGNORABLE_INDICATOR continue sortable_value_list.append(idx_or_lex) return sortable_value_list
def Accessor(art): """Return a list of label values on the given artifact.""" result = [ label.lower() for label in tracker_bizobj.GetLabels(art) if label.lower().startswith(prefix) ] return result
def MakeLabelValuesDict(art): """Return a dict of label values and a list of one-word labels. Args: art: artifact object, e.g., an issue PB. Returns: A dict {prefix: [suffix,...], ...} for each key-value label. """ label_values = collections.defaultdict(list) for label_name in tracker_bizobj.GetLabels(art): if '-' in label_name: key, value = label_name.split('-', 1) label_values[key.lower()].append(value) return label_values
def __init__(self, issue): # List of restrictions that don't map to a known action kind. self.other = [] restrictions_by_action = collections.defaultdict(list) # We can't use GetRestrictions here, as we prefer to preserve # the case of the label when showing restrictions in the UI. for label in tracker_bizobj.GetLabels(issue): if permissions.IsRestrictLabel(label): _kw, action_kind, needed_perm = label.split('-', 2) action_kind = action_kind.lower() if action_kind in self._KNOWN_ACTION_KINDS: restrictions_by_action[action_kind].append(needed_perm) else: self.other.append(label) self.view = ' and '.join(restrictions_by_action[self._VIEW]) self.add_comment = ' and '.join(restrictions_by_action[self._ADD_COMMENT]) self.edit = ' and '.join(restrictions_by_action[self._EDIT]) self.has_restrictions = ezt.boolean( self.view or self.add_comment or self.edit or self.other)
def ComputeUnshownColumns(results, shown_columns, config, built_in_cols): """Return a list of unshown columns that the user could add. Args: results: list of search result PBs. Each must have labels. shown_columns: list of column names to be used in results table. config: harmonized config for the issue search, including all well known labels and custom fields. built_in_cols: list of other column names that are built into the tool. E.g., star count, or creation date. Returns: List of column names to append to the "..." menu. """ unshown_set = set() # lowercases column names unshown_list = [] # original-case column names shown_set = {col.lower() for col in shown_columns} labels_already_seen = set() # whole labels, original case def _MaybeAddLabel(label_name): """Add the key part of the given label if needed.""" if label_name.lower() in labels_already_seen: return labels_already_seen.add(label_name.lower()) if '-' in label_name: col, _value = label_name.split('-', 1) _MaybeAddCol(col) def _MaybeAddCol(col): if col.lower() not in shown_set and col.lower() not in unshown_set: unshown_list.append(col) unshown_set.add(col.lower()) # The user can always add any of the default columns. for col in config.default_col_spec.split(): _MaybeAddCol(col) # The user can always add any of the built-in columns. for col in built_in_cols: _MaybeAddCol(col) # The user can add a column for any well-known labels for wkl in config.well_known_labels: _MaybeAddLabel(wkl.label) phase_names = set( itertools.chain.from_iterable((phase.name.lower() for phase in result.phases) for result in results)) # The user can add a column for any custom field field_ids_alread_seen = set() for fd in config.field_defs: field_lower = fd.field_name.lower() field_ids_alread_seen.add(fd.field_id) if fd.is_phase_field: for name in phase_names: phase_field_col = name + '.' + field_lower if (phase_field_col not in shown_set and phase_field_col not in unshown_set): unshown_list.append(phase_field_col) unshown_set.add(phase_field_col) elif field_lower not in shown_set and field_lower not in unshown_set: unshown_list.append(fd.field_name) unshown_set.add(field_lower) if fd.field_type == tracker_pb2.FieldTypes.APPROVAL_TYPE: approval_lower_approver = (field_lower + tracker_constants.APPROVER_COL_SUFFIX) if (approval_lower_approver not in shown_set and approval_lower_approver not in unshown_set): unshown_list.append(fd.field_name + tracker_constants.APPROVER_COL_SUFFIX) unshown_set.add(approval_lower_approver) # The user can add a column for any key-value label or field in the results. for r in results: for label_name in tracker_bizobj.GetLabels(r): _MaybeAddLabel(label_name) for field_value in r.field_values: if field_value.field_id not in field_ids_alread_seen: field_ids_alread_seen.add(field_value.field_id) fd = tracker_bizobj.FindFieldDefByID(field_value.field_id, config) if fd: # could be None for a foreign field, which we don't display. field_lower = fd.field_name.lower() if field_lower not in shown_set and field_lower not in unshown_set: unshown_list.append(fd.field_name) unshown_set.add(field_lower) return sorted(unshown_list)
def ExtractUniqueValues(columns, artifact_list, users_by_id, config, related_issues, hotlist_context_dict=None): """Build a nested list of unique values so the user can auto-filter. Args: columns: a list of lowercase column name strings, which may contain combined columns like "priority/pri". artifact_list: a list of artifacts in the complete set of search results. users_by_id: dict mapping user_ids to UserViews. config: ProjectIssueConfig PB for the current project. related_issues: dict {issue_id: issue} of pre-fetched related issues. hotlist_context_dict: dict for building a hotlist grid table Returns: [EZTItem(col1, colname1, [val11, val12,...]), ...] A list of EZTItems, each of which has a col_index, column_name, and a list of unique values that appear in that column. """ column_values = {col_name: {} for col_name in columns} # For each combined column "a/b/c", add entries that point from "a" back # to "a/b/c", from "b" back to "a/b/c", and from "c" back to "a/b/c". combined_column_parts = collections.defaultdict(list) for col in columns: if '/' in col: for col_part in col.split('/'): combined_column_parts[col_part].append(col) unique_labels = set() for art in artifact_list: unique_labels.update(tracker_bizobj.GetLabels(art)) for label in unique_labels: if '-' in label: col, val = label.split('-', 1) col = col.lower() if col in column_values: column_values[col][val.lower()] = val if col in combined_column_parts: for combined_column in combined_column_parts[col]: column_values[combined_column][val.lower()] = val else: if 'summary' in column_values: column_values['summary'][label.lower()] = label # TODO(jrobbins): Consider refacting some of this to tracker_bizobj # or a new builtins.py to reduce duplication. if 'reporter' in column_values: for art in artifact_list: reporter_id = art.reporter_id if reporter_id and reporter_id in users_by_id: reporter_username = users_by_id[reporter_id].display_name column_values['reporter'][ reporter_username] = reporter_username if 'owner' in column_values: for art in artifact_list: owner_id = tracker_bizobj.GetOwnerId(art) if owner_id and owner_id in users_by_id: owner_username = users_by_id[owner_id].display_name column_values['owner'][owner_username] = owner_username if 'cc' in column_values: for art in artifact_list: cc_ids = tracker_bizobj.GetCcIds(art) for cc_id in cc_ids: if cc_id and cc_id in users_by_id: cc_username = users_by_id[cc_id].display_name column_values['cc'][cc_username] = cc_username if 'component' in column_values: for art in artifact_list: all_comp_ids = list(art.component_ids) + list( art.derived_component_ids) for component_id in all_comp_ids: cd = tracker_bizobj.FindComponentDefByID(component_id, config) if cd: column_values['component'][cd.path] = cd.path if 'stars' in column_values: for art in artifact_list: star_count = art.star_count column_values['stars'][star_count] = star_count if 'status' in column_values: for art in artifact_list: status = tracker_bizobj.GetStatus(art) if status: column_values['status'][status.lower()] = status if 'project' in column_values: for art in artifact_list: project_name = art.project_name column_values['project'][project_name] = project_name if 'mergedinto' in column_values: for art in artifact_list: if art.merged_into and art.merged_into != 0: merged_issue = related_issues[art.merged_into] merged_issue_ref = tracker_bizobj.FormatIssueRef( (merged_issue.project_name, merged_issue.local_id)) column_values['mergedinto'][ merged_issue_ref] = merged_issue_ref if 'blocked' in column_values: for art in artifact_list: if art.blocked_on_iids: column_values['blocked']['is_blocked'] = 'Yes' else: column_values['blocked']['is_not_blocked'] = 'No' if 'blockedon' in column_values: for art in artifact_list: if art.blocked_on_iids: for blocked_on_iid in art.blocked_on_iids: blocked_on_issue = related_issues[blocked_on_iid] blocked_on_ref = tracker_bizobj.FormatIssueRef( (blocked_on_issue.project_name, blocked_on_issue.local_id)) column_values['blockedon'][blocked_on_ref] = blocked_on_ref if 'blocking' in column_values: for art in artifact_list: if art.blocking_iids: for blocking_iid in art.blocking_iids: blocking_issue = related_issues[blocking_iid] blocking_ref = tracker_bizobj.FormatIssueRef( (blocking_issue.project_name, blocking_issue.local_id)) column_values['blocking'][blocking_ref] = blocking_ref if 'added' in column_values: for art in artifact_list: if hotlist_context_dict and hotlist_context_dict[art.issue_id]: issue_dict = hotlist_context_dict[art.issue_id] date_added = issue_dict['date_added'] column_values['added'][date_added] = date_added if 'adder' in column_values: for art in artifact_list: if hotlist_context_dict and hotlist_context_dict[art.issue_id]: issue_dict = hotlist_context_dict[art.issue_id] adder_id = issue_dict['adder_id'] adder = users_by_id[adder_id].display_name column_values['adder'][adder] = adder if 'note' in column_values: for art in artifact_list: if hotlist_context_dict and hotlist_context_dict[art.issue_id]: issue_dict = hotlist_context_dict[art.issue_id] note = issue_dict['note'] if issue_dict['note']: column_values['note'][note] = note if 'attachments' in column_values: for art in artifact_list: attachment_count = art.attachment_count column_values['attachments'][attachment_count] = attachment_count # Add all custom field values if the custom field name is a shown column. field_id_to_col = {} for art in artifact_list: for fv in art.field_values: field_col, field_type = field_id_to_col.get( fv.field_id, (None, None)) if field_col == 'NOT_SHOWN': continue if field_col is None: fd = tracker_bizobj.FindFieldDefByID(fv.field_id, config) if not fd: field_id_to_col[fv.field_id] = 'NOT_SHOWN', None continue field_col = fd.field_name.lower() field_type = fd.field_type if field_col not in column_values: field_id_to_col[fv.field_id] = 'NOT_SHOWN', None continue field_id_to_col[fv.field_id] = field_col, field_type if field_type == tracker_pb2.FieldTypes.ENUM_TYPE: continue # Already handled by label parsing elif field_type == tracker_pb2.FieldTypes.INT_TYPE: val = fv.int_value elif field_type == tracker_pb2.FieldTypes.STR_TYPE: val = fv.str_value elif field_type == tracker_pb2.FieldTypes.USER_TYPE: user = users_by_id.get(fv.user_id) val = user.email if user else framework_constants.NO_USER_NAME elif field_type == tracker_pb2.FieldTypes.DATE_TYPE: val = fv.int_value # TODO(jrobbins): convert to date elif field_type == tracker_pb2.FieldTypes.BOOL_TYPE: val = 'Yes' if fv.int_value else 'No' column_values[field_col][val] = val # TODO(jrobbins): make the capitalization of well-known unique label and # status values match the way it is written in the issue config. # Return EZTItems for each column in left-to-right display order. result = [] for i, col_name in enumerate(columns): # TODO(jrobbins): sort each set of column values top-to-bottom, by the # order specified in the project artifact config. For now, just sort # lexicographically to make expected output defined. sorted_col_values = sorted(column_values[col_name].values()) result.append( template_helpers.EZTItem(col_index=i, column_name=col_name, filter_values=sorted_col_values)) return result
def _CreateIssueSearchDocuments(issues, comments_dict, users_by_id, config_dict): """Make the GAE search index documents for the given issue batch. Args: issues: list of issues to index. comments_dict: prefetched dictionary of comments on those issues. users_by_id: dictionary {user_id: UserView} so that the email addresses of users who left comments can be found via search. config_dict: dict {project_id: config} for all the projects that the given issues are in. """ documents_by_shard = collections.defaultdict(list) for issue in issues: summary = issue.summary # TODO(jrobbins): allow search specifically on explicit vs derived # fields. owner_id = tracker_bizobj.GetOwnerId(issue) owner_email = users_by_id[owner_id].email config = config_dict[issue.project_id] component_paths = [] for component_id in issue.component_ids: cd = tracker_bizobj.FindComponentDefByID(component_id, config) if cd: component_paths.append(cd.path) field_values = [ tracker_bizobj.GetFieldValue(fv, users_by_id) for fv in issue.field_values ] # Convert to string only the values that are not strings already. # This is done because the default encoding in appengine seems to be 'ascii' # and string values might contain unicode characters, so str will fail to # encode them. field_values = [ value if isinstance(value, string_types) else str(value) for value in field_values ] metadata = '%s %s %s %s %s %s' % ( tracker_bizobj.GetStatus(issue), owner_email, [ users_by_id[cc_id].email for cc_id in tracker_bizobj.GetCcIds(issue) ], ' '.join(component_paths), ' '.join(field_values), ' '.join( tracker_bizobj.GetLabels(issue))) custom_fields = _BuildCustomFTSFields(issue) comments = comments_dict.get(issue.issue_id, []) room_for_comments = (framework_constants.MAX_FTS_FIELD_SIZE - len(summary) - len(metadata) - sum(len(cf.value) for cf in custom_fields)) comments = _IndexableComments(comments, users_by_id, remaining_chars=room_for_comments) logging.info('len(comments) is %r', len(comments)) if comments: description = _ExtractCommentText(comments[0], users_by_id) description = description[:framework_constants.MAX_FTS_FIELD_SIZE] all_comments = ' '.join( _ExtractCommentText(c, users_by_id) for c in comments[1:]) all_comments = all_comments[:framework_constants. MAX_FTS_FIELD_SIZE] else: description = '' all_comments = '' logging.info('Issue %s:%r has zero indexable comments', issue.project_name, issue.local_id) logging.info('Building document for %s:%d', issue.project_name, issue.local_id) logging.info('len(summary) = %d', len(summary)) logging.info('len(metadata) = %d', len(metadata)) logging.info('len(description) = %d', len(description)) logging.info('len(comment) = %d', len(all_comments)) for cf in custom_fields: logging.info('len(%s) = %d', cf.name, len(cf.value)) doc = search.Document( doc_id=str(issue.issue_id), fields=[ search.NumberField(name='project_id', value=issue.project_id), search.TextField(name='summary', value=summary), search.TextField(name='metadata', value=metadata), search.TextField(name='description', value=description), search.TextField(name='comment', value=all_comments), ] + custom_fields) shard_id = issue.issue_id % settings.num_logical_shards documents_by_shard[shard_id].append(doc) start_time = time.time() promises = [] for shard_id, documents in documents_by_shard.items(): if documents: promises.append( framework_helpers.Promise(_IndexDocsInShard, shard_id, documents)) for promise in promises: promise.WaitAndGetValue() logging.info('Finished %d indexing in shards in %d ms', len(documents_by_shard), int( (time.time() - start_time) * 1000))
def _CreateIssueSearchDocuments(issues, comments_dict, users_by_id, config_dict): """Make the GAE search index documents for the given issue batch. Args: issues: list of issues to index. comments_dict: prefetched dictionary of comments on those issues. users_by_id: dictionary {user_id: UserView} so that the email addresses of users who left comments can be found via search. config_dict: dict {project_id: config} for all the projects that the given issues are in. """ documents_by_shard = collections.defaultdict(list) for issue in issues: comments = comments_dict.get(issue.issue_id, []) comments = _IndexableComments(comments, users_by_id) summary = issue.summary # TODO(jrobbins): allow search specifically on explicit vs derived # fields. owner_id = tracker_bizobj.GetOwnerId(issue) owner_email = users_by_id[owner_id].email config = config_dict[issue.project_id] component_paths = [] for component_id in issue.component_ids: cd = tracker_bizobj.FindComponentDefByID(component_id, config) if cd: component_paths.append(cd.path) field_values = [ str(tracker_bizobj.GetFieldValue(fv, users_by_id)) for fv in issue.field_values ] metadata = '%s %s %s %s %s %s' % ( tracker_bizobj.GetStatus(issue), owner_email, [ users_by_id[cc_id].email for cc_id in tracker_bizobj.GetCcIds(issue) ], ' '.join(component_paths), ' '.join(field_values), ' '.join( tracker_bizobj.GetLabels(issue))) assert comments, 'issues should always have at least the description' description = _ExtractCommentText(comments[0], users_by_id) description = description[:framework_constants.MAX_FTS_FIELD_SIZE] all_comments = ' '.join( _ExtractCommentText(c, users_by_id) for c in comments[1:]) all_comments = all_comments[:framework_constants.MAX_FTS_FIELD_SIZE] custom_fields = _BuildCustomFTSFields(issue) doc = search.Document( doc_id=str(issue.issue_id), fields=[ search.NumberField(name='project_id', value=issue.project_id), search.TextField(name='summary', value=summary), search.TextField(name='metadata', value=metadata), search.TextField(name='description', value=description), search.TextField(name='comment', value=all_comments), ] + custom_fields) shard_id = issue.issue_id % settings.num_logical_shards documents_by_shard[shard_id].append(doc) start_time = time.time() promises = [] for shard_id, documents in documents_by_shard.iteritems(): if documents: promises.append( framework_helpers.Promise(_IndexDocsInShard, shard_id, documents)) for promise in promises: promise.WaitAndGetValue() logging.info('Finished %d indexing in shards in %d ms', len(documents_by_shard), int( (time.time() - start_time) * 1000))
def StoreIssueSnapshots(self, cnxn, issues, commit=True): """Adds an IssueSnapshot and updates the previous one for each issue.""" for issue in issues: right_now = self._currentTime() # Look for an existing (latest) IssueSnapshot with this issue_id. previous_snapshots = self.issuesnapshot_tbl.Select( cnxn, cols=ISSUESNAPSHOT_COLS, issue_id=issue.issue_id, limit=1, order_by=[('period_start DESC', [])]) if len(previous_snapshots) > 0: previous_snapshot_id = previous_snapshots[0][0] logging.info('Found previous IssueSnapshot with id: %s', previous_snapshot_id) # Update previous snapshot's end time to right now. delta = {'period_end': right_now} where = [('IssueSnapshot.id = %s', [previous_snapshot_id])] self.issuesnapshot_tbl.Update(cnxn, delta, commit=commit, where=where) config = self.config_service.GetProjectConfig( cnxn, issue.project_id) period_end = settings.maximum_snapshot_period_end is_open = tracker_helpers.MeansOpenInProject( tracker_bizobj.GetStatus(issue), config) shard = issue.issue_id % settings.num_logical_shards status = tracker_bizobj.GetStatus(issue) status_id = self.config_service.LookupStatusID( cnxn, issue.project_id, status) or None owner_id = tracker_bizobj.GetOwnerId(issue) or None issuesnapshot_rows = [(issue.issue_id, shard, issue.project_id, issue.local_id, issue.reporter_id, owner_id, status_id, right_now, period_end, is_open)] ids = self.issuesnapshot_tbl.InsertRows(cnxn, ISSUESNAPSHOT_COLS[1:], issuesnapshot_rows, replace=True, commit=commit, return_generated_ids=True) issuesnapshot_id = ids[0] # Add all labels to IssueSnapshot2Label. label_rows = [ (issuesnapshot_id, self.config_service.LookupLabelID(cnxn, issue.project_id, label)) for label in tracker_bizobj.GetLabels(issue) ] self.issuesnapshot2label_tbl.InsertRows(cnxn, ISSUESNAPSHOT2LABEL_COLS, label_rows, replace=True, commit=commit) # Add all CCs to IssueSnapshot2Cc. cc_rows = [(issuesnapshot_id, cc_id) for cc_id in tracker_bizobj.GetCcIds(issue)] self.issuesnapshot2cc_tbl.InsertRows(cnxn, ISSUESNAPSHOT2CC_COLS, cc_rows, replace=True, commit=commit) # Add all components to IssueSnapshot2Component. component_rows = [(issuesnapshot_id, component_id) for component_id in issue.component_ids] self.issuesnapshot2component_tbl.InsertRows( cnxn, ISSUESNAPSHOT2COMPONENT_COLS, component_rows, replace=True, commit=commit) # Add all components to IssueSnapshot2Hotlist. # This is raw SQL to obviate passing FeaturesService down through # the call stack wherever this function is called. # TODO(jrobbins): sort out dependencies between service classes. cnxn.Execute( ''' INSERT INTO IssueSnapshot2Hotlist (issuesnapshot_id, hotlist_id) SELECT %s, hotlist_id FROM Hotlist2Issue WHERE issue_id = %s ''', [issuesnapshot_id, issue.issue_id])