def testFindComponentDefByID_MatchFound(self): config = tracker_bizobj.MakeDefaultProjectIssueConfig(789) cd = tracker_pb2.ComponentDef(component_id=1, path='UI>Splash') config.component_defs.append(cd) config.component_defs.append(tracker_pb2.ComponentDef( component_id=2, path='UI>AboutBox')) actual = tracker_bizobj.FindComponentDefByID(1, config) self.assertEqual(cd, actual)
def testFindComponentDefByID_NoMatch(self): config = tracker_bizobj.MakeDefaultProjectIssueConfig(789) config.component_defs.append(tracker_pb2.ComponentDef( component_id=1, path='UI>Splash')) config.component_defs.append(tracker_pb2.ComponentDef( component_id=2, path='UI>AboutBox')) actual = tracker_bizobj.FindComponentDefByID(999, config) self.assertIsNone(actual)
def ConvertComponentRef(component_id, config, is_derived=False): """Make a ComponentRef from the component_id and project config.""" component_def = tracker_bizobj.FindComponentDefByID(component_id, config) if not component_def: logging.info('Ignoring non-existing component id %s', component_id) return None result = common_pb2.ComponentRef(path=component_def.path, is_derived=is_derived) return result
def GetComponentCcIDs(issue, config): """Return auto-cc'd users for any component or ancestor the issue is in.""" result = set() for component_id in issue.component_ids: cd = tracker_bizobj.FindComponentDefByID(component_id, config) if cd: result.update(GetCcIDsForComponentAndAncestors(config, cd)) return result
def __init__(self, issue, config=None, **_kw): explicit_paths = [] for component_id in issue.component_ids: cd = tracker_bizobj.FindComponentDefByID(component_id, config) if cd: explicit_paths.append(cd.path) derived_paths = [] for component_id in issue.derived_component_ids: cd = tracker_bizobj.FindComponentDefByID(component_id, config) if cd: derived_paths.append(cd.path) table_view_helpers.TableCell.__init__( self, table_view_helpers.CELL_TYPE_ATTR, explicit_paths, derived_values=derived_paths)
def __init__(self, component_id, config, derived): """Make the component name and docstring available as attrs. Args: component_id: int component_id to look up in the config config: ProjectIssueConfig PB for the issue's project. derived: True if this component was derived. """ cd = tracker_bizobj.FindComponentDefByID(component_id, config) self.path = cd.path self.docstring = cd.docstring self.docstring_short = template_helpers.FitUnsafeText(cd.docstring, 60) self.derived = ezt.boolean(derived)
def PredictComponent(raw_text, config): """Get the component ID predicted for the given text. Args: raw_text: The raw text for which we want to predict a component. config: The config of the project. Used to decide if the predicted component is valid. Returns: The component ID predicted for the provided component, or None if no component was predicted. """ # Set-up ML engine. ml_engine = ml_helpers.setup_ml_engine() # Gets the timestamp number from the folder containing the model's trainer # in order to get the correct files for mappings and features. request = ml_engine.projects().models().get(name=MODEL_NAME) response = request.execute() version = re.search(r'v_(\d+)', response['defaultVersion']['name']).group(1) trainer_name = 'component_trainer_%s' % version top_words = _GetTopWords(trainer_name) components_by_index = _GetComponentsByIndex(trainer_name) logging.info('Length of top words list: %s', len(top_words)) clean_text = generate_dataset.CleanText(raw_text) instance = ml_helpers.GenerateFeaturesRaw( [clean_text], settings.component_features, top_words) # Get the component id with the highest prediction score. Component ids are # stored in GCS as strings, but represented in the app as longs. best_score_index = _GetComponentPrediction(ml_engine, instance) component_id = components_by_index.get(str(best_score_index)) if component_id: component_id = int(component_id) # The predicted component id might not exist. if tracker_bizobj.FindComponentDefByID(component_id, config) is None: return None return component_id
def testFindComponentDefByID_Empty(self): config = tracker_bizobj.MakeDefaultProjectIssueConfig(789) actual = tracker_bizobj.FindComponentDefByID(999, config) self.assertIsNone(actual)
def __init__(self, mr, template, user_service, config): super(IssueTemplateView, self).__init__(template) self.ownername = '' try: self.owner_view = framework_views.MakeUserView( mr.cnxn, user_service, template.owner_id) except exceptions.NoSuchUserException: self.owner_view = None if self.owner_view: self.ownername = self.owner_view.email self.admin_views = list( framework_views.MakeAllUserViews(mr.cnxn, user_service, template.admin_ids).values()) self.admin_names = ', '.join( sorted([admin_view.email for admin_view in self.admin_views])) self.summary_must_be_edited = ezt.boolean( template.summary_must_be_edited) self.members_only = ezt.boolean(template.members_only) self.owner_defaults_to_member = ezt.boolean( template.owner_defaults_to_member) self.component_required = ezt.boolean(template.component_required) component_paths = [] for component_id in template.component_ids: component_paths.append( tracker_bizobj.FindComponentDefByID(component_id, config).path) self.components = ', '.join(component_paths) self.can_view = ezt.boolean( permissions.CanViewTemplate(mr.auth.effective_ids, mr.perms, mr.project, template)) self.can_edit = ezt.boolean( permissions.CanEditTemplate(mr.auth.effective_ids, mr.perms, mr.project, template)) field_name_set = { fd.field_name.lower() for fd in config.field_defs if fd.field_type is tracker_pb2.FieldTypes.ENUM_TYPE and not fd.is_deleted } # TODO(jrobbins): restrictions non_masked_labels = [ lab for lab in template.labels if not tracker_bizobj.LabelIsMaskedByField(lab, field_name_set) ] for i, label in enumerate(non_masked_labels): setattr(self, 'label%d' % i, label) for i in range(len(non_masked_labels), framework_constants.MAX_LABELS): setattr(self, 'label%d' % i, '') field_user_views = MakeFieldUserViews(mr.cnxn, template, user_service) self.field_values = [] for fv in template.field_values: self.field_values.append( template_helpers.EZTItem(field_id=fv.field_id, val=tracker_bizobj.GetFieldValue( fv, field_user_views), idx=len(self.field_values))) self.complete_field_values = MakeAllFieldValueViews( config, template.labels, [], template.field_values, field_user_views) # Templates only display and edit the first value of multi-valued fields, so # expose a single value, if any. # TODO(jrobbins): Fully support multi-valued fields in templates. for idx, field_value_view in enumerate(self.complete_field_values): field_value_view.idx = idx if field_value_view.values: field_value_view.val = field_value_view.values[0].val else: field_value_view.val = None
def GatherPageData(self, mr): """Build up a dictionary of data values to use when rendering the page. Args: mr: commonly used info parsed from the request. Returns: Dict of values used by EZT for rendering the page. """ with self.profiler.Phase('getting config'): config = self.services.config.GetProjectConfig(mr.cnxn, mr.project_id) # In addition to checking perms, we adjust some default field values for # project members. is_member = framework_bizobj.UserIsInProject( mr.project, mr.auth.effective_ids) page_perms = self.MakePagePerms( mr, None, permissions.CREATE_ISSUE, permissions.SET_STAR, permissions.EDIT_ISSUE, permissions.EDIT_ISSUE_SUMMARY, permissions.EDIT_ISSUE_STATUS, permissions.EDIT_ISSUE_OWNER, permissions.EDIT_ISSUE_CC) wkp = _SelectTemplate(mr.template_name, config, is_member) if wkp.summary: initial_summary = wkp.summary initial_summary_must_be_edited = wkp.summary_must_be_edited else: initial_summary = PLACEHOLDER_SUMMARY initial_summary_must_be_edited = True if wkp.status: initial_status = wkp.status elif is_member: initial_status = 'Accepted' else: initial_status = 'New' # not offering meta, only used in hidden field. component_paths = [] for component_id in wkp.component_ids: component_paths.append( tracker_bizobj.FindComponentDefByID(component_id, config).path) initial_components = ', '.join(component_paths) if wkp.owner_id: initial_owner = framework_views.MakeUserView( mr.cnxn, self.services.user, wkp.owner_id) elif wkp.owner_defaults_to_member and page_perms.EditIssue: initial_owner = mr.auth.user_view else: initial_owner = None if initial_owner: initial_owner_name = initial_owner.email owner_avail_state = initial_owner.avail_state owner_avail_message_short = initial_owner.avail_message_short else: initial_owner_name = '' owner_avail_state = None owner_avail_message_short = None # Check whether to allow attachments from the entry page allow_attachments = tracker_helpers.IsUnderSoftAttachmentQuota(mr.project) config_view = tracker_views.ConfigView(mr, self.services, config) # If the user followed a link that specified the template name, make sure # that it is also in the menu as the current choice. for template_view in config_view.templates: if template_view.name == mr.template_name: template_view.can_view = ezt.boolean(True) offer_templates = len(list( tmpl for tmpl in config_view.templates if tmpl.can_view)) > 1 restrict_to_known = config.restrict_to_known field_name_set = {fd.field_name.lower() for fd in config.field_defs if not fd.is_deleted} # TODO(jrobbins): restrictions link_or_template_labels = mr.GetListParam('labels', wkp.labels) labels = [lab for lab in link_or_template_labels if not tracker_bizobj.LabelIsMaskedByField(lab, field_name_set)] field_user_views = tracker_views.MakeFieldUserViews( mr.cnxn, wkp, self.services.user) field_views = [ tracker_views.MakeFieldValueView( fd, config, link_or_template_labels, [], wkp.field_values, field_user_views) # TODO(jrobbins): field-level view restrictions, display options for fd in config.field_defs if not fd.is_deleted] page_data = { 'issue_tab_mode': 'issueEntry', 'initial_summary': initial_summary, 'template_summary': initial_summary, 'clear_summary_on_click': ezt.boolean( initial_summary_must_be_edited and 'initial_summary' not in mr.form_overrides), 'must_edit_summary': ezt.boolean(initial_summary_must_be_edited), 'initial_description': wkp.content, 'template_name': wkp.name, 'component_required': ezt.boolean(wkp.component_required), 'initial_status': initial_status, 'initial_owner': initial_owner_name, 'owner_avail_state': owner_avail_state, 'owner_avail_message_short': owner_avail_message_short, 'initial_components': initial_components, 'initial_cc': '', 'initial_blocked_on': '', 'initial_blocking': '', 'labels': labels, 'fields': field_views, 'any_errors': ezt.boolean(mr.errors.AnyErrors()), 'page_perms': page_perms, 'allow_attachments': ezt.boolean(allow_attachments), 'max_attach_size': template_helpers.BytesKbOrMb( framework_constants.MAX_POST_BODY_SIZE), 'offer_templates': ezt.boolean(offer_templates), 'config': config_view, 'restrict_to_known': ezt.boolean(restrict_to_known), } return page_data
def ExtractUniqueValues(columns, artifact_list, users_by_id, config, related_issues, hotlist_context_dict=None): """Build a nested list of unique values so the user can auto-filter. Args: columns: a list of lowercase column name strings, which may contain combined columns like "priority/pri". artifact_list: a list of artifacts in the complete set of search results. users_by_id: dict mapping user_ids to UserViews. config: ProjectIssueConfig PB for the current project. related_issues: dict {issue_id: issue} of pre-fetched related issues. hotlist_context_dict: dict for building a hotlist grid table Returns: [EZTItem(col1, colname1, [val11, val12,...]), ...] A list of EZTItems, each of which has a col_index, column_name, and a list of unique values that appear in that column. """ column_values = {col_name: {} for col_name in columns} # For each combined column "a/b/c", add entries that point from "a" back # to "a/b/c", from "b" back to "a/b/c", and from "c" back to "a/b/c". combined_column_parts = collections.defaultdict(list) for col in columns: if '/' in col: for col_part in col.split('/'): combined_column_parts[col_part].append(col) unique_labels = set() for art in artifact_list: unique_labels.update(tracker_bizobj.GetLabels(art)) for label in unique_labels: if '-' in label: col, val = label.split('-', 1) col = col.lower() if col in column_values: column_values[col][val.lower()] = val if col in combined_column_parts: for combined_column in combined_column_parts[col]: column_values[combined_column][val.lower()] = val else: if 'summary' in column_values: column_values['summary'][label.lower()] = label # TODO(jrobbins): Consider refacting some of this to tracker_bizobj # or a new builtins.py to reduce duplication. if 'reporter' in column_values: for art in artifact_list: reporter_id = art.reporter_id if reporter_id and reporter_id in users_by_id: reporter_username = users_by_id[reporter_id].display_name column_values['reporter'][ reporter_username] = reporter_username if 'owner' in column_values: for art in artifact_list: owner_id = tracker_bizobj.GetOwnerId(art) if owner_id and owner_id in users_by_id: owner_username = users_by_id[owner_id].display_name column_values['owner'][owner_username] = owner_username if 'cc' in column_values: for art in artifact_list: cc_ids = tracker_bizobj.GetCcIds(art) for cc_id in cc_ids: if cc_id and cc_id in users_by_id: cc_username = users_by_id[cc_id].display_name column_values['cc'][cc_username] = cc_username if 'component' in column_values: for art in artifact_list: all_comp_ids = list(art.component_ids) + list( art.derived_component_ids) for component_id in all_comp_ids: cd = tracker_bizobj.FindComponentDefByID(component_id, config) if cd: column_values['component'][cd.path] = cd.path if 'stars' in column_values: for art in artifact_list: star_count = art.star_count column_values['stars'][star_count] = star_count if 'status' in column_values: for art in artifact_list: status = tracker_bizobj.GetStatus(art) if status: column_values['status'][status.lower()] = status if 'project' in column_values: for art in artifact_list: project_name = art.project_name column_values['project'][project_name] = project_name if 'mergedinto' in column_values: for art in artifact_list: if art.merged_into and art.merged_into != 0: merged_issue = related_issues[art.merged_into] merged_issue_ref = tracker_bizobj.FormatIssueRef( (merged_issue.project_name, merged_issue.local_id)) column_values['mergedinto'][ merged_issue_ref] = merged_issue_ref if 'blocked' in column_values: for art in artifact_list: if art.blocked_on_iids: column_values['blocked']['is_blocked'] = 'Yes' else: column_values['blocked']['is_not_blocked'] = 'No' if 'blockedon' in column_values: for art in artifact_list: if art.blocked_on_iids: for blocked_on_iid in art.blocked_on_iids: blocked_on_issue = related_issues[blocked_on_iid] blocked_on_ref = tracker_bizobj.FormatIssueRef( (blocked_on_issue.project_name, blocked_on_issue.local_id)) column_values['blockedon'][blocked_on_ref] = blocked_on_ref if 'blocking' in column_values: for art in artifact_list: if art.blocking_iids: for blocking_iid in art.blocking_iids: blocking_issue = related_issues[blocking_iid] blocking_ref = tracker_bizobj.FormatIssueRef( (blocking_issue.project_name, blocking_issue.local_id)) column_values['blocking'][blocking_ref] = blocking_ref if 'added' in column_values: for art in artifact_list: if hotlist_context_dict and hotlist_context_dict[art.issue_id]: issue_dict = hotlist_context_dict[art.issue_id] date_added = issue_dict['date_added'] column_values['added'][date_added] = date_added if 'adder' in column_values: for art in artifact_list: if hotlist_context_dict and hotlist_context_dict[art.issue_id]: issue_dict = hotlist_context_dict[art.issue_id] adder_id = issue_dict['adder_id'] adder = users_by_id[adder_id].display_name column_values['adder'][adder] = adder if 'note' in column_values: for art in artifact_list: if hotlist_context_dict and hotlist_context_dict[art.issue_id]: issue_dict = hotlist_context_dict[art.issue_id] note = issue_dict['note'] if issue_dict['note']: column_values['note'][note] = note if 'attachments' in column_values: for art in artifact_list: attachment_count = art.attachment_count column_values['attachments'][attachment_count] = attachment_count # Add all custom field values if the custom field name is a shown column. field_id_to_col = {} for art in artifact_list: for fv in art.field_values: field_col, field_type = field_id_to_col.get( fv.field_id, (None, None)) if field_col == 'NOT_SHOWN': continue if field_col is None: fd = tracker_bizobj.FindFieldDefByID(fv.field_id, config) if not fd: field_id_to_col[fv.field_id] = 'NOT_SHOWN', None continue field_col = fd.field_name.lower() field_type = fd.field_type if field_col not in column_values: field_id_to_col[fv.field_id] = 'NOT_SHOWN', None continue field_id_to_col[fv.field_id] = field_col, field_type if field_type == tracker_pb2.FieldTypes.ENUM_TYPE: continue # Already handled by label parsing elif field_type == tracker_pb2.FieldTypes.INT_TYPE: val = fv.int_value elif field_type == tracker_pb2.FieldTypes.STR_TYPE: val = fv.str_value elif field_type == tracker_pb2.FieldTypes.USER_TYPE: user = users_by_id.get(fv.user_id) val = user.email if user else framework_constants.NO_USER_NAME elif field_type == tracker_pb2.FieldTypes.DATE_TYPE: val = fv.int_value # TODO(jrobbins): convert to date elif field_type == tracker_pb2.FieldTypes.BOOL_TYPE: val = 'Yes' if fv.int_value else 'No' column_values[field_col][val] = val # TODO(jrobbins): make the capitalization of well-known unique label and # status values match the way it is written in the issue config. # Return EZTItems for each column in left-to-right display order. result = [] for i, col_name in enumerate(columns): # TODO(jrobbins): sort each set of column values top-to-bottom, by the # order specified in the project artifact config. For now, just sort # lexicographically to make expected output defined. sorted_col_values = sorted(column_values[col_name].values()) result.append( template_helpers.EZTItem(col_index=i, column_name=col_name, filter_values=sorted_col_values)) return result
def _ProcessEditComponent(self, mr, post_data, config, component_def): """The user wants to edit this component definition.""" parsed = component_helpers.ParseComponentRequest( mr, post_data, self.services) if not tracker_constants.COMPONENT_NAME_RE.match(parsed.leaf_name): mr.errors.leaf_name = 'Invalid component name' original_path = component_def.path if mr.component_path and '>' in mr.component_path: parent_path = mr.component_path[:mr.component_path.rindex('>')] new_path = '%s>%s' % (parent_path, parsed.leaf_name) else: new_path = parsed.leaf_name conflict = tracker_bizobj.FindComponentDef(new_path, config) if conflict and conflict.component_id != component_def.component_id: mr.errors.leaf_name = 'That name is already in use.' creator, created = self._GetUserViewAndFormattedTime( mr, component_def.creator_id, component_def.created) modifier, modified = self._GetUserViewAndFormattedTime( mr, component_def.modifier_id, component_def.modified) if mr.errors.AnyErrors(): self.PleaseCorrect( mr, initial_leaf_name=parsed.leaf_name, initial_docstring=parsed.docstring, initial_deprecated=ezt.boolean(parsed.deprecated), initial_admins=parsed.admin_usernames, initial_cc=parsed.cc_usernames, initial_labels=parsed.label_strs, created=created, creator=creator, modified=modified, modifier=modifier, ) return None new_modified = int(time.time()) new_modifier_id = self.services.user.LookupUserID( mr.cnxn, mr.auth.email, autocreate=False) self.services.config.UpdateComponentDef( mr.cnxn, mr.project_id, component_def.component_id, path=new_path, docstring=parsed.docstring, deprecated=parsed.deprecated, admin_ids=parsed.admin_ids, cc_ids=parsed.cc_ids, modified=new_modified, modifier_id=new_modifier_id, label_ids=parsed.label_ids) update_rule = False if new_path != original_path: update_rule = True # If the name changed then update all of its subcomponents as well. subcomponent_ids = tracker_bizobj.FindMatchingComponentIDs( original_path, config, exact=False) for subcomponent_id in subcomponent_ids: if subcomponent_id == component_def.component_id: continue subcomponent_def = tracker_bizobj.FindComponentDefByID( subcomponent_id, config) subcomponent_new_path = subcomponent_def.path.replace( original_path, new_path, 1) self.services.config.UpdateComponentDef( mr.cnxn, mr.project_id, subcomponent_def.component_id, path=subcomponent_new_path) if (set(parsed.cc_ids) != set(component_def.cc_ids) or set(parsed.label_ids) != set(component_def.label_ids)): update_rule = True if update_rule: filterrules_helpers.RecomputeAllDerivedFields( mr.cnxn, self.services, mr.project, config) return framework_helpers.FormatAbsoluteURL( mr, urls.COMPONENT_DETAIL, component=new_path, saved=1, ts=int(time.time()))
def _CreateIssueSearchDocuments(issues, comments_dict, users_by_id, config_dict): """Make the GAE search index documents for the given issue batch. Args: issues: list of issues to index. comments_dict: prefetched dictionary of comments on those issues. users_by_id: dictionary {user_id: UserView} so that the email addresses of users who left comments can be found via search. config_dict: dict {project_id: config} for all the projects that the given issues are in. """ documents_by_shard = collections.defaultdict(list) for issue in issues: comments = comments_dict.get(issue.issue_id, []) comments = _IndexableComments(comments, users_by_id) summary = issue.summary # TODO(jrobbins): allow search specifically on explicit vs derived # fields. owner_id = tracker_bizobj.GetOwnerId(issue) owner_email = users_by_id[owner_id].email config = config_dict[issue.project_id] component_paths = [] for component_id in issue.component_ids: cd = tracker_bizobj.FindComponentDefByID(component_id, config) if cd: component_paths.append(cd.path) field_values = [ str(tracker_bizobj.GetFieldValue(fv, users_by_id)) for fv in issue.field_values ] metadata = '%s %s %s %s %s %s' % ( tracker_bizobj.GetStatus(issue), owner_email, [ users_by_id[cc_id].email for cc_id in tracker_bizobj.GetCcIds(issue) ], ' '.join(component_paths), ' '.join(field_values), ' '.join( tracker_bizobj.GetLabels(issue))) assert comments, 'issues should always have at least the description' description = _ExtractCommentText(comments[0], users_by_id) description = description[:framework_constants.MAX_FTS_FIELD_SIZE] all_comments = ' '.join( _ExtractCommentText(c, users_by_id) for c in comments[1:]) all_comments = all_comments[:framework_constants.MAX_FTS_FIELD_SIZE] custom_fields = _BuildCustomFTSFields(issue) doc = search.Document( doc_id=str(issue.issue_id), fields=[ search.NumberField(name='project_id', value=issue.project_id), search.TextField(name='summary', value=summary), search.TextField(name='metadata', value=metadata), search.TextField(name='description', value=description), search.TextField(name='comment', value=all_comments), ] + custom_fields) shard_id = issue.issue_id % settings.num_logical_shards documents_by_shard[shard_id].append(doc) start_time = time.time() promises = [] for shard_id, documents in documents_by_shard.iteritems(): if documents: promises.append( framework_helpers.Promise(_IndexDocsInShard, shard_id, documents)) for promise in promises: promise.WaitAndGetValue() logging.info('Finished %d indexing in shards in %d ms', len(documents_by_shard), int( (time.time() - start_time) * 1000))
def __init__( self, issue, users_by_id, config, open_related=None, closed_related=None, all_related=None): """Store relevant values for later display by EZT. Args: issue: An Issue protocol buffer. users_by_id: dict {user_id: UserViews} for all users mentioned in issue. config: ProjectIssueConfig for this issue. open_related: dict of visible open issues that are related to this issue. closed_related: dict {issue_id: issue} of visible closed issues that are related to this issue. all_related: dict {issue_id: issue} of all blocked-on, blocking, or merged-into issues referenced from this issue, regardless of perms. """ super(IssueView, self).__init__(issue) # The users involved in this issue must be present in users_by_id if # this IssueView is to be used on the issue detail or peek pages. But, # they can be absent from users_by_id if the IssueView is used as a # tile in the grid view. self.owner = users_by_id.get(issue.owner_id) self.derived_owner = users_by_id.get(issue.derived_owner_id) self.cc = [users_by_id.get(cc_id) for cc_id in issue.cc_ids if cc_id] self.derived_cc = [users_by_id.get(cc_id) for cc_id in issue.derived_cc_ids if cc_id] self.status = framework_views.StatusView(issue.status, config) self.derived_status = framework_views.StatusView( issue.derived_status, config) # If we don't have a config available, we don't need to access is_open, so # let it be True. self.is_open = ezt.boolean( not config or tracker_helpers.MeansOpenInProject( tracker_bizobj.GetStatus(issue), config)) self.components = sorted( [ComponentValueView(component_id, config, False) for component_id in issue.component_ids if tracker_bizobj.FindComponentDefByID(component_id, config)] + [ComponentValueView(component_id, config, True) for component_id in issue.derived_component_ids if tracker_bizobj.FindComponentDefByID(component_id, config)], key=lambda cvv: cvv.path) self.fields = [ MakeFieldValueView( fd, config, issue.labels, issue.derived_labels, issue.field_values, users_by_id) # TODO(jrobbins): field-level view restrictions, display options for fd in config.field_defs if not fd.is_deleted] self.fields = sorted( self.fields, key=lambda f: (f.applicable_type, f.field_name)) field_names = [fd.field_name.lower() for fd in config.field_defs if not fd.is_deleted] # TODO(jrobbins): restricts self.labels = [ framework_views.LabelView(label, config) for label in tracker_bizobj.NonMaskedLabels(issue.labels, field_names)] self.derived_labels = [ framework_views.LabelView(label, config) for label in issue.derived_labels if not tracker_bizobj.LabelIsMaskedByField(label, field_names)] self.restrictions = _RestrictionsView(issue) # TODO(jrobbins): sort by order of labels in project config self.short_summary = issue.summary[:tracker_constants.SHORT_SUMMARY_LENGTH] if issue.closed_timestamp: self.closed = timestr.FormatAbsoluteDate(issue.closed_timestamp) else: self.closed = '' blocked_on_iids = issue.blocked_on_iids blocking_iids = issue.blocking_iids # Note that merged_into_str and blocked_on_str includes all issue # references, even those referring to issues that the user can't view, # so open_related and closed_related cannot be used. if all_related is not None: all_blocked_on_refs = [ (all_related[ref_iid].project_name, all_related[ref_iid].local_id) for ref_iid in issue.blocked_on_iids] all_blocked_on_refs.extend([ (r.project, r.issue_id) for r in issue.dangling_blocked_on_refs]) self.blocked_on_str = ', '.join( tracker_bizobj.FormatIssueRef( ref, default_project_name=issue.project_name) for ref in all_blocked_on_refs) all_blocking_refs = [ (all_related[ref_iid].project_name, all_related[ref_iid].local_id) for ref_iid in issue.blocking_iids] all_blocking_refs.extend([ (r.project, r.issue_id) for r in issue.dangling_blocking_refs]) self.blocking_str = ', '.join( tracker_bizobj.FormatIssueRef( ref, default_project_name=issue.project_name) for ref in all_blocking_refs) if issue.merged_into: merged_issue = all_related[issue.merged_into] merged_into_ref = merged_issue.project_name, merged_issue.local_id else: merged_into_ref = None self.merged_into_str = tracker_bizobj.FormatIssueRef( merged_into_ref, default_project_name=issue.project_name) self.blocked_on = [] self.has_dangling = ezt.boolean(self.dangling_blocked_on_refs) self.blocking = [] current_project_name = issue.project_name if open_related is not None and closed_related is not None: self.merged_into = IssueRefView( current_project_name, issue.merged_into, open_related, closed_related) self.blocked_on = [ IssueRefView(current_project_name, iid, open_related, closed_related) for iid in blocked_on_iids] self.blocked_on.extend( [DanglingIssueRefView(ref.project, ref.issue_id) for ref in issue.dangling_blocked_on_refs]) self.blocked_on = [irv for irv in self.blocked_on if irv.visible] # TODO(jrobbins): sort by irv project_name and local_id self.blocking = [ IssueRefView(current_project_name, iid, open_related, closed_related) for iid in blocking_iids] self.blocking.extend( [DanglingIssueRefView(ref.project, ref.issue_id) for ref in issue.dangling_blocking_refs]) self.blocking = [irv for irv in self.blocking if irv.visible] # TODO(jrobbins): sort by irv project_name and local_id self.multiple_blocked_on = ezt.boolean(len(self.blocked_on) >= 2) self.detail_relative_url = tracker_helpers.FormatRelativeIssueURL( issue.project_name, urls.ISSUE_DETAIL, id=issue.local_id)
def _CreateIssueSearchDocuments(issues, comments_dict, users_by_id, config_dict): """Make the GAE search index documents for the given issue batch. Args: issues: list of issues to index. comments_dict: prefetched dictionary of comments on those issues. users_by_id: dictionary {user_id: UserView} so that the email addresses of users who left comments can be found via search. config_dict: dict {project_id: config} for all the projects that the given issues are in. """ documents_by_shard = collections.defaultdict(list) for issue in issues: summary = issue.summary # TODO(jrobbins): allow search specifically on explicit vs derived # fields. owner_id = tracker_bizobj.GetOwnerId(issue) owner_email = users_by_id[owner_id].email config = config_dict[issue.project_id] component_paths = [] for component_id in issue.component_ids: cd = tracker_bizobj.FindComponentDefByID(component_id, config) if cd: component_paths.append(cd.path) field_values = [ tracker_bizobj.GetFieldValue(fv, users_by_id) for fv in issue.field_values ] # Convert to string only the values that are not strings already. # This is done because the default encoding in appengine seems to be 'ascii' # and string values might contain unicode characters, so str will fail to # encode them. field_values = [ value if isinstance(value, string_types) else str(value) for value in field_values ] metadata = '%s %s %s %s %s %s' % ( tracker_bizobj.GetStatus(issue), owner_email, [ users_by_id[cc_id].email for cc_id in tracker_bizobj.GetCcIds(issue) ], ' '.join(component_paths), ' '.join(field_values), ' '.join( tracker_bizobj.GetLabels(issue))) custom_fields = _BuildCustomFTSFields(issue) comments = comments_dict.get(issue.issue_id, []) room_for_comments = (framework_constants.MAX_FTS_FIELD_SIZE - len(summary) - len(metadata) - sum(len(cf.value) for cf in custom_fields)) comments = _IndexableComments(comments, users_by_id, remaining_chars=room_for_comments) logging.info('len(comments) is %r', len(comments)) if comments: description = _ExtractCommentText(comments[0], users_by_id) description = description[:framework_constants.MAX_FTS_FIELD_SIZE] all_comments = ' '.join( _ExtractCommentText(c, users_by_id) for c in comments[1:]) all_comments = all_comments[:framework_constants. MAX_FTS_FIELD_SIZE] else: description = '' all_comments = '' logging.info('Issue %s:%r has zero indexable comments', issue.project_name, issue.local_id) logging.info('Building document for %s:%d', issue.project_name, issue.local_id) logging.info('len(summary) = %d', len(summary)) logging.info('len(metadata) = %d', len(metadata)) logging.info('len(description) = %d', len(description)) logging.info('len(comment) = %d', len(all_comments)) for cf in custom_fields: logging.info('len(%s) = %d', cf.name, len(cf.value)) doc = search.Document( doc_id=str(issue.issue_id), fields=[ search.NumberField(name='project_id', value=issue.project_id), search.TextField(name='summary', value=summary), search.TextField(name='metadata', value=metadata), search.TextField(name='description', value=description), search.TextField(name='comment', value=all_comments), ] + custom_fields) shard_id = issue.issue_id % settings.num_logical_shards documents_by_shard[shard_id].append(doc) start_time = time.time() promises = [] for shard_id, documents in documents_by_shard.items(): if documents: promises.append( framework_helpers.Promise(_IndexDocsInShard, shard_id, documents)) for promise in promises: promise.WaitAndGetValue() logging.info('Finished %d indexing in shards in %d ms', len(documents_by_shard), int( (time.time() - start_time) * 1000))
def GetArtifactAttr(art, attribute_name, users_by_id, label_attr_values_dict, config, related_issues, hotlist_issue_context=None): """Return the requested attribute values of the given artifact. Args: art: a tracked artifact with labels, local_id, summary, stars, and owner. attribute_name: lowercase string name of attribute to get. users_by_id: dictionary of UserViews already created. label_attr_values_dict: dictionary {'key': [value, ...], }. config: ProjectIssueConfig PB for the current project. related_issues: dict {issue_id: issue} of pre-fetched related issues. hotlist_issue_context: dict of {hotlist_issue_field: field_value,..} Returns: A list of string attribute values, or [framework_constants.NO_VALUES] if the artifact has no value for that attribute. """ if attribute_name == '--': return [] if attribute_name == 'id': return [art.local_id] if attribute_name == 'summary': return [art.summary] if attribute_name == 'status': return [tracker_bizobj.GetStatus(art)] if attribute_name == 'stars': return [art.star_count] if attribute_name == 'attachments': return [art.attachment_count] # TODO(jrobbins): support blocking if attribute_name == 'project': return [art.project_name] if attribute_name == 'mergedinto': if art.merged_into and art.merged_into != 0: return [ tracker_bizobj.FormatIssueRef( (related_issues[art.merged_into].project_name, related_issues[art.merged_into].local_id)) ] else: return [framework_constants.NO_VALUES] if attribute_name == 'blocked': return ['Yes' if art.blocked_on_iids else 'No'] if attribute_name == 'blockedon': if not art.blocked_on_iids: return [framework_constants.NO_VALUES] else: return [ tracker_bizobj.FormatIssueRef( (related_issues[blocked_on_iid].project_name, related_issues[blocked_on_iid].local_id)) for blocked_on_iid in art.blocked_on_iids ] if attribute_name == 'adder': if hotlist_issue_context: adder_id = hotlist_issue_context['adder_id'] return [users_by_id[adder_id].display_name] else: return [framework_constants.NO_VALUES] if attribute_name == 'added': if hotlist_issue_context: return [hotlist_issue_context['date_added']] else: return [framework_constants.NO_VALUES] if attribute_name == 'reporter': return [users_by_id[art.reporter_id].display_name] if attribute_name == 'owner': owner_id = tracker_bizobj.GetOwnerId(art) if not owner_id: return [framework_constants.NO_VALUES] else: return [users_by_id[owner_id].display_name] if attribute_name == 'cc': cc_ids = tracker_bizobj.GetCcIds(art) if not cc_ids: return [framework_constants.NO_VALUES] else: return [users_by_id[cc_id].display_name for cc_id in cc_ids] if attribute_name == 'component': comp_ids = list(art.component_ids) + list(art.derived_component_ids) if not comp_ids: return [framework_constants.NO_VALUES] else: paths = [] for comp_id in comp_ids: cd = tracker_bizobj.FindComponentDefByID(comp_id, config) if cd: paths.append(cd.path) return paths # Check to see if it is a field. Process as field only if it is not an enum # type because enum types are stored as key-value labels. fd = tracker_bizobj.FindFieldDef(attribute_name, config) if fd and fd.field_type != tracker_pb2.FieldTypes.ENUM_TYPE: values = [] for fv in art.field_values: if fv.field_id == fd.field_id: value = tracker_bizobj.GetFieldValueWithRawValue( fd.field_type, fv, users_by_id, None) values.append(value) return values # Since it is not a built-in attribute or a field, it must be a key-value # label. return label_attr_values_dict.get(attribute_name, [framework_constants.NO_VALUES])
def __init__(self, issue, users_by_id, config): """Store relevant values for later display by EZT. Args: issue: An Issue protocol buffer. users_by_id: dict {user_id: UserViews} for all users mentioned in issue. config: ProjectIssueConfig for this issue. """ super(IssueView, self).__init__(issue) # The users involved in this issue must be present in users_by_id if # this IssueView is to be used on the issue detail or peek pages. But, # they can be absent from users_by_id if the IssueView is used as a # tile in the grid view. self.owner = users_by_id.get(issue.owner_id) self.derived_owner = users_by_id.get(issue.derived_owner_id) self.cc = [users_by_id.get(cc_id) for cc_id in issue.cc_ids if cc_id] self.derived_cc = [ users_by_id.get(cc_id) for cc_id in issue.derived_cc_ids if cc_id ] self.status = framework_views.StatusView(issue.status, config) self.derived_status = framework_views.StatusView( issue.derived_status, config) # If we don't have a config available, we don't need to access is_open, so # let it be True. self.is_open = ezt.boolean( not config or tracker_helpers.MeansOpenInProject( tracker_bizobj.GetStatus(issue), config)) self.components = sorted([ ComponentValueView(component_id, config, False) for component_id in issue.component_ids if tracker_bizobj.FindComponentDefByID(component_id, config) ] + [ ComponentValueView(component_id, config, True) for component_id in issue.derived_component_ids if tracker_bizobj.FindComponentDefByID(component_id, config) ], key=lambda cvv: cvv.path) self.fields = MakeAllFieldValueViews(config, issue.labels, issue.derived_labels, issue.field_values, users_by_id) labels, derived_labels = tracker_bizobj.ExplicitAndDerivedNonMaskedLabels( issue, config) self.labels = [ framework_views.LabelView(label, config) for label in labels ] self.derived_labels = [ framework_views.LabelView(label, config) for label in derived_labels ] self.restrictions = _RestrictionsView(issue) # TODO(jrobbins): sort by order of labels in project config self.short_summary = issue.summary[:tracker_constants. SHORT_SUMMARY_LENGTH] if issue.closed_timestamp: self.closed = timestr.FormatAbsoluteDate(issue.closed_timestamp) else: self.closed = '' self.blocked_on = [] self.has_dangling = ezt.boolean(self.dangling_blocked_on_refs) self.blocking = [] self.detail_relative_url = tracker_helpers.FormatRelativeIssueURL( issue.project_name, urls.ISSUE_DETAIL, id=issue.local_id) self.crbug_url = tracker_helpers.FormatCrBugURL( issue.project_name, issue.local_id)
def GatherPageData(self, mr): """Build up a dictionary of data values to use when rendering the page. Args: mr: commonly used info parsed from the request. Returns: Dict of values used by EZT for rendering the page. """ with mr.profiler.Phase('getting config'): config = self.services.config.GetProjectConfig( mr.cnxn, mr.project_id) # In addition to checking perms, we adjust some default field values for # project members. is_member = framework_bizobj.UserIsInProject(mr.project, mr.auth.effective_ids) page_perms = self.MakePagePerms( mr, None, permissions.CREATE_ISSUE, permissions.SET_STAR, permissions.EDIT_ISSUE, permissions.EDIT_ISSUE_SUMMARY, permissions.EDIT_ISSUE_STATUS, permissions.EDIT_ISSUE_OWNER, permissions.EDIT_ISSUE_CC) with work_env.WorkEnv(mr, self.services) as we: userprefs = we.GetUserPrefs(mr.auth.user_id) code_font = any( pref for pref in userprefs.prefs if pref.name == 'code_font' and pref.value == 'true') template = self._GetTemplate(mr.cnxn, config, mr.template_name, is_member) if template.summary: initial_summary = template.summary initial_summary_must_be_edited = template.summary_must_be_edited else: initial_summary = PLACEHOLDER_SUMMARY initial_summary_must_be_edited = True if template.status: initial_status = template.status elif is_member: initial_status = 'Accepted' else: initial_status = 'New' # not offering meta, only used in hidden field. component_paths = [] for component_id in template.component_ids: component_paths.append( tracker_bizobj.FindComponentDefByID(component_id, config).path) initial_components = ', '.join(component_paths) if template.owner_id: initial_owner = framework_views.MakeUserView( mr.cnxn, self.services.user, template.owner_id) elif template.owner_defaults_to_member and page_perms.EditIssue: initial_owner = mr.auth.user_view else: initial_owner = None if initial_owner: initial_owner_name = initial_owner.email owner_avail_state = initial_owner.avail_state owner_avail_message_short = initial_owner.avail_message_short else: initial_owner_name = '' owner_avail_state = None owner_avail_message_short = None # Check whether to allow attachments from the entry page allow_attachments = tracker_helpers.IsUnderSoftAttachmentQuota( mr.project) config_view = tracker_views.ConfigView(mr, self.services, config, template) # If the user followed a link that specified the template name, make sure # that it is also in the menu as the current choice. # TODO(jeffcarp): Unit test this. config_view.template_view.can_view = ezt.boolean(True) # TODO(jeffcarp): Unit test this. offer_templates = len(config_view.template_names) > 1 restrict_to_known = config.restrict_to_known enum_field_name_set = { fd.field_name.lower() for fd in config.field_defs if fd.field_type is tracker_pb2.FieldTypes.ENUM_TYPE and not fd.is_deleted } # TODO(jrobbins): restrictions link_or_template_labels = mr.GetListParam('labels', template.labels) labels = [ lab for lab in link_or_template_labels if not tracker_bizobj.LabelIsMaskedByField(lab, enum_field_name_set) ] # Corp-mode users automatically add R-V-G. with work_env.WorkEnv(mr, self.services) as we: userprefs = we.GetUserPrefs(mr.auth.user_id) corp_mode = any( up.name == 'restrict_new_issues' and up.value == 'true' for up in userprefs.prefs) if corp_mode: if not any(lab.lower().startswith('restrict-view-') for lab in labels): labels.append(CORP_RESTRICTION_LABEL) field_user_views = tracker_views.MakeFieldUserViews( mr.cnxn, template, self.services.user) approval_ids = [av.approval_id for av in template.approval_values] field_views = tracker_views.MakeAllFieldValueViews( config, link_or_template_labels, [], template.field_values, field_user_views, parent_approval_ids=approval_ids, phases=template.phases) # TODO(jojwang): monorail:6305, remove this hack when Edit perms for field # values are implemented. field_views = [ view for view in field_views if view.field_name.lower() not in RESTRICTED_FLT_FIELDS ] # TODO(jrobbins): remove "or []" after next release. (prechecked_approvals, required_approval_ids, phases) = issue_tmpl_helpers.GatherApprovalsPageData( template.approval_values or [], template.phases, config) approvals = [ view for view in field_views if view.field_id in approval_ids ] page_data = { 'issue_tab_mode': 'issueEntry', 'initial_summary': initial_summary, 'template_summary': initial_summary, 'clear_summary_on_click': ezt.boolean(initial_summary_must_be_edited and 'initial_summary' not in mr.form_overrides), 'must_edit_summary': ezt.boolean(initial_summary_must_be_edited), 'initial_description': template.content, 'template_name': template.name, 'component_required': ezt.boolean(template.component_required), 'initial_status': initial_status, 'initial_owner': initial_owner_name, 'owner_avail_state': owner_avail_state, 'owner_avail_message_short': owner_avail_message_short, 'initial_components': initial_components, 'initial_cc': '', 'initial_blocked_on': '', 'initial_blocking': '', 'initial_hotlists': '', 'labels': labels, 'fields': field_views, 'any_errors': ezt.boolean(mr.errors.AnyErrors()), 'page_perms': page_perms, 'allow_attachments': ezt.boolean(allow_attachments), 'max_attach_size': template_helpers.BytesKbOrMb( framework_constants.MAX_POST_BODY_SIZE), 'offer_templates': ezt.boolean(offer_templates), 'config': config_view, 'restrict_to_known': ezt.boolean(restrict_to_known), 'is_member': ezt.boolean(is_member), 'code_font': ezt.boolean(code_font), # The following are necessary for displaying phases that come with # this template. These are read-only. 'allow_edit': ezt.boolean(False), 'initial_phases': phases, 'approvals': approvals, 'prechecked_approvals': prechecked_approvals, 'required_approval_ids': required_approval_ids, # See monorail:4692 and the use of PHASES_WITH_MILESTONES # in elements/flt/mr-launch-overview/mr-phase.js 'issue_phase_names': list({ phase.name.lower() for phase in phases if phase.name in PHASES_WITH_MILESTONES }), } return page_data
def components_update(self, request): """Update a component.""" mar = self.mar_factory(request) config = self._services.config.GetProjectConfig(mar.cnxn, mar.project_id) component_path = request.componentPath component_def = tracker_bizobj.FindComponentDef( component_path, config) if not component_def: raise config_svc.NoSuchComponentException( 'The component %s does not exist.' % component_path) if not permissions.CanViewComponentDef( mar.auth.effective_ids, mar.perms, mar.project, component_def): raise permissions.PermissionException( 'User is not allowed to view this component %s' % component_path) if not permissions.CanEditComponentDef( mar.auth.effective_ids, mar.perms, mar.project, component_def, config): raise permissions.PermissionException( 'User is not allowed to edit this component %s' % component_path) original_path = component_def.path new_path = component_def.path new_docstring = component_def.docstring new_deprecated = component_def.deprecated new_admin_ids = component_def.admin_ids new_cc_ids = component_def.cc_ids update_filterrule = False for update in request.updates: if update.field == api_pb2_v1.ComponentUpdateFieldID.LEAF_NAME: leaf_name = update.leafName if not tracker_constants.COMPONENT_NAME_RE.match(leaf_name): raise config_svc.InvalidComponentNameException( 'The component name %s is invalid.' % leaf_name) if '>' in original_path: parent_path = original_path[:original_path.rindex('>')] new_path = '%s>%s' % (parent_path, leaf_name) else: new_path = leaf_name conflict = tracker_bizobj.FindComponentDef(new_path, config) if conflict and conflict.component_id != component_def.component_id: raise config_svc.InvalidComponentNameException( 'The name %s is already in use.' % new_path) update_filterrule = True elif update.field == api_pb2_v1.ComponentUpdateFieldID.DESCRIPTION: new_docstring = update.description elif update.field == api_pb2_v1.ComponentUpdateFieldID.ADMIN: user_ids_dict = self._services.user.LookupUserIDs( mar.cnxn, list(update.admin), autocreate=True) new_admin_ids = [user_ids_dict[email] for email in update.admin] elif update.field == api_pb2_v1.ComponentUpdateFieldID.CC: user_ids_dict = self._services.user.LookupUserIDs( mar.cnxn, list(update.cc), autocreate=True) new_cc_ids = [user_ids_dict[email] for email in update.cc] update_filterrule = True elif update.field == api_pb2_v1.ComponentUpdateFieldID.DEPRECATED: new_deprecated = update.deprecated else: logging.error('Unknown component field %r', update.field) new_modified = int(time.time()) new_modifier_id = self._services.user.LookupUserID( mar.cnxn, mar.auth.email, autocreate=False) logging.info( 'Updating component id %d: path-%s, docstring-%s, deprecated-%s,' ' admin_ids-%s, cc_ids-%s modified by %s', component_def.component_id, new_path, new_docstring, new_deprecated, new_admin_ids, new_cc_ids, new_modifier_id) self._services.config.UpdateComponentDef( mar.cnxn, mar.project_id, component_def.component_id, path=new_path, docstring=new_docstring, deprecated=new_deprecated, admin_ids=new_admin_ids, cc_ids=new_cc_ids, modified=new_modified, modifier_id=new_modifier_id) # TODO(sheyang): reuse the code in componentdetails if original_path != new_path: # If the name changed then update all of its subcomponents as well. subcomponent_ids = tracker_bizobj.FindMatchingComponentIDs( original_path, config, exact=False) for subcomponent_id in subcomponent_ids: if subcomponent_id == component_def.component_id: continue subcomponent_def = tracker_bizobj.FindComponentDefByID( subcomponent_id, config) subcomponent_new_path = subcomponent_def.path.replace( original_path, new_path, 1) self._services.config.UpdateComponentDef( mar.cnxn, mar.project_id, subcomponent_def.component_id, path=subcomponent_new_path) if update_filterrule: filterrules_helpers.RecomputeAllDerivedFields( mar.cnxn, self._services, mar.project, config) return message_types.VoidMessage()