def testGetStatus(self):
    issue = tracker_pb2.Issue()
    self.assertEquals(tracker_bizobj.GetStatus(issue), '')

    issue.derived_status = 'InReview'
    self.assertEquals(tracker_bizobj.GetStatus(issue), 'InReview')

    issue.status = 'Forgotten'
    self.assertEquals(tracker_bizobj.GetStatus(issue), 'Forgotten')
Exemple #2
0
def EvaluateSubscriptions(
    cnxn, issue, users_to_queries, services, config):
  """Determine subscribers who have subs that match the given issue."""
  # Note: unlike filter rule, subscriptions see explicit & derived values.
  lower_labels = [lab.lower() for lab in tracker_bizobj.GetLabels(issue)]
  label_set = set(lower_labels)

  subscribers_to_notify = []
  for uid, saved_queries in users_to_queries.items():
    for sq in saved_queries:
      if sq.subscription_mode != 'immediate':
        continue
      if issue.project_id not in sq.executes_in_project_ids:
        continue
      cond = savedqueries_helpers.SavedQueryToCond(sq)
      # TODO(jrobbins): Support linked accounts me_user_ids.
      cond, _warnings = searchpipeline.ReplaceKeywordsWithUserIDs([uid], cond)
      cond_ast = query2ast.ParseUserQuery(
        cond, '', query2ast.BUILTIN_ISSUE_FIELDS, config)

      if filterrules_helpers.EvalPredicate(
          cnxn, services, cond_ast, issue, label_set, config,
          tracker_bizobj.GetOwnerId(issue), tracker_bizobj.GetCcIds(issue),
          tracker_bizobj.GetStatus(issue)):
        subscribers_to_notify.append(uid)
        break  # Don't bother looking at the user's other saved quereies.

  return subscribers_to_notify
Exemple #3
0
    def _CalculateIssuePings(self, issue, config):
        """Return a list of (field, timestamp) pairs for dates that should ping."""
        timestamp_min, timestamp_max = _GetTimestampRange(int(time.time()))
        arrived_dates_by_field_id = {
            fv.field_id: fv.date_value
            for fv in issue.field_values
            if timestamp_min <= fv.date_value < timestamp_max
        }
        logging.info('arrived_dates_by_field_id = %r',
                     arrived_dates_by_field_id)
        # TODO(jrobbins): Lookup field defs regardless of project_id to better
        # handle foreign fields in issues that have been moved between projects.
        pings = [(field, arrived_dates_by_field_id[field.field_id])
                 for field in config.field_defs
                 if (field.field_id in arrived_dates_by_field_id and field.
                     date_action in (tracker_pb2.DateAction.PING_OWNER_ONLY,
                                     tracker_pb2.DateAction.PING_PARTICIPANTS))
                 ]

        # TODO(jrobbins): For now, assume all pings apply only to open issues.
        # Later, allow each date action to specify whether it applies to open
        # issues or all issues.
        means_open = tracker_helpers.MeansOpenInProject(
            tracker_bizobj.GetStatus(issue), config)
        pings = [ping for ping in pings if means_open]

        pings = sorted(pings, key=lambda ping: ping[0].field_name)
        return pings
def ExtractUniqueValues(columns,
                        artifact_list,
                        users_by_id,
                        config,
                        related_issues,
                        hotlist_context_dict=None):
    """Build a nested list of unique values so the user can auto-filter.

  Args:
    columns: a list of lowercase column name strings, which may contain
        combined columns like "priority/pri".
    artifact_list: a list of artifacts in the complete set of search results.
    users_by_id: dict mapping user_ids to UserViews.
    config: ProjectIssueConfig PB for the current project.
    related_issues: dict {issue_id: issue} of pre-fetched related issues.
    hotlist_context_dict: dict for building a hotlist grid table

  Returns:
    [EZTItem(col1, colname1, [val11, val12,...]), ...]
    A list of EZTItems, each of which has a col_index, column_name,
    and a list of unique values that appear in that column.
  """
    column_values = {col_name: {} for col_name in columns}

    # For each combined column "a/b/c", add entries that point from "a" back
    # to "a/b/c", from "b" back to "a/b/c", and from "c" back to "a/b/c".
    combined_column_parts = collections.defaultdict(list)
    for col in columns:
        if '/' in col:
            for col_part in col.split('/'):
                combined_column_parts[col_part].append(col)

    unique_labels = set()
    for art in artifact_list:
        unique_labels.update(tracker_bizobj.GetLabels(art))

    for label in unique_labels:
        if '-' in label:
            col, val = label.split('-', 1)
            col = col.lower()
            if col in column_values:
                column_values[col][val.lower()] = val
            if col in combined_column_parts:
                for combined_column in combined_column_parts[col]:
                    column_values[combined_column][val.lower()] = val
        else:
            if 'summary' in column_values:
                column_values['summary'][label.lower()] = label

    # TODO(jrobbins): Consider refacting some of this to tracker_bizobj
    # or a new builtins.py to reduce duplication.
    if 'reporter' in column_values:
        for art in artifact_list:
            reporter_id = art.reporter_id
            if reporter_id and reporter_id in users_by_id:
                reporter_username = users_by_id[reporter_id].display_name
                column_values['reporter'][
                    reporter_username] = reporter_username

    if 'owner' in column_values:
        for art in artifact_list:
            owner_id = tracker_bizobj.GetOwnerId(art)
            if owner_id and owner_id in users_by_id:
                owner_username = users_by_id[owner_id].display_name
                column_values['owner'][owner_username] = owner_username

    if 'cc' in column_values:
        for art in artifact_list:
            cc_ids = tracker_bizobj.GetCcIds(art)
            for cc_id in cc_ids:
                if cc_id and cc_id in users_by_id:
                    cc_username = users_by_id[cc_id].display_name
                    column_values['cc'][cc_username] = cc_username

    if 'component' in column_values:
        for art in artifact_list:
            all_comp_ids = list(art.component_ids) + list(
                art.derived_component_ids)
            for component_id in all_comp_ids:
                cd = tracker_bizobj.FindComponentDefByID(component_id, config)
                if cd:
                    column_values['component'][cd.path] = cd.path

    if 'stars' in column_values:
        for art in artifact_list:
            star_count = art.star_count
            column_values['stars'][star_count] = star_count

    if 'status' in column_values:
        for art in artifact_list:
            status = tracker_bizobj.GetStatus(art)
            if status:
                column_values['status'][status.lower()] = status

    if 'project' in column_values:
        for art in artifact_list:
            project_name = art.project_name
            column_values['project'][project_name] = project_name

    if 'mergedinto' in column_values:
        for art in artifact_list:
            if art.merged_into and art.merged_into != 0:
                merged_issue = related_issues[art.merged_into]
                merged_issue_ref = tracker_bizobj.FormatIssueRef(
                    (merged_issue.project_name, merged_issue.local_id))
                column_values['mergedinto'][
                    merged_issue_ref] = merged_issue_ref

    if 'blocked' in column_values:
        for art in artifact_list:
            if art.blocked_on_iids:
                column_values['blocked']['is_blocked'] = 'Yes'
            else:
                column_values['blocked']['is_not_blocked'] = 'No'

    if 'blockedon' in column_values:
        for art in artifact_list:
            if art.blocked_on_iids:
                for blocked_on_iid in art.blocked_on_iids:
                    blocked_on_issue = related_issues[blocked_on_iid]
                    blocked_on_ref = tracker_bizobj.FormatIssueRef(
                        (blocked_on_issue.project_name,
                         blocked_on_issue.local_id))
                    column_values['blockedon'][blocked_on_ref] = blocked_on_ref

    if 'blocking' in column_values:
        for art in artifact_list:
            if art.blocking_iids:
                for blocking_iid in art.blocking_iids:
                    blocking_issue = related_issues[blocking_iid]
                    blocking_ref = tracker_bizobj.FormatIssueRef(
                        (blocking_issue.project_name, blocking_issue.local_id))
                    column_values['blocking'][blocking_ref] = blocking_ref

    if 'added' in column_values:
        for art in artifact_list:
            if hotlist_context_dict and hotlist_context_dict[art.issue_id]:
                issue_dict = hotlist_context_dict[art.issue_id]
                date_added = issue_dict['date_added']
                column_values['added'][date_added] = date_added

    if 'adder' in column_values:
        for art in artifact_list:
            if hotlist_context_dict and hotlist_context_dict[art.issue_id]:
                issue_dict = hotlist_context_dict[art.issue_id]
                adder_id = issue_dict['adder_id']
                adder = users_by_id[adder_id].display_name
                column_values['adder'][adder] = adder

    if 'note' in column_values:
        for art in artifact_list:
            if hotlist_context_dict and hotlist_context_dict[art.issue_id]:
                issue_dict = hotlist_context_dict[art.issue_id]
                note = issue_dict['note']
                if issue_dict['note']:
                    column_values['note'][note] = note

    if 'attachments' in column_values:
        for art in artifact_list:
            attachment_count = art.attachment_count
            column_values['attachments'][attachment_count] = attachment_count

    # Add all custom field values if the custom field name is a shown column.
    field_id_to_col = {}
    for art in artifact_list:
        for fv in art.field_values:
            field_col, field_type = field_id_to_col.get(
                fv.field_id, (None, None))
            if field_col == 'NOT_SHOWN':
                continue
            if field_col is None:
                fd = tracker_bizobj.FindFieldDefByID(fv.field_id, config)
                if not fd:
                    field_id_to_col[fv.field_id] = 'NOT_SHOWN', None
                    continue
                field_col = fd.field_name.lower()
                field_type = fd.field_type
                if field_col not in column_values:
                    field_id_to_col[fv.field_id] = 'NOT_SHOWN', None
                    continue
                field_id_to_col[fv.field_id] = field_col, field_type

            if field_type == tracker_pb2.FieldTypes.ENUM_TYPE:
                continue  # Already handled by label parsing
            elif field_type == tracker_pb2.FieldTypes.INT_TYPE:
                val = fv.int_value
            elif field_type == tracker_pb2.FieldTypes.STR_TYPE:
                val = fv.str_value
            elif field_type == tracker_pb2.FieldTypes.USER_TYPE:
                user = users_by_id.get(fv.user_id)
                val = user.email if user else framework_constants.NO_USER_NAME
            elif field_type == tracker_pb2.FieldTypes.DATE_TYPE:
                val = fv.int_value  # TODO(jrobbins): convert to date
            elif field_type == tracker_pb2.FieldTypes.BOOL_TYPE:
                val = 'Yes' if fv.int_value else 'No'

            column_values[field_col][val] = val

    # TODO(jrobbins): make the capitalization of well-known unique label and
    # status values match the way it is written in the issue config.

    # Return EZTItems for each column in left-to-right display order.
    result = []
    for i, col_name in enumerate(columns):
        # TODO(jrobbins): sort each set of column values top-to-bottom, by the
        # order specified in the project artifact config. For now, just sort
        # lexicographically to make expected output defined.
        sorted_col_values = sorted(column_values[col_name].values())
        result.append(
            template_helpers.EZTItem(col_index=i,
                                     column_name=col_name,
                                     filter_values=sorted_col_values))

    return result
Exemple #5
0
def GetArtifactAttr(art,
                    attribute_name,
                    users_by_id,
                    label_attr_values_dict,
                    config,
                    related_issues,
                    hotlist_issue_context=None):
    """Return the requested attribute values of the given artifact.

  Args:
    art: a tracked artifact with labels, local_id, summary, stars, and owner.
    attribute_name: lowercase string name of attribute to get.
    users_by_id: dictionary of UserViews already created.
    label_attr_values_dict: dictionary {'key': [value, ...], }.
    config: ProjectIssueConfig PB for the current project.
    related_issues: dict {issue_id: issue} of pre-fetched related issues.
    hotlist_issue_context: dict of {hotlist_issue_field: field_value,..}

  Returns:
    A list of string attribute values, or [framework_constants.NO_VALUES]
    if the artifact has no value for that attribute.
  """
    if attribute_name == '--':
        return []
    if attribute_name == 'id':
        return [art.local_id]
    if attribute_name == 'summary':
        return [art.summary]
    if attribute_name == 'status':
        return [tracker_bizobj.GetStatus(art)]
    if attribute_name == 'stars':
        return [art.star_count]
    if attribute_name == 'attachments':
        return [art.attachment_count]
    # TODO(jrobbins): support blocking
    if attribute_name == 'project':
        return [art.project_name]
    if attribute_name == 'mergedinto':
        if art.merged_into and art.merged_into != 0:
            return [
                tracker_bizobj.FormatIssueRef(
                    (related_issues[art.merged_into].project_name,
                     related_issues[art.merged_into].local_id))
            ]
        else:
            return [framework_constants.NO_VALUES]
    if attribute_name == 'blocked':
        return ['Yes' if art.blocked_on_iids else 'No']
    if attribute_name == 'blockedon':
        if not art.blocked_on_iids:
            return [framework_constants.NO_VALUES]
        else:
            return [
                tracker_bizobj.FormatIssueRef(
                    (related_issues[blocked_on_iid].project_name,
                     related_issues[blocked_on_iid].local_id))
                for blocked_on_iid in art.blocked_on_iids
            ]
    if attribute_name == 'adder':
        if hotlist_issue_context:
            adder_id = hotlist_issue_context['adder_id']
            return [users_by_id[adder_id].display_name]
        else:
            return [framework_constants.NO_VALUES]
    if attribute_name == 'added':
        if hotlist_issue_context:
            return [hotlist_issue_context['date_added']]
        else:
            return [framework_constants.NO_VALUES]
    if attribute_name == 'reporter':
        return [users_by_id[art.reporter_id].display_name]
    if attribute_name == 'owner':
        owner_id = tracker_bizobj.GetOwnerId(art)
        if not owner_id:
            return [framework_constants.NO_VALUES]
        else:
            return [users_by_id[owner_id].display_name]
    if attribute_name == 'cc':
        cc_ids = tracker_bizobj.GetCcIds(art)
        if not cc_ids:
            return [framework_constants.NO_VALUES]
        else:
            return [users_by_id[cc_id].display_name for cc_id in cc_ids]
    if attribute_name == 'component':
        comp_ids = list(art.component_ids) + list(art.derived_component_ids)
        if not comp_ids:
            return [framework_constants.NO_VALUES]
        else:
            paths = []
            for comp_id in comp_ids:
                cd = tracker_bizobj.FindComponentDefByID(comp_id, config)
                if cd:
                    paths.append(cd.path)
            return paths

    # Check to see if it is a field. Process as field only if it is not an enum
    # type because enum types are stored as key-value labels.
    fd = tracker_bizobj.FindFieldDef(attribute_name, config)
    if fd and fd.field_type != tracker_pb2.FieldTypes.ENUM_TYPE:
        values = []
        for fv in art.field_values:
            if fv.field_id == fd.field_id:
                value = tracker_bizobj.GetFieldValueWithRawValue(
                    fd.field_type, fv, users_by_id, None)
                values.append(value)
        return values

    # Since it is not a built-in attribute or a field, it must be a key-value
    # label.
    return label_attr_values_dict.get(attribute_name,
                                      [framework_constants.NO_VALUES])
Exemple #6
0
def _CreateIssueSearchDocuments(issues, comments_dict, users_by_id,
                                config_dict):
    """Make the GAE search index documents for the given issue batch.

  Args:
    issues: list of issues to index.
    comments_dict: prefetched dictionary of comments on those issues.
    users_by_id: dictionary {user_id: UserView} so that the email
        addresses of users who left comments can be found via search.
    config_dict: dict {project_id: config} for all the projects that
        the given issues are in.
  """
    documents_by_shard = collections.defaultdict(list)
    for issue in issues:
        summary = issue.summary
        # TODO(jrobbins): allow search specifically on explicit vs derived
        # fields.
        owner_id = tracker_bizobj.GetOwnerId(issue)
        owner_email = users_by_id[owner_id].email
        config = config_dict[issue.project_id]
        component_paths = []
        for component_id in issue.component_ids:
            cd = tracker_bizobj.FindComponentDefByID(component_id, config)
            if cd:
                component_paths.append(cd.path)

        field_values = [
            tracker_bizobj.GetFieldValue(fv, users_by_id)
            for fv in issue.field_values
        ]
        # Convert to string only the values that are not strings already.
        # This is done because the default encoding in appengine seems to be 'ascii'
        # and string values might contain unicode characters, so str will fail to
        # encode them.
        field_values = [
            value if isinstance(value, string_types) else str(value)
            for value in field_values
        ]

        metadata = '%s %s %s %s %s %s' % (
            tracker_bizobj.GetStatus(issue), owner_email, [
                users_by_id[cc_id].email
                for cc_id in tracker_bizobj.GetCcIds(issue)
            ], ' '.join(component_paths), ' '.join(field_values), ' '.join(
                tracker_bizobj.GetLabels(issue)))
        custom_fields = _BuildCustomFTSFields(issue)

        comments = comments_dict.get(issue.issue_id, [])
        room_for_comments = (framework_constants.MAX_FTS_FIELD_SIZE -
                             len(summary) - len(metadata) -
                             sum(len(cf.value) for cf in custom_fields))
        comments = _IndexableComments(comments,
                                      users_by_id,
                                      remaining_chars=room_for_comments)
        logging.info('len(comments) is %r', len(comments))
        if comments:
            description = _ExtractCommentText(comments[0], users_by_id)
            description = description[:framework_constants.MAX_FTS_FIELD_SIZE]
            all_comments = ' '.join(
                _ExtractCommentText(c, users_by_id) for c in comments[1:])
            all_comments = all_comments[:framework_constants.
                                        MAX_FTS_FIELD_SIZE]
        else:
            description = ''
            all_comments = ''
            logging.info('Issue %s:%r has zero indexable comments',
                         issue.project_name, issue.local_id)

        logging.info('Building document for %s:%d', issue.project_name,
                     issue.local_id)
        logging.info('len(summary) = %d', len(summary))
        logging.info('len(metadata) = %d', len(metadata))
        logging.info('len(description) = %d', len(description))
        logging.info('len(comment) = %d', len(all_comments))
        for cf in custom_fields:
            logging.info('len(%s) = %d', cf.name, len(cf.value))

        doc = search.Document(
            doc_id=str(issue.issue_id),
            fields=[
                search.NumberField(name='project_id', value=issue.project_id),
                search.TextField(name='summary', value=summary),
                search.TextField(name='metadata', value=metadata),
                search.TextField(name='description', value=description),
                search.TextField(name='comment', value=all_comments),
            ] + custom_fields)

        shard_id = issue.issue_id % settings.num_logical_shards
        documents_by_shard[shard_id].append(doc)

    start_time = time.time()
    promises = []
    for shard_id, documents in documents_by_shard.items():
        if documents:
            promises.append(
                framework_helpers.Promise(_IndexDocsInShard, shard_id,
                                          documents))

    for promise in promises:
        promise.WaitAndGetValue()

    logging.info('Finished %d indexing in shards in %d ms',
                 len(documents_by_shard), int(
                     (time.time() - start_time) * 1000))
Exemple #7
0
    def __init__(self, issue, users_by_id, config):
        """Store relevant values for later display by EZT.

    Args:
      issue: An Issue protocol buffer.
      users_by_id: dict {user_id: UserViews} for all users mentioned in issue.
      config: ProjectIssueConfig for this issue.
    """
        super(IssueView, self).__init__(issue)

        # The users involved in this issue must be present in users_by_id if
        # this IssueView is to be used on the issue detail or peek pages. But,
        # they can be absent from users_by_id if the IssueView is used as a
        # tile in the grid view.
        self.owner = users_by_id.get(issue.owner_id)
        self.derived_owner = users_by_id.get(issue.derived_owner_id)
        self.cc = [users_by_id.get(cc_id) for cc_id in issue.cc_ids if cc_id]
        self.derived_cc = [
            users_by_id.get(cc_id) for cc_id in issue.derived_cc_ids if cc_id
        ]
        self.status = framework_views.StatusView(issue.status, config)
        self.derived_status = framework_views.StatusView(
            issue.derived_status, config)
        # If we don't have a config available, we don't need to access is_open, so
        # let it be True.
        self.is_open = ezt.boolean(
            not config or tracker_helpers.MeansOpenInProject(
                tracker_bizobj.GetStatus(issue), config))

        self.components = sorted([
            ComponentValueView(component_id, config, False)
            for component_id in issue.component_ids
            if tracker_bizobj.FindComponentDefByID(component_id, config)
        ] + [
            ComponentValueView(component_id, config, True)
            for component_id in issue.derived_component_ids
            if tracker_bizobj.FindComponentDefByID(component_id, config)
        ],
                                 key=lambda cvv: cvv.path)

        self.fields = MakeAllFieldValueViews(config, issue.labels,
                                             issue.derived_labels,
                                             issue.field_values, users_by_id)

        labels, derived_labels = tracker_bizobj.ExplicitAndDerivedNonMaskedLabels(
            issue, config)
        self.labels = [
            framework_views.LabelView(label, config) for label in labels
        ]
        self.derived_labels = [
            framework_views.LabelView(label, config)
            for label in derived_labels
        ]
        self.restrictions = _RestrictionsView(issue)

        # TODO(jrobbins): sort by order of labels in project config

        self.short_summary = issue.summary[:tracker_constants.
                                           SHORT_SUMMARY_LENGTH]

        if issue.closed_timestamp:
            self.closed = timestr.FormatAbsoluteDate(issue.closed_timestamp)
        else:
            self.closed = ''

        self.blocked_on = []
        self.has_dangling = ezt.boolean(self.dangling_blocked_on_refs)
        self.blocking = []

        self.detail_relative_url = tracker_helpers.FormatRelativeIssueURL(
            issue.project_name, urls.ISSUE_DETAIL, id=issue.local_id)
        self.crbug_url = tracker_helpers.FormatCrBugURL(
            issue.project_name, issue.local_id)
Exemple #8
0
def convert_issue(cls, issue, mar, services):
    """Convert Monorail Issue PB to API IssuesGetInsertResponse."""

    config = services.config.GetProjectConfig(mar.cnxn, issue.project_id)
    granted_perms = tracker_bizobj.GetGrantedPerms(issue,
                                                   mar.auth.effective_ids,
                                                   config)
    issue_project = services.project.GetProject(mar.cnxn, issue.project_id)
    component_list = []
    for cd in config.component_defs:
        cid = cd.component_id
        if cid in issue.component_ids:
            component_list.append(cd.path)
    cc_list = [convert_person(p, mar.cnxn, services) for p in issue.cc_ids]
    cc_list = [p for p in cc_list if p is not None]
    field_values_list = []
    fds_by_id = {fd.field_id: fd for fd in config.field_defs}
    phases_by_id = {phase.phase_id: phase for phase in issue.phases}
    for fv in issue.field_values:
        fd = fds_by_id.get(fv.field_id)
        if not fd:
            logging.warning('Custom field %d of project %s does not exist',
                            fv.field_id, issue_project.project_name)
            continue
        val = None
        if fv.user_id:
            val = _get_user_email(services.user, mar.cnxn, fv.user_id)
        else:
            val = tracker_bizobj.GetFieldValue(fv, {})
            if not isinstance(val, string_types):
                val = str(val)
        new_fv = api_pb2_v1.FieldValue(fieldName=fd.field_name,
                                       fieldValue=val,
                                       derived=fv.derived)
        if fd.approval_id:  # Attach parent approval name
            approval_fd = fds_by_id.get(fd.approval_id)
            if not approval_fd:
                logging.warning(
                    'Parent approval field %d of field %s does not exist',
                    fd.approval_id, fd.field_name)
            else:
                new_fv.approvalName = approval_fd.field_name
        elif fv.phase_id:  # Attach phase name
            phase = phases_by_id.get(fv.phase_id)
            if not phase:
                logging.warning('Phase %d for field %s does not exist',
                                fv.phase_id, fd.field_name)
            else:
                new_fv.phaseName = phase.name
        field_values_list.append(new_fv)
    approval_values_list = convert_approvals(mar.cnxn, issue.approval_values,
                                             services, config, issue.phases)
    phases_list = convert_phases(issue.phases)
    with work_env.WorkEnv(mar, services) as we:
        starred = we.IsIssueStarred(issue)
    resp = cls(
        kind='monorail#issue',
        id=issue.local_id,
        title=issue.summary,
        summary=issue.summary,
        projectId=issue_project.project_name,
        stars=issue.star_count,
        starred=starred,
        status=issue.status,
        state=(api_pb2_v1.IssueState.open
               if tracker_helpers.MeansOpenInProject(
                   tracker_bizobj.GetStatus(issue), config) else
               api_pb2_v1.IssueState.closed),
        labels=issue.labels,
        components=component_list,
        author=convert_person(issue.reporter_id, mar.cnxn, services),
        owner=convert_person(issue.owner_id, mar.cnxn, services),
        cc=cc_list,
        updated=datetime.datetime.fromtimestamp(issue.modified_timestamp),
        published=datetime.datetime.fromtimestamp(issue.opened_timestamp),
        blockedOn=convert_issue_ids(issue.blocked_on_iids, mar, services),
        blocking=convert_issue_ids(issue.blocking_iids, mar, services),
        canComment=permissions.CanCommentIssue(mar.auth.effective_ids,
                                               mar.perms,
                                               issue_project,
                                               issue,
                                               granted_perms=granted_perms),
        canEdit=permissions.CanEditIssue(mar.auth.effective_ids,
                                         mar.perms,
                                         issue_project,
                                         issue,
                                         granted_perms=granted_perms),
        fieldValues=field_values_list,
        approvalValues=approval_values_list,
        phases=phases_list)
    if issue.closed_timestamp > 0:
        resp.closed = datetime.datetime.fromtimestamp(issue.closed_timestamp)
    if issue.merged_into:
        resp.mergedInto = convert_issue_ids([issue.merged_into], mar,
                                            services)[0]
    if issue.owner_modified_timestamp:
        resp.owner_modified = datetime.datetime.fromtimestamp(
            issue.owner_modified_timestamp)
    if issue.status_modified_timestamp:
        resp.status_modified = datetime.datetime.fromtimestamp(
            issue.status_modified_timestamp)
    if issue.component_modified_timestamp:
        resp.component_modified = datetime.datetime.fromtimestamp(
            issue.component_modified_timestamp)
    return resp
def _CreateIssueSearchDocuments(issues, comments_dict, users_by_id,
                                config_dict):
    """Make the GAE search index documents for the given issue batch.

  Args:
    issues: list of issues to index.
    comments_dict: prefetched dictionary of comments on those issues.
    users_by_id: dictionary {user_id: UserView} so that the email
        addresses of users who left comments can be found via search.
    config_dict: dict {project_id: config} for all the projects that
        the given issues are in.
  """
    documents_by_shard = collections.defaultdict(list)
    for issue in issues:
        comments = comments_dict.get(issue.issue_id, [])
        comments = _IndexableComments(comments, users_by_id)
        summary = issue.summary
        # TODO(jrobbins): allow search specifically on explicit vs derived
        # fields.
        owner_id = tracker_bizobj.GetOwnerId(issue)
        owner_email = users_by_id[owner_id].email
        config = config_dict[issue.project_id]
        component_paths = []
        for component_id in issue.component_ids:
            cd = tracker_bizobj.FindComponentDefByID(component_id, config)
            if cd:
                component_paths.append(cd.path)

        field_values = [
            str(tracker_bizobj.GetFieldValue(fv, users_by_id))
            for fv in issue.field_values
        ]

        metadata = '%s %s %s %s %s %s' % (
            tracker_bizobj.GetStatus(issue), owner_email, [
                users_by_id[cc_id].email
                for cc_id in tracker_bizobj.GetCcIds(issue)
            ], ' '.join(component_paths), ' '.join(field_values), ' '.join(
                tracker_bizobj.GetLabels(issue)))
        assert comments, 'issues should always have at least the description'
        description = _ExtractCommentText(comments[0], users_by_id)
        description = description[:framework_constants.MAX_FTS_FIELD_SIZE]
        all_comments = ' '.join(
            _ExtractCommentText(c, users_by_id) for c in comments[1:])
        all_comments = all_comments[:framework_constants.MAX_FTS_FIELD_SIZE]

        custom_fields = _BuildCustomFTSFields(issue)
        doc = search.Document(
            doc_id=str(issue.issue_id),
            fields=[
                search.NumberField(name='project_id', value=issue.project_id),
                search.TextField(name='summary', value=summary),
                search.TextField(name='metadata', value=metadata),
                search.TextField(name='description', value=description),
                search.TextField(name='comment', value=all_comments),
            ] + custom_fields)

        shard_id = issue.issue_id % settings.num_logical_shards
        documents_by_shard[shard_id].append(doc)

    start_time = time.time()
    promises = []
    for shard_id, documents in documents_by_shard.iteritems():
        if documents:
            promises.append(
                framework_helpers.Promise(_IndexDocsInShard, shard_id,
                                          documents))

    for promise in promises:
        promise.WaitAndGetValue()

    logging.info('Finished %d indexing in shards in %d ms',
                 len(documents_by_shard), int(
                     (time.time() - start_time) * 1000))
def convert_issue(cls, issue, mar, services):
  """Convert Monorail Issue PB to API IssuesGetInsertResponse."""

  config = services.config.GetProjectConfig(mar.cnxn, issue.project_id)
  granted_perms = tracker_bizobj.GetGrantedPerms(
      issue, mar.auth.effective_ids, config)
  issue_project = services.project.GetProject(mar.cnxn, issue.project_id)
  component_list = []
  for cd in config.component_defs:
    cid = cd.component_id
    if cid in issue.component_ids:
      component_list.append(cd.path)
  cc_list = [convert_person(p, mar.cnxn, services) for p in issue.cc_ids]
  cc_list = [p for p in cc_list if p is not None]
  field_values_list = []
  field_id_dict = {
      fd.field_id: fd.field_name for fd in config.field_defs}
  for fv in issue.field_values:
    field_name = field_id_dict.get(fv.field_id)
    if not field_name:
      logging.warning('Custom field %d of project %s does not exist',
                      fv.field_id, issue_project.project_name)
      continue
    val = None
    if fv.user_id:
      val = _get_user_email(
          services.user, mar.cnxn, fv.user_id)
    elif fv.str_value:
      val = fv.str_value
    elif fv.int_value:
      val = str(fv.int_value)
    new_fv = api_pb2_v1.FieldValue(
        fieldName=field_name,
        fieldValue=val,
        derived=fv.derived)
    field_values_list.append(new_fv)
  resp = cls(
      kind='monorail#issue',
      id=issue.local_id,
      title=issue.summary,
      summary=issue.summary,
      projectId=issue_project.project_name,
      stars=issue.star_count,
      starred=services.issue_star.IsItemStarredBy(
          mar.cnxn, issue.issue_id, mar.auth.user_id),
      status=issue.status,
      state=(api_pb2_v1.IssueState.open if
             tracker_helpers.MeansOpenInProject(
                 tracker_bizobj.GetStatus(issue), config)
             else api_pb2_v1.IssueState.closed),
      labels=issue.labels,
      components=component_list,
      author=convert_person(issue.reporter_id, mar.cnxn, services),
      owner=convert_person(issue.owner_id, mar.cnxn, services),
      cc=cc_list,
      updated=datetime.datetime.fromtimestamp(issue.modified_timestamp),
      published=datetime.datetime.fromtimestamp(issue.opened_timestamp),
      blockedOn=convert_issue_ids(issue.blocked_on_iids, mar, services),
      blocking=convert_issue_ids(issue.blocking_iids, mar, services),
      canComment=permissions.CanCommentIssue(
          mar.auth.effective_ids, mar.perms, issue_project, issue,
          granted_perms=granted_perms),
      canEdit=permissions.CanEditIssue(
          mar.auth.effective_ids, mar.perms, issue_project, issue,
          granted_perms=granted_perms),
      fieldValues=field_values_list)
  if issue.closed_timestamp > 0:
    resp.closed = datetime.datetime.fromtimestamp(issue.closed_timestamp)
  if issue.merged_into:
    resp.mergedInto=convert_issue_ids([issue.merged_into], mar, services)[0]
  if issue.owner_modified_timestamp:
    resp.owner_modified = datetime.datetime.fromtimestamp(
        issue.owner_modified_timestamp)
  if issue.status_modified_timestamp:
    resp.status_modified = datetime.datetime.fromtimestamp(
        issue.status_modified_timestamp)
  if issue.component_modified_timestamp:
    resp.component_modified = datetime.datetime.fromtimestamp(
        issue.component_modified_timestamp)
  return resp
Exemple #11
0
  def __init__(
      self, issue, users_by_id, config, open_related=None,
      closed_related=None, all_related=None):
    """Store relevant values for later display by EZT.

    Args:
      issue: An Issue protocol buffer.
      users_by_id: dict {user_id: UserViews} for all users mentioned in issue.
      config: ProjectIssueConfig for this issue.
      open_related: dict of visible open issues that are related to this issue.
      closed_related: dict {issue_id: issue} of visible closed issues that
          are related to this issue.
      all_related: dict {issue_id: issue} of all blocked-on, blocking,
          or merged-into issues referenced from this issue, regardless of
          perms.
    """
    super(IssueView, self).__init__(issue)

    # The users involved in this issue must be present in users_by_id if
    # this IssueView is to be used on the issue detail or peek pages. But,
    # they can be absent from users_by_id if the IssueView is used as a
    # tile in the grid view.
    self.owner = users_by_id.get(issue.owner_id)
    self.derived_owner = users_by_id.get(issue.derived_owner_id)
    self.cc = [users_by_id.get(cc_id) for cc_id in issue.cc_ids
               if cc_id]
    self.derived_cc = [users_by_id.get(cc_id)
                       for cc_id in issue.derived_cc_ids
                       if cc_id]
    self.status = framework_views.StatusView(issue.status, config)
    self.derived_status = framework_views.StatusView(
        issue.derived_status, config)
    # If we don't have a config available, we don't need to access is_open, so
    # let it be True.
    self.is_open = ezt.boolean(
        not config or
        tracker_helpers.MeansOpenInProject(
            tracker_bizobj.GetStatus(issue), config))

    self.components = sorted(
        [ComponentValueView(component_id, config, False)
         for component_id in issue.component_ids
         if tracker_bizobj.FindComponentDefByID(component_id, config)] +
        [ComponentValueView(component_id, config, True)
         for component_id in issue.derived_component_ids
         if tracker_bizobj.FindComponentDefByID(component_id, config)],
        key=lambda cvv: cvv.path)

    self.fields = [
        MakeFieldValueView(
            fd, config, issue.labels, issue.derived_labels, issue.field_values,
            users_by_id)
        # TODO(jrobbins): field-level view restrictions, display options
        for fd in config.field_defs
        if not fd.is_deleted]
    self.fields = sorted(
        self.fields, key=lambda f: (f.applicable_type, f.field_name))

    field_names = [fd.field_name.lower() for fd in config.field_defs
                   if not fd.is_deleted]  # TODO(jrobbins): restricts
    self.labels = [
        framework_views.LabelView(label, config)
        for label in tracker_bizobj.NonMaskedLabels(issue.labels, field_names)]
    self.derived_labels = [
        framework_views.LabelView(label, config)
        for label in issue.derived_labels
        if not tracker_bizobj.LabelIsMaskedByField(label, field_names)]
    self.restrictions = _RestrictionsView(issue)

    # TODO(jrobbins): sort by order of labels in project config

    self.short_summary = issue.summary[:tracker_constants.SHORT_SUMMARY_LENGTH]

    if issue.closed_timestamp:
      self.closed = timestr.FormatAbsoluteDate(issue.closed_timestamp)
    else:
      self.closed = ''

    blocked_on_iids = issue.blocked_on_iids
    blocking_iids = issue.blocking_iids

    # Note that merged_into_str and blocked_on_str includes all issue
    # references, even those referring to issues that the user can't view,
    # so open_related and closed_related cannot be used.
    if all_related is not None:
      all_blocked_on_refs = [
          (all_related[ref_iid].project_name, all_related[ref_iid].local_id)
          for ref_iid in issue.blocked_on_iids]
      all_blocked_on_refs.extend([
          (r.project, r.issue_id) for r in issue.dangling_blocked_on_refs])
      self.blocked_on_str = ', '.join(
          tracker_bizobj.FormatIssueRef(
              ref, default_project_name=issue.project_name)
          for ref in all_blocked_on_refs)
      all_blocking_refs = [
          (all_related[ref_iid].project_name, all_related[ref_iid].local_id)
          for ref_iid in issue.blocking_iids]
      all_blocking_refs.extend([
          (r.project, r.issue_id) for r in issue.dangling_blocking_refs])
      self.blocking_str = ', '.join(
          tracker_bizobj.FormatIssueRef(
              ref, default_project_name=issue.project_name)
          for ref in all_blocking_refs)
      if issue.merged_into:
        merged_issue = all_related[issue.merged_into]
        merged_into_ref = merged_issue.project_name, merged_issue.local_id
      else:
        merged_into_ref = None
      self.merged_into_str = tracker_bizobj.FormatIssueRef(
          merged_into_ref, default_project_name=issue.project_name)

    self.blocked_on = []
    self.has_dangling = ezt.boolean(self.dangling_blocked_on_refs)
    self.blocking = []
    current_project_name = issue.project_name

    if open_related is not None and closed_related is not None:
      self.merged_into = IssueRefView(
          current_project_name, issue.merged_into,
          open_related, closed_related)

      self.blocked_on = [
          IssueRefView(current_project_name, iid, open_related, closed_related)
          for iid in blocked_on_iids]
      self.blocked_on.extend(
          [DanglingIssueRefView(ref.project, ref.issue_id)
           for ref in issue.dangling_blocked_on_refs])
      self.blocked_on = [irv for irv in self.blocked_on if irv.visible]
      # TODO(jrobbins): sort by irv project_name and local_id

      self.blocking = [
          IssueRefView(current_project_name, iid, open_related, closed_related)
          for iid in blocking_iids]
      self.blocking.extend(
          [DanglingIssueRefView(ref.project, ref.issue_id)
           for ref in issue.dangling_blocking_refs])
      self.blocking = [irv for irv in self.blocking if irv.visible]
      # TODO(jrobbins): sort by irv project_name and local_id

    self.multiple_blocked_on = ezt.boolean(len(self.blocked_on) >= 2)
    self.detail_relative_url = tracker_helpers.FormatRelativeIssueURL(
        issue.project_name, urls.ISSUE_DETAIL, id=issue.local_id)
Exemple #12
0
    def StoreIssueSnapshots(self, cnxn, issues, commit=True):
        """Adds an IssueSnapshot and updates the previous one for each issue."""
        for issue in issues:
            right_now = self._currentTime()

            # Look for an existing (latest) IssueSnapshot with this issue_id.
            previous_snapshots = self.issuesnapshot_tbl.Select(
                cnxn,
                cols=ISSUESNAPSHOT_COLS,
                issue_id=issue.issue_id,
                limit=1,
                order_by=[('period_start DESC', [])])

            if len(previous_snapshots) > 0:
                previous_snapshot_id = previous_snapshots[0][0]
                logging.info('Found previous IssueSnapshot with id: %s',
                             previous_snapshot_id)

                # Update previous snapshot's end time to right now.
                delta = {'period_end': right_now}
                where = [('IssueSnapshot.id = %s', [previous_snapshot_id])]
                self.issuesnapshot_tbl.Update(cnxn,
                                              delta,
                                              commit=commit,
                                              where=where)

            config = self.config_service.GetProjectConfig(
                cnxn, issue.project_id)
            period_end = settings.maximum_snapshot_period_end
            is_open = tracker_helpers.MeansOpenInProject(
                tracker_bizobj.GetStatus(issue), config)
            shard = issue.issue_id % settings.num_logical_shards
            status = tracker_bizobj.GetStatus(issue)
            status_id = self.config_service.LookupStatusID(
                cnxn, issue.project_id, status) or None
            owner_id = tracker_bizobj.GetOwnerId(issue) or None

            issuesnapshot_rows = [(issue.issue_id, shard, issue.project_id,
                                   issue.local_id, issue.reporter_id, owner_id,
                                   status_id, right_now, period_end, is_open)]

            ids = self.issuesnapshot_tbl.InsertRows(cnxn,
                                                    ISSUESNAPSHOT_COLS[1:],
                                                    issuesnapshot_rows,
                                                    replace=True,
                                                    commit=commit,
                                                    return_generated_ids=True)
            issuesnapshot_id = ids[0]

            # Add all labels to IssueSnapshot2Label.
            label_rows = [
                (issuesnapshot_id,
                 self.config_service.LookupLabelID(cnxn, issue.project_id,
                                                   label))
                for label in tracker_bizobj.GetLabels(issue)
            ]
            self.issuesnapshot2label_tbl.InsertRows(cnxn,
                                                    ISSUESNAPSHOT2LABEL_COLS,
                                                    label_rows,
                                                    replace=True,
                                                    commit=commit)

            # Add all CCs to IssueSnapshot2Cc.
            cc_rows = [(issuesnapshot_id, cc_id)
                       for cc_id in tracker_bizobj.GetCcIds(issue)]
            self.issuesnapshot2cc_tbl.InsertRows(cnxn,
                                                 ISSUESNAPSHOT2CC_COLS,
                                                 cc_rows,
                                                 replace=True,
                                                 commit=commit)

            # Add all components to IssueSnapshot2Component.
            component_rows = [(issuesnapshot_id, component_id)
                              for component_id in issue.component_ids]
            self.issuesnapshot2component_tbl.InsertRows(
                cnxn,
                ISSUESNAPSHOT2COMPONENT_COLS,
                component_rows,
                replace=True,
                commit=commit)

            # Add all components to IssueSnapshot2Hotlist.
            # This is raw SQL to obviate passing FeaturesService down through
            #   the call stack wherever this function is called.
            # TODO(jrobbins): sort out dependencies between service classes.
            cnxn.Execute(
                '''
        INSERT INTO IssueSnapshot2Hotlist (issuesnapshot_id, hotlist_id)
        SELECT %s, hotlist_id FROM Hotlist2Issue WHERE issue_id = %s
      ''', [issuesnapshot_id, issue.issue_id])