def _MakeFieldValueItems(field_values, users_by_id):
"""Make appropriate int, string, or user values in the given fields."""
result = []
for fv in field_values:
val = tracker_bizobj.GetFieldValue(fv, users_by_id)
result.append(
template_helpers.EZTItem(val=val, docstring=val, idx=len(result)))
return result
def _SortableFieldValues(art, fd_list, users_by_id):
"""Return a list of field values relevant to one UI table column."""
sortable_value_list = []
for fd in fd_list:
for fv in art.field_values:
if fv.field_id == fd.field_id:
sortable_value_list.append(
tracker_bizobj.GetFieldValue(fv, users_by_id))
return sortable_value_list
def FindFieldValues(field_values, field_id, users_by_id):
"""Accumulate appropriate int, string, or user values in the given fields."""
result = []
for fv in field_values:
if fv.field_id != field_id:
continue
val = tracker_bizobj.GetFieldValue(fv, users_by_id)
result.append(template_helpers.EZTItem(
val=val, docstring=val, idx=len(result)))
return result
def _SortableFieldValues(art, fd_list, users_by_id, phase_name):
"""Return a list of field values relevant to one UI table column."""
phase_id = None
if phase_name:
phase_id = next((
phase.phase_id for phase in art.phases
if phase.name.lower() == phase_name), None)
sortable_value_list = []
for fd in fd_list:
for fv in art.field_values:
if fv.field_id == fd.field_id and fv.phase_id == phase_id:
sortable_value_list.append(
tracker_bizobj.GetFieldValue(fv, users_by_id))
return sortable_value_list
def __init__(self, art, col=None, users_by_id=None, config=None, **_kw):
explicit_values = []
derived_values = []
for fv in art.field_values:
# TODO(jrobbins): for cross-project search this could be a list.
fd = tracker_bizobj.FindFieldDefByID(fv.field_id, config)
if not fd:
# TODO(jrobbins): This can happen if an issue with a custom
# field value is moved to a different project.
logging.warn('Issue ID %r has undefined field value %r',
art.issue_id, fv)
elif fd.field_name.lower() == col:
val = tracker_bizobj.GetFieldValue(fv, users_by_id)
if fv.derived:
derived_values.append(val)
else:
explicit_values.append(val)
TableCell.__init__(self, CELL_TYPE_ATTR, explicit_values,
derived_values=derived_values)
def GetTemplateInfoFromParsed(mr, services, parsed, config):
"""Get Template field info and PBs from a ParsedTemplate."""
admin_ids, _ = tracker_helpers.ParseAdminUsers(mr.cnxn, parsed.admin_str,
services.user)
owner_id = 0
if parsed.owner_str:
try:
user_id = services.user.LookupUserID(mr.cnxn, parsed.owner_str)
auth = authdata.AuthData.FromUserID(mr.cnxn, user_id, services)
if framework_bizobj.UserIsInProject(mr.project,
auth.effective_ids):
owner_id = user_id
else:
mr.errors.owner = 'User is not a member of this project.'
except exceptions.NoSuchUserException:
mr.errors.owner = 'Owner not found.'
component_ids = tracker_helpers.LookupComponentIDs(parsed.component_paths,
config, mr.errors)
# TODO(jojwang): monorail:4678 Process phase field values.
phase_field_val_strs = {}
field_values = field_helpers.ParseFieldValues(mr.cnxn, services.user,
parsed.field_val_strs,
phase_field_val_strs, config)
for fv in field_values:
logging.info('field_value is %r: %r', fv.field_id,
tracker_bizobj.GetFieldValue(fv, {}))
phases = []
approvals = []
if parsed.add_approvals:
phases, approvals = _GetPhasesAndApprovalsFromParsed(
mr, parsed.phase_names, parsed.approvals_to_phase_idx,
parsed.required_approval_ids)
return admin_ids, owner_id, component_ids, field_values, phases, approvals
def __init__(self, art, col=None, users_by_id=None, config=None, **_kw):
explicit_values = []
derived_values = []
cell_type = CELL_TYPE_ATTR
phase_names_by_id = {
phase.phase_id: phase.name.lower()
for phase in art.phases
}
phase_name = None
# Check if col represents a phase field value in the form <phase>.<field>
if '.' in col:
phase_name, col = col.split('.', 1)
for fv in art.field_values:
# TODO(jrobbins): for cross-project search this could be a list.
fd = tracker_bizobj.FindFieldDefByID(fv.field_id, config)
if not fd:
# TODO(jrobbins): This can happen if an issue with a custom
# field value is moved to a different project.
logging.warn('Issue ID %r has undefined field value %r',
art.issue_id, fv)
elif fd.field_name.lower() == col and (phase_names_by_id.get(
fv.phase_id) == phase_name):
if fd.field_type == tracker_pb2.FieldTypes.URL_TYPE:
cell_type = CELL_TYPE_URL
if fd.field_type == tracker_pb2.FieldTypes.STR_TYPE:
self.NOWRAP = ezt.boolean(False)
val = tracker_bizobj.GetFieldValue(fv, users_by_id)
if fv.derived:
derived_values.append(val)
else:
explicit_values.append(val)
TableCell.__init__(self,
cell_type,
explicit_values,
derived_values=derived_values)
def _CreateIssueSearchDocuments(issues, comments_dict, users_by_id,
config_dict):
"""Make the GAE search index documents for the given issue batch.
Args:
issues: list of issues to index.
comments_dict: prefetched dictionary of comments on those issues.
users_by_id: dictionary {user_id: UserView} so that the email
addresses of users who left comments can be found via search.
config_dict: dict {project_id: config} for all the projects that
the given issues are in.
"""
documents_by_shard = collections.defaultdict(list)
for issue in issues:
summary = issue.summary
# TODO(jrobbins): allow search specifically on explicit vs derived
# fields.
owner_id = tracker_bizobj.GetOwnerId(issue)
owner_email = users_by_id[owner_id].email
config = config_dict[issue.project_id]
component_paths = []
for component_id in issue.component_ids:
cd = tracker_bizobj.FindComponentDefByID(component_id, config)
if cd:
component_paths.append(cd.path)
field_values = [
tracker_bizobj.GetFieldValue(fv, users_by_id)
for fv in issue.field_values
]
# Convert to string only the values that are not strings already.
# This is done because the default encoding in appengine seems to be 'ascii'
# and string values might contain unicode characters, so str will fail to
# encode them.
field_values = [
value if isinstance(value, string_types) else str(value)
for value in field_values
]
metadata = '%s %s %s %s %s %s' % (
tracker_bizobj.GetStatus(issue), owner_email, [
users_by_id[cc_id].email
for cc_id in tracker_bizobj.GetCcIds(issue)
], ' '.join(component_paths), ' '.join(field_values), ' '.join(
tracker_bizobj.GetLabels(issue)))
custom_fields = _BuildCustomFTSFields(issue)
comments = comments_dict.get(issue.issue_id, [])
room_for_comments = (framework_constants.MAX_FTS_FIELD_SIZE -
len(summary) - len(metadata) -
sum(len(cf.value) for cf in custom_fields))
comments = _IndexableComments(comments,
users_by_id,
remaining_chars=room_for_comments)
logging.info('len(comments) is %r', len(comments))
if comments:
description = _ExtractCommentText(comments[0], users_by_id)
description = description[:framework_constants.MAX_FTS_FIELD_SIZE]
all_comments = ' '.join(
_ExtractCommentText(c, users_by_id) for c in comments[1:])
all_comments = all_comments[:framework_constants.
MAX_FTS_FIELD_SIZE]
else:
description = ''
all_comments = ''
logging.info('Issue %s:%r has zero indexable comments',
issue.project_name, issue.local_id)
logging.info('Building document for %s:%d', issue.project_name,
issue.local_id)
logging.info('len(summary) = %d', len(summary))
logging.info('len(metadata) = %d', len(metadata))
logging.info('len(description) = %d', len(description))
logging.info('len(comment) = %d', len(all_comments))
for cf in custom_fields:
logging.info('len(%s) = %d', cf.name, len(cf.value))
doc = search.Document(
doc_id=str(issue.issue_id),
fields=[
search.NumberField(name='project_id', value=issue.project_id),
search.TextField(name='summary', value=summary),
search.TextField(name='metadata', value=metadata),
search.TextField(name='description', value=description),
search.TextField(name='comment', value=all_comments),
] + custom_fields)
shard_id = issue.issue_id % settings.num_logical_shards
documents_by_shard[shard_id].append(doc)
start_time = time.time()
promises = []
for shard_id, documents in documents_by_shard.items():
if documents:
promises.append(
framework_helpers.Promise(_IndexDocsInShard, shard_id,
documents))
for promise in promises:
promise.WaitAndGetValue()
logging.info('Finished %d indexing in shards in %d ms',
len(documents_by_shard), int(
(time.time() - start_time) * 1000))
def ConvertFieldValues(config,
labels,
derived_labels,
field_values,
users_by_id,
phases=None):
"""Convert lists of labels and field_values to protoc FieldValues."""
fvs = []
phase_names_by_id = {phase.phase_id: phase.name for phase in phases or []}
fds_by_id = {fd.field_id: fd for fd in config.field_defs}
fids_by_name = {fd.field_name: fd.field_id for fd in config.field_defs}
enum_names_by_lower = {
fd.field_name.lower(): fd.field_name
for fd in config.field_defs
if fd.field_type == tracker_pb2.FieldTypes.ENUM_TYPE
}
labels_by_prefix = tracker_bizobj.LabelsByPrefix(
labels, list(enum_names_by_lower.keys()))
der_labels_by_prefix = tracker_bizobj.LabelsByPrefix(
derived_labels, list(enum_names_by_lower.keys()))
for lower_field_name, values in labels_by_prefix.items():
field_name = enum_names_by_lower.get(lower_field_name)
if not field_name:
continue
fvs.extend([
ConvertFieldValue(fids_by_name.get(field_name), field_name, value,
tracker_pb2.FieldTypes.ENUM_TYPE)
for value in values
])
for lower_field_name, values in der_labels_by_prefix.items():
field_name = enum_names_by_lower.get(lower_field_name)
if not field_name:
continue
fvs.extend([
ConvertFieldValue(fids_by_name.get(field_name),
field_name,
value,
tracker_pb2.FieldTypes.ENUM_TYPE,
is_derived=True) for value in values
])
for fv in field_values:
field_def = fds_by_id.get(fv.field_id)
if not field_def:
logging.info(
'Ignoring field value referencing a non-existent field: %r',
fv)
continue
value = tracker_bizobj.GetFieldValue(fv, users_by_id)
field_name = field_def.field_name
field_type = field_def.field_type
approval_name = None
if field_def.approval_id is not None:
approval_def = fds_by_id.get(field_def.approval_id)
if approval_def:
approval_name = approval_def.field_name
fvs.append(
ConvertFieldValue(fv.field_id,
field_name,
value,
field_type,
approval_name=approval_name,
phase_name=phase_names_by_id.get(fv.phase_id),
is_derived=fv.derived))
return fvs
def __init__(self, mr, template, user_service, config):
super(IssueTemplateView, self).__init__(template)
self.ownername = ''
try:
self.owner_view = framework_views.MakeUserView(
mr.cnxn, user_service, template.owner_id)
except exceptions.NoSuchUserException:
self.owner_view = None
if self.owner_view:
self.ownername = self.owner_view.email
self.admin_views = list(
framework_views.MakeAllUserViews(mr.cnxn, user_service,
template.admin_ids).values())
self.admin_names = ', '.join(
sorted([admin_view.email for admin_view in self.admin_views]))
self.summary_must_be_edited = ezt.boolean(
template.summary_must_be_edited)
self.members_only = ezt.boolean(template.members_only)
self.owner_defaults_to_member = ezt.boolean(
template.owner_defaults_to_member)
self.component_required = ezt.boolean(template.component_required)
component_paths = []
for component_id in template.component_ids:
component_paths.append(
tracker_bizobj.FindComponentDefByID(component_id, config).path)
self.components = ', '.join(component_paths)
self.can_view = ezt.boolean(
permissions.CanViewTemplate(mr.auth.effective_ids, mr.perms,
mr.project, template))
self.can_edit = ezt.boolean(
permissions.CanEditTemplate(mr.auth.effective_ids, mr.perms,
mr.project, template))
field_name_set = {
fd.field_name.lower()
for fd in config.field_defs
if fd.field_type is tracker_pb2.FieldTypes.ENUM_TYPE
and not fd.is_deleted
} # TODO(jrobbins): restrictions
non_masked_labels = [
lab for lab in template.labels
if not tracker_bizobj.LabelIsMaskedByField(lab, field_name_set)
]
for i, label in enumerate(non_masked_labels):
setattr(self, 'label%d' % i, label)
for i in range(len(non_masked_labels), framework_constants.MAX_LABELS):
setattr(self, 'label%d' % i, '')
field_user_views = MakeFieldUserViews(mr.cnxn, template, user_service)
self.field_values = []
for fv in template.field_values:
self.field_values.append(
template_helpers.EZTItem(field_id=fv.field_id,
val=tracker_bizobj.GetFieldValue(
fv, field_user_views),
idx=len(self.field_values)))
self.complete_field_values = MakeAllFieldValueViews(
config, template.labels, [], template.field_values,
field_user_views)
# Templates only display and edit the first value of multi-valued fields, so
# expose a single value, if any.
# TODO(jrobbins): Fully support multi-valued fields in templates.
for idx, field_value_view in enumerate(self.complete_field_values):
field_value_view.idx = idx
if field_value_view.values:
field_value_view.val = field_value_view.values[0].val
else:
field_value_view.val = None
def _ParseTemplate(self, post_data, mr, i, orig_template, config):
"""Parse an issue template. Return orig_template if cannot edit."""
if not self._CanEditTemplate(mr, orig_template):
return orig_template
name = post_data['name_%s' % i]
if name == tracker_constants.DELETED_TEMPLATE_NAME:
return None
members_only = False
if ('members_only_%s' % i) in post_data:
members_only = (post_data['members_only_%s' % i] == 'yes')
summary = ''
if ('summary_%s' % i) in post_data:
summary = post_data['summary_%s' % i]
summary_must_be_edited = False
if ('summary_must_be_edited_%s' % i) in post_data:
summary_must_be_edited = (post_data['summary_must_be_edited_%s' %
i] == 'yes')
content = ''
if ('content_%s' % i) in post_data:
content = post_data['content_%s' % i]
# wrap="hard" has no effect on the content because we copy it to
# a hidden form field before submission. So, server-side word wrap.
content = framework_helpers.WordWrapSuperLongLines(content,
max_cols=75)
status = ''
if ('status_%s' % i) in post_data:
status = post_data['status_%s' % i]
owner_id = 0
if ('owner_%s' % i) in post_data:
owner = post_data['owner_%s' % i]
if owner:
user_id = self.services.user.LookupUserID(mr.cnxn, owner)
auth = monorailrequest.AuthData.FromUserID(
mr.cnxn, user_id, self.services)
if framework_bizobj.UserIsInProject(mr.project,
auth.effective_ids):
owner_id = user_id
labels = post_data.getall('label_%s' % i)
labels_remove = []
field_val_strs = collections.defaultdict(list)
for fd in config.field_defs:
field_value_key = 'field_value_%d_%d' % (i, fd.field_id)
if post_data.get(field_value_key):
field_val_strs[fd.field_id].append(post_data[field_value_key])
field_helpers.ShiftEnumFieldsIntoLabels(labels, labels_remove,
field_val_strs, {}, config)
field_values = field_helpers.ParseFieldValues(mr.cnxn,
self.services.user,
field_val_strs, config)
for fv in field_values:
logging.info('field_value is %r: %r', fv.field_id,
tracker_bizobj.GetFieldValue(fv, {}))
admin_ids = []
if ('admin_names_%s' % i) in post_data:
admin_ids, _admin_str = tracker_helpers.ParseAdminUsers(
mr.cnxn, post_data['admin_names_%s' % i], self.services.user)
component_ids = []
if ('components_%s' % i) in post_data:
component_paths = []
for component_path in post_data['components_%s' % i].split(','):
if component_path.strip() not in component_paths:
component_paths.append(component_path.strip())
component_ids = tracker_helpers.LookupComponentIDs(
component_paths, config, mr.errors)
owner_defaults_to_member = False
if ('owner_defaults_to_member_%s' % i) in post_data:
owner_defaults_to_member = (
post_data['owner_defaults_to_member_%s' % i] == 'yes')
component_required = False
if ('component_required_%s' % i) in post_data:
component_required = post_data['component_required_%s' %
i] == 'yes'
template = tracker_bizobj.MakeIssueTemplate(
name,
summary,
status,
owner_id,
content,
labels,
field_values,
admin_ids,
component_ids,
summary_must_be_edited=summary_must_be_edited,
owner_defaults_to_member=owner_defaults_to_member,
component_required=component_required,
members_only=members_only)
template_id = int(post_data['template_id_%s' % i])
if template_id: # new templates have ID 0, so leave that None in PB.
template.template_id = template_id
logging.info('template is %r', template)
return template
def convert_issue(cls, issue, mar, services):
"""Convert Monorail Issue PB to API IssuesGetInsertResponse."""
config = services.config.GetProjectConfig(mar.cnxn, issue.project_id)
granted_perms = tracker_bizobj.GetGrantedPerms(issue,
mar.auth.effective_ids,
config)
issue_project = services.project.GetProject(mar.cnxn, issue.project_id)
component_list = []
for cd in config.component_defs:
cid = cd.component_id
if cid in issue.component_ids:
component_list.append(cd.path)
cc_list = [convert_person(p, mar.cnxn, services) for p in issue.cc_ids]
cc_list = [p for p in cc_list if p is not None]
field_values_list = []
fds_by_id = {fd.field_id: fd for fd in config.field_defs}
phases_by_id = {phase.phase_id: phase for phase in issue.phases}
for fv in issue.field_values:
fd = fds_by_id.get(fv.field_id)
if not fd:
logging.warning('Custom field %d of project %s does not exist',
fv.field_id, issue_project.project_name)
continue
val = None
if fv.user_id:
val = _get_user_email(services.user, mar.cnxn, fv.user_id)
else:
val = tracker_bizobj.GetFieldValue(fv, {})
if not isinstance(val, string_types):
val = str(val)
new_fv = api_pb2_v1.FieldValue(fieldName=fd.field_name,
fieldValue=val,
derived=fv.derived)
if fd.approval_id: # Attach parent approval name
approval_fd = fds_by_id.get(fd.approval_id)
if not approval_fd:
logging.warning(
'Parent approval field %d of field %s does not exist',
fd.approval_id, fd.field_name)
else:
new_fv.approvalName = approval_fd.field_name
elif fv.phase_id: # Attach phase name
phase = phases_by_id.get(fv.phase_id)
if not phase:
logging.warning('Phase %d for field %s does not exist',
fv.phase_id, fd.field_name)
else:
new_fv.phaseName = phase.name
field_values_list.append(new_fv)
approval_values_list = convert_approvals(mar.cnxn, issue.approval_values,
services, config, issue.phases)
phases_list = convert_phases(issue.phases)
with work_env.WorkEnv(mar, services) as we:
starred = we.IsIssueStarred(issue)
resp = cls(
kind='monorail#issue',
id=issue.local_id,
title=issue.summary,
summary=issue.summary,
projectId=issue_project.project_name,
stars=issue.star_count,
starred=starred,
status=issue.status,
state=(api_pb2_v1.IssueState.open
if tracker_helpers.MeansOpenInProject(
tracker_bizobj.GetStatus(issue), config) else
api_pb2_v1.IssueState.closed),
labels=issue.labels,
components=component_list,
author=convert_person(issue.reporter_id, mar.cnxn, services),
owner=convert_person(issue.owner_id, mar.cnxn, services),
cc=cc_list,
updated=datetime.datetime.fromtimestamp(issue.modified_timestamp),
published=datetime.datetime.fromtimestamp(issue.opened_timestamp),
blockedOn=convert_issue_ids(issue.blocked_on_iids, mar, services),
blocking=convert_issue_ids(issue.blocking_iids, mar, services),
canComment=permissions.CanCommentIssue(mar.auth.effective_ids,
mar.perms,
issue_project,
issue,
granted_perms=granted_perms),
canEdit=permissions.CanEditIssue(mar.auth.effective_ids,
mar.perms,
issue_project,
issue,
granted_perms=granted_perms),
fieldValues=field_values_list,
approvalValues=approval_values_list,
phases=phases_list)
if issue.closed_timestamp > 0:
resp.closed = datetime.datetime.fromtimestamp(issue.closed_timestamp)
if issue.merged_into:
resp.mergedInto = convert_issue_ids([issue.merged_into], mar,
services)[0]
if issue.owner_modified_timestamp:
resp.owner_modified = datetime.datetime.fromtimestamp(
issue.owner_modified_timestamp)
if issue.status_modified_timestamp:
resp.status_modified = datetime.datetime.fromtimestamp(
issue.status_modified_timestamp)
if issue.component_modified_timestamp:
resp.component_modified = datetime.datetime.fromtimestamp(
issue.component_modified_timestamp)
return resp
def _CreateIssueSearchDocuments(issues, comments_dict, users_by_id,
config_dict):
"""Make the GAE search index documents for the given issue batch.
Args:
issues: list of issues to index.
comments_dict: prefetched dictionary of comments on those issues.
users_by_id: dictionary {user_id: UserView} so that the email
addresses of users who left comments can be found via search.
config_dict: dict {project_id: config} for all the projects that
the given issues are in.
"""
documents_by_shard = collections.defaultdict(list)
for issue in issues:
comments = comments_dict.get(issue.issue_id, [])
comments = _IndexableComments(comments, users_by_id)
summary = issue.summary
# TODO(jrobbins): allow search specifically on explicit vs derived
# fields.
owner_id = tracker_bizobj.GetOwnerId(issue)
owner_email = users_by_id[owner_id].email
config = config_dict[issue.project_id]
component_paths = []
for component_id in issue.component_ids:
cd = tracker_bizobj.FindComponentDefByID(component_id, config)
if cd:
component_paths.append(cd.path)
field_values = [
str(tracker_bizobj.GetFieldValue(fv, users_by_id))
for fv in issue.field_values
]
metadata = '%s %s %s %s %s %s' % (
tracker_bizobj.GetStatus(issue), owner_email, [
users_by_id[cc_id].email
for cc_id in tracker_bizobj.GetCcIds(issue)
], ' '.join(component_paths), ' '.join(field_values), ' '.join(
tracker_bizobj.GetLabels(issue)))
assert comments, 'issues should always have at least the description'
description = _ExtractCommentText(comments[0], users_by_id)
description = description[:framework_constants.MAX_FTS_FIELD_SIZE]
all_comments = ' '.join(
_ExtractCommentText(c, users_by_id) for c in comments[1:])
all_comments = all_comments[:framework_constants.MAX_FTS_FIELD_SIZE]
custom_fields = _BuildCustomFTSFields(issue)
doc = search.Document(
doc_id=str(issue.issue_id),
fields=[
search.NumberField(name='project_id', value=issue.project_id),
search.TextField(name='summary', value=summary),
search.TextField(name='metadata', value=metadata),
search.TextField(name='description', value=description),
search.TextField(name='comment', value=all_comments),
] + custom_fields)
shard_id = issue.issue_id % settings.num_logical_shards
documents_by_shard[shard_id].append(doc)
start_time = time.time()
promises = []
for shard_id, documents in documents_by_shard.iteritems():
if documents:
promises.append(
framework_helpers.Promise(_IndexDocsInShard, shard_id,
documents))
for promise in promises:
promise.WaitAndGetValue()
logging.info('Finished %d indexing in shards in %d ms',
len(documents_by_shard), int(
(time.time() - start_time) * 1000))