def _PreprocessExactUsers(cnxn, cond, user_service, id_fields):
"""Preprocess a foo=emails cond into foo_id=IDs, if exact user match.
This preprocesing step converts string conditions to int ID conditions.
E.g., [owner=email] to [owner_id=ID]. It only does it in cases
where (a) the email was "me", so it was already converted to an string of
digits in the search pipeline, or (b) it is "user@domain" which resolves to
a known Monorail user. It is also possible to search for, e.g.,
[owner:substring], but such searches remain 'owner' field searches rather
than 'owner_id', and they cannot be combined with the "me" keyword.
Args:
cnxn: connection to the DB.
cond: original parsed query Condition PB.
user_service: connection to user persistence layer.
id_fields: list of the search fields to use if the conversion to IDs
succeeds.
Returns:
A new Condition PB that checks the id_field. Or, the original cond.
"""
op = _TextOpToIntOp(cond.op)
if _IsDefinedOp(op):
# No need to look up any IDs if we are just testing for any defined value.
return ast_pb2.Condition(op=op, field_defs=id_fields)
# This preprocessing step is only for ops that compare whole values, not
# substrings.
if not _IsEqualityOp(op):
logging.info('could not convert to IDs because op is %r', op)
return cond
user_ids = []
for val in cond.str_values:
try:
user_ids.append(int(val))
except ValueError:
try:
user_ids.append(user_service.LookupUserID(cnxn, val))
except user_svc.NoSuchUserException:
logging.info('could not convert user %r to int ID', val)
return cond # preprocessing failed, stick with the original cond.
return ast_pb2.Condition(op=op, field_defs=id_fields, int_values=user_ids)
def _PreprocessMergedIntoCond(cnxn, cond, project_ids, services,
_harmonized_config, _is_member):
"""Preprocess mergedinto=xyz and has:mergedinto conds.
Preprocesses mergedinto=xyz cond into mergedinto_id:issue_ids.
Preprocesses has:mergedinto cond into has:mergedinto_id.
"""
issue_ids = _GetIssueIDsFromLocalIdsCond(cnxn, cond, project_ids, services)
return ast_pb2.Condition(
op=_TextOpToIntOp(cond.op),
field_defs=[query2ast.BUILTIN_ISSUE_FIELDS['mergedinto_id']],
int_values=issue_ids)
def _PreprocessBlockingCond(cnxn, cond, project_ids, services,
_harmonized_config, _is_member):
"""Preprocess blocking=xyz and has:blocking conds.
Preprocesses blocking=xyz cond into blocking_id:issue_ids.
Preprocesses has:blocking cond into issues that are blocking other issues.
"""
issue_ids = _GetIssueIDsFromLocalIdsCond(cnxn, cond, project_ids, services)
return ast_pb2.Condition(
op=_TextOpToIntOp(cond.op),
field_defs=[query2ast.BUILTIN_ISSUE_FIELDS['blocking_id']],
int_values=issue_ids)
def _PreprocessIsBlockedCond(_cnxn, cond, _project_ids, _services,
_harmonized_config, _is_member):
"""Preprocess an is:blocked cond into issues that are blocked."""
if cond.op == ast_pb2.QueryOp.EQ:
op = ast_pb2.QueryOp.IS_DEFINED
elif cond.op == ast_pb2.QueryOp.NE:
op = ast_pb2.QueryOp.IS_NOT_DEFINED
else:
raise MalformedQuery('Blocked condition got nonsensical op %r' %
cond.op)
return ast_pb2.Condition(
op=op, field_defs=[query2ast.BUILTIN_ISSUE_FIELDS['blockedon_id']])
def _PreprocessIsSpamCond(_cnxn, cond, _project_ids, _services,
_harmonized_config, _is_member):
"""Preprocess an is:spam cond into is_spam == 1."""
if cond.op == ast_pb2.QueryOp.EQ:
int_values = [1]
elif cond.op == ast_pb2.QueryOp.NE:
int_values = [0]
else:
raise MalformedQuery('Spam condition got nonsensical op %r' % cond.op)
return ast_pb2.Condition(
op=ast_pb2.QueryOp.EQ,
field_defs=[query2ast.BUILTIN_ISSUE_FIELDS['is_spam']],
int_values=int_values)
def testSearchIssueFullText_Normal(self):
self.SetUpSearchIssueFullText()
self.mox.ReplayAll()
summary_fd = tracker_pb2.FieldDef(
field_name='summary', field_type=tracker_pb2.FieldTypes.STR_TYPE)
query_ast_conj = ast_pb2.Conjunction(conds=[
ast_pb2.Condition(op=ast_pb2.QueryOp.TEXT_HAS,
field_defs=[summary_fd],
str_values=['test'])
])
issue_ids, capped = tracker_fulltext.SearchIssueFullText(
[789], query_ast_conj, 1)
self.mox.VerifyAll()
self.assertItemsEqual([123, 234], issue_ids)
self.assertFalse(capped)
def _PreprocessStatusCond(cnxn, cond, project_ids, services,
_harmonized_config, _is_member):
"""Preprocess a status=names cond into status_id=IDs."""
if project_ids:
status_ids = []
for project_id in project_ids:
status_ids.extend(
services.config.LookupStatusIDs(cnxn, project_id,
cond.str_values))
else:
status_ids = services.config.LookupStatusIDsAnyProject(
cnxn, cond.str_values)
return ast_pb2.Condition(
op=_TextOpToIntOp(cond.op),
field_defs=[query2ast.BUILTIN_ISSUE_FIELDS['status_id']],
int_values=status_ids)
def _PreprocessHotlistCond(cnxn, cond, _project_ids, services,
_harmonized_config, _is_member):
"""Preprocess hotlist query
Preprocesses a hotlist query in the form:
'hotlist=<user_email>:<hotlist-name>,<hotlist-name>,<user2_email>:...
into hotlist_id=IDs, if exact.
"""
# TODO(jojwang): add support for searches that don't contain domain names.
# eg jojwang:hotlist-name
users_to_hotlists = collections.defaultdict(list)
cur_user = ''
for val in cond.str_values:
if ':' in val:
cur_user, hotlists_str = val.split(':', 1)
else:
hotlists_str = val
try:
users_to_hotlists[int(cur_user)].append(hotlists_str)
except ValueError:
try:
user_id = services.user.LookupUserID(cnxn, cur_user)
users_to_hotlists[user_id].append(hotlists_str)
except exceptions.NoSuchUserException:
logging.info('could not convert user %r to int ID', val)
return cond
hotlist_ids = set()
for user_id, hotlists in users_to_hotlists.items():
if not hotlists[0]:
user_hotlists = services.features.GetHotlistsByUserID(
cnxn, user_id)
user_hotlist_ids = [
hotlist.hotlist_id for hotlist in user_hotlists
if user_id in hotlist.owner_ids
]
else:
user_hotlist_ids = list(
services.features.LookupHotlistIDs(cnxn, hotlists,
[user_id]).values())
for hotlist_id in user_hotlist_ids:
hotlist_ids.add(hotlist_id)
return ast_pb2.Condition(
op=_TextOpToIntOp(cond.op),
field_defs=[query2ast.BUILTIN_ISSUE_FIELDS['hotlist_id']],
int_values=list(hotlist_ids))
def testSearchIssueFullText_CrossProject(self):
self.mox.StubOutWithMock(fulltext_helpers, 'ComprehensiveSearch')
fulltext_helpers.ComprehensiveSearch(
'(project_id:789 OR project_id:678) (summary:"test")',
settings.search_index_name_format % 1).AndReturn([123, 234])
self.mox.ReplayAll()
summary_fd = tracker_pb2.FieldDef(
field_name='summary', field_type=tracker_pb2.FieldTypes.STR_TYPE)
query_ast_conj = ast_pb2.Conjunction(conds=[
ast_pb2.Condition(op=ast_pb2.QueryOp.TEXT_HAS,
field_defs=[summary_fd],
str_values=['test'])
])
issue_ids, capped = tracker_fulltext.SearchIssueFullText(
[789, 678], query_ast_conj, 1)
self.mox.VerifyAll()
self.assertItemsEqual([123, 234], issue_ids)
self.assertFalse(capped)
def testSearchIssueFullText_Capped(self):
try:
orig = settings.fulltext_limit_per_shard
settings.fulltext_limit_per_shard = 1
self.SetUpSearchIssueFullText()
self.mox.ReplayAll()
summary_fd = tracker_pb2.FieldDef(
field_name='summary',
field_type=tracker_pb2.FieldTypes.STR_TYPE)
query_ast_conj = ast_pb2.Conjunction(conds=[
ast_pb2.Condition(op=ast_pb2.QueryOp.TEXT_HAS,
field_defs=[summary_fd],
str_values=['test'])
])
issue_ids, capped = tracker_fulltext.SearchIssueFullText(
[789], query_ast_conj, 1)
self.mox.VerifyAll()
self.assertItemsEqual([123, 234], issue_ids)
self.assertTrue(capped)
finally:
settings.fulltext_limit_per_shard = orig
def _PreprocessLabelCond(cnxn, cond, project_ids, services, _harmonized_config,
_is_member):
"""Preprocess a label=names cond into label_id=IDs."""
if project_ids:
label_ids = []
for project_id in project_ids:
if _IsEqualityOp(cond.op):
label_ids.extend(
services.config.LookupLabelIDs(cnxn, project_id,
cond.str_values))
elif _IsDefinedOp(cond.op):
label_ids.extend(
services.config.LookupIDsOfLabelsMatching(
cnxn, project_id, _MakePrefixRegex(cond)))
elif cond.op == ast_pb2.QueryOp.KEY_HAS:
label_ids.extend(
services.config.LookupIDsOfLabelsMatching(
cnxn, project_id, _MakeKeyValueRegex(cond)))
else:
label_ids.extend(
services.config.LookupIDsOfLabelsMatching(
cnxn, project_id, _MakeWordBoundaryRegex(cond)))
else:
if _IsEqualityOp(cond.op):
label_ids = services.config.LookupLabelIDsAnyProject(
cnxn, cond.str_values)
elif _IsDefinedOp(cond.op):
label_ids = services.config.LookupIDsOfLabelsMatchingAnyProject(
cnxn, _MakePrefixRegex(cond))
elif cond.op == ast_pb2.QueryOp.KEY_HAS:
label_ids = services.config.LookupIDsOfLabelsMatchingAnyProject(
cnxn, _MakeKeyValueRegex(cond))
else:
label_ids = services.config.LookupIDsOfLabelsMatchingAnyProject(
cnxn, _MakeWordBoundaryRegex(cond))
return ast_pb2.Condition(
op=_TextOpToIntOp(cond.op),
field_defs=[query2ast.BUILTIN_ISSUE_FIELDS['label_id']],
int_values=label_ids)
def _PreprocessComponentCond(cnxn, cond, project_ids, services,
harmonized_config, _is_member):
"""Preprocess a component= or component:name cond into component_id=IDs."""
exact = _IsEqualityOp(cond.op)
component_ids = []
if project_ids:
# We are searching within specific projects, so harmonized_config
# holds the config data for all those projects.
for comp_path in cond.str_values:
component_ids.extend(
tracker_bizobj.FindMatchingComponentIDs(comp_path,
harmonized_config,
exact=exact))
else:
# We are searching across the whole site, so we have no harmonized_config
# to use.
component_ids = services.config.FindMatchingComponentIDsAnyProject(
cnxn, cond.str_values, exact=exact)
return ast_pb2.Condition(
op=_TextOpToIntOp(cond.op),
field_defs=[query2ast.BUILTIN_ISSUE_FIELDS['component_id']],
int_values=component_ids)
def _PreprocessIsOpenCond(cnxn, cond, project_ids, services,
_harmonized_config, _is_member):
"""Preprocess an is:open cond into status_id != closed_status_ids."""
if project_ids:
closed_status_ids = []
for project_id in project_ids:
closed_status_ids.extend(
services.config.LookupClosedStatusIDs(cnxn, project_id))
else:
closed_status_ids = services.config.LookupClosedStatusIDsAnyProject(
cnxn)
# Invert the operator, because we're comparing against *closed* statuses.
if cond.op == ast_pb2.QueryOp.EQ:
op = ast_pb2.QueryOp.NE
elif cond.op == ast_pb2.QueryOp.NE:
op = ast_pb2.QueryOp.EQ
else:
raise MalformedQuery('Open condition got nonsensical op %r' % cond.op)
return ast_pb2.Condition(
op=op,
field_defs=[query2ast.BUILTIN_ISSUE_FIELDS['status_id']],
int_values=closed_status_ids)
def _PreprocessExactUsers(cnxn, cond, user_service, id_fields, is_member):
"""Preprocess a foo=emails cond into foo_id=IDs, if exact user match.
This preprocesing step converts string conditions to int ID conditions.
E.g., [owner=email] to [owner_id=ID]. It only does it in cases
where (a) the email was "me", so it was already converted to an string of
digits in the search pipeline, or (b) it is "user@domain" which resolves to
a known Monorail user. It is also possible to search for, e.g.,
[owner:substring], but such searches remain 'owner' field searches rather
than 'owner_id', and they cannot be combined with the "me" keyword.
Args:
cnxn: connection to the DB.
cond: original parsed query Condition PB.
user_service: connection to user persistence layer.
id_fields: list of the search fields to use if the conversion to IDs
succeed.
is_member: True if user is a member of all the projects being searchers,
so they can do user substring searches.
Returns:
A new Condition PB that checks the id_field. Or, the original cond.
Raises:
MalformedQuery: A non-member used a query term that could be used to
guess full user email addresses.
"""
op = _TextOpToIntOp(cond.op)
if _IsDefinedOp(op):
# No need to look up any IDs if we are just testing for any defined value.
return ast_pb2.Condition(op=op,
field_defs=id_fields,
key_suffix=cond.key_suffix,
phase_name=cond.phase_name)
# This preprocessing step is only for ops that compare whole values, not
# substrings.
if not _IsEqualityOp(op):
logging.info('could not convert to IDs because op is %r', op)
if not is_member:
raise MalformedQuery(
'Only project members may compare user strings')
return cond
user_ids = []
for val in cond.str_values:
try:
user_ids.append(int(val))
except ValueError:
try:
user_ids.append(user_service.LookupUserID(cnxn, val))
except exceptions.NoSuchUserException:
if not is_member and val != 'me' and not val.startswith('@'):
logging.info('could not convert user %r to int ID', val)
if '@' in val:
raise MalformedQuery('User email address not found')
else:
raise MalformedQuery(
'Only project members may search for user substrings'
)
return cond # preprocessing failed, stick with the original cond.
return ast_pb2.MakeCond(op,
id_fields, [],
user_ids,
key_suffix=cond.key_suffix,
phase_name=cond.phase_name)
def testQueryIssueSnapshots_WithQueryStringAndCannedQuery(self):
"""Test the query param is parsed and used."""
project = fake.Project(project_id=789)
perms = permissions.USER_PERMISSIONSET
search_helpers.GetPersonalAtRiskLabelIDs(self.cnxn, None,
self.config_service, [10, 20],
project, perms).AndReturn([])
cols = [
'Lab.label',
'IssueSnapshot.issue_id',
]
left_joins = [
('Issue ON IssueSnapshot.issue_id = Issue.id', []),
('Issue2Cc AS I2cc'
' ON Issue.id = I2cc.issue_id'
' AND I2cc.cc_id IN (%s,%s)', [10, 20]),
('IssueSnapshot2Label AS Is2l'
' ON Is2l.issuesnapshot_id = IssueSnapshot.id', []),
('LabelDef AS Lab ON Lab.id = Is2l.label_id', []),
('IssueSnapshot2Label AS Cond0 '
'ON IssueSnapshot.id = Cond0.issuesnapshot_id '
'AND Cond0.label_id = %s', [15]),
]
where = [
('IssueSnapshot.period_start <= %s', [1514764800]),
('IssueSnapshot.period_end > %s', [1514764800]),
('IssueSnapshot.project_id = %s', [789]),
('Issue.is_spam = %s', [False]),
('Issue.deleted = %s', [False]),
('(Issue.reporter_id IN (%s,%s)'
' OR Issue.owner_id IN (%s,%s)'
' OR I2cc.cc_id IS NOT NULL)', [10, 20, 10, 20]),
('LOWER(Lab.label) LIKE %s', ['foo-%']),
('Cond0.label_id IS NULL', []),
('IssueSnapshot.is_open = %s', [True]),
]
group_by = ['Lab.label']
query_left_joins = [('IssueSnapshot2Label AS Cond0 '
'ON IssueSnapshot.id = Cond0.issuesnapshot_id '
'AND Cond0.label_id = %s', [15])]
query_where = [
('Cond0.label_id IS NULL', []),
('IssueSnapshot.is_open = %s', [True]),
]
unsupported_field_names = ['ownerbouncing']
unsupported_conds = [
ast_pb2.Condition(
op=ast_pb2.QueryOp(1),
field_defs=[
tracker_pb2.FieldDef(
field_name='ownerbouncing',
field_type=tracker_pb2.FieldTypes.BOOL_TYPE),
])
]
stmt, stmt_args = self.services.chart._BuildSnapshotQuery(cols,
where,
left_joins,
group_by,
shard_id=0)
self.services.chart._QueryToWhere(mox.IgnoreArg(), mox.IgnoreArg(),
mox.IgnoreArg(), mox.IgnoreArg(),
mox.IgnoreArg(),
mox.IgnoreArg()).AndReturn(
(query_left_joins, query_where,
unsupported_conds))
self.cnxn.Execute(stmt, stmt_args, shard_id=0).AndReturn([])
self._verifySQL(cols, left_joins, where, group_by)
self.mox.ReplayAll()
_, unsupported, limit_reached = self.services.chart.QueryIssueSnapshots(
self.cnxn,
self.services,
unixtime=1514764800,
effective_ids=[10, 20],
project=project,
perms=perms,
group_by='label',
label_prefix='Foo',
query='-label:Performance%20is:ownerbouncing',
canned_query='is:open')
self.mox.VerifyAll()
self.assertEqual(unsupported_field_names, unsupported)
self.assertFalse(limit_reached)
