def testBlockingIDCond_MultiValue(self):
fd = BUILTIN_ISSUE_FIELDS['blocking_id']
txt_cond = ast_pb2.MakeCond(ast_pb2.QueryOp.EQ, [fd], ['1', '2', '3'],
[])
num_cond = ast_pb2.MakeCond(ast_pb2.QueryOp.EQ, [fd], [], [1L, 2L, 3L])
for cond, expected in ((txt_cond, ['1', '2',
'3']), (num_cond, [1L, 2L, 3L])):
left_joins, where = ast2select._ProcessBlockingIDCond(
cond, 'Cond1', 'Issue1')
self.assertEqual([
('IssueRelation AS Cond1 ON Issue.id = Cond1.dst_issue_id AND '
'Cond1.kind = %s AND Cond1.issue_id IN (%s,%s,%s)',
['blockedon'] + expected)
], left_joins)
self.assertTrue(sql._IsValidJoin(left_joins[0][0]))
self.assertEqual([('Cond1.dst_issue_id IS NOT NULL', [])], where)
self.assertTrue(sql._IsValidWhereCond(where[0][0]))
def testProcessCustomFieldCond_UserType_ByEmail(self):
fd = tracker_pb2.FieldDef(field_id=1,
project_id=789,
field_name='ExecutiveProducer',
field_type=tracker_pb2.FieldTypes.USER_TYPE)
val = '*****@*****.**'
cond = ast_pb2.MakeCond(ast_pb2.QueryOp.EQ, [fd], [val], [])
left_joins, where = ast2select._ProcessCustomFieldCond(
cond, 'Cond1', 'User1')
self.assertEqual(
[('User AS User1 ON Cond1.user_id = User1.user_id AND '
'LOWER(User1.email) = %s', [val]),
('Issue2FieldValue AS Cond1 ON Issue.id = Cond1.issue_id AND '
'Issue.shard = Cond1.issue_shard AND '
'Cond1.field_id = %s', [1])], left_joins)
self.assertTrue(sql._IsValidJoin(left_joins[0][0]))
self.assertEqual([('Cond1.field_id IS NOT NULL', [])], where)
self.assertTrue(sql._IsValidWhereCond(where[0][0]))
def testBuildSQLQuery_Normal(self):
owner_field = BUILTIN_ISSUE_FIELDS['owner']
reporter_id_field = BUILTIN_ISSUE_FIELDS['reporter_id']
conds = [
ast_pb2.MakeCond(ast_pb2.QueryOp.TEXT_HAS, [owner_field],
['example.com'], []),
ast_pb2.MakeCond(ast_pb2.QueryOp.EQ, [reporter_id_field], [],
[111L])
]
ast = ast_pb2.QueryAST(conjunctions=[ast_pb2.Conjunction(conds=conds)])
left_joins, where = ast2select.BuildSQLQuery(ast)
self.assertEqual([('User AS Cond0 ON (Issue.owner_id = Cond0.user_id '
'OR Issue.derived_owner_id = Cond0.user_id)', [])],
left_joins)
self.assertTrue(sql._IsValidJoin(left_joins[0][0]))
self.assertEqual([('(LOWER(Cond0.email) LIKE %s)', ['%example.com%']),
('Issue.reporter_id = %s', [111L])], where)
self.assertTrue(sql._IsValidWhereCond(where[0][0]))
def _FilterSpam(query_ast):
uses_spam = False
# TODO(jrobbins): Handle "OR" in queries. For now, we just modify the
# first conjunction.
conjunction = query_ast.conjunctions[0]
for condition in conjunction.conds:
for field in condition.field_defs:
if field.field_name == 'spam':
uses_spam = True
if not uses_spam:
query_ast.conjunctions[0].conds.append(
ast_pb2.MakeCond(ast_pb2.QueryOp.NE, [
tracker_pb2.FieldDef(
field_name='spam',
field_type=tracker_pb2.FieldTypes.BOOL_TYPE)
], [], []))
return query_ast
def _ParseCond(cond_str, fields, warnings, now=None):
"""Parse one user query condition string into a Condition PB."""
op_match = OP_RE.match(cond_str)
# Do not treat as key:value search terms if any of the special prefixes match.
special_prefixes_match = any(
cond_str.startswith(p) for p in NON_OP_PREFIXES)
if op_match and not special_prefixes_match:
prefix = op_match.group('prefix')
op = op_match.group('op')
val = op_match.group('value')
# Special case handling to continue to support old date query terms from
# code.google.com. See monorail:151 for more details.
if prefix.startswith(_DATE_FIELDS):
for date_suffix in _DATE_FIELD_SUFFIX_TO_OP:
if prefix.endswith(date_suffix):
prefix = prefix.rstrip(date_suffix)
op = _DATE_FIELD_SUFFIX_TO_OP[date_suffix]
return _ParseStructuredTerm(prefix, op, val, fields, now=now)
# Treat the cond as a full-text search term, which might be negated.
if cond_str.startswith('-'):
op = NOT_TEXT_HAS
cond_str = cond_str[1:]
else:
op = TEXT_HAS
# Construct a full-text Query object as a dry-run to validate that
# the syntax is acceptable.
try:
_fts_query = search.Query(cond_str)
except search.QueryError:
warnings.append('Ignoring full-text term: %s' % cond_str)
return None
# Flag a potential user misunderstanding.
if cond_str.lower() in ('and', 'or', 'not'):
warnings.append(
'The only supported boolean operator is OR (all capitals).')
return ast_pb2.MakeCond(
op, [BUILTIN_ISSUE_FIELDS[ast_pb2.ANY_FIELD]], [cond_str], [])
def testPreprocessHotlistCond(self):
hotlist_field = BUILTIN_ISSUE_FIELDS['hotlist']
hotlist_id_field = BUILTIN_ISSUE_FIELDS['hotlist_id']
self.services.user.TestAddUser('*****@*****.**', 111)
self.services.user.TestAddUser('*****@*****.**', 222)
self.services.user.TestAddUser('*****@*****.**', 333)
# Setup hotlists
self.services.features.TestAddHotlist('Hotlist1',
owner_ids=[111],
hotlist_id=10)
self.services.features.TestAddHotlist('Hotlist2',
owner_ids=[111],
hotlist_id=20)
self.services.features.TestAddHotlist('Hotlist3',
owner_ids=[222],
hotlist_id=30)
self.services.features.TestAddHotlist('Hotlist4',
owner_ids=[222],
hotlist_id=40)
self.services.features.TestAddHotlist('Hotlist5',
owner_ids=[333],
hotlist_id=50)
self.services.features.TestAddHotlist('Hotlist6',
owner_ids=[333],
hotlist_id=60)
hotlist_query_vals = [
'[email protected]:Hotlist1', '[email protected]:',
'[email protected]:Hotlist3', 'Hotlist4'
]
cond = ast_pb2.MakeCond(ast_pb2.QueryOp.TEXT_HAS, [hotlist_field],
hotlist_query_vals, [])
actual = ast2ast._PreprocessHotlistCond(self.cnxn, cond, [1],
self.services, None, True)
self.assertEqual(ast_pb2.QueryOp.EQ, actual.op)
self.assertEqual([hotlist_id_field], actual.field_defs)
self.assertItemsEqual([10, 30, 40, 50, 60], actual.int_values)
def testPreprocessBlockingCond_WithExternalIssues(self):
blocking_field = BUILTIN_ISSUE_FIELDS['blocking']
blocking_id_field = BUILTIN_ISSUE_FIELDS['blocking_id']
self.services.project.TestAddProject('Project1', project_id=1)
issue1 = fake.MakeTestIssue(project_id=1,
local_id=1,
summary='sum',
status='new',
owner_id=2,
issue_id=101)
issue2 = fake.MakeTestIssue(project_id=1,
local_id=2,
summary='sum',
status='new',
owner_id=2,
issue_id=102)
self.services.issue.TestAddIssue(issue1)
self.services.issue.TestAddIssue(issue2)
for local_ids, expected_issues, expected_ext_issues in (([
'b/1234'
], [], ['b/1234']), (['Project1:1', 'b/1234'], [101],
['b/1234'
]), (['1', 'b/1234', 'b/1551',
'Project1:2'], [101,
102], ['b/1234',
'b/1551'])):
cond = ast_pb2.MakeCond(ast_pb2.QueryOp.TEXT_HAS, [blocking_field],
local_ids, [])
new_cond = ast2ast._PreprocessBlockingCond(self.cnxn, cond, [1],
self.services, None,
True)
self.assertEqual(ast_pb2.QueryOp.EQ, new_cond.op)
self.assertEqual([blocking_id_field], new_cond.field_defs)
self.assertEqual(expected_issues, new_cond.int_values)
self.assertEqual(expected_ext_issues, new_cond.str_values)
def _PreprocessExactUsers(cnxn, cond, user_service, id_fields, is_member):
"""Preprocess a foo=emails cond into foo_id=IDs, if exact user match.
This preprocesing step converts string conditions to int ID conditions.
E.g., [owner=email] to [owner_id=ID]. It only does it in cases
where (a) the email was "me", so it was already converted to an string of
digits in the search pipeline, or (b) it is "user@domain" which resolves to
a known Monorail user. It is also possible to search for, e.g.,
[owner:substring], but such searches remain 'owner' field searches rather
than 'owner_id', and they cannot be combined with the "me" keyword.
Args:
cnxn: connection to the DB.
cond: original parsed query Condition PB.
user_service: connection to user persistence layer.
id_fields: list of the search fields to use if the conversion to IDs
succeed.
is_member: True if user is a member of all the projects being searchers,
so they can do user substring searches.
Returns:
A new Condition PB that checks the id_field. Or, the original cond.
Raises:
MalformedQuery: A non-member used a query term that could be used to
guess full user email addresses.
"""
op = _TextOpToIntOp(cond.op)
if _IsDefinedOp(op):
# No need to look up any IDs if we are just testing for any defined value.
return ast_pb2.Condition(op=op,
field_defs=id_fields,
key_suffix=cond.key_suffix,
phase_name=cond.phase_name)
# This preprocessing step is only for ops that compare whole values, not
# substrings.
if not _IsEqualityOp(op):
logging.info('could not convert to IDs because op is %r', op)
if not is_member:
raise MalformedQuery(
'Only project members may compare user strings')
return cond
user_ids = []
for val in cond.str_values:
try:
user_ids.append(int(val))
except ValueError:
try:
user_ids.append(user_service.LookupUserID(cnxn, val))
except exceptions.NoSuchUserException:
if not is_member and val != 'me' and not val.startswith('@'):
logging.info('could not convert user %r to int ID', val)
if '@' in val:
raise MalformedQuery('User email address not found')
else:
raise MalformedQuery(
'Only project members may search for user substrings'
)
return cond # preprocessing failed, stick with the original cond.
return ast_pb2.MakeCond(op,
id_fields, [],
user_ids,
key_suffix=cond.key_suffix,
phase_name=cond.phase_name)
def testPreprocessCond_NoChange(self):
cond = ast_pb2.MakeCond(ast_pb2.QueryOp.TEXT_HAS, [ANY_FIELD], ['foo'],
[])
self.assertEqual(
cond, ast2ast._PreprocessCond(self.cnxn, cond, [], None, None,
True))
def testKeyValueRegex_multipleKeys(self):
cond = ast_pb2.MakeCond(ast_pb2.QueryOp.KEY_HAS,
[BUILTIN_ISSUE_FIELDS['label']],
['Type-Bug', 'Security-Bug'], [])
with self.assertRaises(ValueError):
ast2ast._MakeKeyValueRegex(cond)
def _ParseStructuredTerm(prefix, op_str, value, fields, now=None):
"""Parse one user structured query term into an internal representation.
Args:
prefix: The query operator, usually a field name. E.g., summary. It can
also be special operators like "is" to test boolean fields.
op_str: the comparison operator. Usually ":" or "=", but can be any OPS.
value: the value to compare against, e.g., term to find in that field.
fields: dict {name_lower: [FieldDef, ...]} for built-in and custom fields.
now: optional timestamp for tests, otherwise time.time() is used.
Returns:
A Condition PB.
"""
unquoted_value = value.strip('"')
# Quick-OR is a convenient way to write one condition that matches any one of
# multiple values, like set membership. E.g., [Priority=High,Critical].
quick_or_vals = [v.strip() for v in unquoted_value.split(',')]
op = OPS[op_str]
negate = False
if prefix.startswith('-'):
negate = True
op = NEGATED_OPS.get(op, op)
prefix = prefix[1:]
if prefix == 'is' and unquoted_value in [
'open', 'blocked', 'spam', 'ownerbouncing']:
return ast_pb2.MakeCond(
NE if negate else EQ, fields[unquoted_value], [], [])
# Search entries with or without any value in the specified field.
if prefix == 'has':
op = IS_NOT_DEFINED if negate else IS_DEFINED
if unquoted_value in fields: # Look for that field with any value.
return ast_pb2.MakeCond(op, fields[unquoted_value], [], [])
else: # Look for any label with that prefix.
return ast_pb2.MakeCond(op, fields['label'], [unquoted_value], [])
if prefix in fields: # search built-in and custom fields. E.g., summary.
# Note: if first matching field is date-type, we assume they all are.
# TODO(jrobbins): better handling for rare case where multiple projects
# define the same custom field name, and one is a date and another is not.
first_field = fields[prefix][0]
if first_field.field_type == DATE:
date_values = [_ParseDateValue(val, now=now) for val in quick_or_vals]
return ast_pb2.MakeCond(op, fields[prefix], [], date_values)
else:
quick_or_ints = []
for qov in quick_or_vals:
try:
quick_or_ints.append(int(qov))
except ValueError:
pass
return ast_pb2.MakeCond(op, fields[prefix], quick_or_vals, quick_or_ints)
# Since it is not a field, treat it as labels, E.g., Priority.
quick_or_labels = ['%s-%s' % (prefix, v) for v in quick_or_vals]
# Convert substring match to key-value match if user typed 'foo:bar'.
if op == TEXT_HAS:
op = KEY_HAS
return ast_pb2.MakeCond(op, fields['label'], quick_or_labels, [])
def testBuildFTSCondition_NegatedAnyField(self):
query_cond = ast_pb2.MakeCond(NOT_TEXT_HAS, [self.any_field_fd],
['needle'], [])
fulltext_query_clause = fulltext_helpers._BuildFTSCondition(
query_cond, self.fulltext_fields)
self.assertEqual('NOT ("needle")', fulltext_query_clause)
def testBuildFTSCondition_BuiltinField(self):
query_cond = ast_pb2.MakeCond(TEXT_HAS, [self.summary_fd], ['needle'],
[])
fulltext_query_clause = fulltext_helpers._BuildFTSCondition(
query_cond, self.fulltext_fields)
self.assertEqual('(summary:"needle")', fulltext_query_clause)
def testBuildFTSCondition_IgnoredOperator(self):
query_cond = ast_pb2.MakeCond(GE, [self.summary_fd], ['needle'], [])
fulltext_query_clause = fulltext_helpers._BuildFTSCondition(
query_cond, self.fulltext_fields)
self.assertEqual('', fulltext_query_clause)
def _ParseStructuredTerm(prefix, op_str, value, fields, now=None):
"""Parse one user structured query term into an internal representation.
Args:
prefix: The query operator, usually a field name. E.g., summary. It can
also be special operators like "is" to test boolean fields.
op_str: the comparison operator. Usually ":" or "=", but can be any OPS.
value: the value to compare against, e.g., term to find in that field.
fields: dict {name_lower: [FieldDef, ...]} for built-in and custom fields.
now: optional timestamp for tests, otherwise time.time() is used.
Returns:
A Condition PB.
"""
unquoted_value = value.strip('"')
# Quick-OR is a convenient way to write one condition that matches any one of
# multiple values, like set membership. E.g., [Priority=High,Critical].
# Ignore empty values caused by duplicated or trailing commas. E.g.,
# [Priority=High,,Critical,] is equivalent to [Priority=High,Critical].
quick_or_vals = [v.strip() for v in unquoted_value.split(',') if v.strip()]
op = OPS[op_str]
negate = False
if prefix.startswith('-'):
negate = True
op = NEGATED_OPS.get(op, op)
prefix = prefix[1:]
if prefix == 'is' and unquoted_value in [
'open', 'blocked', 'spam', 'ownerbouncing'
]:
return ast_pb2.MakeCond(NE if negate else EQ, fields[unquoted_value],
[], [])
# Search entries with or without any value in the specified field.
if prefix == 'has':
op = IS_NOT_DEFINED if negate else IS_DEFINED
if '.' in unquoted_value: # Possible search for phase field with any value.
phase_name, possible_field = unquoted_value.split('.', 1)
if possible_field in fields:
return ast_pb2.MakeCond(op,
fields[possible_field], [], [],
phase_name=phase_name)
elif unquoted_value in fields: # Look for that field with any value.
return ast_pb2.MakeCond(op, fields[unquoted_value], [], [])
else: # Look for any label with that prefix.
return ast_pb2.MakeCond(op, fields['label'], [unquoted_value], [])
# Search entries with certain gates.
if prefix == 'gate':
return ast_pb2.MakeCond(op, fields['gate'], quick_or_vals, [])
# Determine hotlist query type.
# If prefix is not 'hotlist', quick_or_vals is empty, or qov
# does not contain ':', is_fields will remain True
is_fields = True
if prefix == 'hotlist':
try:
if ':' not in quick_or_vals[0]:
is_fields = False
except IndexError:
is_fields = False
phase_name = None
if '.' in prefix and is_fields:
split_prefix = prefix.split('.', 1)
if split_prefix[1] in fields:
phase_name, prefix = split_prefix
# search built-in and custom fields. E.g., summary.
if prefix in fields and is_fields:
# Note: if first matching field is date-type, we assume they all are.
# TODO(jrobbins): better handling for rare case where multiple projects
# define the same custom field name, and one is a date and another is not.
first_field = fields[prefix][0]
if first_field.field_type == DATE:
date_values = [
_ParseDateValue(val, now=now) for val in quick_or_vals
]
return ast_pb2.MakeCond(op, fields[prefix], [], date_values)
else:
quick_or_ints = []
for qov in quick_or_vals:
try:
quick_or_ints.append(int(qov))
except ValueError:
pass
if first_field.field_type == APPROVAL:
for approval_suffix in _APPROVAL_SUFFIXES:
if prefix.endswith(approval_suffix):
return ast_pb2.MakeCond(op,
fields[prefix],
quick_or_vals,
quick_or_ints,
key_suffix=approval_suffix,
phase_name=phase_name)
return ast_pb2.MakeCond(op,
fields[prefix],
quick_or_vals,
quick_or_ints,
phase_name=phase_name)
# Since it is not a field, treat it as labels, E.g., Priority.
quick_or_labels = ['%s-%s' % (prefix, v) for v in quick_or_vals]
# Convert substring match to key-value match if user typed 'foo:bar'.
if op == TEXT_HAS:
op = KEY_HAS
return ast_pb2.MakeCond(op, fields['label'], quick_or_labels, [])
def testProcessLabelIDCond_NoValue(self):
fd = BUILTIN_ISSUE_FIELDS['label_id']
cond = ast_pb2.MakeCond(ast_pb2.QueryOp.EQ, [fd], [], [])
with self.assertRaises(ast2select.NoPossibleResults):
ast2select._ProcessLabelIDCond(cond, 'Cond1', 'User1')
