def PreprocessAST(cnxn,
query_ast,
project_ids,
services,
harmonized_config,
is_member=True):
"""Preprocess the query by doing lookups so that the SQL query is simpler.
Args:
cnxn: connection to SQL database.
query_ast: user query abstract syntax tree parsed by query2ast.py.
project_ids: collection of int project IDs to use to look up status values
and labels.
services: Connections to persistence layer for users and configs.
harmonized_config: harmonized config for all projects being searched.
is_member: True if user is a member of all the projects being searched,
so they can do user substring searches.
Returns:
A new QueryAST PB with simplified conditions. Specifically, string values
for labels, statuses, and components are replaced with the int IDs of
those items. Also, is:open is distilled down to
status_id != closed_status_ids.
"""
new_conjs = []
for conj in query_ast.conjunctions:
new_conds = [
_PreprocessCond(cnxn, cond, project_ids, services,
harmonized_config, is_member)
for cond in conj.conds
]
new_conjs.append(ast_pb2.Conjunction(conds=new_conds))
return ast_pb2.QueryAST(conjunctions=new_conjs)
def ParseUserQuery(
query, scope, builtin_fields, harmonized_config, warnings=None,
now=None):
"""Parse a user query and return a set of structure terms.
Args:
query: string with user's query. E.g., 'Priority=High'.
scope: string search terms that define the scope in which the
query should be executed. They are expressed in the same
user query language. E.g., adding the canned query.
builtin_fields: dict {field_name: FieldDef(field_name, type)}
mapping field names to FieldDef objects for built-in fields.
harmonized_config: config for all the projects being searched.
@@@ custom field name is not unique in cross project search.
- custom_fields = {field_name: [fd, ...]}
- query build needs to OR each possible interpretation
- could be label in one project and field in another project.
@@@ what about searching across all projects?
warnings: optional list to accumulate warning messages.
now: optional timestamp for tests, otherwise time.time() is used.
Returns:
A QueryAST with conjunctions (usually just one), where each has a list of
Condition PBs with op, fields, str_values and int_values. E.g., the query
[priority=high leak OR stars>100] over open issues would return
QueryAST(
Conjunction(Condition(EQ, [open_fd], [], [1]),
Condition(EQ, [label_fd], ['priority-high'], []),
Condition(TEXT_HAS, any_field_fd, ['leak'], [])),
Conjunction(Condition(EQ, [open_fd], [], [1]),
Condition(GT, [stars_fd], [], [100])))
Raises:
InvalidQueryError: If a problem was detected in the user's query.
"""
if warnings is None:
warnings = []
if _HasParens(query):
warnings.append('Parentheses are ignored in user queries.')
if _HasParens(scope):
warnings.append('Parentheses are ignored in saved queries.')
# Convert the overall query into one or more OR'd subqueries.
subqueries = query.split(' OR ')
# Make a dictionary of all fields: built-in + custom in each project.
combined_fields = collections.defaultdict(
list, {field_name: [field_def]
for field_name, field_def in builtin_fields.iteritems()})
for fd in harmonized_config.field_defs:
if fd.field_type != tracker_pb2.FieldTypes.ENUM_TYPE:
# Only do non-enum fields because enums are stored as labels
combined_fields[fd.field_name.lower()].append(fd)
conjunctions = [
_ParseConjunction(sq, scope, combined_fields, warnings, now=now)
for sq in subqueries]
logging.info('search warnings: %r', warnings)
return ast_pb2.QueryAST(conjunctions=conjunctions)
def testBuildSQLQuery_Normal(self):
owner_field = BUILTIN_ISSUE_FIELDS['owner']
reporter_id_field = BUILTIN_ISSUE_FIELDS['reporter_id']
conds = [
ast_pb2.MakeCond(ast_pb2.QueryOp.TEXT_HAS, [owner_field],
['example.com'], []),
ast_pb2.MakeCond(ast_pb2.QueryOp.EQ, [reporter_id_field], [],
[111L])
]
ast = ast_pb2.QueryAST(conjunctions=[ast_pb2.Conjunction(conds=conds)])
left_joins, where = ast2select.BuildSQLQuery(ast)
self.assertEqual([('User AS Cond0 ON (Issue.owner_id = Cond0.user_id '
'OR Issue.derived_owner_id = Cond0.user_id)', [])],
left_joins)
self.assertTrue(sql._IsValidJoin(left_joins[0][0]))
self.assertEqual([('(LOWER(Cond0.email) LIKE %s)', ['%example.com%']),
('Issue.reporter_id = %s', [111L])], where)
self.assertTrue(sql._IsValidWhereCond(where[0][0]))
def testPreprocessAST_Normal(self):
open_field = BUILTIN_ISSUE_FIELDS['open']
label_field = BUILTIN_ISSUE_FIELDS['label']
label_id_field = BUILTIN_ISSUE_FIELDS['label_id']
status_id_field = BUILTIN_ISSUE_FIELDS['status_id']
conds = [
ast_pb2.MakeCond(ast_pb2.QueryOp.EQ, [open_field], [], []),
ast_pb2.MakeCond(ast_pb2.QueryOp.EQ, [label_field], ['Hot'], [])
]
ast = ast_pb2.QueryAST()
ast.conjunctions.append(ast_pb2.Conjunction(conds=conds))
new_ast = ast2ast.PreprocessAST(self.cnxn, ast, [789], self.services,
self.config)
self.assertEqual(2, len(new_ast.conjunctions[0].conds))
new_cond_1, new_cond_2 = new_ast.conjunctions[0].conds
self.assertEqual(ast_pb2.QueryOp.NE, new_cond_1.op)
self.assertEqual([status_id_field], new_cond_1.field_defs)
self.assertEqual([7, 8, 9], new_cond_1.int_values)
self.assertEqual([], new_cond_1.str_values)
self.assertEqual(ast_pb2.QueryOp.EQ, new_cond_2.op)
self.assertEqual([label_id_field], new_cond_2.field_defs)
self.assertEqual([0], new_cond_2.int_values)
self.assertEqual([], new_cond_2.str_values)
def testPreprocessAST_EmptyAST(self):
ast = ast_pb2.QueryAST() # No conjunctions in it.
new_ast = ast2ast.PreprocessAST(self.cnxn, ast, [789], self.services,
self.config)
self.assertEqual(ast, new_ast)
def testBuildSQLQuery_EmptyAST(self):
ast = ast_pb2.QueryAST(conjunctions=[ast_pb2.Conjunction()
]) # No conds
left_joins, where = ast2select.BuildSQLQuery(ast)
self.assertEqual([], left_joins)
self.assertEqual([], where)