def _QueryToWhere(self, cnxn, services, project_config, query,
canned_query, project):
"""Parses a query string into LEFT JOIN and WHERE conditions.
Args:
cnxn: A MonorailConnection instance.
services: A Services instance.
project_config: The configuration for the given project.
query (string): The query to parse.
canned_query (string): The supplied canned query.
project: The current project.
Returns:
1. A list of LEFT JOIN clauses for the SQL query.
2. A list of WHERE clases for the SQL query.
3. A list of query conditions that are unsupported with snapshots.
"""
if not (query or canned_query):
return [], [], []
query = query or ''
scope = canned_query or ''
query_ast = query2ast.ParseUserQuery(query, scope,
query2ast.BUILTIN_ISSUE_FIELDS,
project_config)
query_ast = ast2ast.PreprocessAST(cnxn, query_ast,
[project.project_id], services,
project_config)
left_joins, where, unsupported = ast2select.BuildSQLQuery(
query_ast, snapshot_mode=True)
return left_joins, where, unsupported
def testSearchProjectCan_FTSCapped(self):
query_ast = query2ast.ParseUserQuery(
'Priority:High', 'is:open', query2ast.BUILTIN_ISSUE_FIELDS,
self.config)
simplified_query_ast = ast2ast.PreprocessAST(
self.cnxn, query_ast, [789], self.services, self.config)
conj = simplified_query_ast.conjunctions[0]
self.mox.StubOutWithMock(tracker_fulltext, 'SearchIssueFullText')
tracker_fulltext.SearchIssueFullText(
[789], conj, 2).AndReturn(([10002, 10052], True))
self.mox.StubOutWithMock(self.services.issue, 'RunIssueQuery')
self.services.issue.RunIssueQuery(
self.cnxn, mox.IsA(list), mox.IsA(list), mox.IsA(list),
shard_id=2).AndReturn(([10002, 10052], False))
self.mox.ReplayAll()
result, capped, err = backendsearchpipeline.SearchProjectCan(
self.cnxn, self.services, [789], query_ast, 2, self.config)
self.mox.VerifyAll()
self.assertEqual([10002, 10052], result)
self.assertTrue(capped)
self.assertEqual(None, err)
def testPreprocessAST_Normal(self):
open_field = BUILTIN_ISSUE_FIELDS['open']
label_field = BUILTIN_ISSUE_FIELDS['label']
label_id_field = BUILTIN_ISSUE_FIELDS['label_id']
status_id_field = BUILTIN_ISSUE_FIELDS['status_id']
conds = [
ast_pb2.MakeCond(ast_pb2.QueryOp.EQ, [open_field], [], []),
ast_pb2.MakeCond(ast_pb2.QueryOp.EQ, [label_field], ['Hot'], [])
]
ast = ast_pb2.QueryAST()
ast.conjunctions.append(ast_pb2.Conjunction(conds=conds))
new_ast = ast2ast.PreprocessAST(self.cnxn, ast, [789], self.services,
self.config)
self.assertEqual(2, len(new_ast.conjunctions[0].conds))
new_cond_1, new_cond_2 = new_ast.conjunctions[0].conds
self.assertEqual(ast_pb2.QueryOp.NE, new_cond_1.op)
self.assertEqual([status_id_field], new_cond_1.field_defs)
self.assertEqual([7, 8, 9], new_cond_1.int_values)
self.assertEqual([], new_cond_1.str_values)
self.assertEqual(ast_pb2.QueryOp.EQ, new_cond_2.op)
self.assertEqual([label_id_field], new_cond_2.field_defs)
self.assertEqual([0], new_cond_2.int_values)
self.assertEqual([], new_cond_2.str_values)
def testPreprocessAST_EmptyAST(self):
ast = ast_pb2.QueryAST() # No conjunctions in it.
new_ast = ast2ast.PreprocessAST(self.cnxn, ast, [789], self.services,
self.config)
self.assertEqual(ast, new_ast)
def SearchProjectCan(cnxn,
services,
project_ids,
query_ast,
shard_id,
harmonized_config,
left_joins=None,
where=None,
sort_directives=None,
query_desc=''):
"""Return a list of issue global IDs in the projects that satisfy the query.
Args:
cnxn: Regular database connection to the master DB.
services: interface to issue storage backends.
project_ids: list of int IDs of the project to search
query_ast: A QueryAST PB with conjunctions and conditions.
shard_id: limit search to the specified shard ID int.
harmonized_config: harmonized config for all projects being searched.
left_joins: SQL LEFT JOIN clauses that are needed in addition to
anything generated from the query_ast.
where: SQL WHERE clauses that are needed in addition to
anything generated from the query_ast.
sort_directives: list of strings specifying the columns to sort on.
query_desc: descriptive string for debugging.
Returns:
(issue_ids, capped, error) where issue_ids is a list of issue issue_ids
that satisfy the query, capped is True if the number of results were
capped due to an implementation limit, and error is any well-known error
(probably a query parsing error) encountered during search.
"""
logging.info('searching projects %r for AST %r', project_ids, query_ast)
start_time = time.time()
left_joins = left_joins or []
where = where or []
if project_ids:
cond_str = 'Issue.project_id IN (%s)' % sql.PlaceHolders(project_ids)
where.append((cond_str, project_ids))
try:
query_ast = ast2ast.PreprocessAST(cnxn, query_ast, project_ids,
services, harmonized_config)
logging.info('simplified AST is %r', query_ast)
query_left_joins, query_where, _ = ast2select.BuildSQLQuery(query_ast)
left_joins.extend(query_left_joins)
where.extend(query_where)
except ast2ast.MalformedQuery as e:
# TODO(jrobbins): inform the user that their query had invalid tokens.
logging.info('Invalid query tokens %s.\n %r\n\n', e.message, query_ast)
return [], False, e
except ast2select.NoPossibleResults as e:
# TODO(jrobbins): inform the user that their query was impossible.
logging.info('Impossible query %s.\n %r\n\n', e.message, query_ast)
return [], False, e
logging.info('translated to left_joins %r', left_joins)
logging.info('translated to where %r', where)
fts_capped = False
if query_ast.conjunctions:
# TODO(jrobbins): Handle "OR" in queries. For now, we just process the
# first conjunction.
assert len(query_ast.conjunctions) == 1
conj = query_ast.conjunctions[0]
full_text_iids, fts_capped = tracker_fulltext.SearchIssueFullText(
project_ids, conj, shard_id)
if full_text_iids is not None:
if not full_text_iids:
return [], False, None # No match on fulltext, so don't bother DB.
cond_str = 'Issue.id IN (%s)' % sql.PlaceHolders(full_text_iids)
where.append((cond_str, full_text_iids))
label_def_rows = []
status_def_rows = []
if sort_directives:
if project_ids:
for pid in project_ids:
label_def_rows.extend(
services.config.GetLabelDefRows(cnxn, pid))
status_def_rows.extend(
services.config.GetStatusDefRows(cnxn, pid))
else:
label_def_rows = services.config.GetLabelDefRowsAnyProject(cnxn)
status_def_rows = services.config.GetStatusDefRowsAnyProject(cnxn)
harmonized_labels = tracker_bizobj.HarmonizeLabelOrStatusRows(
label_def_rows)
harmonized_statuses = tracker_bizobj.HarmonizeLabelOrStatusRows(
status_def_rows)
harmonized_fields = harmonized_config.field_defs
sort_left_joins, order_by = ast2sort.BuildSortClauses(
sort_directives, harmonized_labels, harmonized_statuses,
harmonized_fields)
logging.info('translated to sort left_joins %r', sort_left_joins)
logging.info('translated to order_by %r', order_by)
issue_ids, db_capped = services.issue.RunIssueQuery(cnxn,
left_joins +
sort_left_joins,
where,
order_by,
shard_id=shard_id)
logging.warn('executed "%s" query %r for %d issues in %dms', query_desc,
query_ast, len(issue_ids),
int((time.time() - start_time) * 1000))
capped = fts_capped or db_capped
return issue_ids, capped, None