def testSearchIssueFullText_Normal(self):
self.SetUpSearchIssueFullText()
self.mox.ReplayAll()
summary_fd = tracker_pb2.FieldDef(
field_name='summary', field_type=tracker_pb2.FieldTypes.STR_TYPE)
query_ast_conj = ast_pb2.Conjunction(conds=[
ast_pb2.Condition(op=ast_pb2.QueryOp.TEXT_HAS,
field_defs=[summary_fd],
str_values=['test'])
])
issue_ids, capped = tracker_fulltext.SearchIssueFullText(
[789], query_ast_conj, 1)
self.mox.VerifyAll()
self.assertItemsEqual([123, 234], issue_ids)
self.assertFalse(capped)
def testSearchIssueFullText_CrossProject(self):
self.mox.StubOutWithMock(fulltext_helpers, 'ComprehensiveSearch')
fulltext_helpers.ComprehensiveSearch(
'(project_id:789 OR project_id:678) (summary:"test")',
settings.search_index_name_format % 1).AndReturn([123, 234])
self.mox.ReplayAll()
summary_fd = tracker_pb2.FieldDef(
field_name='summary', field_type=tracker_pb2.FieldTypes.STR_TYPE)
query_ast_conj = ast_pb2.Conjunction(conds=[
ast_pb2.Condition(op=ast_pb2.QueryOp.TEXT_HAS,
field_defs=[summary_fd],
str_values=['test'])
])
issue_ids, capped = tracker_fulltext.SearchIssueFullText(
[789, 678], query_ast_conj, 1)
self.mox.VerifyAll()
self.assertItemsEqual([123, 234], issue_ids)
self.assertFalse(capped)
def testSearchIssueFullText_Capped(self):
try:
orig = settings.fulltext_limit_per_shard
settings.fulltext_limit_per_shard = 1
self.SetUpSearchIssueFullText()
self.mox.ReplayAll()
summary_fd = tracker_pb2.FieldDef(
field_name='summary',
field_type=tracker_pb2.FieldTypes.STR_TYPE)
query_ast_conj = ast_pb2.Conjunction(conds=[
ast_pb2.Condition(op=ast_pb2.QueryOp.TEXT_HAS,
field_defs=[summary_fd],
str_values=['test'])
])
issue_ids, capped = tracker_fulltext.SearchIssueFullText(
[789], query_ast_conj, 1)
self.mox.VerifyAll()
self.assertItemsEqual([123, 234], issue_ids)
self.assertTrue(capped)
finally:
settings.fulltext_limit_per_shard = orig
def testSearchProjectCan_FTSCapped(self):
query_ast = query2ast.ParseUserQuery(
'Priority:High', 'is:open', query2ast.BUILTIN_ISSUE_FIELDS,
self.config)
simplified_query_ast = ast2ast.PreprocessAST(
self.cnxn, query_ast, [789], self.services, self.config)
conj = simplified_query_ast.conjunctions[0]
self.mox.StubOutWithMock(tracker_fulltext, 'SearchIssueFullText')
tracker_fulltext.SearchIssueFullText(
[789], conj, 2).AndReturn(([10002, 10052], True))
self.mox.StubOutWithMock(self.services.issue, 'RunIssueQuery')
self.services.issue.RunIssueQuery(
self.cnxn, mox.IsA(list), mox.IsA(list), mox.IsA(list),
shard_id=2).AndReturn(([10002, 10052], False))
self.mox.ReplayAll()
result, capped, err = backendsearchpipeline.SearchProjectCan(
self.cnxn, self.services, [789], query_ast, 2, self.config)
self.mox.VerifyAll()
self.assertEqual([10002, 10052], result)
self.assertTrue(capped)
self.assertEqual(None, err)
def SearchProjectCan(cnxn,
services,
project_ids,
query_ast,
shard_id,
harmonized_config,
left_joins=None,
where=None,
sort_directives=None,
query_desc=''):
"""Return a list of issue global IDs in the projects that satisfy the query.
Args:
cnxn: Regular database connection to the master DB.
services: interface to issue storage backends.
project_ids: list of int IDs of the project to search
query_ast: A QueryAST PB with conjunctions and conditions.
shard_id: limit search to the specified shard ID int.
harmonized_config: harmonized config for all projects being searched.
left_joins: SQL LEFT JOIN clauses that are needed in addition to
anything generated from the query_ast.
where: SQL WHERE clauses that are needed in addition to
anything generated from the query_ast.
sort_directives: list of strings specifying the columns to sort on.
query_desc: descriptive string for debugging.
Returns:
(issue_ids, capped, error) where issue_ids is a list of issue issue_ids
that satisfy the query, capped is True if the number of results were
capped due to an implementation limit, and error is any well-known error
(probably a query parsing error) encountered during search.
"""
logging.info('searching projects %r for AST %r', project_ids, query_ast)
start_time = time.time()
left_joins = left_joins or []
where = where or []
if project_ids:
cond_str = 'Issue.project_id IN (%s)' % sql.PlaceHolders(project_ids)
where.append((cond_str, project_ids))
try:
query_ast = ast2ast.PreprocessAST(cnxn, query_ast, project_ids,
services, harmonized_config)
logging.info('simplified AST is %r', query_ast)
query_left_joins, query_where, _ = ast2select.BuildSQLQuery(query_ast)
left_joins.extend(query_left_joins)
where.extend(query_where)
except ast2ast.MalformedQuery as e:
# TODO(jrobbins): inform the user that their query had invalid tokens.
logging.info('Invalid query tokens %s.\n %r\n\n', e.message, query_ast)
return [], False, e
except ast2select.NoPossibleResults as e:
# TODO(jrobbins): inform the user that their query was impossible.
logging.info('Impossible query %s.\n %r\n\n', e.message, query_ast)
return [], False, e
logging.info('translated to left_joins %r', left_joins)
logging.info('translated to where %r', where)
fts_capped = False
if query_ast.conjunctions:
# TODO(jrobbins): Handle "OR" in queries. For now, we just process the
# first conjunction.
assert len(query_ast.conjunctions) == 1
conj = query_ast.conjunctions[0]
full_text_iids, fts_capped = tracker_fulltext.SearchIssueFullText(
project_ids, conj, shard_id)
if full_text_iids is not None:
if not full_text_iids:
return [], False, None # No match on fulltext, so don't bother DB.
cond_str = 'Issue.id IN (%s)' % sql.PlaceHolders(full_text_iids)
where.append((cond_str, full_text_iids))
label_def_rows = []
status_def_rows = []
if sort_directives:
if project_ids:
for pid in project_ids:
label_def_rows.extend(
services.config.GetLabelDefRows(cnxn, pid))
status_def_rows.extend(
services.config.GetStatusDefRows(cnxn, pid))
else:
label_def_rows = services.config.GetLabelDefRowsAnyProject(cnxn)
status_def_rows = services.config.GetStatusDefRowsAnyProject(cnxn)
harmonized_labels = tracker_bizobj.HarmonizeLabelOrStatusRows(
label_def_rows)
harmonized_statuses = tracker_bizobj.HarmonizeLabelOrStatusRows(
status_def_rows)
harmonized_fields = harmonized_config.field_defs
sort_left_joins, order_by = ast2sort.BuildSortClauses(
sort_directives, harmonized_labels, harmonized_statuses,
harmonized_fields)
logging.info('translated to sort left_joins %r', sort_left_joins)
logging.info('translated to order_by %r', order_by)
issue_ids, db_capped = services.issue.RunIssueQuery(cnxn,
left_joins +
sort_left_joins,
where,
order_by,
shard_id=shard_id)
logging.warn('executed "%s" query %r for %d issues in %dms', query_desc,
query_ast, len(issue_ids),
int((time.time() - start_time) * 1000))
capped = fts_capped or db_capped
return issue_ids, capped, None
