def testBuildFTSQuery_SpecialPrefixQuery(self):
special_prefix = query2ast.NON_OP_PREFIXES[0]
# Test with summary field.
query_ast_conj = ast_pb2.Conjunction(conds=[
ast_pb2.MakeCond(TEXT_HAS, [self.summary_fd],
['%s//google.com' % special_prefix], []),
ast_pb2.MakeCond(TEXT_HAS, [self.milestone_fd], ['Q3', 'Q4'], [])
])
fulltext_query = fulltext_helpers.BuildFTSQuery(
query_ast_conj, self.fulltext_fields)
self.assertEqual(
'(summary:"%s//google.com") (custom_123:"Q3" OR custom_123:"Q4")' %
(special_prefix), fulltext_query)
# Test with any field.
any_fd = tracker_pb2.FieldDef(
field_name=ast_pb2.ANY_FIELD,
field_type=tracker_pb2.FieldTypes.STR_TYPE)
query_ast_conj = ast_pb2.Conjunction(conds=[
ast_pb2.MakeCond(TEXT_HAS, [any_fd],
['%s//google.com' % special_prefix], []),
ast_pb2.MakeCond(TEXT_HAS, [self.milestone_fd], ['Q3', 'Q4'], [])
])
fulltext_query = fulltext_helpers.BuildFTSQuery(
query_ast_conj, self.fulltext_fields)
self.assertEqual(
'("%s//google.com") (custom_123:"Q3" OR custom_123:"Q4")' %
(special_prefix), fulltext_query)
def PreprocessAST(cnxn,
query_ast,
project_ids,
services,
harmonized_config,
is_member=True):
"""Preprocess the query by doing lookups so that the SQL query is simpler.
Args:
cnxn: connection to SQL database.
query_ast: user query abstract syntax tree parsed by query2ast.py.
project_ids: collection of int project IDs to use to look up status values
and labels.
services: Connections to persistence layer for users and configs.
harmonized_config: harmonized config for all projects being searched.
is_member: True if user is a member of all the projects being searched,
so they can do user substring searches.
Returns:
A new QueryAST PB with simplified conditions. Specifically, string values
for labels, statuses, and components are replaced with the int IDs of
those items. Also, is:open is distilled down to
status_id != closed_status_ids.
"""
new_conjs = []
for conj in query_ast.conjunctions:
new_conds = [
_PreprocessCond(cnxn, cond, project_ids, services,
harmonized_config, is_member)
for cond in conj.conds
]
new_conjs.append(ast_pb2.Conjunction(conds=new_conds))
return ast_pb2.QueryAST(conjunctions=new_conjs)
def testBuildFTSQuery_WithQuotes(self):
query_ast_conj = ast_pb2.Conjunction(conds=[
ast_pb2.MakeCond(TEXT_HAS, [self.summary_fd],
['"needle haystack"'], [])
])
fulltext_query = fulltext_helpers.BuildFTSQuery(
query_ast_conj, self.fulltext_fields)
self.assertEqual('(summary:"needle haystack")', fulltext_query)
def _ParseConjunction(subquery, scope, fields, warnings, now=None):
"""Parse part of a user query into a Conjunction PB."""
logging.info('Parsing sub query: %r in scope %r', subquery, scope)
scoped_query = ('%s %s' % (scope, subquery)).lower()
cond_strs = _ExtractConds(scoped_query, warnings)
conds = [_ParseCond(cond_str, fields, warnings, now=now)
for cond_str in cond_strs]
conds = [cond for cond in conds if cond]
return ast_pb2.Conjunction(conds=conds)
def testBuildFTSQuery_InvalidQuery(self):
query_ast_conj = ast_pb2.Conjunction(conds=[
ast_pb2.MakeCond(TEXT_HAS, [self.summary_fd], ['haystack"needle'],
[]),
ast_pb2.MakeCond(TEXT_HAS, [self.milestone_fd], ['Q3', 'Q4'], [])
])
with self.assertRaises(AssertionError):
fulltext_helpers.BuildFTSQuery(query_ast_conj,
self.fulltext_fields)
def testBuildFTSQuery_Normal(self):
query_ast_conj = ast_pb2.Conjunction(conds=[
ast_pb2.MakeCond(TEXT_HAS, [self.summary_fd], ['needle'], []),
ast_pb2.MakeCond(TEXT_HAS, [self.milestone_fd], ['Q3', 'Q4'], [])
])
fulltext_query = fulltext_helpers.BuildFTSQuery(
query_ast_conj, self.fulltext_fields)
self.assertEqual(
'(summary:"needle") (custom_123:"Q3" OR custom_123:"Q4")',
fulltext_query)
def testBuildFTSQuery_NoFullTextConditions(self):
estimated_hours_fd = tracker_pb2.FieldDef(
field_name='estimate',
field_type=tracker_pb2.FieldTypes.INT_TYPE,
field_id=124)
query_ast_conj = ast_pb2.Conjunction(
conds=[ast_pb2.MakeCond(TEXT_HAS, [estimated_hours_fd], [], [40])])
fulltext_query = fulltext_helpers.BuildFTSQuery(
query_ast_conj, self.fulltext_fields)
self.assertEqual(None, fulltext_query)
def testSearchIssueFullText_Normal(self):
self.SetUpSearchIssueFullText()
self.mox.ReplayAll()
summary_fd = tracker_pb2.FieldDef(
field_name='summary', field_type=tracker_pb2.FieldTypes.STR_TYPE)
query_ast_conj = ast_pb2.Conjunction(conds=[
ast_pb2.Condition(op=ast_pb2.QueryOp.TEXT_HAS,
field_defs=[summary_fd],
str_values=['test'])
])
issue_ids, capped = tracker_fulltext.SearchIssueFullText(
[789], query_ast_conj, 1)
self.mox.VerifyAll()
self.assertItemsEqual([123, 234], issue_ids)
self.assertFalse(capped)
def testBuildSQLQuery_Normal(self):
owner_field = BUILTIN_ISSUE_FIELDS['owner']
reporter_id_field = BUILTIN_ISSUE_FIELDS['reporter_id']
conds = [
ast_pb2.MakeCond(ast_pb2.QueryOp.TEXT_HAS, [owner_field],
['example.com'], []),
ast_pb2.MakeCond(ast_pb2.QueryOp.EQ, [reporter_id_field], [],
[111L])
]
ast = ast_pb2.QueryAST(conjunctions=[ast_pb2.Conjunction(conds=conds)])
left_joins, where = ast2select.BuildSQLQuery(ast)
self.assertEqual([('User AS Cond0 ON (Issue.owner_id = Cond0.user_id '
'OR Issue.derived_owner_id = Cond0.user_id)', [])],
left_joins)
self.assertTrue(sql._IsValidJoin(left_joins[0][0]))
self.assertEqual([('(LOWER(Cond0.email) LIKE %s)', ['%example.com%']),
('Issue.reporter_id = %s', [111L])], where)
self.assertTrue(sql._IsValidWhereCond(where[0][0]))
def testSearchIssueFullText_CrossProject(self):
self.mox.StubOutWithMock(fulltext_helpers, 'ComprehensiveSearch')
fulltext_helpers.ComprehensiveSearch(
'(project_id:789 OR project_id:678) (summary:"test")',
settings.search_index_name_format % 1).AndReturn([123, 234])
self.mox.ReplayAll()
summary_fd = tracker_pb2.FieldDef(
field_name='summary', field_type=tracker_pb2.FieldTypes.STR_TYPE)
query_ast_conj = ast_pb2.Conjunction(conds=[
ast_pb2.Condition(op=ast_pb2.QueryOp.TEXT_HAS,
field_defs=[summary_fd],
str_values=['test'])
])
issue_ids, capped = tracker_fulltext.SearchIssueFullText(
[789, 678], query_ast_conj, 1)
self.mox.VerifyAll()
self.assertItemsEqual([123, 234], issue_ids)
self.assertFalse(capped)
def testSearchIssueFullText_Capped(self):
try:
orig = settings.fulltext_limit_per_shard
settings.fulltext_limit_per_shard = 1
self.SetUpSearchIssueFullText()
self.mox.ReplayAll()
summary_fd = tracker_pb2.FieldDef(
field_name='summary',
field_type=tracker_pb2.FieldTypes.STR_TYPE)
query_ast_conj = ast_pb2.Conjunction(conds=[
ast_pb2.Condition(op=ast_pb2.QueryOp.TEXT_HAS,
field_defs=[summary_fd],
str_values=['test'])
])
issue_ids, capped = tracker_fulltext.SearchIssueFullText(
[789], query_ast_conj, 1)
self.mox.VerifyAll()
self.assertItemsEqual([123, 234], issue_ids)
self.assertTrue(capped)
finally:
settings.fulltext_limit_per_shard = orig
def testPreprocessAST_Normal(self):
open_field = BUILTIN_ISSUE_FIELDS['open']
label_field = BUILTIN_ISSUE_FIELDS['label']
label_id_field = BUILTIN_ISSUE_FIELDS['label_id']
status_id_field = BUILTIN_ISSUE_FIELDS['status_id']
conds = [
ast_pb2.MakeCond(ast_pb2.QueryOp.EQ, [open_field], [], []),
ast_pb2.MakeCond(ast_pb2.QueryOp.EQ, [label_field], ['Hot'], [])
]
ast = ast_pb2.QueryAST()
ast.conjunctions.append(ast_pb2.Conjunction(conds=conds))
new_ast = ast2ast.PreprocessAST(self.cnxn, ast, [789], self.services,
self.config)
self.assertEqual(2, len(new_ast.conjunctions[0].conds))
new_cond_1, new_cond_2 = new_ast.conjunctions[0].conds
self.assertEqual(ast_pb2.QueryOp.NE, new_cond_1.op)
self.assertEqual([status_id_field], new_cond_1.field_defs)
self.assertEqual([7, 8, 9], new_cond_1.int_values)
self.assertEqual([], new_cond_1.str_values)
self.assertEqual(ast_pb2.QueryOp.EQ, new_cond_2.op)
self.assertEqual([label_id_field], new_cond_2.field_defs)
self.assertEqual([0], new_cond_2.int_values)
self.assertEqual([], new_cond_2.str_values)
def testBuildFTSQuery_EmptyQueryConjunction(self):
query_ast_conj = ast_pb2.Conjunction()
fulltext_query = fulltext_helpers.BuildFTSQuery(
query_ast_conj, self.fulltext_fields)
self.assertEqual(None, fulltext_query)
def testBuildSQLQuery_EmptyAST(self):
ast = ast_pb2.QueryAST(conjunctions=[ast_pb2.Conjunction()
]) # No conds
left_joins, where = ast2select.BuildSQLQuery(ast)
self.assertEqual([], left_joins)
self.assertEqual([], where)
