def testBuildFTSQuery_SpecialPrefixQuery(self): special_prefix = query2ast.NON_OP_PREFIXES[0] # Test with summary field. query_ast_conj = ast_pb2.Conjunction(conds=[ ast_pb2.MakeCond(TEXT_HAS, [self.summary_fd], ['%s//google.com' % special_prefix], []), ast_pb2.MakeCond(TEXT_HAS, [self.milestone_fd], ['Q3', 'Q4'], []) ]) fulltext_query = fulltext_helpers.BuildFTSQuery( query_ast_conj, self.fulltext_fields) self.assertEqual( '(summary:"%s//google.com") (custom_123:"Q3" OR custom_123:"Q4")' % (special_prefix), fulltext_query) # Test with any field. any_fd = tracker_pb2.FieldDef( field_name=ast_pb2.ANY_FIELD, field_type=tracker_pb2.FieldTypes.STR_TYPE) query_ast_conj = ast_pb2.Conjunction(conds=[ ast_pb2.MakeCond(TEXT_HAS, [any_fd], ['%s//google.com' % special_prefix], []), ast_pb2.MakeCond(TEXT_HAS, [self.milestone_fd], ['Q3', 'Q4'], []) ]) fulltext_query = fulltext_helpers.BuildFTSQuery( query_ast_conj, self.fulltext_fields) self.assertEqual( '("%s//google.com") (custom_123:"Q3" OR custom_123:"Q4")' % (special_prefix), fulltext_query)
def PreprocessAST(cnxn, query_ast, project_ids, services, harmonized_config, is_member=True): """Preprocess the query by doing lookups so that the SQL query is simpler. Args: cnxn: connection to SQL database. query_ast: user query abstract syntax tree parsed by query2ast.py. project_ids: collection of int project IDs to use to look up status values and labels. services: Connections to persistence layer for users and configs. harmonized_config: harmonized config for all projects being searched. is_member: True if user is a member of all the projects being searched, so they can do user substring searches. Returns: A new QueryAST PB with simplified conditions. Specifically, string values for labels, statuses, and components are replaced with the int IDs of those items. Also, is:open is distilled down to status_id != closed_status_ids. """ new_conjs = [] for conj in query_ast.conjunctions: new_conds = [ _PreprocessCond(cnxn, cond, project_ids, services, harmonized_config, is_member) for cond in conj.conds ] new_conjs.append(ast_pb2.Conjunction(conds=new_conds)) return ast_pb2.QueryAST(conjunctions=new_conjs)
def testBuildFTSQuery_WithQuotes(self): query_ast_conj = ast_pb2.Conjunction(conds=[ ast_pb2.MakeCond(TEXT_HAS, [self.summary_fd], ['"needle haystack"'], []) ]) fulltext_query = fulltext_helpers.BuildFTSQuery( query_ast_conj, self.fulltext_fields) self.assertEqual('(summary:"needle haystack")', fulltext_query)
def _ParseConjunction(subquery, scope, fields, warnings, now=None): """Parse part of a user query into a Conjunction PB.""" logging.info('Parsing sub query: %r in scope %r', subquery, scope) scoped_query = ('%s %s' % (scope, subquery)).lower() cond_strs = _ExtractConds(scoped_query, warnings) conds = [_ParseCond(cond_str, fields, warnings, now=now) for cond_str in cond_strs] conds = [cond for cond in conds if cond] return ast_pb2.Conjunction(conds=conds)
def testBuildFTSQuery_InvalidQuery(self): query_ast_conj = ast_pb2.Conjunction(conds=[ ast_pb2.MakeCond(TEXT_HAS, [self.summary_fd], ['haystack"needle'], []), ast_pb2.MakeCond(TEXT_HAS, [self.milestone_fd], ['Q3', 'Q4'], []) ]) with self.assertRaises(AssertionError): fulltext_helpers.BuildFTSQuery(query_ast_conj, self.fulltext_fields)
def testBuildFTSQuery_Normal(self): query_ast_conj = ast_pb2.Conjunction(conds=[ ast_pb2.MakeCond(TEXT_HAS, [self.summary_fd], ['needle'], []), ast_pb2.MakeCond(TEXT_HAS, [self.milestone_fd], ['Q3', 'Q4'], []) ]) fulltext_query = fulltext_helpers.BuildFTSQuery( query_ast_conj, self.fulltext_fields) self.assertEqual( '(summary:"needle") (custom_123:"Q3" OR custom_123:"Q4")', fulltext_query)
def testBuildFTSQuery_NoFullTextConditions(self): estimated_hours_fd = tracker_pb2.FieldDef( field_name='estimate', field_type=tracker_pb2.FieldTypes.INT_TYPE, field_id=124) query_ast_conj = ast_pb2.Conjunction( conds=[ast_pb2.MakeCond(TEXT_HAS, [estimated_hours_fd], [], [40])]) fulltext_query = fulltext_helpers.BuildFTSQuery( query_ast_conj, self.fulltext_fields) self.assertEqual(None, fulltext_query)
def testSearchIssueFullText_Normal(self): self.SetUpSearchIssueFullText() self.mox.ReplayAll() summary_fd = tracker_pb2.FieldDef( field_name='summary', field_type=tracker_pb2.FieldTypes.STR_TYPE) query_ast_conj = ast_pb2.Conjunction(conds=[ ast_pb2.Condition(op=ast_pb2.QueryOp.TEXT_HAS, field_defs=[summary_fd], str_values=['test']) ]) issue_ids, capped = tracker_fulltext.SearchIssueFullText( [789], query_ast_conj, 1) self.mox.VerifyAll() self.assertItemsEqual([123, 234], issue_ids) self.assertFalse(capped)
def testBuildSQLQuery_Normal(self): owner_field = BUILTIN_ISSUE_FIELDS['owner'] reporter_id_field = BUILTIN_ISSUE_FIELDS['reporter_id'] conds = [ ast_pb2.MakeCond(ast_pb2.QueryOp.TEXT_HAS, [owner_field], ['example.com'], []), ast_pb2.MakeCond(ast_pb2.QueryOp.EQ, [reporter_id_field], [], [111L]) ] ast = ast_pb2.QueryAST(conjunctions=[ast_pb2.Conjunction(conds=conds)]) left_joins, where = ast2select.BuildSQLQuery(ast) self.assertEqual([('User AS Cond0 ON (Issue.owner_id = Cond0.user_id ' 'OR Issue.derived_owner_id = Cond0.user_id)', [])], left_joins) self.assertTrue(sql._IsValidJoin(left_joins[0][0])) self.assertEqual([('(LOWER(Cond0.email) LIKE %s)', ['%example.com%']), ('Issue.reporter_id = %s', [111L])], where) self.assertTrue(sql._IsValidWhereCond(where[0][0]))
def testSearchIssueFullText_CrossProject(self): self.mox.StubOutWithMock(fulltext_helpers, 'ComprehensiveSearch') fulltext_helpers.ComprehensiveSearch( '(project_id:789 OR project_id:678) (summary:"test")', settings.search_index_name_format % 1).AndReturn([123, 234]) self.mox.ReplayAll() summary_fd = tracker_pb2.FieldDef( field_name='summary', field_type=tracker_pb2.FieldTypes.STR_TYPE) query_ast_conj = ast_pb2.Conjunction(conds=[ ast_pb2.Condition(op=ast_pb2.QueryOp.TEXT_HAS, field_defs=[summary_fd], str_values=['test']) ]) issue_ids, capped = tracker_fulltext.SearchIssueFullText( [789, 678], query_ast_conj, 1) self.mox.VerifyAll() self.assertItemsEqual([123, 234], issue_ids) self.assertFalse(capped)
def testSearchIssueFullText_Capped(self): try: orig = settings.fulltext_limit_per_shard settings.fulltext_limit_per_shard = 1 self.SetUpSearchIssueFullText() self.mox.ReplayAll() summary_fd = tracker_pb2.FieldDef( field_name='summary', field_type=tracker_pb2.FieldTypes.STR_TYPE) query_ast_conj = ast_pb2.Conjunction(conds=[ ast_pb2.Condition(op=ast_pb2.QueryOp.TEXT_HAS, field_defs=[summary_fd], str_values=['test']) ]) issue_ids, capped = tracker_fulltext.SearchIssueFullText( [789], query_ast_conj, 1) self.mox.VerifyAll() self.assertItemsEqual([123, 234], issue_ids) self.assertTrue(capped) finally: settings.fulltext_limit_per_shard = orig
def testPreprocessAST_Normal(self): open_field = BUILTIN_ISSUE_FIELDS['open'] label_field = BUILTIN_ISSUE_FIELDS['label'] label_id_field = BUILTIN_ISSUE_FIELDS['label_id'] status_id_field = BUILTIN_ISSUE_FIELDS['status_id'] conds = [ ast_pb2.MakeCond(ast_pb2.QueryOp.EQ, [open_field], [], []), ast_pb2.MakeCond(ast_pb2.QueryOp.EQ, [label_field], ['Hot'], []) ] ast = ast_pb2.QueryAST() ast.conjunctions.append(ast_pb2.Conjunction(conds=conds)) new_ast = ast2ast.PreprocessAST(self.cnxn, ast, [789], self.services, self.config) self.assertEqual(2, len(new_ast.conjunctions[0].conds)) new_cond_1, new_cond_2 = new_ast.conjunctions[0].conds self.assertEqual(ast_pb2.QueryOp.NE, new_cond_1.op) self.assertEqual([status_id_field], new_cond_1.field_defs) self.assertEqual([7, 8, 9], new_cond_1.int_values) self.assertEqual([], new_cond_1.str_values) self.assertEqual(ast_pb2.QueryOp.EQ, new_cond_2.op) self.assertEqual([label_id_field], new_cond_2.field_defs) self.assertEqual([0], new_cond_2.int_values) self.assertEqual([], new_cond_2.str_values)
def testBuildFTSQuery_EmptyQueryConjunction(self): query_ast_conj = ast_pb2.Conjunction() fulltext_query = fulltext_helpers.BuildFTSQuery( query_ast_conj, self.fulltext_fields) self.assertEqual(None, fulltext_query)
def testBuildSQLQuery_EmptyAST(self): ast = ast_pb2.QueryAST(conjunctions=[ast_pb2.Conjunction() ]) # No conds left_joins, where = ast2select.BuildSQLQuery(ast) self.assertEqual([], left_joins) self.assertEqual([], where)