def testSearchIssueFullText_Normal(self): self.SetUpSearchIssueFullText() self.mox.ReplayAll() summary_fd = tracker_pb2.FieldDef( field_name='summary', field_type=tracker_pb2.FieldTypes.STR_TYPE) query_ast_conj = ast_pb2.Conjunction(conds=[ ast_pb2.Condition(op=ast_pb2.QueryOp.TEXT_HAS, field_defs=[summary_fd], str_values=['test']) ]) issue_ids, capped = tracker_fulltext.SearchIssueFullText( [789], query_ast_conj, 1) self.mox.VerifyAll() self.assertItemsEqual([123, 234], issue_ids) self.assertFalse(capped)
def testSearchIssueFullText_CrossProject(self): self.mox.StubOutWithMock(fulltext_helpers, 'ComprehensiveSearch') fulltext_helpers.ComprehensiveSearch( '(project_id:789 OR project_id:678) (summary:"test")', settings.search_index_name_format % 1).AndReturn([123, 234]) self.mox.ReplayAll() summary_fd = tracker_pb2.FieldDef( field_name='summary', field_type=tracker_pb2.FieldTypes.STR_TYPE) query_ast_conj = ast_pb2.Conjunction(conds=[ ast_pb2.Condition(op=ast_pb2.QueryOp.TEXT_HAS, field_defs=[summary_fd], str_values=['test']) ]) issue_ids, capped = tracker_fulltext.SearchIssueFullText( [789, 678], query_ast_conj, 1) self.mox.VerifyAll() self.assertItemsEqual([123, 234], issue_ids) self.assertFalse(capped)
def testSearchIssueFullText_Capped(self): try: orig = settings.fulltext_limit_per_shard settings.fulltext_limit_per_shard = 1 self.SetUpSearchIssueFullText() self.mox.ReplayAll() summary_fd = tracker_pb2.FieldDef( field_name='summary', field_type=tracker_pb2.FieldTypes.STR_TYPE) query_ast_conj = ast_pb2.Conjunction(conds=[ ast_pb2.Condition(op=ast_pb2.QueryOp.TEXT_HAS, field_defs=[summary_fd], str_values=['test']) ]) issue_ids, capped = tracker_fulltext.SearchIssueFullText( [789], query_ast_conj, 1) self.mox.VerifyAll() self.assertItemsEqual([123, 234], issue_ids) self.assertTrue(capped) finally: settings.fulltext_limit_per_shard = orig
def testSearchProjectCan_FTSCapped(self): query_ast = query2ast.ParseUserQuery( 'Priority:High', 'is:open', query2ast.BUILTIN_ISSUE_FIELDS, self.config) simplified_query_ast = ast2ast.PreprocessAST( self.cnxn, query_ast, [789], self.services, self.config) conj = simplified_query_ast.conjunctions[0] self.mox.StubOutWithMock(tracker_fulltext, 'SearchIssueFullText') tracker_fulltext.SearchIssueFullText( [789], conj, 2).AndReturn(([10002, 10052], True)) self.mox.StubOutWithMock(self.services.issue, 'RunIssueQuery') self.services.issue.RunIssueQuery( self.cnxn, mox.IsA(list), mox.IsA(list), mox.IsA(list), shard_id=2).AndReturn(([10002, 10052], False)) self.mox.ReplayAll() result, capped, err = backendsearchpipeline.SearchProjectCan( self.cnxn, self.services, [789], query_ast, 2, self.config) self.mox.VerifyAll() self.assertEqual([10002, 10052], result) self.assertTrue(capped) self.assertEqual(None, err)
def SearchProjectCan(cnxn, services, project_ids, query_ast, shard_id, harmonized_config, left_joins=None, where=None, sort_directives=None, query_desc=''): """Return a list of issue global IDs in the projects that satisfy the query. Args: cnxn: Regular database connection to the master DB. services: interface to issue storage backends. project_ids: list of int IDs of the project to search query_ast: A QueryAST PB with conjunctions and conditions. shard_id: limit search to the specified shard ID int. harmonized_config: harmonized config for all projects being searched. left_joins: SQL LEFT JOIN clauses that are needed in addition to anything generated from the query_ast. where: SQL WHERE clauses that are needed in addition to anything generated from the query_ast. sort_directives: list of strings specifying the columns to sort on. query_desc: descriptive string for debugging. Returns: (issue_ids, capped, error) where issue_ids is a list of issue issue_ids that satisfy the query, capped is True if the number of results were capped due to an implementation limit, and error is any well-known error (probably a query parsing error) encountered during search. """ logging.info('searching projects %r for AST %r', project_ids, query_ast) start_time = time.time() left_joins = left_joins or [] where = where or [] if project_ids: cond_str = 'Issue.project_id IN (%s)' % sql.PlaceHolders(project_ids) where.append((cond_str, project_ids)) try: query_ast = ast2ast.PreprocessAST(cnxn, query_ast, project_ids, services, harmonized_config) logging.info('simplified AST is %r', query_ast) query_left_joins, query_where, _ = ast2select.BuildSQLQuery(query_ast) left_joins.extend(query_left_joins) where.extend(query_where) except ast2ast.MalformedQuery as e: # TODO(jrobbins): inform the user that their query had invalid tokens. logging.info('Invalid query tokens %s.\n %r\n\n', e.message, query_ast) return [], False, e except ast2select.NoPossibleResults as e: # TODO(jrobbins): inform the user that their query was impossible. logging.info('Impossible query %s.\n %r\n\n', e.message, query_ast) return [], False, e logging.info('translated to left_joins %r', left_joins) logging.info('translated to where %r', where) fts_capped = False if query_ast.conjunctions: # TODO(jrobbins): Handle "OR" in queries. For now, we just process the # first conjunction. assert len(query_ast.conjunctions) == 1 conj = query_ast.conjunctions[0] full_text_iids, fts_capped = tracker_fulltext.SearchIssueFullText( project_ids, conj, shard_id) if full_text_iids is not None: if not full_text_iids: return [], False, None # No match on fulltext, so don't bother DB. cond_str = 'Issue.id IN (%s)' % sql.PlaceHolders(full_text_iids) where.append((cond_str, full_text_iids)) label_def_rows = [] status_def_rows = [] if sort_directives: if project_ids: for pid in project_ids: label_def_rows.extend( services.config.GetLabelDefRows(cnxn, pid)) status_def_rows.extend( services.config.GetStatusDefRows(cnxn, pid)) else: label_def_rows = services.config.GetLabelDefRowsAnyProject(cnxn) status_def_rows = services.config.GetStatusDefRowsAnyProject(cnxn) harmonized_labels = tracker_bizobj.HarmonizeLabelOrStatusRows( label_def_rows) harmonized_statuses = tracker_bizobj.HarmonizeLabelOrStatusRows( status_def_rows) harmonized_fields = harmonized_config.field_defs sort_left_joins, order_by = ast2sort.BuildSortClauses( sort_directives, harmonized_labels, harmonized_statuses, harmonized_fields) logging.info('translated to sort left_joins %r', sort_left_joins) logging.info('translated to order_by %r', order_by) issue_ids, db_capped = services.issue.RunIssueQuery(cnxn, left_joins + sort_left_joins, where, order_by, shard_id=shard_id) logging.warn('executed "%s" query %r for %d issues in %dms', query_desc, query_ast, len(issue_ids), int((time.time() - start_time) * 1000)) capped = fts_capped or db_capped return issue_ids, capped, None