def _PreprocessExactUsers(cnxn, cond, user_service, id_fields): """Preprocess a foo=emails cond into foo_id=IDs, if exact user match. This preprocesing step converts string conditions to int ID conditions. E.g., [owner=email] to [owner_id=ID]. It only does it in cases where (a) the email was "me", so it was already converted to an string of digits in the search pipeline, or (b) it is "user@domain" which resolves to a known Monorail user. It is also possible to search for, e.g., [owner:substring], but such searches remain 'owner' field searches rather than 'owner_id', and they cannot be combined with the "me" keyword. Args: cnxn: connection to the DB. cond: original parsed query Condition PB. user_service: connection to user persistence layer. id_fields: list of the search fields to use if the conversion to IDs succeeds. Returns: A new Condition PB that checks the id_field. Or, the original cond. """ op = _TextOpToIntOp(cond.op) if _IsDefinedOp(op): # No need to look up any IDs if we are just testing for any defined value. return ast_pb2.Condition(op=op, field_defs=id_fields) # This preprocessing step is only for ops that compare whole values, not # substrings. if not _IsEqualityOp(op): logging.info('could not convert to IDs because op is %r', op) return cond user_ids = [] for val in cond.str_values: try: user_ids.append(int(val)) except ValueError: try: user_ids.append(user_service.LookupUserID(cnxn, val)) except user_svc.NoSuchUserException: logging.info('could not convert user %r to int ID', val) return cond # preprocessing failed, stick with the original cond. return ast_pb2.Condition(op=op, field_defs=id_fields, int_values=user_ids)
def _PreprocessMergedIntoCond(cnxn, cond, project_ids, services, _harmonized_config, _is_member): """Preprocess mergedinto=xyz and has:mergedinto conds. Preprocesses mergedinto=xyz cond into mergedinto_id:issue_ids. Preprocesses has:mergedinto cond into has:mergedinto_id. """ issue_ids = _GetIssueIDsFromLocalIdsCond(cnxn, cond, project_ids, services) return ast_pb2.Condition( op=_TextOpToIntOp(cond.op), field_defs=[query2ast.BUILTIN_ISSUE_FIELDS['mergedinto_id']], int_values=issue_ids)
def _PreprocessBlockingCond(cnxn, cond, project_ids, services, _harmonized_config, _is_member): """Preprocess blocking=xyz and has:blocking conds. Preprocesses blocking=xyz cond into blocking_id:issue_ids. Preprocesses has:blocking cond into issues that are blocking other issues. """ issue_ids = _GetIssueIDsFromLocalIdsCond(cnxn, cond, project_ids, services) return ast_pb2.Condition( op=_TextOpToIntOp(cond.op), field_defs=[query2ast.BUILTIN_ISSUE_FIELDS['blocking_id']], int_values=issue_ids)
def _PreprocessIsBlockedCond(_cnxn, cond, _project_ids, _services, _harmonized_config, _is_member): """Preprocess an is:blocked cond into issues that are blocked.""" if cond.op == ast_pb2.QueryOp.EQ: op = ast_pb2.QueryOp.IS_DEFINED elif cond.op == ast_pb2.QueryOp.NE: op = ast_pb2.QueryOp.IS_NOT_DEFINED else: raise MalformedQuery('Blocked condition got nonsensical op %r' % cond.op) return ast_pb2.Condition( op=op, field_defs=[query2ast.BUILTIN_ISSUE_FIELDS['blockedon_id']])
def _PreprocessIsSpamCond(_cnxn, cond, _project_ids, _services, _harmonized_config, _is_member): """Preprocess an is:spam cond into is_spam == 1.""" if cond.op == ast_pb2.QueryOp.EQ: int_values = [1] elif cond.op == ast_pb2.QueryOp.NE: int_values = [0] else: raise MalformedQuery('Spam condition got nonsensical op %r' % cond.op) return ast_pb2.Condition( op=ast_pb2.QueryOp.EQ, field_defs=[query2ast.BUILTIN_ISSUE_FIELDS['is_spam']], int_values=int_values)
def testSearchIssueFullText_Normal(self): self.SetUpSearchIssueFullText() self.mox.ReplayAll() summary_fd = tracker_pb2.FieldDef( field_name='summary', field_type=tracker_pb2.FieldTypes.STR_TYPE) query_ast_conj = ast_pb2.Conjunction(conds=[ ast_pb2.Condition(op=ast_pb2.QueryOp.TEXT_HAS, field_defs=[summary_fd], str_values=['test']) ]) issue_ids, capped = tracker_fulltext.SearchIssueFullText( [789], query_ast_conj, 1) self.mox.VerifyAll() self.assertItemsEqual([123, 234], issue_ids) self.assertFalse(capped)
def _PreprocessStatusCond(cnxn, cond, project_ids, services, _harmonized_config, _is_member): """Preprocess a status=names cond into status_id=IDs.""" if project_ids: status_ids = [] for project_id in project_ids: status_ids.extend( services.config.LookupStatusIDs(cnxn, project_id, cond.str_values)) else: status_ids = services.config.LookupStatusIDsAnyProject( cnxn, cond.str_values) return ast_pb2.Condition( op=_TextOpToIntOp(cond.op), field_defs=[query2ast.BUILTIN_ISSUE_FIELDS['status_id']], int_values=status_ids)
def _PreprocessHotlistCond(cnxn, cond, _project_ids, services, _harmonized_config, _is_member): """Preprocess hotlist query Preprocesses a hotlist query in the form: 'hotlist=<user_email>:<hotlist-name>,<hotlist-name>,<user2_email>:... into hotlist_id=IDs, if exact. """ # TODO(jojwang): add support for searches that don't contain domain names. # eg jojwang:hotlist-name users_to_hotlists = collections.defaultdict(list) cur_user = '' for val in cond.str_values: if ':' in val: cur_user, hotlists_str = val.split(':', 1) else: hotlists_str = val try: users_to_hotlists[int(cur_user)].append(hotlists_str) except ValueError: try: user_id = services.user.LookupUserID(cnxn, cur_user) users_to_hotlists[user_id].append(hotlists_str) except exceptions.NoSuchUserException: logging.info('could not convert user %r to int ID', val) return cond hotlist_ids = set() for user_id, hotlists in users_to_hotlists.items(): if not hotlists[0]: user_hotlists = services.features.GetHotlistsByUserID( cnxn, user_id) user_hotlist_ids = [ hotlist.hotlist_id for hotlist in user_hotlists if user_id in hotlist.owner_ids ] else: user_hotlist_ids = list( services.features.LookupHotlistIDs(cnxn, hotlists, [user_id]).values()) for hotlist_id in user_hotlist_ids: hotlist_ids.add(hotlist_id) return ast_pb2.Condition( op=_TextOpToIntOp(cond.op), field_defs=[query2ast.BUILTIN_ISSUE_FIELDS['hotlist_id']], int_values=list(hotlist_ids))
def testSearchIssueFullText_CrossProject(self): self.mox.StubOutWithMock(fulltext_helpers, 'ComprehensiveSearch') fulltext_helpers.ComprehensiveSearch( '(project_id:789 OR project_id:678) (summary:"test")', settings.search_index_name_format % 1).AndReturn([123, 234]) self.mox.ReplayAll() summary_fd = tracker_pb2.FieldDef( field_name='summary', field_type=tracker_pb2.FieldTypes.STR_TYPE) query_ast_conj = ast_pb2.Conjunction(conds=[ ast_pb2.Condition(op=ast_pb2.QueryOp.TEXT_HAS, field_defs=[summary_fd], str_values=['test']) ]) issue_ids, capped = tracker_fulltext.SearchIssueFullText( [789, 678], query_ast_conj, 1) self.mox.VerifyAll() self.assertItemsEqual([123, 234], issue_ids) self.assertFalse(capped)
def testSearchIssueFullText_Capped(self): try: orig = settings.fulltext_limit_per_shard settings.fulltext_limit_per_shard = 1 self.SetUpSearchIssueFullText() self.mox.ReplayAll() summary_fd = tracker_pb2.FieldDef( field_name='summary', field_type=tracker_pb2.FieldTypes.STR_TYPE) query_ast_conj = ast_pb2.Conjunction(conds=[ ast_pb2.Condition(op=ast_pb2.QueryOp.TEXT_HAS, field_defs=[summary_fd], str_values=['test']) ]) issue_ids, capped = tracker_fulltext.SearchIssueFullText( [789], query_ast_conj, 1) self.mox.VerifyAll() self.assertItemsEqual([123, 234], issue_ids) self.assertTrue(capped) finally: settings.fulltext_limit_per_shard = orig
def _PreprocessLabelCond(cnxn, cond, project_ids, services, _harmonized_config, _is_member): """Preprocess a label=names cond into label_id=IDs.""" if project_ids: label_ids = [] for project_id in project_ids: if _IsEqualityOp(cond.op): label_ids.extend( services.config.LookupLabelIDs(cnxn, project_id, cond.str_values)) elif _IsDefinedOp(cond.op): label_ids.extend( services.config.LookupIDsOfLabelsMatching( cnxn, project_id, _MakePrefixRegex(cond))) elif cond.op == ast_pb2.QueryOp.KEY_HAS: label_ids.extend( services.config.LookupIDsOfLabelsMatching( cnxn, project_id, _MakeKeyValueRegex(cond))) else: label_ids.extend( services.config.LookupIDsOfLabelsMatching( cnxn, project_id, _MakeWordBoundaryRegex(cond))) else: if _IsEqualityOp(cond.op): label_ids = services.config.LookupLabelIDsAnyProject( cnxn, cond.str_values) elif _IsDefinedOp(cond.op): label_ids = services.config.LookupIDsOfLabelsMatchingAnyProject( cnxn, _MakePrefixRegex(cond)) elif cond.op == ast_pb2.QueryOp.KEY_HAS: label_ids = services.config.LookupIDsOfLabelsMatchingAnyProject( cnxn, _MakeKeyValueRegex(cond)) else: label_ids = services.config.LookupIDsOfLabelsMatchingAnyProject( cnxn, _MakeWordBoundaryRegex(cond)) return ast_pb2.Condition( op=_TextOpToIntOp(cond.op), field_defs=[query2ast.BUILTIN_ISSUE_FIELDS['label_id']], int_values=label_ids)
def _PreprocessComponentCond(cnxn, cond, project_ids, services, harmonized_config, _is_member): """Preprocess a component= or component:name cond into component_id=IDs.""" exact = _IsEqualityOp(cond.op) component_ids = [] if project_ids: # We are searching within specific projects, so harmonized_config # holds the config data for all those projects. for comp_path in cond.str_values: component_ids.extend( tracker_bizobj.FindMatchingComponentIDs(comp_path, harmonized_config, exact=exact)) else: # We are searching across the whole site, so we have no harmonized_config # to use. component_ids = services.config.FindMatchingComponentIDsAnyProject( cnxn, cond.str_values, exact=exact) return ast_pb2.Condition( op=_TextOpToIntOp(cond.op), field_defs=[query2ast.BUILTIN_ISSUE_FIELDS['component_id']], int_values=component_ids)
def _PreprocessIsOpenCond(cnxn, cond, project_ids, services, _harmonized_config, _is_member): """Preprocess an is:open cond into status_id != closed_status_ids.""" if project_ids: closed_status_ids = [] for project_id in project_ids: closed_status_ids.extend( services.config.LookupClosedStatusIDs(cnxn, project_id)) else: closed_status_ids = services.config.LookupClosedStatusIDsAnyProject( cnxn) # Invert the operator, because we're comparing against *closed* statuses. if cond.op == ast_pb2.QueryOp.EQ: op = ast_pb2.QueryOp.NE elif cond.op == ast_pb2.QueryOp.NE: op = ast_pb2.QueryOp.EQ else: raise MalformedQuery('Open condition got nonsensical op %r' % cond.op) return ast_pb2.Condition( op=op, field_defs=[query2ast.BUILTIN_ISSUE_FIELDS['status_id']], int_values=closed_status_ids)
def _PreprocessExactUsers(cnxn, cond, user_service, id_fields, is_member): """Preprocess a foo=emails cond into foo_id=IDs, if exact user match. This preprocesing step converts string conditions to int ID conditions. E.g., [owner=email] to [owner_id=ID]. It only does it in cases where (a) the email was "me", so it was already converted to an string of digits in the search pipeline, or (b) it is "user@domain" which resolves to a known Monorail user. It is also possible to search for, e.g., [owner:substring], but such searches remain 'owner' field searches rather than 'owner_id', and they cannot be combined with the "me" keyword. Args: cnxn: connection to the DB. cond: original parsed query Condition PB. user_service: connection to user persistence layer. id_fields: list of the search fields to use if the conversion to IDs succeed. is_member: True if user is a member of all the projects being searchers, so they can do user substring searches. Returns: A new Condition PB that checks the id_field. Or, the original cond. Raises: MalformedQuery: A non-member used a query term that could be used to guess full user email addresses. """ op = _TextOpToIntOp(cond.op) if _IsDefinedOp(op): # No need to look up any IDs if we are just testing for any defined value. return ast_pb2.Condition(op=op, field_defs=id_fields, key_suffix=cond.key_suffix, phase_name=cond.phase_name) # This preprocessing step is only for ops that compare whole values, not # substrings. if not _IsEqualityOp(op): logging.info('could not convert to IDs because op is %r', op) if not is_member: raise MalformedQuery( 'Only project members may compare user strings') return cond user_ids = [] for val in cond.str_values: try: user_ids.append(int(val)) except ValueError: try: user_ids.append(user_service.LookupUserID(cnxn, val)) except exceptions.NoSuchUserException: if not is_member and val != 'me' and not val.startswith('@'): logging.info('could not convert user %r to int ID', val) if '@' in val: raise MalformedQuery('User email address not found') else: raise MalformedQuery( 'Only project members may search for user substrings' ) return cond # preprocessing failed, stick with the original cond. return ast_pb2.MakeCond(op, id_fields, [], user_ids, key_suffix=cond.key_suffix, phase_name=cond.phase_name)
def testQueryIssueSnapshots_WithQueryStringAndCannedQuery(self): """Test the query param is parsed and used.""" project = fake.Project(project_id=789) perms = permissions.USER_PERMISSIONSET search_helpers.GetPersonalAtRiskLabelIDs(self.cnxn, None, self.config_service, [10, 20], project, perms).AndReturn([]) cols = [ 'Lab.label', 'IssueSnapshot.issue_id', ] left_joins = [ ('Issue ON IssueSnapshot.issue_id = Issue.id', []), ('Issue2Cc AS I2cc' ' ON Issue.id = I2cc.issue_id' ' AND I2cc.cc_id IN (%s,%s)', [10, 20]), ('IssueSnapshot2Label AS Is2l' ' ON Is2l.issuesnapshot_id = IssueSnapshot.id', []), ('LabelDef AS Lab ON Lab.id = Is2l.label_id', []), ('IssueSnapshot2Label AS Cond0 ' 'ON IssueSnapshot.id = Cond0.issuesnapshot_id ' 'AND Cond0.label_id = %s', [15]), ] where = [ ('IssueSnapshot.period_start <= %s', [1514764800]), ('IssueSnapshot.period_end > %s', [1514764800]), ('IssueSnapshot.project_id = %s', [789]), ('Issue.is_spam = %s', [False]), ('Issue.deleted = %s', [False]), ('(Issue.reporter_id IN (%s,%s)' ' OR Issue.owner_id IN (%s,%s)' ' OR I2cc.cc_id IS NOT NULL)', [10, 20, 10, 20]), ('LOWER(Lab.label) LIKE %s', ['foo-%']), ('Cond0.label_id IS NULL', []), ('IssueSnapshot.is_open = %s', [True]), ] group_by = ['Lab.label'] query_left_joins = [('IssueSnapshot2Label AS Cond0 ' 'ON IssueSnapshot.id = Cond0.issuesnapshot_id ' 'AND Cond0.label_id = %s', [15])] query_where = [ ('Cond0.label_id IS NULL', []), ('IssueSnapshot.is_open = %s', [True]), ] unsupported_field_names = ['ownerbouncing'] unsupported_conds = [ ast_pb2.Condition( op=ast_pb2.QueryOp(1), field_defs=[ tracker_pb2.FieldDef( field_name='ownerbouncing', field_type=tracker_pb2.FieldTypes.BOOL_TYPE), ]) ] stmt, stmt_args = self.services.chart._BuildSnapshotQuery(cols, where, left_joins, group_by, shard_id=0) self.services.chart._QueryToWhere(mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg()).AndReturn( (query_left_joins, query_where, unsupported_conds)) self.cnxn.Execute(stmt, stmt_args, shard_id=0).AndReturn([]) self._verifySQL(cols, left_joins, where, group_by) self.mox.ReplayAll() _, unsupported, limit_reached = self.services.chart.QueryIssueSnapshots( self.cnxn, self.services, unixtime=1514764800, effective_ids=[10, 20], project=project, perms=perms, group_by='label', label_prefix='Foo', query='-label:Performance%20is:ownerbouncing', canned_query='is:open') self.mox.VerifyAll() self.assertEqual(unsupported_field_names, unsupported) self.assertFalse(limit_reached)