def testBlockingIDCond_MultiValue(self): fd = BUILTIN_ISSUE_FIELDS['blocking_id'] txt_cond = ast_pb2.MakeCond(ast_pb2.QueryOp.EQ, [fd], ['1', '2', '3'], []) num_cond = ast_pb2.MakeCond(ast_pb2.QueryOp.EQ, [fd], [], [1L, 2L, 3L]) for cond, expected in ((txt_cond, ['1', '2', '3']), (num_cond, [1L, 2L, 3L])): left_joins, where = ast2select._ProcessBlockingIDCond( cond, 'Cond1', 'Issue1') self.assertEqual([ ('IssueRelation AS Cond1 ON Issue.id = Cond1.dst_issue_id AND ' 'Cond1.kind = %s AND Cond1.issue_id IN (%s,%s,%s)', ['blockedon'] + expected) ], left_joins) self.assertTrue(sql._IsValidJoin(left_joins[0][0])) self.assertEqual([('Cond1.dst_issue_id IS NOT NULL', [])], where) self.assertTrue(sql._IsValidWhereCond(where[0][0]))
def testProcessCustomFieldCond_UserType_ByEmail(self): fd = tracker_pb2.FieldDef(field_id=1, project_id=789, field_name='ExecutiveProducer', field_type=tracker_pb2.FieldTypes.USER_TYPE) val = '*****@*****.**' cond = ast_pb2.MakeCond(ast_pb2.QueryOp.EQ, [fd], [val], []) left_joins, where = ast2select._ProcessCustomFieldCond( cond, 'Cond1', 'User1') self.assertEqual( [('User AS User1 ON Cond1.user_id = User1.user_id AND ' 'LOWER(User1.email) = %s', [val]), ('Issue2FieldValue AS Cond1 ON Issue.id = Cond1.issue_id AND ' 'Issue.shard = Cond1.issue_shard AND ' 'Cond1.field_id = %s', [1])], left_joins) self.assertTrue(sql._IsValidJoin(left_joins[0][0])) self.assertEqual([('Cond1.field_id IS NOT NULL', [])], where) self.assertTrue(sql._IsValidWhereCond(where[0][0]))
def testBuildSQLQuery_Normal(self): owner_field = BUILTIN_ISSUE_FIELDS['owner'] reporter_id_field = BUILTIN_ISSUE_FIELDS['reporter_id'] conds = [ ast_pb2.MakeCond(ast_pb2.QueryOp.TEXT_HAS, [owner_field], ['example.com'], []), ast_pb2.MakeCond(ast_pb2.QueryOp.EQ, [reporter_id_field], [], [111L]) ] ast = ast_pb2.QueryAST(conjunctions=[ast_pb2.Conjunction(conds=conds)]) left_joins, where = ast2select.BuildSQLQuery(ast) self.assertEqual([('User AS Cond0 ON (Issue.owner_id = Cond0.user_id ' 'OR Issue.derived_owner_id = Cond0.user_id)', [])], left_joins) self.assertTrue(sql._IsValidJoin(left_joins[0][0])) self.assertEqual([('(LOWER(Cond0.email) LIKE %s)', ['%example.com%']), ('Issue.reporter_id = %s', [111L])], where) self.assertTrue(sql._IsValidWhereCond(where[0][0]))
def _FilterSpam(query_ast): uses_spam = False # TODO(jrobbins): Handle "OR" in queries. For now, we just modify the # first conjunction. conjunction = query_ast.conjunctions[0] for condition in conjunction.conds: for field in condition.field_defs: if field.field_name == 'spam': uses_spam = True if not uses_spam: query_ast.conjunctions[0].conds.append( ast_pb2.MakeCond(ast_pb2.QueryOp.NE, [ tracker_pb2.FieldDef( field_name='spam', field_type=tracker_pb2.FieldTypes.BOOL_TYPE) ], [], [])) return query_ast
def _ParseCond(cond_str, fields, warnings, now=None): """Parse one user query condition string into a Condition PB.""" op_match = OP_RE.match(cond_str) # Do not treat as key:value search terms if any of the special prefixes match. special_prefixes_match = any( cond_str.startswith(p) for p in NON_OP_PREFIXES) if op_match and not special_prefixes_match: prefix = op_match.group('prefix') op = op_match.group('op') val = op_match.group('value') # Special case handling to continue to support old date query terms from # code.google.com. See monorail:151 for more details. if prefix.startswith(_DATE_FIELDS): for date_suffix in _DATE_FIELD_SUFFIX_TO_OP: if prefix.endswith(date_suffix): prefix = prefix.rstrip(date_suffix) op = _DATE_FIELD_SUFFIX_TO_OP[date_suffix] return _ParseStructuredTerm(prefix, op, val, fields, now=now) # Treat the cond as a full-text search term, which might be negated. if cond_str.startswith('-'): op = NOT_TEXT_HAS cond_str = cond_str[1:] else: op = TEXT_HAS # Construct a full-text Query object as a dry-run to validate that # the syntax is acceptable. try: _fts_query = search.Query(cond_str) except search.QueryError: warnings.append('Ignoring full-text term: %s' % cond_str) return None # Flag a potential user misunderstanding. if cond_str.lower() in ('and', 'or', 'not'): warnings.append( 'The only supported boolean operator is OR (all capitals).') return ast_pb2.MakeCond( op, [BUILTIN_ISSUE_FIELDS[ast_pb2.ANY_FIELD]], [cond_str], [])
def testPreprocessHotlistCond(self): hotlist_field = BUILTIN_ISSUE_FIELDS['hotlist'] hotlist_id_field = BUILTIN_ISSUE_FIELDS['hotlist_id'] self.services.user.TestAddUser('*****@*****.**', 111) self.services.user.TestAddUser('*****@*****.**', 222) self.services.user.TestAddUser('*****@*****.**', 333) # Setup hotlists self.services.features.TestAddHotlist('Hotlist1', owner_ids=[111], hotlist_id=10) self.services.features.TestAddHotlist('Hotlist2', owner_ids=[111], hotlist_id=20) self.services.features.TestAddHotlist('Hotlist3', owner_ids=[222], hotlist_id=30) self.services.features.TestAddHotlist('Hotlist4', owner_ids=[222], hotlist_id=40) self.services.features.TestAddHotlist('Hotlist5', owner_ids=[333], hotlist_id=50) self.services.features.TestAddHotlist('Hotlist6', owner_ids=[333], hotlist_id=60) hotlist_query_vals = [ '[email protected]:Hotlist1', '[email protected]:', '[email protected]:Hotlist3', 'Hotlist4' ] cond = ast_pb2.MakeCond(ast_pb2.QueryOp.TEXT_HAS, [hotlist_field], hotlist_query_vals, []) actual = ast2ast._PreprocessHotlistCond(self.cnxn, cond, [1], self.services, None, True) self.assertEqual(ast_pb2.QueryOp.EQ, actual.op) self.assertEqual([hotlist_id_field], actual.field_defs) self.assertItemsEqual([10, 30, 40, 50, 60], actual.int_values)
def testPreprocessBlockingCond_WithExternalIssues(self): blocking_field = BUILTIN_ISSUE_FIELDS['blocking'] blocking_id_field = BUILTIN_ISSUE_FIELDS['blocking_id'] self.services.project.TestAddProject('Project1', project_id=1) issue1 = fake.MakeTestIssue(project_id=1, local_id=1, summary='sum', status='new', owner_id=2, issue_id=101) issue2 = fake.MakeTestIssue(project_id=1, local_id=2, summary='sum', status='new', owner_id=2, issue_id=102) self.services.issue.TestAddIssue(issue1) self.services.issue.TestAddIssue(issue2) for local_ids, expected_issues, expected_ext_issues in (([ 'b/1234' ], [], ['b/1234']), (['Project1:1', 'b/1234'], [101], ['b/1234' ]), (['1', 'b/1234', 'b/1551', 'Project1:2'], [101, 102], ['b/1234', 'b/1551'])): cond = ast_pb2.MakeCond(ast_pb2.QueryOp.TEXT_HAS, [blocking_field], local_ids, []) new_cond = ast2ast._PreprocessBlockingCond(self.cnxn, cond, [1], self.services, None, True) self.assertEqual(ast_pb2.QueryOp.EQ, new_cond.op) self.assertEqual([blocking_id_field], new_cond.field_defs) self.assertEqual(expected_issues, new_cond.int_values) self.assertEqual(expected_ext_issues, new_cond.str_values)
def _PreprocessExactUsers(cnxn, cond, user_service, id_fields, is_member): """Preprocess a foo=emails cond into foo_id=IDs, if exact user match. This preprocesing step converts string conditions to int ID conditions. E.g., [owner=email] to [owner_id=ID]. It only does it in cases where (a) the email was "me", so it was already converted to an string of digits in the search pipeline, or (b) it is "user@domain" which resolves to a known Monorail user. It is also possible to search for, e.g., [owner:substring], but such searches remain 'owner' field searches rather than 'owner_id', and they cannot be combined with the "me" keyword. Args: cnxn: connection to the DB. cond: original parsed query Condition PB. user_service: connection to user persistence layer. id_fields: list of the search fields to use if the conversion to IDs succeed. is_member: True if user is a member of all the projects being searchers, so they can do user substring searches. Returns: A new Condition PB that checks the id_field. Or, the original cond. Raises: MalformedQuery: A non-member used a query term that could be used to guess full user email addresses. """ op = _TextOpToIntOp(cond.op) if _IsDefinedOp(op): # No need to look up any IDs if we are just testing for any defined value. return ast_pb2.Condition(op=op, field_defs=id_fields, key_suffix=cond.key_suffix, phase_name=cond.phase_name) # This preprocessing step is only for ops that compare whole values, not # substrings. if not _IsEqualityOp(op): logging.info('could not convert to IDs because op is %r', op) if not is_member: raise MalformedQuery( 'Only project members may compare user strings') return cond user_ids = [] for val in cond.str_values: try: user_ids.append(int(val)) except ValueError: try: user_ids.append(user_service.LookupUserID(cnxn, val)) except exceptions.NoSuchUserException: if not is_member and val != 'me' and not val.startswith('@'): logging.info('could not convert user %r to int ID', val) if '@' in val: raise MalformedQuery('User email address not found') else: raise MalformedQuery( 'Only project members may search for user substrings' ) return cond # preprocessing failed, stick with the original cond. return ast_pb2.MakeCond(op, id_fields, [], user_ids, key_suffix=cond.key_suffix, phase_name=cond.phase_name)
def testPreprocessCond_NoChange(self): cond = ast_pb2.MakeCond(ast_pb2.QueryOp.TEXT_HAS, [ANY_FIELD], ['foo'], []) self.assertEqual( cond, ast2ast._PreprocessCond(self.cnxn, cond, [], None, None, True))
def testKeyValueRegex_multipleKeys(self): cond = ast_pb2.MakeCond(ast_pb2.QueryOp.KEY_HAS, [BUILTIN_ISSUE_FIELDS['label']], ['Type-Bug', 'Security-Bug'], []) with self.assertRaises(ValueError): ast2ast._MakeKeyValueRegex(cond)
def _ParseStructuredTerm(prefix, op_str, value, fields, now=None): """Parse one user structured query term into an internal representation. Args: prefix: The query operator, usually a field name. E.g., summary. It can also be special operators like "is" to test boolean fields. op_str: the comparison operator. Usually ":" or "=", but can be any OPS. value: the value to compare against, e.g., term to find in that field. fields: dict {name_lower: [FieldDef, ...]} for built-in and custom fields. now: optional timestamp for tests, otherwise time.time() is used. Returns: A Condition PB. """ unquoted_value = value.strip('"') # Quick-OR is a convenient way to write one condition that matches any one of # multiple values, like set membership. E.g., [Priority=High,Critical]. quick_or_vals = [v.strip() for v in unquoted_value.split(',')] op = OPS[op_str] negate = False if prefix.startswith('-'): negate = True op = NEGATED_OPS.get(op, op) prefix = prefix[1:] if prefix == 'is' and unquoted_value in [ 'open', 'blocked', 'spam', 'ownerbouncing']: return ast_pb2.MakeCond( NE if negate else EQ, fields[unquoted_value], [], []) # Search entries with or without any value in the specified field. if prefix == 'has': op = IS_NOT_DEFINED if negate else IS_DEFINED if unquoted_value in fields: # Look for that field with any value. return ast_pb2.MakeCond(op, fields[unquoted_value], [], []) else: # Look for any label with that prefix. return ast_pb2.MakeCond(op, fields['label'], [unquoted_value], []) if prefix in fields: # search built-in and custom fields. E.g., summary. # Note: if first matching field is date-type, we assume they all are. # TODO(jrobbins): better handling for rare case where multiple projects # define the same custom field name, and one is a date and another is not. first_field = fields[prefix][0] if first_field.field_type == DATE: date_values = [_ParseDateValue(val, now=now) for val in quick_or_vals] return ast_pb2.MakeCond(op, fields[prefix], [], date_values) else: quick_or_ints = [] for qov in quick_or_vals: try: quick_or_ints.append(int(qov)) except ValueError: pass return ast_pb2.MakeCond(op, fields[prefix], quick_or_vals, quick_or_ints) # Since it is not a field, treat it as labels, E.g., Priority. quick_or_labels = ['%s-%s' % (prefix, v) for v in quick_or_vals] # Convert substring match to key-value match if user typed 'foo:bar'. if op == TEXT_HAS: op = KEY_HAS return ast_pb2.MakeCond(op, fields['label'], quick_or_labels, [])
def testBuildFTSCondition_NegatedAnyField(self): query_cond = ast_pb2.MakeCond(NOT_TEXT_HAS, [self.any_field_fd], ['needle'], []) fulltext_query_clause = fulltext_helpers._BuildFTSCondition( query_cond, self.fulltext_fields) self.assertEqual('NOT ("needle")', fulltext_query_clause)
def testBuildFTSCondition_BuiltinField(self): query_cond = ast_pb2.MakeCond(TEXT_HAS, [self.summary_fd], ['needle'], []) fulltext_query_clause = fulltext_helpers._BuildFTSCondition( query_cond, self.fulltext_fields) self.assertEqual('(summary:"needle")', fulltext_query_clause)
def testBuildFTSCondition_IgnoredOperator(self): query_cond = ast_pb2.MakeCond(GE, [self.summary_fd], ['needle'], []) fulltext_query_clause = fulltext_helpers._BuildFTSCondition( query_cond, self.fulltext_fields) self.assertEqual('', fulltext_query_clause)
def _ParseStructuredTerm(prefix, op_str, value, fields, now=None): """Parse one user structured query term into an internal representation. Args: prefix: The query operator, usually a field name. E.g., summary. It can also be special operators like "is" to test boolean fields. op_str: the comparison operator. Usually ":" or "=", but can be any OPS. value: the value to compare against, e.g., term to find in that field. fields: dict {name_lower: [FieldDef, ...]} for built-in and custom fields. now: optional timestamp for tests, otherwise time.time() is used. Returns: A Condition PB. """ unquoted_value = value.strip('"') # Quick-OR is a convenient way to write one condition that matches any one of # multiple values, like set membership. E.g., [Priority=High,Critical]. # Ignore empty values caused by duplicated or trailing commas. E.g., # [Priority=High,,Critical,] is equivalent to [Priority=High,Critical]. quick_or_vals = [v.strip() for v in unquoted_value.split(',') if v.strip()] op = OPS[op_str] negate = False if prefix.startswith('-'): negate = True op = NEGATED_OPS.get(op, op) prefix = prefix[1:] if prefix == 'is' and unquoted_value in [ 'open', 'blocked', 'spam', 'ownerbouncing' ]: return ast_pb2.MakeCond(NE if negate else EQ, fields[unquoted_value], [], []) # Search entries with or without any value in the specified field. if prefix == 'has': op = IS_NOT_DEFINED if negate else IS_DEFINED if '.' in unquoted_value: # Possible search for phase field with any value. phase_name, possible_field = unquoted_value.split('.', 1) if possible_field in fields: return ast_pb2.MakeCond(op, fields[possible_field], [], [], phase_name=phase_name) elif unquoted_value in fields: # Look for that field with any value. return ast_pb2.MakeCond(op, fields[unquoted_value], [], []) else: # Look for any label with that prefix. return ast_pb2.MakeCond(op, fields['label'], [unquoted_value], []) # Search entries with certain gates. if prefix == 'gate': return ast_pb2.MakeCond(op, fields['gate'], quick_or_vals, []) # Determine hotlist query type. # If prefix is not 'hotlist', quick_or_vals is empty, or qov # does not contain ':', is_fields will remain True is_fields = True if prefix == 'hotlist': try: if ':' not in quick_or_vals[0]: is_fields = False except IndexError: is_fields = False phase_name = None if '.' in prefix and is_fields: split_prefix = prefix.split('.', 1) if split_prefix[1] in fields: phase_name, prefix = split_prefix # search built-in and custom fields. E.g., summary. if prefix in fields and is_fields: # Note: if first matching field is date-type, we assume they all are. # TODO(jrobbins): better handling for rare case where multiple projects # define the same custom field name, and one is a date and another is not. first_field = fields[prefix][0] if first_field.field_type == DATE: date_values = [ _ParseDateValue(val, now=now) for val in quick_or_vals ] return ast_pb2.MakeCond(op, fields[prefix], [], date_values) else: quick_or_ints = [] for qov in quick_or_vals: try: quick_or_ints.append(int(qov)) except ValueError: pass if first_field.field_type == APPROVAL: for approval_suffix in _APPROVAL_SUFFIXES: if prefix.endswith(approval_suffix): return ast_pb2.MakeCond(op, fields[prefix], quick_or_vals, quick_or_ints, key_suffix=approval_suffix, phase_name=phase_name) return ast_pb2.MakeCond(op, fields[prefix], quick_or_vals, quick_or_ints, phase_name=phase_name) # Since it is not a field, treat it as labels, E.g., Priority. quick_or_labels = ['%s-%s' % (prefix, v) for v in quick_or_vals] # Convert substring match to key-value match if user typed 'foo:bar'. if op == TEXT_HAS: op = KEY_HAS return ast_pb2.MakeCond(op, fields['label'], quick_or_labels, [])
def testProcessLabelIDCond_NoValue(self): fd = BUILTIN_ISSUE_FIELDS['label_id'] cond = ast_pb2.MakeCond(ast_pb2.QueryOp.EQ, [fd], [], []) with self.assertRaises(ast2select.NoPossibleResults): ast2select._ProcessLabelIDCond(cond, 'Cond1', 'User1')