def testParseUserQuery_QuickOr(self):
# quick-or searches
ast = query2ast.ParseUserQuery('milestone:2008,2009,2010', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(KEY_HAS, [BUILTIN_ISSUE_FIELDS['label']],
['milestone-2008', 'milestone-2009', 'milestone-2010'],
[]), cond1)
ast = query2ast.ParseUserQuery(
'label:milestone-2008,milestone-2009,milestone-2010', '',
BUILTIN_ISSUE_FIELDS, self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(TEXT_HAS, [BUILTIN_ISSUE_FIELDS['label']],
['milestone-2008', 'milestone-2009', 'milestone-2010'],
[]), cond1)
ast = query2ast.ParseUserQuery('milestone=2008,2009,2010', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(EQ, [BUILTIN_ISSUE_FIELDS['label']],
['milestone-2008', 'milestone-2009', 'milestone-2010'],
[]), cond1)
def testParseUserQuery_CodeSyntaxThatWeNeedToCopeWith(self):
# positive phrases
ast = query2ast.ParseUserQuery('Base::Tuple', '', BUILTIN_ISSUE_FIELDS,
self.default_config)
cond = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(TEXT_HAS, [ANY_FIELD], ['"base::tuple"'], []), cond)
# stuff we just ignore
ast = query2ast.ParseUserQuery(':: - -- .', '', BUILTIN_ISSUE_FIELDS,
self.default_config)
self.assertEqual([], ast.conjunctions[0].conds)
def testParseUserQuery_IsOperator(self):
"""Test is:open, is:spam, and is:blocked."""
for keyword in ['open', 'spam', 'blocked']:
ast = query2ast.ParseUserQuery('is:' + keyword, '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(EQ, [BUILTIN_ISSUE_FIELDS[keyword]], [], []), cond1)
ast = query2ast.ParseUserQuery('-is:' + keyword, '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(NE, [BUILTIN_ISSUE_FIELDS[keyword]], [], []), cond1)
def EvaluateSubscriptions(
cnxn, issue, users_to_queries, services, config):
"""Determine subscribers who have subs that match the given issue."""
# Note: unlike filter rule, subscriptions see explicit & derived values.
lower_labels = [lab.lower() for lab in tracker_bizobj.GetLabels(issue)]
label_set = set(lower_labels)
subscribers_to_notify = []
for uid, saved_queries in users_to_queries.items():
for sq in saved_queries:
if sq.subscription_mode != 'immediate':
continue
if issue.project_id not in sq.executes_in_project_ids:
continue
cond = savedqueries_helpers.SavedQueryToCond(sq)
# TODO(jrobbins): Support linked accounts me_user_ids.
cond, _warnings = searchpipeline.ReplaceKeywordsWithUserIDs([uid], cond)
cond_ast = query2ast.ParseUserQuery(
cond, '', query2ast.BUILTIN_ISSUE_FIELDS, config)
if filterrules_helpers.EvalPredicate(
cnxn, services, cond_ast, issue, label_set, config,
tracker_bizobj.GetOwnerId(issue), tracker_bizobj.GetCcIds(issue),
tracker_bizobj.GetStatus(issue)):
subscribers_to_notify.append(uid)
break # Don't bother looking at the user's other saved quereies.
return subscribers_to_notify
def testParseUserQuery_Phase(self):
ast = query2ast.ParseUserQuery(
'gate:Canary,Stable', '', BUILTIN_ISSUE_FIELDS, self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(TEXT_HAS, [BUILTIN_ISSUE_FIELDS['gate']],
['canary', 'stable'], []),
cond1)
ast = query2ast.ParseUserQuery(
'-gate:Canary,Stable', '', BUILTIN_ISSUE_FIELDS, self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(NOT_TEXT_HAS, [BUILTIN_ISSUE_FIELDS['gate']],
['canary', 'stable'], []),
cond1)
def testGetSpamQueryResultIIDs(self):
sd = ['project', 'id']
slice_term = ('Issue.shard = %s', [2])
query_ast = query2ast.ParseUserQuery(
'Priority:High is:spam', 'is:open', query2ast.BUILTIN_ISSUE_FIELDS,
self.config)
query_ast = backendsearchpipeline._FilterSpam(query_ast)
self.mox.StubOutWithMock(backendsearchpipeline, 'SearchProjectCan')
backendsearchpipeline.SearchProjectCan(
self.cnxn, self.services, [789], query_ast, 2, self.config,
sort_directives=sd, where=[slice_term],
query_desc='getting query issue IDs'
).AndReturn(([10002, 10052], False, None))
self.mox.ReplayAll()
result, capped, err = backendsearchpipeline._GetQueryResultIIDs(
self.cnxn, self.services, 'is:open', 'Priority:High is:spam',
[789], self.config, sd, slice_term, 2, 12345)
self.mox.VerifyAll()
self.assertEqual([10002, 10052], result)
self.assertFalse(capped)
self.assertEqual(None, err)
self.assertEqual(
([10002, 10052], 12345),
memcache.get('789;is:open;Priority:High is:spam;project id;2'))
def _QueryToWhere(self, cnxn, services, project_config, query,
canned_query, project):
"""Parses a query string into LEFT JOIN and WHERE conditions.
Args:
cnxn: A MonorailConnection instance.
services: A Services instance.
project_config: The configuration for the given project.
query (string): The query to parse.
canned_query (string): The supplied canned query.
project: The current project.
Returns:
1. A list of LEFT JOIN clauses for the SQL query.
2. A list of WHERE clases for the SQL query.
3. A list of query conditions that are unsupported with snapshots.
"""
if not (query or canned_query):
return [], [], []
query = query or ''
scope = canned_query or ''
query_ast = query2ast.ParseUserQuery(query, scope,
query2ast.BUILTIN_ISSUE_FIELDS,
project_config)
query_ast = ast2ast.PreprocessAST(cnxn, query_ast,
[project.project_id], services,
project_config)
left_joins, where, unsupported = ast2select.BuildSQLQuery(
query_ast, snapshot_mode=True)
return left_joins, where, unsupported
def testParseUserQuery_Components(self):
"""Parse user queries for components"""
ast = query2ast.ParseUserQuery('component:UI', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(TEXT_HAS, [BUILTIN_ISSUE_FIELDS['component']], ['ui'],
[]), cond1)
ast = query2ast.ParseUserQuery('Component:UI>AboutBox', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(TEXT_HAS, [BUILTIN_ISSUE_FIELDS['component']],
['ui>aboutbox'], []), cond1)
def testParseUserQuery_SyntaxErrors(self):
"""Bad queries should report warnings."""
warnings = []
ast = query2ast.ParseUserQuery('one two (three four)', '',
BUILTIN_ISSUE_FIELDS,
self.default_config, warnings)
self.assertEqual(4, len(ast.conjunctions[0].conds))
self.assertEqual(['Parentheses are ignored in user queries.'],
warnings)
warnings = []
ast = query2ast.ParseUserQuery('', 'one two (three four)',
BUILTIN_ISSUE_FIELDS,
self.default_config, warnings)
self.assertEqual(4, len(ast.conjunctions[0].conds))
self.assertEqual(['Parentheses are ignored in saved queries.'],
warnings)
def testParseUserQuery_BadDates(self):
bad_dates = ['today-13h', 'yesterday', '2/2', 'm/y/d',
'99/99/1999', '0-0-0']
for val in bad_dates:
with self.assertRaises(query2ast.InvalidQueryError) as cm:
query2ast.ParseUserQuery(
'modified>=' + val, '', BUILTIN_ISSUE_FIELDS,
self.default_config)
self.assertEqual('Could not parse date: ' + val, cm.exception.message)
def testParseUserQuery_HasOperator(self):
# Search for issues with at least one attachment
ast = query2ast.ParseUserQuery('has:attachment', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(IS_DEFINED, [BUILTIN_ISSUE_FIELDS['attachment']], [], []),
cond1)
ast = query2ast.ParseUserQuery('-has:attachment', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(IS_NOT_DEFINED, [BUILTIN_ISSUE_FIELDS['attachment']], [],
[]), cond1)
ast = query2ast.ParseUserQuery('has=attachment', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(IS_DEFINED, [BUILTIN_ISSUE_FIELDS['attachment']], [], []),
cond1)
ast = query2ast.ParseUserQuery('-has=attachment', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(IS_NOT_DEFINED, [BUILTIN_ISSUE_FIELDS['attachment']], [],
[]), cond1)
# Search for numeric fields for searches with 'has' prefix
ast = query2ast.ParseUserQuery('has:attachments', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(IS_DEFINED, [BUILTIN_ISSUE_FIELDS['attachments']], [],
[]), cond1)
ast = query2ast.ParseUserQuery('-has:attachments', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(IS_NOT_DEFINED, [BUILTIN_ISSUE_FIELDS['attachments']], [],
[]), cond1)
# If it is not a field, look for any key-value label.
ast = query2ast.ParseUserQuery('has:Size', '', BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(IS_DEFINED, [BUILTIN_ISSUE_FIELDS['label']], ['size'],
[]), cond1)
def testParseUserQuery_OrClause(self):
# an "OR" query, which should look like two separate simple querys
# joined together by a pipe.
ast = query2ast.ParseUserQuery(
'ham OR fancy', '', BUILTIN_ISSUE_FIELDS, self.default_config)
conj1 = ast.conjunctions[0]
conj2 = ast.conjunctions[1]
self.assertEqual([MakeCond(TEXT_HAS, [ANY_FIELD], ['ham'], [])],
conj1.conds)
self.assertEqual([MakeCond(TEXT_HAS, [ANY_FIELD], ['fancy'], [])],
conj2.conds)
def ParsePredicateASTs(rules, config, me_user_ids):
"""Parse the given rules in QueryAST PBs."""
predicates = [rule.predicate for rule in rules]
if me_user_ids:
predicates = [
searchpipeline.ReplaceKeywordsWithUserIDs(me_user_ids, pred)[0]
for pred in predicates]
predicate_asts = [
query2ast.ParseUserQuery(pred, '', query2ast.BUILTIN_ISSUE_FIELDS, config)
for pred in predicates]
return predicate_asts
def testParseUserQuery_PhaseFields(self):
fd = tracker_bizobj.MakeFieldDef(
1, self.project_id, 'EstDays', tracker_pb2.FieldTypes.INT_TYPE,
'applic', 'applic', False, False, False, None, None, None, False, None,
None, None, 'no_action', 'doc', False, is_phase_field=True)
self.default_config.field_defs.append(fd)
ast = query2ast.ParseUserQuery(
'UXReview.EstDays>3', '', BUILTIN_ISSUE_FIELDS, self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(GT, [fd], ['3'], [3], phase_name='uxreview'),
cond1)
def testParseUserQuery_Phrases(self):
# positive phrases
ast = query2ast.ParseUserQuery(
'"one two"', '-label:deprecated', BUILTIN_ISSUE_FIELDS,
self.default_config)
scope_cond1, fulltext_cond = ast.conjunctions[0].conds
self.assertEqual(
MakeCond(NOT_TEXT_HAS, [BUILTIN_ISSUE_FIELDS['label']],
['deprecated'], []),
scope_cond1)
self.assertEqual(
MakeCond(TEXT_HAS, [ANY_FIELD], ['"one two"'], []), fulltext_cond)
# negative phrases
ast = query2ast.ParseUserQuery(
'-"one two"', '-label:deprecated', BUILTIN_ISSUE_FIELDS,
self.default_config)
scope_cond1, fulltext_cond = ast.conjunctions[0].conds
self.assertEqual(
MakeCond(NOT_TEXT_HAS, [BUILTIN_ISSUE_FIELDS['label']],
['deprecated'], []),
scope_cond1)
self.assertEqual(
MakeCond(NOT_TEXT_HAS, [ANY_FIELD], ['"one two"'], []), fulltext_cond)
# multiple phrases
ast = query2ast.ParseUserQuery(
'-"a b" "x y"', '-label:deprecated', BUILTIN_ISSUE_FIELDS,
self.default_config)
scope_cond1, ft_cond1, ft_cond2 = ast.conjunctions[0].conds
self.assertEqual(
MakeCond(NOT_TEXT_HAS, [BUILTIN_ISSUE_FIELDS['label']],
['deprecated'], []),
scope_cond1)
self.assertEqual(
MakeCond(NOT_TEXT_HAS, [ANY_FIELD], ['"a b"'], []), ft_cond1)
self.assertEqual(
MakeCond(TEXT_HAS, [ANY_FIELD], ['"x y"'], []), ft_cond2)
def testParseUserQuery_OwnersReportersAndCc(self):
"""Parse user queries for owner:, reporter: and cc:."""
ast = query2ast.ParseUserQuery(
'owner:user', '', BUILTIN_ISSUE_FIELDS, self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(TEXT_HAS, [BUILTIN_ISSUE_FIELDS['owner']],
['user'], []),
cond1)
ast = query2ast.ParseUserQuery(
'owner:[email protected]', '', BUILTIN_ISSUE_FIELDS, self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(TEXT_HAS, [BUILTIN_ISSUE_FIELDS['owner']],
['*****@*****.**'], []),
cond1)
ast = query2ast.ParseUserQuery(
'[email protected]', '', BUILTIN_ISSUE_FIELDS, self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(EQ, [BUILTIN_ISSUE_FIELDS['owner']],
['*****@*****.**'], []),
cond1)
ast = query2ast.ParseUserQuery(
'[email protected]', '', BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(NE, [BUILTIN_ISSUE_FIELDS['reporter']],
['*****@*****.**'], []),
cond1)
ast = query2ast.ParseUserQuery(
'[email protected],[email protected]', '', BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(EQ, [BUILTIN_ISSUE_FIELDS['cc']],
['*****@*****.**', '*****@*****.**'], []),
cond1)
ast = query2ast.ParseUserQuery(
'cc:user,user2', '', BUILTIN_ISSUE_FIELDS, self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(TEXT_HAS, [BUILTIN_ISSUE_FIELDS['cc']],
['user', 'user2'], []),
cond1)
def testSearchProjectCan_FTSCapped(self):
query_ast = query2ast.ParseUserQuery(
'Priority:High', 'is:open', query2ast.BUILTIN_ISSUE_FIELDS,
self.config)
simplified_query_ast = ast2ast.PreprocessAST(
self.cnxn, query_ast, [789], self.services, self.config)
conj = simplified_query_ast.conjunctions[0]
self.mox.StubOutWithMock(tracker_fulltext, 'SearchIssueFullText')
tracker_fulltext.SearchIssueFullText(
[789], conj, 2).AndReturn(([10002, 10052], True))
self.mox.StubOutWithMock(self.services.issue, 'RunIssueQuery')
self.services.issue.RunIssueQuery(
self.cnxn, mox.IsA(list), mox.IsA(list), mox.IsA(list),
shard_id=2).AndReturn(([10002, 10052], False))
self.mox.ReplayAll()
result, capped, err = backendsearchpipeline.SearchProjectCan(
self.cnxn, self.services, [789], query_ast, 2, self.config)
self.mox.VerifyAll()
self.assertEqual([10002, 10052], result)
self.assertTrue(capped)
self.assertEqual(None, err)
def testParseUserQuery_SearchWithinCustomFields(self):
"""Enums are treated as labels, other fields are kept as fields."""
fd1 = tracker_bizobj.MakeFieldDef(
1, self.project_id, 'Size', tracker_pb2.FieldTypes.ENUM_TYPE,
'applic', 'applic', False, False, False, None, None, None, False, None,
None, None, 'no_action', 'doc', False)
fd2 = tracker_bizobj.MakeFieldDef(
1, self.project_id, 'EstDays', tracker_pb2.FieldTypes.INT_TYPE,
'applic', 'applic', False, False, False, None, None, None, False, None,
None, None, 'no_action', 'doc', False)
self.default_config.field_defs.extend([fd1, fd2])
ast = query2ast.ParseUserQuery(
'Size:Small EstDays>3', '', BUILTIN_ISSUE_FIELDS, self.default_config)
cond1 = ast.conjunctions[0].conds[0]
cond2 = ast.conjunctions[0].conds[1]
self.assertEqual(
MakeCond(KEY_HAS, [BUILTIN_ISSUE_FIELDS['label']],
['size-small'], []),
cond1)
self.assertEqual(
MakeCond(GT, [fd2], ['3'], [3]),
cond2)
def ParseQuery(mr, config, services):
"""Parse the user's query.
Args:
mr: commonly used info parsed from the request.
config: The ProjectConfig PB for the project.
services: connections to backends.
Returns:
A pair (ast, is_fulltext) with the parsed query abstract syntax tree
and a boolean that is True if the query included any fulltext terms.
"""
canned_query = savedqueries_helpers.SavedQueryIDToCond(
mr.cnxn, services.features, mr.can)
query_ast = query2ast.ParseUserQuery(
mr.query, canned_query, query2ast.BUILTIN_ISSUE_FIELDS, config)
is_fulltext_query = bool(
query_ast.conjunctions and
fulltext_helpers.BuildFTSQuery(
query_ast.conjunctions[0], tracker_fulltext.ISSUE_FULLTEXT_FIELDS))
return query_ast, is_fulltext_query
def testParseUserQuery_Approvals(self):
"""Test approval queries are parsed correctly."""
fd1 = tracker_bizobj.MakeFieldDef(
1, self.project_id, 'UIReview', tracker_pb2.FieldTypes.APPROVAL_TYPE,
'applic', 'applic', False, False, False, None, None, None, False, None,
None, None, 'no_action', 'doc', False)
fd2 = tracker_bizobj.MakeFieldDef(
2, self.project_id, 'EstDays', tracker_pb2.FieldTypes.INT_TYPE,
'applic', 'applic', False, False, False, None, None, None, False, None,
None, None, 'no_action', 'doc', False)
fd3 = tracker_bizobj.MakeFieldDef(
3, self.project_id, 'UXReview', tracker_pb2.FieldTypes.APPROVAL_TYPE,
'applic', 'applic', False, False, False, None, None, None, False, None,
None, None, 'no_action', 'doc', False)
self.default_config.field_defs.extend([fd1, fd2, fd3])
ast = query2ast.ParseUserQuery(
'UXReview-approver:[email protected],[email protected] UIReview:Approved',
'', BUILTIN_ISSUE_FIELDS, self.default_config)
cond1 = ast.conjunctions[0].conds[0]
cond2 = ast.conjunctions[0].conds[1]
self.assertEqual(MakeCond(TEXT_HAS, [fd3],
['*****@*****.**', '*****@*****.**'], [],
key_suffix='-approver'), cond1)
self.assertEqual(MakeCond(TEXT_HAS, [fd1], ['approved'], []), cond2)
def _GetQueryResultIIDs(cnxn, services, canned_query, user_query,
query_project_ids, harmonized_config, sd, slice_term,
shard_id, invalidation_timestep):
"""Do a search and return a list of matching issue IDs.
Args:
cnxn: connection to the database.
services: interface to issue storage backends.
canned_query: string part of the query from the drop-down menu.
user_query: string part of the query that the user typed in.
query_project_ids: list of project IDs to search.
harmonized_config: combined configs for all the queried projects.
sd: list of sort directives.
slice_term: additional query term to narrow results to a logical shard
within a physical shard.
shard_id: int number of the database shard to search.
invalidation_timestep: int timestep to use keep memcached items fresh.
Returns:
Tuple consisting of:
A list of issue issue_ids that match the user's query. An empty list, [],
is returned if no issues match the query.
Boolean that is set to True if the search results limit of this shard is
hit.
An error (subclass of Exception) encountered during query processing. None
means that no error was encountered.
"""
query_ast = _FilterSpam(
query2ast.ParseUserQuery(user_query, canned_query,
query2ast.BUILTIN_ISSUE_FIELDS,
harmonized_config))
logging.info('query_project_ids is %r', query_project_ids)
is_fulltext_query = bool(
query_ast.conjunctions and fulltext_helpers.BuildFTSQuery(
query_ast.conjunctions[0], tracker_fulltext.ISSUE_FULLTEXT_FIELDS))
expiration = framework_constants.MEMCACHE_EXPIRATION
if is_fulltext_query:
expiration = framework_constants.FULLTEXT_MEMCACHE_EXPIRATION
# Might raise ast2ast.MalformedQuery or ast2select.NoPossibleResults.
result_iids, search_limit_reached, error = SearchProjectCan(
cnxn,
services,
query_project_ids,
query_ast,
shard_id,
harmonized_config,
sort_directives=sd,
where=[slice_term],
query_desc='getting query issue IDs')
logging.info('Found %d result_iids', len(result_iids))
if error:
logging.warn('Got error %r', error)
projects_str = ','.join(str(pid) for pid in sorted(query_project_ids))
projects_str = projects_str or 'all'
memcache_key = ';'.join(
[projects_str, canned_query, user_query, ' '.join(sd),
str(shard_id)])
memcache.set(memcache_key, (result_iids, invalidation_timestep),
time=expiration,
namespace=settings.memcache_namespace)
logging.info('set memcache key %r', memcache_key)
search_limit_memcache_key = ';'.join([
projects_str, canned_query, user_query, ' '.join(sd),
'search_limit_reached',
str(shard_id)
])
memcache.set(search_limit_memcache_key,
(search_limit_reached, invalidation_timestep),
time=expiration,
namespace=settings.memcache_namespace)
logging.info('set search limit memcache key %r', search_limit_memcache_key)
timestamps_for_projects = memcache.get_multi(
keys=(['%d;%d' % (pid, shard_id)
for pid in query_project_ids] + ['all:%d' % shard_id]),
namespace=settings.memcache_namespace)
if query_project_ids:
for pid in query_project_ids:
key = '%d;%d' % (pid, shard_id)
if key not in timestamps_for_projects:
memcache.set(key,
invalidation_timestep,
time=framework_constants.MEMCACHE_EXPIRATION,
namespace=settings.memcache_namespace)
else:
key = 'all;%d' % shard_id
if key not in timestamps_for_projects:
memcache.set(key,
invalidation_timestep,
time=framework_constants.MEMCACHE_EXPIRATION,
namespace=settings.memcache_namespace)
return result_iids, search_limit_reached, error
def testParseUserQuery_Words(self):
# an "ORTerm" is actually anything appearing on either side of an
# "OR" operator. So this could be thought of as "simple" query parsing.
# a simple query with no spaces
ast = query2ast.ParseUserQuery('hamfancy', '', BUILTIN_ISSUE_FIELDS,
self.default_config)
fulltext_cond = ast.conjunctions[0].conds[0]
self.assertEqual(MakeCond(TEXT_HAS, [ANY_FIELD], ['hamfancy'], []),
fulltext_cond)
# negative word
ast = query2ast.ParseUserQuery('-hamfancy', '', BUILTIN_ISSUE_FIELDS,
self.default_config)
fulltext_cond = ast.conjunctions[0].conds[0]
self.assertEqual(
# note: not NOT_TEXT_HAS.
MakeCond(NOT_TEXT_HAS, [ANY_FIELD], ['hamfancy'], []),
fulltext_cond)
# invalid fulltext term
ast = query2ast.ParseUserQuery('ham=fancy\\', '', BUILTIN_ISSUE_FIELDS,
self.default_config)
self.assertEqual([], ast.conjunctions[0].conds)
# an explicit "AND" query in the "featured" context
warnings = []
query2ast.ParseUserQuery('ham AND fancy',
'label:featured',
BUILTIN_ISSUE_FIELDS,
self.default_config,
warnings=warnings)
self.assertEqual(
['The only supported boolean operator is OR (all capitals).'],
warnings)
# an implicit "AND" query
ast = query2ast.ParseUserQuery('ham fancy', '-label:deprecated',
BUILTIN_ISSUE_FIELDS,
self.default_config)
scope_cond1, ft_cond1, ft_cond2 = ast.conjunctions[0].conds
self.assertEqual(
MakeCond(NOT_TEXT_HAS, [BUILTIN_ISSUE_FIELDS['label']],
['deprecated'], []), scope_cond1)
self.assertEqual(MakeCond(TEXT_HAS, [ANY_FIELD], ['ham'], []),
ft_cond1)
self.assertEqual(MakeCond(TEXT_HAS, [ANY_FIELD], ['fancy'], []),
ft_cond2)
# Use word with non-operator prefix.
word_with_non_op_prefix = '%stest' % query2ast.NON_OP_PREFIXES[0]
ast = query2ast.ParseUserQuery(word_with_non_op_prefix, '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
fulltext_cond = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(TEXT_HAS, [ANY_FIELD], ['"%s"' % word_with_non_op_prefix],
[]), fulltext_cond)
# mix positive and negative words
ast = query2ast.ParseUserQuery('ham -fancy', '-label:deprecated',
BUILTIN_ISSUE_FIELDS,
self.default_config)
scope_cond1, ft_cond1, ft_cond2 = ast.conjunctions[0].conds
self.assertEqual(
MakeCond(NOT_TEXT_HAS, [BUILTIN_ISSUE_FIELDS['label']],
['deprecated'], []), scope_cond1)
self.assertEqual(MakeCond(TEXT_HAS, [ANY_FIELD], ['ham'], []),
ft_cond1)
self.assertEqual(MakeCond(NOT_TEXT_HAS, [ANY_FIELD], ['fancy'], []),
ft_cond2)
# converts terms to lower case
ast = query2ast.ParseUserQuery('AmDude', '-label:deprecated',
BUILTIN_ISSUE_FIELDS,
self.default_config)
scope_cond1, fulltext_cond = ast.conjunctions[0].conds
self.assertEqual(
MakeCond(NOT_TEXT_HAS, [BUILTIN_ISSUE_FIELDS['label']],
['deprecated'], []), scope_cond1)
self.assertEqual(MakeCond(TEXT_HAS, [ANY_FIELD], ['amdude'], []),
fulltext_cond)
def testParseUserQuery_Dates(self):
# query with a daterange
ast = query2ast.ParseUserQuery('modified>=2009-5-12', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
ts1 = int(time.mktime(datetime.datetime(2009, 5, 12).timetuple()))
self.assertEqual(
MakeCond(GE, [BUILTIN_ISSUE_FIELDS['modified']], [], [ts1]), cond1)
# query with quick-or
ast = query2ast.ParseUserQuery('modified=2009-5-12,2009-5-13', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
ts1 = int(time.mktime(datetime.datetime(2009, 5, 12).timetuple()))
ts2 = int(time.mktime(datetime.datetime(2009, 5, 13).timetuple()))
self.assertEqual(
MakeCond(EQ, [BUILTIN_ISSUE_FIELDS['modified']], [], [ts1, ts2]),
cond1)
# query with multiple dateranges
ast = query2ast.ParseUserQuery('modified>=2009-5-12 opened<2008/1/1',
'', BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1, cond2 = ast.conjunctions[0].conds
ts1 = int(time.mktime(datetime.datetime(2009, 5, 12).timetuple()))
self.assertEqual(
MakeCond(GE, [BUILTIN_ISSUE_FIELDS['modified']], [], [ts1]), cond1)
ts2 = int(time.mktime(datetime.datetime(2008, 1, 1).timetuple()))
self.assertEqual(
MakeCond(LT, [BUILTIN_ISSUE_FIELDS['opened']], [], [ts2]), cond2)
# query with multiple dateranges plus a search term
ast = query2ast.ParseUserQuery(
'one two modified>=2009-5-12 opened<2008/1/1', '',
BUILTIN_ISSUE_FIELDS, self.default_config)
ft_cond1, ft_cond2, cond1, cond2 = ast.conjunctions[0].conds
ts1 = int(time.mktime(datetime.datetime(2009, 5, 12).timetuple()))
self.assertEqual(MakeCond(TEXT_HAS, [ANY_FIELD], ['one'], []),
ft_cond1)
self.assertEqual(MakeCond(TEXT_HAS, [ANY_FIELD], ['two'], []),
ft_cond2)
self.assertEqual(
MakeCond(GE, [BUILTIN_ISSUE_FIELDS['modified']], [], [ts1]), cond1)
ts2 = int(time.mktime(datetime.datetime(2008, 1, 1).timetuple()))
self.assertEqual(
MakeCond(LT, [BUILTIN_ISSUE_FIELDS['opened']], [], [ts2]), cond2)
# query with a date field compared to "today"
ast = query2ast.ParseUserQuery('modified<today',
'',
BUILTIN_ISSUE_FIELDS,
self.default_config,
now=NOW)
cond1 = ast.conjunctions[0].conds[0]
ts1 = query2ast._CalculatePastDate(0, now=NOW)
self.assertEqual(
MakeCond(LT, [BUILTIN_ISSUE_FIELDS['modified']], [], [ts1]), cond1)
# query with a daterange using today-N alias
ast = query2ast.ParseUserQuery('modified>=today-13',
'',
BUILTIN_ISSUE_FIELDS,
self.default_config,
now=NOW)
cond1 = ast.conjunctions[0].conds[0]
ts1 = query2ast._CalculatePastDate(13, now=NOW)
self.assertEqual(
MakeCond(GE, [BUILTIN_ISSUE_FIELDS['modified']], [], [ts1]), cond1)
ast = query2ast.ParseUserQuery('modified>today-13',
'',
BUILTIN_ISSUE_FIELDS,
self.default_config,
now=NOW)
cond1 = ast.conjunctions[0].conds[0]
ts1 = query2ast._CalculatePastDate(13, now=NOW)
self.assertEqual(
MakeCond(GT, [BUILTIN_ISSUE_FIELDS['modified']], [], [ts1]), cond1)
# query with multiple old date query terms.
ast = query2ast.ParseUserQuery(
'modified-after:2009-5-12 opened-before:2008/1/1 '
'closed-after:2007-2-1', '', BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1, cond2, cond3 = ast.conjunctions[0].conds
ts1 = int(time.mktime(datetime.datetime(2009, 5, 12).timetuple()))
self.assertEqual(
MakeCond(GT, [BUILTIN_ISSUE_FIELDS['modified']], [], [ts1]), cond1)
ts2 = int(time.mktime(datetime.datetime(2008, 1, 1).timetuple()))
self.assertEqual(
MakeCond(LT, [BUILTIN_ISSUE_FIELDS['opened']], [], [ts2]), cond2)
ts3 = int(time.mktime(datetime.datetime(2007, 2, 1).timetuple()))
self.assertEqual(
MakeCond(GT, [BUILTIN_ISSUE_FIELDS['closed']], [], [ts3]), cond3)
def testParseUserQuery_SearchWithinFields(self):
# Search for issues with certain filenames
ast = query2ast.ParseUserQuery('attachment:filename', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(TEXT_HAS, [BUILTIN_ISSUE_FIELDS['attachment']],
['filename'], []), cond1)
ast = query2ast.ParseUserQuery('-attachment:filename', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(NOT_TEXT_HAS, [BUILTIN_ISSUE_FIELDS['attachment']],
['filename'], []), cond1)
# Search for issues with a certain number of attachments
ast = query2ast.ParseUserQuery('attachments:2', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(TEXT_HAS, [BUILTIN_ISSUE_FIELDS['attachments']], ['2'],
[2]), cond1)
# Searches with '=' syntax
ast = query2ast.ParseUserQuery('attachment=filename', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(EQ, [BUILTIN_ISSUE_FIELDS['attachment']], ['filename'],
[]), cond1)
ast = query2ast.ParseUserQuery('-attachment=filename', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(NE, [BUILTIN_ISSUE_FIELDS['attachment']], ['filename'],
[]), cond1)
ast = query2ast.ParseUserQuery('milestone=2009', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(EQ, [BUILTIN_ISSUE_FIELDS['label']], ['milestone-2009'],
[]), cond1)
ast = query2ast.ParseUserQuery('-milestone=2009', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(NE, [BUILTIN_ISSUE_FIELDS['label']], ['milestone-2009'],
[]), cond1)
ast = query2ast.ParseUserQuery('milestone=2009-Q1', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(EQ, [BUILTIN_ISSUE_FIELDS['label']],
['milestone-2009-q1'], []), cond1)
ast = query2ast.ParseUserQuery('-milestone=2009-Q1', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(NE, [BUILTIN_ISSUE_FIELDS['label']],
['milestone-2009-q1'], []), cond1)
# Searches with ':' syntax
ast = query2ast.ParseUserQuery('summary:foo', '', BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(TEXT_HAS, [BUILTIN_ISSUE_FIELDS['summary']], ['foo'], []),
cond1)
ast = query2ast.ParseUserQuery('summary:"greetings programs"', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(TEXT_HAS, [BUILTIN_ISSUE_FIELDS['summary']],
['greetings programs'], []), cond1)
ast = query2ast.ParseUserQuery('summary:"Ӓ"', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(TEXT_HAS, [BUILTIN_ISSUE_FIELDS['summary']], ['Ӓ'],
[]), cond1)
ast = query2ast.ParseUserQuery('priority:high', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(KEY_HAS, [BUILTIN_ISSUE_FIELDS['label']],
['priority-high'], []), cond1)
ast = query2ast.ParseUserQuery('type:security', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(KEY_HAS, [BUILTIN_ISSUE_FIELDS['label']],
['type-security'], []), cond1)
ast = query2ast.ParseUserQuery('label:priority-high', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(TEXT_HAS, [BUILTIN_ISSUE_FIELDS['label']],
['priority-high'], []), cond1)
ast = query2ast.ParseUserQuery('blockedon:other:123', '',
BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(TEXT_HAS, [BUILTIN_ISSUE_FIELDS['blockedon']],
['other:123'], []), cond1)
ast = query2ast.ParseUserQuery('cost=-2', '', BUILTIN_ISSUE_FIELDS,
self.default_config)
cond1 = ast.conjunctions[0].conds[0]
self.assertEqual(
MakeCond(EQ, [BUILTIN_ISSUE_FIELDS['label']], ['cost--2'], []),
cond1)
def testApplyRule(self):
cnxn = 'fake sql connection'
issue = fake.MakeTestIssue(789,
1,
ORIG_SUMMARY,
'New',
111L,
labels=ORIG_LABELS)
config = tracker_pb2.ProjectIssueConfig()
# Empty label set cannot satisfy rule looking for labels.
pred = 'label:a label:b'
rule = filterrules_helpers.MakeRule(pred,
default_owner_id=1,
default_status='S')
predicate_ast = query2ast.ParseUserQuery(
pred, '', query2ast.BUILTIN_ISSUE_FIELDS, config)
self.assertEquals(
(None, None, [], [], []),
filterrules_helpers._ApplyRule(cnxn, self.services, rule,
predicate_ast, issue, set(),
config))
pred = 'label:a -label:b'
rule = filterrules_helpers.MakeRule(pred,
default_owner_id=1,
default_status='S')
predicate_ast = query2ast.ParseUserQuery(
pred, '', query2ast.BUILTIN_ISSUE_FIELDS, config)
self.assertEquals(
(None, None, [], [], []),
filterrules_helpers._ApplyRule(cnxn, self.services, rule,
predicate_ast, issue, set(),
config))
# Empty label set will satisfy rule looking for missing labels.
pred = '-label:a -label:b'
rule = filterrules_helpers.MakeRule(pred,
default_owner_id=1,
default_status='S')
predicate_ast = query2ast.ParseUserQuery(
pred, '', query2ast.BUILTIN_ISSUE_FIELDS, config)
self.assertEquals(
(1, 'S', [], [], []),
filterrules_helpers._ApplyRule(cnxn, self.services, rule,
predicate_ast, issue, set(),
config))
# Label set has the needed labels.
pred = 'label:a label:b'
rule = filterrules_helpers.MakeRule(pred,
default_owner_id=1,
default_status='S')
predicate_ast = query2ast.ParseUserQuery(
pred, '', query2ast.BUILTIN_ISSUE_FIELDS, config)
self.assertEquals(
(1, 'S', [], [], []),
filterrules_helpers._ApplyRule(cnxn, self.services, rule,
predicate_ast, issue, {'a', 'b'},
config))
# Label set has the needed labels with test for unicode.
pred = 'label:a label:b'
rule = filterrules_helpers.MakeRule(pred,
default_owner_id=1,
default_status='S')
predicate_ast = query2ast.ParseUserQuery(
pred, '', query2ast.BUILTIN_ISSUE_FIELDS, config)
self.assertEquals(
(1, 'S', [], [], []),
filterrules_helpers._ApplyRule(cnxn, self.services, rule,
predicate_ast, issue, {u'a', u'b'},
config))
# Label set has the needed labels, capitalization irrelevant.
pred = 'label:A label:B'
rule = filterrules_helpers.MakeRule(pred,
default_owner_id=1,
default_status='S')
predicate_ast = query2ast.ParseUserQuery(
pred, '', query2ast.BUILTIN_ISSUE_FIELDS, config)
self.assertEquals(
(1, 'S', [], [], []),
filterrules_helpers._ApplyRule(cnxn, self.services, rule,
predicate_ast, issue, {'a', 'b'},
config))
# Label set has a label, the rule negates.
pred = 'label:a -label:b'
rule = filterrules_helpers.MakeRule(pred,
default_owner_id=1,
default_status='S')
predicate_ast = query2ast.ParseUserQuery(
pred, '', query2ast.BUILTIN_ISSUE_FIELDS, config)
self.assertEquals(
(None, None, [], [], []),
filterrules_helpers._ApplyRule(cnxn, self.services, rule,
predicate_ast, issue, {'a', 'b'},
config))
