def testBuildFTSQuery_SpecialPrefixQuery(self):
special_prefix = query2ast.NON_OP_PREFIXES[0]
# Test with summary field.
query_ast_conj = ast_pb2.Conjunction(conds=[
ast_pb2.MakeCond(TEXT_HAS, [self.summary_fd],
['%s//google.com' % special_prefix], []),
ast_pb2.MakeCond(TEXT_HAS, [self.milestone_fd], ['Q3', 'Q4'], [])
])
fulltext_query = fulltext_helpers.BuildFTSQuery(
query_ast_conj, self.fulltext_fields)
self.assertEqual(
'(summary:"%s//google.com") (custom_123:"Q3" OR custom_123:"Q4")' %
(special_prefix), fulltext_query)
# Test with any field.
any_fd = tracker_pb2.FieldDef(
field_name=ast_pb2.ANY_FIELD,
field_type=tracker_pb2.FieldTypes.STR_TYPE)
query_ast_conj = ast_pb2.Conjunction(conds=[
ast_pb2.MakeCond(TEXT_HAS, [any_fd],
['%s//google.com' % special_prefix], []),
ast_pb2.MakeCond(TEXT_HAS, [self.milestone_fd], ['Q3', 'Q4'], [])
])
fulltext_query = fulltext_helpers.BuildFTSQuery(
query_ast_conj, self.fulltext_fields)
self.assertEqual(
'("%s//google.com") (custom_123:"Q3" OR custom_123:"Q4")' %
(special_prefix), fulltext_query)
def SearchIssueFullText(project_ids, query_ast_conj, shard_id):
"""Do full-text search in GAE FTS.
Args:
project_ids: list of project ID numbers to consider.
query_ast_conj: One conjuctive clause from the AST parsed
from the user's query.
shard_id: int shard ID for the shard to consider.
Returns:
(issue_ids, capped) where issue_ids is a list of issue issue_ids that match
the full-text query. And, capped is True if the results were capped due to
an implementation limitation. Or, return (None, False) if the given AST
conjunction contains no full-text conditions.
"""
fulltext_query = fulltext_helpers.BuildFTSQuery(query_ast_conj,
ISSUE_FULLTEXT_FIELDS)
if fulltext_query is None:
return None, False
if project_ids:
project_clause = ' OR '.join('project_id:%d' % pid
for pid in project_ids)
fulltext_query = '(%s) %s' % (project_clause, fulltext_query)
# TODO(jrobbins): it would be good to also include some other
# structured search terms to narrow down the set of index
# documents considered. E.g., most queries are only over the
# open issues.
logging.info('FTS query is %r', fulltext_query)
issue_ids = fulltext_helpers.ComprehensiveSearch(
fulltext_query, settings.search_index_name_format % shard_id)
capped = len(issue_ids) >= settings.fulltext_limit_per_shard
return issue_ids, capped
def testBuildFTSQuery_WithQuotes(self):
query_ast_conj = ast_pb2.Conjunction(conds=[
ast_pb2.MakeCond(TEXT_HAS, [self.summary_fd],
['"needle haystack"'], [])
])
fulltext_query = fulltext_helpers.BuildFTSQuery(
query_ast_conj, self.fulltext_fields)
self.assertEqual('(summary:"needle haystack")', fulltext_query)
def testBuildFTSQuery_InvalidQuery(self):
query_ast_conj = ast_pb2.Conjunction(conds=[
ast_pb2.MakeCond(TEXT_HAS, [self.summary_fd], ['haystack"needle'],
[]),
ast_pb2.MakeCond(TEXT_HAS, [self.milestone_fd], ['Q3', 'Q4'], [])
])
with self.assertRaises(AssertionError):
fulltext_helpers.BuildFTSQuery(query_ast_conj,
self.fulltext_fields)
def testBuildFTSQuery_Normal(self):
query_ast_conj = ast_pb2.Conjunction(conds=[
ast_pb2.MakeCond(TEXT_HAS, [self.summary_fd], ['needle'], []),
ast_pb2.MakeCond(TEXT_HAS, [self.milestone_fd], ['Q3', 'Q4'], [])
])
fulltext_query = fulltext_helpers.BuildFTSQuery(
query_ast_conj, self.fulltext_fields)
self.assertEqual(
'(summary:"needle") (custom_123:"Q3" OR custom_123:"Q4")',
fulltext_query)
def testBuildFTSQuery_NoFullTextConditions(self):
estimated_hours_fd = tracker_pb2.FieldDef(
field_name='estimate',
field_type=tracker_pb2.FieldTypes.INT_TYPE,
field_id=124)
query_ast_conj = ast_pb2.Conjunction(
conds=[ast_pb2.MakeCond(TEXT_HAS, [estimated_hours_fd], [], [40])])
fulltext_query = fulltext_helpers.BuildFTSQuery(
query_ast_conj, self.fulltext_fields)
self.assertEqual(None, fulltext_query)
def ParseQuery(mr, config, services):
"""Parse the user's query.
Args:
mr: commonly used info parsed from the request.
config: The ProjectConfig PB for the project.
services: connections to backends.
Returns:
A pair (ast, is_fulltext) with the parsed query abstract syntax tree
and a boolean that is True if the query included any fulltext terms.
"""
canned_query = savedqueries_helpers.SavedQueryIDToCond(
mr.cnxn, services.features, mr.can)
query_ast = query2ast.ParseUserQuery(
mr.query, canned_query, query2ast.BUILTIN_ISSUE_FIELDS, config)
is_fulltext_query = bool(
query_ast.conjunctions and
fulltext_helpers.BuildFTSQuery(
query_ast.conjunctions[0], tracker_fulltext.ISSUE_FULLTEXT_FIELDS))
return query_ast, is_fulltext_query
def testBuildFTSQuery_EmptyQueryConjunction(self):
query_ast_conj = ast_pb2.Conjunction()
fulltext_query = fulltext_helpers.BuildFTSQuery(
query_ast_conj, self.fulltext_fields)
self.assertEqual(None, fulltext_query)
def _GetQueryResultIIDs(cnxn, services, canned_query, user_query,
query_project_ids, harmonized_config, sd, slice_term,
shard_id, invalidation_timestep):
"""Do a search and return a list of matching issue IDs.
Args:
cnxn: connection to the database.
services: interface to issue storage backends.
canned_query: string part of the query from the drop-down menu.
user_query: string part of the query that the user typed in.
query_project_ids: list of project IDs to search.
harmonized_config: combined configs for all the queried projects.
sd: list of sort directives.
slice_term: additional query term to narrow results to a logical shard
within a physical shard.
shard_id: int number of the database shard to search.
invalidation_timestep: int timestep to use keep memcached items fresh.
Returns:
Tuple consisting of:
A list of issue issue_ids that match the user's query. An empty list, [],
is returned if no issues match the query.
Boolean that is set to True if the search results limit of this shard is
hit.
An error (subclass of Exception) encountered during query processing. None
means that no error was encountered.
"""
query_ast = _FilterSpam(
query2ast.ParseUserQuery(user_query, canned_query,
query2ast.BUILTIN_ISSUE_FIELDS,
harmonized_config))
logging.info('query_project_ids is %r', query_project_ids)
is_fulltext_query = bool(
query_ast.conjunctions and fulltext_helpers.BuildFTSQuery(
query_ast.conjunctions[0], tracker_fulltext.ISSUE_FULLTEXT_FIELDS))
expiration = framework_constants.MEMCACHE_EXPIRATION
if is_fulltext_query:
expiration = framework_constants.FULLTEXT_MEMCACHE_EXPIRATION
# Might raise ast2ast.MalformedQuery or ast2select.NoPossibleResults.
result_iids, search_limit_reached, error = SearchProjectCan(
cnxn,
services,
query_project_ids,
query_ast,
shard_id,
harmonized_config,
sort_directives=sd,
where=[slice_term],
query_desc='getting query issue IDs')
logging.info('Found %d result_iids', len(result_iids))
if error:
logging.warn('Got error %r', error)
projects_str = ','.join(str(pid) for pid in sorted(query_project_ids))
projects_str = projects_str or 'all'
memcache_key = ';'.join(
[projects_str, canned_query, user_query, ' '.join(sd),
str(shard_id)])
memcache.set(memcache_key, (result_iids, invalidation_timestep),
time=expiration,
namespace=settings.memcache_namespace)
logging.info('set memcache key %r', memcache_key)
search_limit_memcache_key = ';'.join([
projects_str, canned_query, user_query, ' '.join(sd),
'search_limit_reached',
str(shard_id)
])
memcache.set(search_limit_memcache_key,
(search_limit_reached, invalidation_timestep),
time=expiration,
namespace=settings.memcache_namespace)
logging.info('set search limit memcache key %r', search_limit_memcache_key)
timestamps_for_projects = memcache.get_multi(
keys=(['%d;%d' % (pid, shard_id)
for pid in query_project_ids] + ['all:%d' % shard_id]),
namespace=settings.memcache_namespace)
if query_project_ids:
for pid in query_project_ids:
key = '%d;%d' % (pid, shard_id)
if key not in timestamps_for_projects:
memcache.set(key,
invalidation_timestep,
time=framework_constants.MEMCACHE_EXPIRATION,
namespace=settings.memcache_namespace)
else:
key = 'all;%d' % shard_id
if key not in timestamps_for_projects:
memcache.set(key,
invalidation_timestep,
time=framework_constants.MEMCACHE_EXPIRATION,
namespace=settings.memcache_namespace)
return result_iids, search_limit_reached, error