def _Dynamic_ListDocuments(self, request, response):
"""A local implementation of SearchService.ListDocuments RPC.
Args:
request: A search_service_pb.ListDocumentsRequest.
response: An search_service_pb.ListDocumentsResponse.
"""
params = request.params()
index = self._GetIndex(params.index_spec())
if index is None:
response.mutable_status().set_code(
search_service_pb.SearchServiceError.OK)
return
num_docs = 0
start = not params.has_start_doc_id()
for document in sorted(index.Documents(), key=lambda doc: doc.id()):
if start:
if num_docs < params.limit():
self._AddDocument(response, document, params.keys_only())
num_docs += 1
else:
if document.id() >= params.start_doc_id():
start = True
if (document.id() != params.start_doc_id() or
params.include_start_doc()):
self._AddDocument(response, document, params.keys_only())
num_docs += 1
response.mutable_status().set_code(
search_service_pb.SearchServiceError.OK)
def _Dynamic_DeleteSchema(self, request, response):
"""A local implementation of SearchService.DeleteSchema RPC.
Args:
request: A search_service_pb.DeleteSchemaRequest.
response: An search_service_pb.DeleteSchemaResponse.
"""
params = request.params()
for index_spec in params.index_spec_list():
response.add_status().set_code(search_service_pb.SearchServiceError.OK)
def _Dynamic_IndexDocument(self, request, response):
"""A local implementation of SearchService.IndexDocument RPC.
Index a new document or update an existing document.
Args:
request: A search_service_pb.IndexDocumentRequest.
response: An search_service_pb.IndexDocumentResponse.
"""
params = request.params()
index = self._GetIndex(params.index_spec(), create=True)
index.IndexDocuments(params.document_list(), response)
def _Dynamic_DeleteDocument(self, request, response):
"""A local implementation of SearchService.DeleteDocument RPC.
Args:
request: A search_service_pb.DeleteDocumentRequest.
response: An search_service_pb.DeleteDocumentResponse.
"""
params = request.params()
index_spec = params.index_spec()
index = self._GetIndex(index_spec)
for document_id in params.doc_id_list():
delete_status = response.add_status()
if index is None:
delete_status.set_code(search_service_pb.SearchServiceError.OK)
delete_status.set_error_detail('Not found')
else:
index.DeleteDocument(document_id, delete_status)
def _Dynamic_Search(self, request, response):
"""A local implementation of SearchService.Search RPC.
Args:
request: A search_service_pb.SearchRequest.
response: An search_service_pb.SearchResponse.
"""
if request.has_app_id():
self._RandomSearchResponse(request, response)
return
index = self._GetIndex(request.params().index_spec())
if index is None:
self._UnknownIndex(response.mutable_status(),
request.params().index_spec())
response.set_matched_count(0)
return
params = request.params()
try:
results = index.Search(params)
except query_parser.QueryException as e:
self._InvalidRequest(response.mutable_status(), e)
response.set_matched_count(0)
return
except expression_evaluator.ExpressionEvaluationError as e:
self._InvalidRequest(response.mutable_status(), e)
response.set_matched_count(0)
return
except document_matcher.ExpressionTreeException as e:
self._InvalidRequest(response.mutable_status(), e)
response.set_matched_count(0)
return
facet_analyzer = simple_facet.SimpleFacet(params)
results = facet_analyzer.RefineResults(results)
response.set_matched_count(len(results))
offset = 0
if params.has_cursor():
try:
doc_id = self._DecodeCursor(params.cursor())
except _InvalidCursorException as e:
self._InvalidRequest(response.mutable_status(), e)
response.set_matched_count(0)
return
for i, result in enumerate(results):
if result.document.id() == doc_id:
offset = i + 1
break
elif params.has_offset():
offset = params.offset()
if offset < len(results):
limit = offset + params.limit()
if limit >= len(results):
range_end = len(results)
else:
range_end = limit
if params.cursor_type() == search_service_pb.SearchParams.SINGLE:
document = results[range_end - 1].document
response.set_cursor(self._EncodeCursor(document))
result_range = list(range(offset, range_end))
else:
result_range = list(range(0))
field_names = params.field_spec().name_list()
self._FillSearchResponse(results, result_range, params.cursor_type(),
_ScoreRequested(params), response, field_names,
params.keys_only())
facet_analyzer.FillFacetResponse(results, response)
response.mutable_status().set_code(search_service_pb.SearchServiceError.OK)
def _RandomSearchResponse(self, request, response):
random.seed()
if random.random() < 0.03:
raise apiproxy_errors.ResponseTooLargeError()
response.mutable_status().set_code(
random.choice([search_service_pb.SearchServiceError.OK] * 30 +
[search_service_pb.SearchServiceError.TRANSIENT_ERROR] +
[search_service_pb.SearchServiceError.INTERNAL_ERROR]))
params = request.params()
random.seed(params.query())
total = random.randint(0, 100)
if random.random() < 0.3:
total = 0
offset = 0
if params.has_offset():
offset = params.offset()
remaining = max(0, total - offset)
nresults = min(remaining, params.limit())
matched_count = offset + nresults
if remaining > nresults:
matched_count += random.randint(1, 100)
def RandomText(charset, min_len, max_len):
return ''.join(random.choice(charset)
for _ in range(random.randint(min_len, max_len)))
for i in range(nresults):
seed = '%s:%s' % (params.query(), i + offset)
random.seed(seed)
result = response.add_result()
doc = result.mutable_document()
doc_id = RandomText(string.letters + string.digits, 8, 10)
doc.set_id(doc_id)
random.seed(doc_id)
for _ in params.sort_spec_list():
result.add_score(random.random())
for name, probability in [('creator', 0.90), ('last_change', 0.40)]:
if random.random() < probability:
field = doc.add_field()
field.set_name(name)
value = field.mutable_value()
value.set_type(document_pb.FieldValue.TEXT)
value.set_string_value(
RandomText(string.letters + string.digits, 2, 10)
+ '@google.com')
field = doc.add_field()
field.set_name('content')
value = field.mutable_value()
value.set_type(document_pb.FieldValue.TEXT)
value.set_string_value(
RandomText(string.printable, 0, 15) + params.query() +
RandomText(string.printable + 10 * string.whitespace, 5, 5000))
for i in range(random.randint(0, 2)):
field = doc.add_field()
field.set_name(RandomText(string.letters, 3, 7))
value = field.mutable_value()
value.set_type(document_pb.FieldValue.TEXT)
value.set_string_value(RandomText(string.printable, 0, 100))
response.set_matched_count(matched_count)
def _Dynamic_ListIndexes(self, request, response):
"""A local implementation of SearchService.ListIndexes RPC.
Args:
request: A search_service_pb.ListIndexesRequest.
response: An search_service_pb.ListIndexesResponse.
Raises:
ResponseTooLargeError: raised for testing admin console.
"""
if request.has_app_id():
if random.choice([True] + [False] * 9):
raise apiproxy_errors.ResponseTooLargeError()
for _ in range(random.randint(0, 2) * random.randint(5, 15)):
new_index_spec = response.add_index_metadata().mutable_index_spec()
new_index_spec.set_name(
random.choice(list(_VISIBLE_PRINTABLE_ASCII - set('!'))) +
''.join(random.choice(list(_VISIBLE_PRINTABLE_ASCII))
for _ in range(random.randint(
0, search.MAXIMUM_INDEX_NAME_LENGTH))))
response.mutable_status().set_code(
random.choice([search_service_pb.SearchServiceError.OK] * 10 +
[search_service_pb.SearchServiceError.TRANSIENT_ERROR] +
[search_service_pb.SearchServiceError.INTERNAL_ERROR]))
return
response.mutable_status().set_code(
search_service_pb.SearchServiceError.OK)
namespace = self._GetNamespace(request.params().namespace())
if namespace not in self.__indexes or not self.__indexes[namespace]:
return
keys, indexes = list(zip(*sorted(
iter(self.__indexes[namespace].items()), key=lambda v: v[0])))
position = 0
params = request.params()
if params.has_start_index_name():
position = bisect.bisect_left(keys, params.start_index_name())
if (not params.include_start_index() and position < len(keys)
and keys[position] == params.start_index_name()):
position += 1
elif params.has_index_name_prefix():
position = bisect.bisect_left(keys, params.index_name_prefix())
if params.has_offset():
position += params.offset()
end_position = position + params.limit()
prefix = params.index_name_prefix()
for index in indexes[min(position, len(keys)):min(end_position, len(keys))]:
index_spec = index.index_spec
if prefix and not index_spec.name().startswith(prefix):
break
metadata = response.add_index_metadata()
new_index_spec = metadata.mutable_index_spec()
new_index_spec.set_name(index_spec.name())
new_index_spec.set_namespace(index_spec.namespace())
if params.fetch_schema():
self._AddSchemaInformation(index, metadata)
self._AddStorageInformation(index, metadata)