def _Dynamic_RunQuery(self, query, query_result, request_id=None):
"""Send a query request to the datastore server. """
if query.has_transaction():
if not query.has_ancestor():
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST,
'Only ancestor queries are allowed inside transactions.')
(filters, orders) = datastore_index.Normalize(query.filter_list(),
query.order_list(), [])
old_datastore_stub_util.FillUsersInQuery(filters)
if not query.has_app():
query.set_app(self.__app_id)
self.__ValidateAppId(query.app())
# Set the composite index if it applies.
indexes = []
if query.has_kind():
kind_indexes = self.__index_cache.get(query.kind())
if kind_indexes:
indexes.extend(kind_indexes)
index_to_use = _FindIndexToUse(query, indexes)
if index_to_use != None:
new_index = query.add_composite_index()
new_index.MergeFrom(index_to_use)
self._RemoteSend(query, query_result, "RunQuery", request_id)
results = query_result.result_list()
for result in results:
old_datastore_stub_util.PrepareSpecialPropertiesForLoad(result)
last_cursor = None
if query_result.has_compiled_cursor():
last_cursor = query_result.compiled_cursor()
if query_result.more_results():
new_cursor = InternalCursor(query, last_cursor, len(results))
cursor_id = self.__getCursorID()
cursor = query_result.mutable_cursor()
cursor.set_app(self.__app_id)
cursor.set_cursor(cursor_id)
self.__queries[cursor_id] = new_cursor
if query.compile():
compiled_query = query_result.mutable_compiled_query()
compiled_query.set_keys_only(query.keys_only())
compiled_query.mutable_primaryscan().set_index_name(query.Encode())
def _Dynamic_Next(self, next_request, query_result, request_id=None):
"""Get the next set of entities from a previously run query. """
self.__ValidateAppId(next_request.cursor().app())
cursor_handle = next_request.cursor().cursor()
if cursor_handle not in self.__queries:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST,
'Cursor %d not found' % cursor_handle)
internal_cursor = self.__queries.get(cursor_handle)
last_cursor = internal_cursor.get_last_cursor()
query = internal_cursor.get_query()
if not last_cursor:
query_result.set_more_results(False)
if next_request.compile():
compiled_query = query_result.mutable_compiled_query()
compiled_query.set_keys_only(query.keys_only())
compiled_query.mutable_primaryscan().set_index_name(
query.Encode())
del self.__queries[cursor_handle]
return
if internal_cursor.get_offset() >= internal_cursor.get_count():
query_result.set_more_results(False)
query_result.mutable_compiled_cursor().CopyFrom(last_cursor)
if next_request.compile():
compiled_query = query_result.mutable_compiled_query()
compiled_query.set_keys_only(query.keys_only())
compiled_query.mutable_primaryscan().set_index_name(
query.Encode())
del self.__queries[cursor_handle]
return
count = _BATCH_SIZE
if next_request.has_count():
count = next_request.count()
query.set_count(count)
if next_request.has_offset():
query.set_offset(next_request.offset())
if next_request.has_compile():
query.set_compile(next_request.compile())
# Remove any offset since first RunQuery deals with it.
query.clear_offset()
query.mutable_compiled_cursor().CopyFrom(last_cursor)
self._RemoteSend(query, query_result, "RunQuery", request_id)
results = query_result.result_list()
for result in results:
old_datastore_stub_util.PrepareSpecialPropertiesForLoad(result)
if len(results) > 0:
if query_result.has_compiled_cursor():
last_cursor = query_result.compiled_cursor()
internal_cursor.set_last_cursor(last_cursor)
offset = internal_cursor.get_offset()
internal_cursor.set_offset(offset + len(results))
query_result.set_more_results(internal_cursor.get_offset() < \
internal_cursor.get_count())
else:
query_result.mutable_compiled_cursor().CopyFrom(last_cursor)
query_result.set_more_results(False)
if query.compile():
compiled_query = query_result.mutable_compiled_query()
compiled_query.set_keys_only(query.keys_only())
compiled_query.mutable_primaryscan().set_index_name(query.Encode())
if not query_result.more_results():
del self.__queries[cursor_handle]
else:
cursor = query_result.mutable_cursor()
cursor.set_app(self.__app_id)
cursor.set_cursor(cursor_handle)
def _Dynamic_RunQuery(self, query, query_result):
"""Send a query request to the datastore server. """
if query.has_transaction():
if not query.has_ancestor():
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST,
'Only ancestor queries are allowed inside transactions.')
(filters, orders) = datastore_index.Normalize(query.filter_list(),
query.order_list(), [])
old_datastore_stub_util.FillUsersInQuery(filters)
query_response = datastore_pb.QueryResult()
if not query.has_app():
query.set_app(self.__app_id)
self.__ValidateAppId(query.app())
self._RemoteSend(query, query_response, "RunQuery")
skipped_results = 0
if query_response.has_skipped_results():
skipped_results = query_response.skipped_results()
def has_prop_indexed(entity, prop):
"""Returns True if prop is in the entity and is indexed."""
if prop in datastore_types._SPECIAL_PROPERTIES:
return True
elif prop in entity.unindexed_properties():
return False
values = entity.get(prop, [])
if not isinstance(values, (tuple, list)):
values = [values]
for value in values:
if type(value) not in datastore_types._RAW_PROPERTY_TYPES:
return True
return False
def order_compare_entities(a, b):
""" Return a negative, zero or positive number depending on whether
entity a is considered smaller than, equal to, or larger than b,
according to the query's orderings. """
cmped = 0
for o in orders:
prop = o.property().decode('utf-8')
reverse = (o.direction() is
datastore_pb.Query_Order.DESCENDING)
a_val = datastore._GetPropertyValue(a, prop)
if isinstance(a_val, list):
a_val = sorted(a_val,
order_compare_properties,
reverse=reverse)[0]
b_val = datastore._GetPropertyValue(b, prop)
if isinstance(b_val, list):
b_val = sorted(b_val,
order_compare_properties,
reverse=reverse)[0]
cmped = order_compare_properties(a_val, b_val)
if o.direction() is datastore_pb.Query_Order.DESCENDING:
cmped = -cmped
if cmped != 0:
return cmped
if cmped == 0:
return cmp(a.key(), b.key())
def order_compare_entities_pb(a, b):
""" Return a negative, zero or positive number depending on whether
entity a is considered smaller than, equal to, or larger than b,
according to the query's orderings. a and b are protobuf-encoded
entities."""
return order_compare_entities(datastore.Entity.FromPb(a),
datastore.Entity.FromPb(b))
def order_compare_properties(x, y):
"""Return a negative, zero or positive number depending on whether
property value x is considered smaller than, equal to, or larger than
property value y. If x and y are different types, they're compared based
on the type ordering used in the real datastore, which is based on the
tag numbers in the PropertyValue PB.
"""
if isinstance(x, datetime.datetime):
x = datastore_types.DatetimeToTimestamp(x)
if isinstance(y, datetime.datetime):
y = datastore_types.DatetimeToTimestamp(y)
x_type = self._PROPERTY_TYPE_TAGS.get(x.__class__)
y_type = self._PROPERTY_TYPE_TAGS.get(y.__class__)
if x_type == y_type:
try:
return cmp(x, y)
except TypeError:
return 0
else:
return cmp(x_type, y_type)
results = query_response.result_list()
for result in results:
old_datastore_stub_util.PrepareSpecialPropertiesForLoad(result)
old_datastore_stub_util.ValidateQuery(query, filters, orders,
_MAX_QUERY_COMPONENTS)
cursor = old_datastore_stub_util.ListCursor(query, results,
order_compare_entities_pb)
self.__cleanup_old_cursors()
self.__queries[cursor.cursor] = cursor, datetime.datetime.now()
if query.has_count():
count = query.count()
elif query.has_limit():
count = query.limit()
else:
count = _BATCH_SIZE
cursor.PopulateQueryResult(query_result,
count,
query.offset(),
compile=query.compile())
query_result.set_skipped_results(skipped_results)
if query.compile():
compiled_query = query_result.mutable_compiled_query()
compiled_query.set_keys_only(query.keys_only())
compiled_query.mutable_primaryscan().set_index_name(query.Encode())
