def _Dynamic_RunQuery(self, query, query_result, request_id=None): """Send a query request to the datastore server. """ if query.has_transaction(): if not query.has_ancestor(): raise apiproxy_errors.ApplicationError( datastore_pb.Error.BAD_REQUEST, 'Only ancestor queries are allowed inside transactions.') (filters, orders) = datastore_index.Normalize(query.filter_list(), query.order_list(), []) old_datastore_stub_util.FillUsersInQuery(filters) if not query.has_app(): query.set_app(self.__app_id) self.__ValidateAppId(query.app()) # Set the composite index if it applies. indexes = [] if query.has_kind(): kind_indexes = self.__index_cache.get(query.kind()) if kind_indexes: indexes.extend(kind_indexes) index_to_use = _FindIndexToUse(query, indexes) if index_to_use != None: new_index = query.add_composite_index() new_index.MergeFrom(index_to_use) self._RemoteSend(query, query_result, "RunQuery", request_id) results = query_result.result_list() for result in results: old_datastore_stub_util.PrepareSpecialPropertiesForLoad(result) last_cursor = None if query_result.has_compiled_cursor(): last_cursor = query_result.compiled_cursor() if query_result.more_results(): new_cursor = InternalCursor(query, last_cursor, len(results)) cursor_id = self.__getCursorID() cursor = query_result.mutable_cursor() cursor.set_app(self.__app_id) cursor.set_cursor(cursor_id) self.__queries[cursor_id] = new_cursor if query.compile(): compiled_query = query_result.mutable_compiled_query() compiled_query.set_keys_only(query.keys_only()) compiled_query.mutable_primaryscan().set_index_name(query.Encode())
def _Dynamic_Next(self, next_request, query_result, request_id=None): """Get the next set of entities from a previously run query. """ self.__ValidateAppId(next_request.cursor().app()) cursor_handle = next_request.cursor().cursor() if cursor_handle not in self.__queries: raise apiproxy_errors.ApplicationError( datastore_pb.Error.BAD_REQUEST, 'Cursor %d not found' % cursor_handle) internal_cursor = self.__queries.get(cursor_handle) last_cursor = internal_cursor.get_last_cursor() query = internal_cursor.get_query() if not last_cursor: query_result.set_more_results(False) if next_request.compile(): compiled_query = query_result.mutable_compiled_query() compiled_query.set_keys_only(query.keys_only()) compiled_query.mutable_primaryscan().set_index_name( query.Encode()) del self.__queries[cursor_handle] return if internal_cursor.get_offset() >= internal_cursor.get_count(): query_result.set_more_results(False) query_result.mutable_compiled_cursor().CopyFrom(last_cursor) if next_request.compile(): compiled_query = query_result.mutable_compiled_query() compiled_query.set_keys_only(query.keys_only()) compiled_query.mutable_primaryscan().set_index_name( query.Encode()) del self.__queries[cursor_handle] return count = _BATCH_SIZE if next_request.has_count(): count = next_request.count() query.set_count(count) if next_request.has_offset(): query.set_offset(next_request.offset()) if next_request.has_compile(): query.set_compile(next_request.compile()) # Remove any offset since first RunQuery deals with it. query.clear_offset() query.mutable_compiled_cursor().CopyFrom(last_cursor) self._RemoteSend(query, query_result, "RunQuery", request_id) results = query_result.result_list() for result in results: old_datastore_stub_util.PrepareSpecialPropertiesForLoad(result) if len(results) > 0: if query_result.has_compiled_cursor(): last_cursor = query_result.compiled_cursor() internal_cursor.set_last_cursor(last_cursor) offset = internal_cursor.get_offset() internal_cursor.set_offset(offset + len(results)) query_result.set_more_results(internal_cursor.get_offset() < \ internal_cursor.get_count()) else: query_result.mutable_compiled_cursor().CopyFrom(last_cursor) query_result.set_more_results(False) if query.compile(): compiled_query = query_result.mutable_compiled_query() compiled_query.set_keys_only(query.keys_only()) compiled_query.mutable_primaryscan().set_index_name(query.Encode()) if not query_result.more_results(): del self.__queries[cursor_handle] else: cursor = query_result.mutable_cursor() cursor.set_app(self.__app_id) cursor.set_cursor(cursor_handle)
def _Dynamic_RunQuery(self, query, query_result): """Send a query request to the datastore server. """ if query.has_transaction(): if not query.has_ancestor(): raise apiproxy_errors.ApplicationError( datastore_pb.Error.BAD_REQUEST, 'Only ancestor queries are allowed inside transactions.') (filters, orders) = datastore_index.Normalize(query.filter_list(), query.order_list(), []) old_datastore_stub_util.FillUsersInQuery(filters) query_response = datastore_pb.QueryResult() if not query.has_app(): query.set_app(self.__app_id) self.__ValidateAppId(query.app()) self._RemoteSend(query, query_response, "RunQuery") skipped_results = 0 if query_response.has_skipped_results(): skipped_results = query_response.skipped_results() def has_prop_indexed(entity, prop): """Returns True if prop is in the entity and is indexed.""" if prop in datastore_types._SPECIAL_PROPERTIES: return True elif prop in entity.unindexed_properties(): return False values = entity.get(prop, []) if not isinstance(values, (tuple, list)): values = [values] for value in values: if type(value) not in datastore_types._RAW_PROPERTY_TYPES: return True return False def order_compare_entities(a, b): """ Return a negative, zero or positive number depending on whether entity a is considered smaller than, equal to, or larger than b, according to the query's orderings. """ cmped = 0 for o in orders: prop = o.property().decode('utf-8') reverse = (o.direction() is datastore_pb.Query_Order.DESCENDING) a_val = datastore._GetPropertyValue(a, prop) if isinstance(a_val, list): a_val = sorted(a_val, order_compare_properties, reverse=reverse)[0] b_val = datastore._GetPropertyValue(b, prop) if isinstance(b_val, list): b_val = sorted(b_val, order_compare_properties, reverse=reverse)[0] cmped = order_compare_properties(a_val, b_val) if o.direction() is datastore_pb.Query_Order.DESCENDING: cmped = -cmped if cmped != 0: return cmped if cmped == 0: return cmp(a.key(), b.key()) def order_compare_entities_pb(a, b): """ Return a negative, zero or positive number depending on whether entity a is considered smaller than, equal to, or larger than b, according to the query's orderings. a and b are protobuf-encoded entities.""" return order_compare_entities(datastore.Entity.FromPb(a), datastore.Entity.FromPb(b)) def order_compare_properties(x, y): """Return a negative, zero or positive number depending on whether property value x is considered smaller than, equal to, or larger than property value y. If x and y are different types, they're compared based on the type ordering used in the real datastore, which is based on the tag numbers in the PropertyValue PB. """ if isinstance(x, datetime.datetime): x = datastore_types.DatetimeToTimestamp(x) if isinstance(y, datetime.datetime): y = datastore_types.DatetimeToTimestamp(y) x_type = self._PROPERTY_TYPE_TAGS.get(x.__class__) y_type = self._PROPERTY_TYPE_TAGS.get(y.__class__) if x_type == y_type: try: return cmp(x, y) except TypeError: return 0 else: return cmp(x_type, y_type) results = query_response.result_list() for result in results: old_datastore_stub_util.PrepareSpecialPropertiesForLoad(result) old_datastore_stub_util.ValidateQuery(query, filters, orders, _MAX_QUERY_COMPONENTS) cursor = old_datastore_stub_util.ListCursor(query, results, order_compare_entities_pb) self.__cleanup_old_cursors() self.__queries[cursor.cursor] = cursor, datetime.datetime.now() if query.has_count(): count = query.count() elif query.has_limit(): count = query.limit() else: count = _BATCH_SIZE cursor.PopulateQueryResult(query_result, count, query.offset(), compile=query.compile()) query_result.set_skipped_results(skipped_results) if query.compile(): compiled_query = query_result.mutable_compiled_query() compiled_query.set_keys_only(query.keys_only()) compiled_query.mutable_primaryscan().set_index_name(query.Encode())