Ejemplo n.º 1
0
    def Query(self, query, filters, orders):
        """Perform a query on this pseudo-kind.

    Args:
      query: the original datastore_pb.Query
      filters: the filters from query
      orders: the orders from query

    Returns:
      A query cursor to iterate over the query results, or None if the query
      is invalid.
    """
        namespace_range = datastore_stub_util.ParseNamespaceQuery(
            query, filters, orders)
        app_str = query.app()

        namespace_entities = []

        namespaces = self.sqlitestub._DatastoreSqliteStub__namespaces
        for app_id, namespace in sorted(namespaces):
            if app_id == app_str and namespace_range.Contains(namespace):
                if namespace:
                    ns_id = namespace
                else:
                    ns_id = datastore_types._EMPTY_NAMESPACE_ID
                namespace_entities.append(
                    MakeEntityForQuery(query, self.name, ns_id))

        return datastore_stub_util.ListCursor(
            query, namespace_entities,
            datastore_stub_util.CompareEntityPbByKey, set())
Ejemplo n.º 2
0
    def Query(self, query, filters, orders):
        """Perform a query on this pseudo-kind.

    Args:
      query: the original datastore_pb.Query
      filters: the filters from query
      orders: the orders from query

    Returns:
      A query cursor to iterate over the query results, or None if the query
      is invalid.
    """
        kind_range = datastore_stub_util.ParseKindQuery(query, filters, orders)
        conn = self.sqlitestub._GetConnection()
        cursor = None
        try:
            prefix = self.sqlitestub._GetTablePrefix(query)
            filters = []

            def AddExtremeFilter(extreme, inclusive, is_end):
                """Add filter for kind start/end."""
                if not is_end:
                    if inclusive:
                        op = datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL
                    else:
                        op = datastore_pb.Query_Filter.GREATER_THAN
                else:
                    if inclusive:
                        op = datastore_pb.Query_Filter.LESS_THAN_OR_EQUAL
                    else:
                        op = datastore_pb.Query_Filter.LESS_THAN
                filters.append(('kind', op, extreme))

            kind_range.MapExtremes(AddExtremeFilter)

            params = []
            sql_filters = self.sqlitestub._CreateFilterString(filters, params)

            sql_stmt = ('SELECT kind FROM "%s!Entities" %s GROUP BY kind' %
                        (prefix, sql_filters))
            c = conn.execute(sql_stmt, params)

            kinds = []
            for row in c.fetchall():
                kinds.append(
                    MakeEntityForQuery(query, self.name, ToUtf8(row[0])))

            cursor = datastore_stub_util.ListCursor(
                query, kinds, datastore_stub_util.CompareEntityPbByKey, set())
        finally:
            self.sqlitestub._ReleaseConnection(conn)

        return cursor
Ejemplo n.º 3
0
  def _GetQueryCursor(self, query, filters, orders, index_list):
    """Returns a query cursor for the provided query.

    Args:
      query: The datastore_pb.Query to run.
      filters: A list of filters that override the ones found on query.
      orders: A list of orders that override the ones found on query.
      index_list: A list of indexes used by the query.

    Returns:
      A QueryCursor object.
    """
    if query.has_kind() and query.kind() in self._pseudo_kinds:

      datastore_stub_util.NormalizeCursors(query,
                                           datastore_pb.Query_Order.ASCENDING)
      cursor = self._pseudo_kinds[query.kind()].Query(query, filters, orders)
      datastore_stub_util.Check(cursor,
                                'Could not create query for pseudo-kind')
    else:
      orders = datastore_stub_util._GuessOrders(filters, orders)


      datastore_stub_util.NormalizeCursors(query, orders[0].direction())
      filter_info = self.__GenerateFilterInfo(filters, query)
      order_info = self.__GenerateOrderInfo(orders)

      for strategy in DatastoreSqliteStub._QUERY_STRATEGIES:
        result = strategy(self, query, filter_info, order_info)
        if result:
          break
      else:
        raise apiproxy_errors.ApplicationError(
            datastore_pb.Error.BAD_REQUEST,
            'No strategy found to satisfy query.')

      sql_stmt, params = result

      conn = self._GetConnection()
      try:
        if query.property_name_list():
          db_cursor = _ProjectionPartialEntityGenerator(
              conn.execute(sql_stmt, params))
        else:
          db_cursor = _DedupingEntityGenerator(conn.execute(sql_stmt, params))
        dsquery = datastore_stub_util._MakeQuery(query, filters, orders)

        cursor = datastore_stub_util.ListCursor(query, dsquery, orders,
                                                index_list, list(db_cursor))
      finally:
        self._ReleaseConnection(conn)
    return cursor
Ejemplo n.º 4
0
    def _GetQueryCursor(self,
                        query,
                        filters,
                        orders,
                        index_list,
                        filter_predicate=None):
        """Returns a query cursor for the provided query.

    Args:
      query: The datastore_pb.Query to run.
      filters: A list of filters that override the ones found on query.
      orders: A list of orders that override the ones found on query.
      index_list: A list of indexes used by the query.
      filter_predicate: an additional filter of type
          datastore_query.FilterPredicate. This is passed along to implement V4
          specific filters without changing the entire stub.

    Returns:
      A QueryCursor object.
    """
        if query.has_kind() and query.kind() in self._pseudo_kinds:
            cursor = self._pseudo_kinds[query.kind()].Query(
                query, filters, orders)
            datastore_stub_util.Check(
                cursor, 'Could not create query for pseudo-kind')
        else:
            orders = datastore_stub_util._GuessOrders(filters, orders)
            filter_info = self.__GenerateFilterInfo(filters, query)
            order_info = self.__GenerateOrderInfo(orders)

            for strategy in DatastoreSqliteStub._QUERY_STRATEGIES:
                result = strategy(self, query, filter_info, order_info)
                if result:
                    break
            else:
                raise apiproxy_errors.ApplicationError(
                    datastore_pb.Error.BAD_REQUEST,
                    'No strategy found to satisfy query.')

            sql_stmt, params = result

            conn = self._GetConnection()
            try:
                if query.property_name_list():
                    db_cursor = _ProjectionPartialEntityGenerator(
                        conn.execute(sql_stmt, params))
                else:
                    db_cursor = _DedupingEntityGenerator(
                        conn.execute(sql_stmt, params))
                dsquery = datastore_stub_util._MakeQuery(
                    query, filters, orders, filter_predicate)

                filtered_entities = [r for r in db_cursor]

                if filter_predicate:
                    filtered_entities = list(
                        filter(filter_predicate, filtered_entities))

                cursor = datastore_stub_util.ListCursor(
                    query, dsquery, orders, index_list, filtered_entities)
            finally:
                self._ReleaseConnection(conn)
        return cursor
    def _Dynamic_RunQuery(self, query, query_result):
        """Send a query request to the datastore server. """
        if query.has_transaction():
            if not query.has_ancestor():
                raise apiproxy_errors.ApplicationError(
                    datastore_pb.Error.BAD_REQUEST,
                    'Only ancestor queries are allowed inside transactions.')
        (filters, orders) = datastore_index.Normalize(query.filter_list(),
                                                      query.order_list(), [])

        datastore_stub_util.FillUsersInQuery(filters)

        query_response = datastore_pb.QueryResult()
        if not query.has_app():
            query.set_app(self.__app_id)
        self.__ValidateAppId(query.app())
        self._RemoteSend(query, query_response, "RunQuery")

        skipped_results = 0
        if query_response.has_skipped_results():
            skipped_results = query_response.skipped_results()

        def has_prop_indexed(entity, prop):
            """Returns True if prop is in the entity and is indexed."""
            if prop in datastore_types._SPECIAL_PROPERTIES:
                return True
            elif prop in entity.unindexed_properties():
                return False

            values = entity.get(prop, [])
            if not isinstance(values, (tuple, list)):
                values = [values]

            for value in values:
                if type(value) not in datastore_types._RAW_PROPERTY_TYPES:
                    return True
            return False

        def order_compare_entities(a, b):
            """ Return a negative, zero or positive number depending on whether
      entity a is considered smaller than, equal to, or larger than b,
      according to the query's orderings. """
            cmped = 0
            for o in orders:
                prop = o.property().decode('utf-8')

                reverse = (o.direction() is
                           datastore_pb.Query_Order.DESCENDING)

                a_val = datastore._GetPropertyValue(a, prop)
                if isinstance(a_val, list):
                    a_val = sorted(a_val,
                                   order_compare_properties,
                                   reverse=reverse)[0]

                b_val = datastore._GetPropertyValue(b, prop)
                if isinstance(b_val, list):
                    b_val = sorted(b_val,
                                   order_compare_properties,
                                   reverse=reverse)[0]

                cmped = order_compare_properties(a_val, b_val)

                if o.direction() is datastore_pb.Query_Order.DESCENDING:
                    cmped = -cmped

                if cmped != 0:
                    return cmped

            if cmped == 0:
                return cmp(a.key(), b.key())

        def order_compare_entities_pb(a, b):
            """ Return a negative, zero or positive number depending on whether
      entity a is considered smaller than, equal to, or larger than b,
      according to the query's orderings. a and b are protobuf-encoded
      entities."""
            return order_compare_entities(datastore.Entity.FromPb(a),
                                          datastore.Entity.FromPb(b))

        def order_compare_properties(x, y):
            """Return a negative, zero or positive number depending on whether
      property value x is considered smaller than, equal to, or larger than
      property value y. If x and y are different types, they're compared based
      on the type ordering used in the real datastore, which is based on the
      tag numbers in the PropertyValue PB.
      """
            if isinstance(x, datetime.datetime):
                x = datastore_types.DatetimeToTimestamp(x)
            if isinstance(y, datetime.datetime):
                y = datastore_types.DatetimeToTimestamp(y)

            x_type = self._PROPERTY_TYPE_TAGS.get(x.__class__)
            y_type = self._PROPERTY_TYPE_TAGS.get(y.__class__)

            if x_type == y_type:
                try:
                    return cmp(x, y)
                except TypeError:
                    return 0
            else:
                return cmp(x_type, y_type)

        results = query_response.result_list()
        results = [datastore.Entity._FromPb(r) for r in results]
        results = [r._ToPb() for r in results]
        for result in results:
            datastore_stub_util.PrepareSpecialPropertiesForLoad(result)

        datastore_stub_util.ValidateQuery(query, filters, orders,
                                          _MAX_QUERY_COMPONENTS)

        cursor = datastore_stub_util.ListCursor(query, results,
                                                order_compare_entities_pb)
        self.__cleanup_old_cursors()
        self.__queries[cursor.cursor] = cursor, datetime.datetime.now()

        if query.has_count():
            count = query.count()
        elif query.has_limit():
            count = query.limit()
        else:
            count = _BATCH_SIZE

        cursor.PopulateQueryResult(query_result,
                                   count,
                                   query.offset(),
                                   compile=query.compile())
        query_result.set_skipped_results(skipped_results)
        if query.compile():
            compiled_query = query_result.mutable_compiled_query()
            compiled_query.set_keys_only(query.keys_only())
            compiled_query.mutable_primaryscan().set_index_name(query.Encode())
Ejemplo n.º 6
0
    def Query(self, query, filters, orders):
        """Perform a query on this pseudo-kind.

    Args:
      query: the original datastore_pb.Query
      filters: the filters from query
      orders: the orders from query

    Returns:
      A query cursor to iterate over the query results, or None if the query
      is invalid.
    """
        property_range = datastore_stub_util.ParsePropertyQuery(
            query, filters, orders)
        keys_only = query.keys_only()
        conn = self.sqlitestub._GetConnection()
        cursor = None
        try:
            prefix = self.sqlitestub._GetTablePrefix(query)
            filters = []

            def AddExtremeFilter(extreme, inclusive, is_end):
                """Add filter for kind start/end."""
                if not is_end:
                    op = datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL
                else:
                    op = datastore_pb.Query_Filter.LESS_THAN_OR_EQUAL
                filters.append(('kind', op, extreme[0]))

            property_range.MapExtremes(AddExtremeFilter)

            for name in datastore_stub_util.GetInvisibleSpecialPropertyNames():
                filters.append(('name', '!=', name))

            params = []
            sql_filters = self.sqlitestub._CreateFilterString(filters, params)
            if not keys_only:

                sql_stmt = (
                    'SELECT kind, name, value FROM "%s!EntitiesByProperty" %s '
                    'GROUP BY kind, name, substr(value, 1, 1) '
                    'ORDER BY kind, name' % (prefix, sql_filters))
            else:

                sql_stmt = (
                    'SELECT kind, name FROM "%s!EntitiesByProperty" %s '
                    'GROUP BY kind, name ORDER BY kind, name' %
                    (prefix, sql_filters))
            c = conn.execute(sql_stmt, params)

            properties = []
            kind = None
            name = None
            property_pb = None
            for row in c.fetchall():
                if not (row[0] == kind and row[1] == name):

                    if not property_range.Contains((row[0], row[1])):
                        continue
                    kind, name = row[:2]

                    if property_pb:
                        properties.append(property_pb)
                    property_pb = MakeEntityForQuery(query,
                                                     KindPseudoKind.name,
                                                     ToUtf8(kind), self.name,
                                                     ToUtf8(name))

                if not keys_only:

                    value_data = row[2]
                    value_decoder = sortable_pb_encoder.Decoder(
                        array.array('B', str(value_data)))
                    raw_value_pb = entity_pb.PropertyValue()
                    raw_value_pb.Merge(value_decoder)
                    tag = datastore_types.GetPropertyValueTag(raw_value_pb)
                    tag_name = datastore_stub_util._PROPERTY_TYPE_NAMES[tag]

                    representation_pb = property_pb.add_property()
                    representation_pb.set_name(u'property_representation')
                    representation_pb.set_multiple(True)
                    representation_pb.mutable_value().set_stringvalue(tag_name)

            if property_pb:
                properties.append(property_pb)

            cursor = datastore_stub_util.ListCursor(
                query, properties, datastore_stub_util.CompareEntityPbByKey,
                set())
        finally:
            self.sqlitestub._ReleaseConnection(conn)

        return cursor
Ejemplo n.º 7
0
    def _Dynamic_RunQuery(self, query, query_result):
        if query.has_transaction():
            self.__ValidateTransaction(query.transaction())
            entities = self.__tx_snapshot
        else:
            entities = self.__entities

        app_id = query.app()
        namespace = query.name_space()
        self.__ValidateAppId(app_id)

        (filters, orders) = datastore_index.Normalize(query.filter_list(),
                                                      query.order_list())
        datastore_stub_util.ValidateQuery(query, filters, orders,
                                          _MAX_QUERY_COMPONENTS)
        datastore_stub_util.FillUsersInQuery(filters)

        pseudo_kind = None
        if query.has_kind() and query.kind() in self.__pseudo_kinds:
            pseudo_kind = self.__pseudo_kinds[query.kind()]

        if not pseudo_kind and self.__require_indexes:
            required, kind, ancestor, props, num_eq_filters = datastore_index.CompositeIndexForQuery(
                query)
            if required:
                required_key = kind, ancestor, props
                indexes = self.__indexes.get(app_id)
                if not indexes:
                    raise apiproxy_errors.ApplicationError(
                        datastore_pb.Error.NEED_INDEX,
                        "This query requires a composite index, but none are defined. "
                        "You must create an index.yaml file in your application root."
                    )
                eq_filters_set = set(props[:num_eq_filters])
                remaining_filters = props[num_eq_filters:]
                for index in indexes:
                    definition = datastore_index.ProtoToIndexDefinition(index)
                    index_key = datastore_index.IndexToKey(definition)
                    if required_key == index_key:
                        break
                    if num_eq_filters > 1 and (kind,
                                               ancestor) == index_key[:2]:
                        this_props = index_key[2]
                        this_eq_filters_set = set(this_props[:num_eq_filters])
                        this_remaining_filters = this_props[num_eq_filters:]
                        if (eq_filters_set == this_eq_filters_set and
                                remaining_filters == this_remaining_filters):
                            break
                else:
                    raise apiproxy_errors.ApplicationError(
                        datastore_pb.Error.NEED_INDEX,
                        "This query requires a composite index that is not defined. "
                        "You must update the index.yaml file in your application root."
                    )

        try:
            query.set_app(app_id)
            datastore_types.SetNamespace(query, namespace)
            encoded = datastore_types.EncodeAppIdNamespace(app_id, namespace)
            if pseudo_kind:
                (results, filters,
                 orders) = pseudo_kind.Query(entities, query, filters, orders)
            elif query.has_kind():
                results = entities[encoded, query.kind()].values()
                results = [entity.native for entity in results]
            else:
                results = []
                for key in entities:
                    if key[0] == encoded:
                        results += [
                            entity.native for entity in entities[key].values()
                        ]
        except KeyError:
            results = []

        if query.has_ancestor():
            ancestor_path = query.ancestor().path().element_list()

            def is_descendant(entity):
                path = entity.key()._Key__reference.path().element_list()
                return path[:len(ancestor_path)] == ancestor_path

            results = filter(is_descendant, results)

        operators = {
            datastore_pb.Query_Filter.LESS_THAN: '<',
            datastore_pb.Query_Filter.LESS_THAN_OR_EQUAL: '<=',
            datastore_pb.Query_Filter.GREATER_THAN: '>',
            datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL: '>=',
            datastore_pb.Query_Filter.EQUAL: '==',
        }

        def has_prop_indexed(entity, prop):
            """Returns True if prop is in the entity and is indexed."""
            if prop in datastore_types._SPECIAL_PROPERTIES:
                return True
            elif prop in entity.unindexed_properties():
                return False

            values = entity.get(prop, [])
            if not isinstance(values, (tuple, list)):
                values = [values]

            for value in values:
                if type(value) not in datastore_types._RAW_PROPERTY_TYPES:
                    return True
            return False

        for filt in filters:
            assert filt.op() != datastore_pb.Query_Filter.IN

            prop = filt.property(0).name().decode('utf-8')
            op = operators[filt.op()]

            filter_val_list = [
                datastore_types.FromPropertyPb(filter_prop)
                for filter_prop in filt.property_list()
            ]

            def passes_filter(entity):
                """Returns True if the entity passes the filter, False otherwise.

        The filter being evaluated is filt, the current filter that we're on
        in the list of filters in the query.
        """
                if not has_prop_indexed(entity, prop):
                    return False

                try:
                    entity_vals = datastore._GetPropertyValue(entity, prop)
                except KeyError:
                    entity_vals = []

                if not isinstance(entity_vals, list):
                    entity_vals = [entity_vals]

                for fixed_entity_val in entity_vals:
                    for filter_val in filter_val_list:
                        fixed_entity_type = self._PROPERTY_TYPE_TAGS.get(
                            fixed_entity_val.__class__)
                        filter_type = self._PROPERTY_TYPE_TAGS.get(
                            filter_val.__class__)
                        if fixed_entity_type == filter_type:
                            comp = u'%r %s %r' % (fixed_entity_val, op,
                                                  filter_val)
                        elif op != '==':
                            comp = '%r %s %r' % (fixed_entity_type, op,
                                                 filter_type)
                        else:
                            continue

                        logging.log(logging.DEBUG - 1,
                                    'Evaling filter expression "%s"', comp)

                        try:
                            ret = eval(comp)
                            if ret and ret != NotImplementedError:
                                return True
                        except TypeError:
                            pass

                return False

            results = filter(passes_filter, results)

        for order in orders:
            prop = order.property().decode('utf-8')
            results = [
                entity for entity in results if has_prop_indexed(entity, prop)
            ]

        def order_compare_entities(a, b):
            """ Return a negative, zero or positive number depending on whether
      entity a is considered smaller than, equal to, or larger than b,
      according to the query's orderings. """
            cmped = 0
            for o in orders:
                prop = o.property().decode('utf-8')

                reverse = (o.direction() is
                           datastore_pb.Query_Order.DESCENDING)

                a_val = datastore._GetPropertyValue(a, prop)
                if isinstance(a_val, list):
                    a_val = sorted(a_val,
                                   order_compare_properties,
                                   reverse=reverse)[0]

                b_val = datastore._GetPropertyValue(b, prop)
                if isinstance(b_val, list):
                    b_val = sorted(b_val,
                                   order_compare_properties,
                                   reverse=reverse)[0]

                cmped = order_compare_properties(a_val, b_val)

                if o.direction() is datastore_pb.Query_Order.DESCENDING:
                    cmped = -cmped

                if cmped != 0:
                    return cmped

            if cmped == 0:
                return cmp(a.key(), b.key())

        def order_compare_entities_pb(a, b):
            """ Return a negative, zero or positive number depending on whether
      entity a is considered smaller than, equal to, or larger than b,
      according to the query's orderings. a and b are protobuf-encoded
      entities."""
            return order_compare_entities(datastore.Entity.FromPb(a),
                                          datastore.Entity.FromPb(b))

        def order_compare_properties(x, y):
            """Return a negative, zero or positive number depending on whether
      property value x is considered smaller than, equal to, or larger than
      property value y. If x and y are different types, they're compared based
      on the type ordering used in the real datastore, which is based on the
      tag numbers in the PropertyValue PB.
      """
            if isinstance(x, datetime.datetime):
                x = datastore_types.DatetimeToTimestamp(x)
            if isinstance(y, datetime.datetime):
                y = datastore_types.DatetimeToTimestamp(y)

            x_type = self._PROPERTY_TYPE_TAGS.get(x.__class__)
            y_type = self._PROPERTY_TYPE_TAGS.get(y.__class__)

            if x_type == y_type:
                try:
                    return cmp(x, y)
                except TypeError:
                    return 0
            else:
                return cmp(x_type, y_type)

        results.sort(order_compare_entities)

        clone = datastore_pb.Query()
        clone.CopyFrom(query)
        clone.clear_hint()
        clone.clear_limit()
        clone.clear_offset()
        if clone in self.__query_history:
            self.__query_history[clone] += 1
        else:
            self.__query_history[clone] = 1

        results = [r._ToPb() for r in results]
        for result in results:
            datastore_stub_util.PrepareSpecialPropertiesForLoad(result)
        cursor = datastore_stub_util.ListCursor(query, results,
                                                order_compare_entities_pb)

        if query.has_count():
            count = query.count()
        elif query.has_limit():
            count = query.limit()
        else:
            count = _BATCH_SIZE

        cursor.PopulateQueryResult(query_result,
                                   count,
                                   query.offset(),
                                   compile=query.compile())

        if query.compile():
            compiled_query = query_result.mutable_compiled_query()
            compiled_query.set_keys_only(query.keys_only())
            compiled_query.mutable_primaryscan().set_index_name(query.Encode())

        if query_result.has_cursor():
            self.__queries[query_result.cursor().cursor()] = cursor
Ejemplo n.º 8
0
    def _Dynamic_RunQuery(self, query, query_result):
        if query.has_transaction():
            if not query.has_ancestor():
                raise apiproxy_errors.ApplicationError(
                    datastore_pb.Error.BAD_REQUEST,
                    'Only ancestor queries are allowed inside transactions.')
        (filters, orders) = datastore_index.Normalize(query.filter_list(),
                                                      query.order_list())

        datastore_stub_util.FillUsersInQuery(filters)

        query_response = datastore_pb.QueryResult()
        query.set_app(self.__app_id)
        self._RemoteSend(query, query_response, "RunQuery")
        results = query_response.result_list()
        results = [datastore.Entity._FromPb(r) for r in results]

        if query.has_ancestor():
            ancestor_path = query.ancestor().path().element_list()

            def is_descendant(entity):
                path = entity.key()._Key__reference.path().element_list()
                return path[:len(ancestor_path)] == ancestor_path

            results = filter(is_descendant, results)

        operators = {
            datastore_pb.Query_Filter.LESS_THAN: '<',
            datastore_pb.Query_Filter.LESS_THAN_OR_EQUAL: '<=',
            datastore_pb.Query_Filter.GREATER_THAN: '>',
            datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL: '>=',
            datastore_pb.Query_Filter.EQUAL: '==',
        }

        def has_prop_indexed(entity, prop):
            """Returns True if prop is in the entity and is indexed."""
            if prop in datastore_types._SPECIAL_PROPERTIES:
                return True
            elif prop in entity.unindexed_properties():
                return False

            values = entity.get(prop, [])
            if not isinstance(values, (tuple, list)):
                values = [values]

            for value in values:
                if type(value) not in datastore_types._RAW_PROPERTY_TYPES:
                    return True
            return False

        for filt in filters:
            assert filt.op() != datastore_pb.Query_Filter.IN

            prop = filt.property(0).name().decode('utf-8')
            op = operators[filt.op()]

            filter_val_list = [
                datastore_types.FromPropertyPb(filter_prop)
                for filter_prop in filt.property_list()
            ]

        for order in orders:
            prop = order.property().decode('utf-8')
            results = [
                entity for entity in results if has_prop_indexed(entity, prop)
            ]

        def order_compare_entities(a, b):
            """ Return a negative, zero or positive number depending on whether
      entity a is considered smaller than, equal to, or larger than b,
      according to the query's orderings. """
            cmped = 0
            for o in orders:
                prop = o.property().decode('utf-8')

                reverse = (o.direction() is
                           datastore_pb.Query_Order.DESCENDING)

                a_val = datastore._GetPropertyValue(a, prop)
                if isinstance(a_val, list):
                    a_val = sorted(a_val,
                                   order_compare_properties,
                                   reverse=reverse)[0]

                b_val = datastore._GetPropertyValue(b, prop)
                if isinstance(b_val, list):
                    b_val = sorted(b_val,
                                   order_compare_properties,
                                   reverse=reverse)[0]

                cmped = order_compare_properties(a_val, b_val)

                if o.direction() is datastore_pb.Query_Order.DESCENDING:
                    cmped = -cmped

                if cmped != 0:
                    return cmped

            if cmped == 0:
                return cmp(a.key(), b.key())

        def order_compare_entities_pb(a, b):
            """ Return a negative, zero or positive number depending on whether
      entity a is considered smaller than, equal to, or larger than b,
      according to the query's orderings. a and b are protobuf-encoded
      entities."""
            return order_compare_entities(datastore.Entity.FromPb(a),
                                          datastore.Entity.FromPb(b))

        def order_compare_properties(x, y):
            """Return a negative, zero or positive number depending on whether
      property value x is considered smaller than, equal to, or larger than
      property value y. If x and y are different types, they're compared based
      on the type ordering used in the real datastore, which is based on the
      tag numbers in the PropertyValue PB.
      """
            if isinstance(x, datetime.datetime):
                x = datastore_types.DatetimeToTimestamp(x)
            if isinstance(y, datetime.datetime):
                y = datastore_types.DatetimeToTimestamp(y)

            x_type = self._PROPERTY_TYPE_TAGS.get(x.__class__)
            y_type = self._PROPERTY_TYPE_TAGS.get(y.__class__)

            if x_type == y_type:
                try:
                    return cmp(x, y)
                except TypeError:
                    return 0
            else:
                return cmp(x_type, y_type)

        results.sort(order_compare_entities)

        clone = datastore_pb.Query()
        clone.CopyFrom(query)
        clone.clear_hint()
        clone.clear_limit()
        clone.clear_offset()
        #if clone in self.__query_history:
        #  self.__query_history[clone] += 1
        #else:
        #  self.__query_history[clone] = 1

        results = [r._ToPb() for r in results]
        for result in results:
            datastore_stub_util.PrepareSpecialPropertiesForLoad(result)

        datastore_stub_util.ValidateQuery(query, filters, orders,
                                          _MAX_QUERY_COMPONENTS)

        cursor = datastore_stub_util.ListCursor(query, results,
                                                order_compare_entities_pb)
        self.__queries[cursor.cursor] = cursor

        if query.has_count():
            count = query.count()
        elif query.has_limit():
            count = query.limit()
        else:
            count = _BATCH_SIZE

        cursor.PopulateQueryResult(query_result,
                                   count,
                                   query.offset(),
                                   compile=query.compile())

        if query.compile():
            compiled_query = query_result.mutable_compiled_query()
            compiled_query.set_keys_only(query.keys_only())
            compiled_query.mutable_primaryscan().set_index_name(query.Encode())