def __init__(self, connection, query, keys_only=False):
self.where = None
self.original_query = query
self.connection = connection
self.limits = (query.low_mark, query.high_mark)
self.results_returned = 0
opts = query.get_meta()
self.distinct = query.distinct
self.distinct_values = set()
self.distinct_on_field = None
self.distinct_field_convertor = None
self.queried_fields = []
self.model = query.model
self.pk_col = opts.pk.column
self.is_count = query.aggregates
self.extra_select = query.extra_select
self._set_db_table()
try:
self._validate_query_is_possible(query)
self.ordering = _convert_ordering(query)
except NotSupportedError as e:
# If we can detect here, or when parsing the WHERE tree that a query is unsupported
# we set this flag, and then throw NotSupportedError when execute is called.
# This will then wrap the exception in Django's NotSupportedError meaning users
# only need to catch that one, and not both Django's and ours
self.unsupported_query_message = str(e)
return
else:
self.unsupported_query_message = ""
# If the query uses defer()/only() then we need to process deferred. We have to get all deferred columns
# for all (concrete) inherited models and then only include columns if they appear in that list
deferred_columns = {}
query.deferred_to_data(deferred_columns, query.deferred_to_columns_cb)
inherited_db_tables = [x._meta.db_table for x in get_concrete_parents(self.model)]
only_load = list(chain(*[list(deferred_columns.get(x, [])) for x in inherited_db_tables]))
if query.select:
for x in query.select:
if hasattr(x, "field"):
# In Django 1.6+ 'x' above is a SelectInfo (which is a tuple subclass), whereas in 1.5 it's a tuple
# in 1.6 x[1] == Field, but 1.5 x[1] == unicode (column name)
if x.field is None:
column = x.col.col[1] # This is the column we are getting
lookup_type = x.col.lookup_type
self.distinct_on_field = column
# This whole section of code is weird, and is probably better implemented as a custom Query type (like QueryByKeys)
# basically, appengine gives back dates as a time since the epoch, we convert it to a date, then floor it, then convert it back
# in our transform function. The transform is applied when the results are read back so that only distinct values are returned.
# this is very hacky...
if lookup_type in DATE_TRANSFORMS:
self.distinct_field_convertor = lambda value: DATE_TRANSFORMS[lookup_type](self.connection, value)
else:
raise CouldBeSupportedError("Unhandled lookup_type %s" % lookup_type)
else:
column = x.field.column
else:
column = x[1]
if only_load and column not in only_load:
continue
self.queried_fields.append(column)
else:
# If no specific fields were specified, select all fields if the query is distinct (as App Engine only supports
# distinct on projection queries) or the ones specified by only_load
self.queried_fields = [x.column for x in opts.fields if (x.column in only_load) or self.distinct]
self.keys_only = keys_only or self.queried_fields == [opts.pk.column]
# Projection queries don't return results unless all projected fields are
# indexed on the model. This means if you add a field, and all fields on the model
# are projectable, you will never get any results until you've resaved all of them.
# Because it's not possible to detect this situation, we only try a projection query if a
# subset of fields was specified (e.g. values_list('bananas')) which makes the behaviour a
# bit more predictable. It would be nice at some point to add some kind of force_projection()
# thing on a queryset that would do this whenever possible, but that's for the future, maybe.
try_projection = (self.keys_only is False) and bool(self.queried_fields)
if not self.queried_fields:
# If we don't have any queried fields yet, it must have been an empty select and not a distinct
# and not an only/defer, so get all the fields
self.queried_fields = [ x.column for x in opts.fields ]
self.excluded_pks = set()
self.has_inequality_filter = False
self.all_filters = []
self.results = None
self.gae_query = None
projection_fields = []
if try_projection:
for field in self.queried_fields:
# We don't include the primary key in projection queries...
if field == self.pk_col:
order_fields = set([ x.strip("-") for x in self.ordering])
if self.pk_col in order_fields or "pk" in order_fields:
# If we were ordering on __key__ we can't do a projection at all
self.projection_fields = []
break
continue
# Text and byte fields aren't indexed, so we can't do a
# projection query
f = get_field_from_column(self.model, field)
if not f:
raise CouldBeSupportedError("Attempting a cross-table select or dates query, or something?!")
assert f # If this happens, we have a cross-table select going on! #FIXME
db_type = f.db_type(connection)
if db_type in ("bytes", "text", "list", "set"):
projection_fields = []
break
projection_fields.append(field)
self.projection = list(set(projection_fields)) or None
if opts.parents:
self.projection = None
if isinstance(query.where, EmptyWhere):
# Empty where means return nothing!
raise EmptyResultSet()
else:
try:
where_tables = _get_tables_from_where(query.where)
except TypeError:
# This exception is thrown by get_group_by_cols if one of the constraints is a SubQueryConstraint
# yeah, we can't do that.
self.unsupported_query_message = "Subquery WHERE constraints aren't supported"
return
if where_tables and where_tables != [ query.model._meta.db_table ]:
# Mark this query as unsupported and return
self.unsupported_query_message = "Cross-join WHERE constraints aren't supported: %s" % _cols_from_where_node(query.where)
return
from dnf import parse_dnf
try:
self.where, columns, self.excluded_pks = parse_dnf(query.where, self.connection, ordering=self.ordering)
except NotSupportedError as e:
# Mark this query as unsupported and return
self.unsupported_query_message = str(e)
return
# DISABLE PROJECTION IF WE ARE FILTERING ON ONE OF THE PROJECTION_FIELDS
for field in self.projection or []:
if field in columns:
self.projection = None
break
try:
# If the PK was queried, we switch it in our queried
# fields store with __key__
pk_index = self.queried_fields.index(self.pk_col)
self.queried_fields[pk_index] = "__key__"
except ValueError:
pass
def _convert_ordering(query):
if not query.default_ordering:
result = query.order_by
else:
result = query.order_by or query.get_meta().ordering
if query.extra_order_by:
# This is a best attempt at ordering by extra select, it covers the cases
# in the Django tests, but use this functionality with care
all_fields = query.get_meta().get_all_field_names()
new_ordering = []
for col in query.extra_order_by:
# If the query in the extra order by is part of the extra select
# and the extra select is just an alias, then use the original column
if col in query.extra_select:
if query.extra_select[col][0] in all_fields:
new_ordering.append(query.extra_select[col][0])
else:
# It wasn't an alias, probably can't support it
raise NotSupportedError("Unsupported extra_order_by: {}".format(query.extra_order_by))
else:
# Not in the extra select, probably just a column so use it if it is
if col in all_fields:
new_ordering.append(col)
else:
raise NotSupportedError("Unsupported extra_order_by: {}".format(query.extra_order_by))
result = tuple(new_ordering)
if result:
# We factor out cross-table orderings (rather than raising NotSupportedError) otherwise we'll break
# the admin which uses them. We log a warning when this happens though
try:
ordering = []
for name in result:
if name == "?":
raise NotSupportedError("Random ordering is not supported on the datastore")
if not (isinstance(name, basestring) and "__" in name):
if isinstance(name, basestring):
if name.lstrip("-") == "pk":
field_column = query.model._meta.pk.column
else:
field = query.model._meta.get_field_by_name(name.lstrip("-"))[0]
field_column = field.column
ordering.append(field_column if not name.startswith("-") else "-{}".format(field_column))
else:
ordering.append(name)
except FieldDoesNotExist:
opts = query.model._meta
available = opts.get_all_field_names()
raise FieldError("Cannot resolve keyword %r into field. "
"Choices are: %s" % (name, ", ".join(available))
)
if len(ordering) < len(result):
diff = set(result) - set(ordering)
log_once(
DJANGAE_LOG.warning if not on_production() else DJANGAE_LOG.debug,
"The following orderings were ignored as cross-table orderings are not supported on the datastore: %s", diff
)
result = ordering
return result
