def remove_empty_in(node):
"""
Once we are normalized, if any of the branches filters
on an empty list, we can remove that entire branch from the
query. If this leaves no branches, then the result set is empty
"""
# This is a bit ugly, but you try and do it more succinctly :)
# We have the following possible situations for IN queries with an empty
# value:
# - Negated: One of the nodes in the and branch will always be true and is therefore
# unnecessary, we leave it alone though
# - Not negated: The entire AND branch will always be false, so that branch can be removed
# if that was the last branch, then the queryset will be empty
# Everything got wiped out!
if node.connector == 'OR' and len(node.children) == 0:
raise EmptyResultSet()
for and_branch in node.children[:]:
if and_branch.is_leaf and and_branch.operator == "IN" and not len(and_branch.value):
node.children.remove(and_branch)
if not node.children:
raise EmptyResultSet()
def process_and_branch(query, and_branch):
for child in and_branch[-1]:
column, op, value = child[1]
# for column, op, value in and_branch[-1]:
if column == self.pk_col:
column = "__key__"
#FIXME: This EmptyResultSet check should happen during normalization so that Django doesn't count it as a query
if op == "=" and "__key__ =" in query and query["__key__ ="] != value:
# We've already done an exact lookup on a key, this query can't return anything!
raise EmptyResultSet()
if not isinstance(value, datastore.Key):
value = get_datastore_key(self.model, value)
key = "%s %s" % (column, op)
try:
if isinstance(value, basestring):
value = coerce_unicode(value)
if key in query:
if type(query[key]) == list:
if value not in query[key]:
query[key].append(value)
else:
if query[key] != value:
query[key] = [ query[key], value ]
else:
query[key] = value
except datastore_errors.BadFilterError as e:
raise NotSupportedError(str(e))
def set_leaf(self, column, operator, value, is_pk_field, negated, target_field=None):
assert column
assert operator
assert isinstance(is_pk_field, bool)
assert isinstance(negated, bool)
if operator == "iexact" and isinstance(target_field, AutoField):
# When new instance is created, automatic primary key 'id' does not generate '_idx_iexact_id'.
# As the primary key 'id' (AutoField) is integer and is always case insensitive,
# we can deal with 'id_iexact=' query by using 'exact' rather than 'iexact'.
operator = "exact"
value = int(value)
if is_pk_field:
# If this is a primary key, we need to make sure that the value
# we pass to the query is a datastore Key. We have to deal with IN queries here
# because they aren't flattened until the DNF stage
model = get_top_concrete_parent(target_field.model)
table = model._meta.db_table
if isinstance(value, (list, tuple)):
value = [
datastore.Key.from_path(table, x)
for x in value if x
]
else:
if operator == "isnull" and value is True:
# FIXME: Strictly, this isn't correct, this could be one of several branches
# but id=None filters are silly anyway. This should be moved to after normalization..
# probably. This fixes a test in Django which does this in get_or_create for some reason
raise EmptyResultSet()
if not value:
# Empty strings and 0 are forbidden as keys
# so make this an impossible filter
# FIXME: This is a hack! It screws with the ordering
# because it's an inequality. Instead we should wipe this
# filter out when preprocessing in the DNF (because it's impossible)
value = datastore.Key.from_path('', 1)
operator = '<'
else:
value = datastore.Key.from_path(table, value)
column = "__key__"
# Do any special index conversions necessary to perform this lookup
if operator in REQUIRES_SPECIAL_INDEXES:
add_special_index(target_field.model, column, operator, value)
indexer = REQUIRES_SPECIAL_INDEXES[operator]
index_type = indexer.prepare_index_type(operator, value)
value = indexer.prep_value_for_query(value)
if not indexer.validate_can_be_indexed(value, negated):
raise NotSupportedError("Unsupported special index or value '%s %s'" % (column, operator))
column = indexer.indexed_column_name(column, value, index_type)
operator = indexer.prep_query_operator(operator)
self.column = column
self.operator = convert_operator(operator)
self.value = value
def _transform_query_18(connection, kind, query):
from django.db.models.sql.where import EmptyWhere
if isinstance(query.where, EmptyWhere):
# Empty where means return nothing!
raise EmptyResultSet()
ret = Query(query.model, kind)
ret.connection = connection
# Add the root concrete table as the source table
root_table = get_top_concrete_parent(query.model)._meta.db_table
ret.add_source_table(root_table)
# Extract the ordering of the query results
for order_col in _extract_ordering_from_query_18(query):
ret.add_order_by(order_col)
# Extract any projected columns (values/values_list/only/defer)
for projected_col in _extract_projected_columns_from_query_18(query):
ret.add_projected_column(projected_col)
# Add any extra selects
for col, select in query.extra_select.items():
ret.add_extra_select(col, select[0])
if query.distinct:
# This must happen after extracting projected cols
ret.set_distinct(list(query.distinct_fields))
# Process annotations!
if query.annotation_select:
for k, v in query.annotation_select.items():
ret.add_annotation(k, v)
# Extract any query offsets and limits
ret.low_mark = query.low_mark
ret.high_mark = query.high_mark
output = WhereNode()
output.connector = query.where.connector
_walk_django_where(
query,
_django_18_query_walk_trunk,
_django_18_query_walk_leaf,
new_parent=output,
connection=connection,
negated=query.where.negated,
model=query.model
)
# If there no child nodes, just wipe out the where
if not output.children:
output = None
ret.where = output
return ret
def _prepare_for_transformation(self):
from django.db.models.sql.where import NothingNode, WhereNode as DjangoWhereNode
query = self.django_query
# It could either be a NothingNode, or a WhereNode(AND NothingNode)
if (isinstance(query.where, NothingNode)
or (isinstance(query.where, DjangoWhereNode)
and len(query.where.children) == 1
and isinstance(query.where.children[0], NothingNode))):
# Empty where means return nothing!
raise EmptyResultSet()
def parse_dnf(node, connection, ordering=None):
should_in_memory_exclude = should_exclude_pks_in_memory(node, ordering)
tree, filtered_columns, excluded_pks = parse_tree(
node,
connection,
excluded_pks=set() if should_in_memory_exclude else None)
if not should_exclude_pks_in_memory:
assert excluded_pks is None
if tree:
tree = tripled(tree)
if tree and tree[0] != 'OR':
tree = ('OR', [tree])
# Filter out impossible branches of the where, if that then results in an empty tree then
# raise an EmptyResultSet, otherwise replace the tree with the now simpler query
if tree:
final = []
for and_branch in tree[-1]:
if and_branch[0] == 'LIT' and and_branch[-1] == IMPOSSIBLE_FILTER:
continue
elif and_branch[0] == 'AND' and IMPOSSIBLE_FILTER in [
x[-1] for x in and_branch[-1]
]:
continue
final.append(and_branch)
if not final:
raise EmptyResultSet()
else:
tree = (tree[0], final)
# If there are more than 30 filters, and not all filters are PK filters
if tree and len(tree[-1]) > 30:
for and_branch in tree[-1]:
if and_branch[0] == 'LIT':
and_branch = [and_branch]
for lit in and_branch[-1] if and_branch[
0] == 'AND' else and_branch: # Go through each literal tuple
if isinstance(lit[-1], datastore.Key) or isinstance(
lit[-1][-1], datastore.Key
): # If the value is a key, then break the loop
break
else:
# If we didn't find a literal with a datastore Key, then raise unsupported
raise NotSupportedError(
"The datastore doesn't support this query, more than 30 filters were needed"
)
return tree, filtered_columns, excluded_pks or set()
def check_query(self):
"""
Checks if the current query is supported by the database.
In general, we expect queries requiring JOINs (many-to-many
relations, abstract model bases, or model spanning filtering),
using DISTINCT (through `QuerySet.distinct()`, which is not
required in most situations) or using the SQL-specific
`QuerySet.extra()` to not work with nonrel back-ends.
"""
if hasattr(self.query, 'is_empty') and self.query.is_empty():
raise EmptyResultSet()
if (len([a for a in self.query.alias_map if self.query.alias_refcount[a]]) > 1
or self.query.distinct or self.query.extra or self.query.having):
raise DatabaseError("This query is not supported by the database.")
def detect_conflicting_key_filter(node):
assert node.connector == "OR"
for and_branch in node.children[:]:
# If we have a Root OR with leaf elements, we don't need to worry
if and_branch.is_leaf:
break
pk_equality_found = None
for child in and_branch.children:
if child.column == "__key__" and child.operator == "=":
if pk_equality_found and pk_equality_found != child.value:
# Remove this AND branch as it's impossible to return anything
node.children.remove(and_branch)
else:
pk_equality_found = child.value
if not node.children:
raise EmptyResultSet()
def process_and_branch(query, and_branch):
for column, op, value in and_branch[-1]:
if column == self.pk_col:
column = "__key__"
#FIXME: This EmptyResultSet check should happen during normalization so that Django doesn't count it as a query
if op == "=" and "__key__ =" in query:
#We've already done an exact lookup on a key, this query can't return anything!
raise EmptyResultSet()
if not isinstance(value, datastore.Key):
value = get_datastore_key(self.model, value)
key = "%s %s" % (column, op)
if key in query:
query[key] = [ query[key], value ]
else:
query[key] = value
def check_query(self):
"""
Checks if the current query is supported by the database.
In general, we expect queries requiring JOINs (many-to-many
relations, abstract model bases, or model spanning filtering),
using DISTINCT (through `QuerySet.distinct()`, which is not
required in most situations) or using the SQL-specific
`QuerySet.extra()` to not work with nonrel back-ends.
"""
if hasattr(self.query, 'is_empty') and self.query.is_empty():
raise EmptyResultSet()
if (len([
a for a in self.query.alias_map if self.query.alias_refcount[a]
]) > 1 or self.query.distinct
or self.query.extra): # or self.having -- Not quite working.
# having is no longer part of the query as of 1.9; It moved to the compiler
# https://github.com/django/django/commit/afe0bb7b13bb8dc4370f32225238012c873b0ee3
raise DatabaseError("This query is not supported by the database.")
def _prepare_for_transformation(self):
from django.db.models.sql.where import NothingNode
query = self.django_query
def where_will_always_be_empty(where):
if isinstance(where, NothingNode):
return True
if where.connector == 'AND' and any(isinstance(x, NothingNode) for x in where.children):
return True
if where.connector == 'OR' and len(where.children) == 1 and isinstance(where.children[0], NothingNode):
return True
return False
# It could either be a NothingNode, or a WhereNode(AND NothingNode)
if where_will_always_be_empty(query.where):
# Empty where means return nothing!
raise EmptyResultSet()
def remove_unnecessary_nodes(top_node):
"""
Sometimes you end up with a branch that has two nodes that
have the same column and operator, but different values. When
this happens we need to simplify, e.g.:
AND:[username<A],[username<B] -> AND:[username<A]
"""
for and_branch in top_node.children[:]:
seen = {}
altered = False
for node in and_branch.children:
key = (node.column, node.operator)
if key in seen:
altered = True
if node.operator in ('<', '<='):
seen[key].value = min(seen[key].value, node.value)
elif node.operator in ('>', '>='):
seen[key].value = max(seen[key].value, node.value)
elif node.operator == "=":
# Impossible filter! remove the AND branch entirely
if and_branch in top_node.children and seen[
key].value != node.value:
top_node.children.remove(and_branch)
break
else:
pass
else:
seen[key] = node
if altered:
and_branch.children = [x for x in seen.values()]
# If all the OR clause are impossible filters we end up with no filters
# at all, which is incorrect
if not top_node.children:
raise EmptyResultSet()
def _remove_impossible_branches(self):
"""
If we mark a child node as never returning results we either need to
remove those nodes, or remove the branches of the tree they are on depending
on the connector of the parent node.
"""
if not self._where:
return
def walk(node, negated):
if node.negated:
negated = not negated
for child in node.children[:]:
walk(child, negated)
if child.will_never_return_results:
if node.connector == 'AND':
if child.negated:
node.children.remove(child)
else:
node.will_never_return_results = True
else:
#OR
if not child.negated:
node.children.remove(child)
if not node.children:
node.will_never_return_results = True
else:
node.children[:] = []
walk(self._where, False)
if self._where.will_never_return_results:
# We removed all the children of the root where node, so no results
raise EmptyResultSet()
def make_atom(self, child, qn, connection):
lvalue, lookup_type, value_annot, param = child
kwargs = {'connection': connection}
if lvalue and lvalue.field and hasattr(
lvalue.field,
'db_type') and lvalue.field.db_type(**kwargs) == 'hstore':
try:
lvalue, params = lvalue.process(lookup_type, param, connection)
except EmptyShortCircuit:
raise EmptyResultSet()
field = self.sql_for_columns(lvalue, qn, connection)
if lookup_type == 'exact':
if isinstance(param, dict):
return ('{0} = %s'.format(field), [param])
raise ValueError('invalid value')
elif lookup_type in ('gt', 'gte', 'lt', 'lte'):
if isinstance(param, dict):
sign = (lookup_type[0] == 'g' and '>%s'
or '<%s') % (lookup_type[-1] == 'e' and '=' or '')
param_keys = list(param.keys())
conditions = []
for key in param_keys:
cast = get_cast_for_param(value_annot, key)
conditions.append('(%s->\'%s\')%s %s %%s' %
(field, key, cast, sign))
return (" AND ".join(conditions), param.values())
raise ValueError('invalid value')
elif lookup_type in ['contains', 'icontains']:
if isinstance(param, dict):
values = list(param.values())
keys = list(param.keys())
if len(values) == 1 and isinstance(values[0],
(list, tuple)):
# Can't cast here because the list could contain multiple types
return ('%s->\'%s\' = ANY(%%s)' % (field, keys[0]),
[[str(x) for x in values[0]]])
elif len(keys) == 1 and len(values) == 1:
# Retrieve key and compare to param instead of using '@>' in order to cast hstore value
cast = get_cast_for_param(value_annot, keys[0])
return ('(%s->\'%s\')%s = %%s' %
(field, keys[0], cast), [values[0]])
return ('%s @> %%s' % field, [param])
elif isinstance(param, (list, tuple)):
if len(param) == 0:
raise ValueError('invalid value')
if len(param) < 2:
return ('%s ? %%s' % field, [param[0]])
if param:
return ('%s ?& %%s' % field, [param])
raise ValueError('invalid value')
elif isinstance(param, six.string_types):
# if looking for a string perform the normal text lookup
# that is: look for occurence of string in all the keys
pass
elif hasattr(child[0].field, 'serializer'):
try:
child[0].field._serialize_value(param)
pass
except Exception:
raise ValueError('invalid value')
else:
raise ValueError('invalid value')
elif lookup_type == 'isnull':
if isinstance(param, dict):
param_keys = list(param.keys())
conditions = []
for key in param_keys:
op = 'IS NULL' if value_annot[key] else 'IS NOT NULL'
conditions.append('(%s->\'%s\') %s' % (field, key, op))
return (" AND ".join(conditions), [])
# do not perform any special format
return super(HStoreWhereNode,
self).make_atom(child, qn, connection)
else:
raise TypeError('invalid lookup type')
return super(HStoreWhereNode, self).make_atom(child, qn, connection)
def __init__(self, connection, query, keys_only=False):
self.original_query = query
self.connection = connection
self.limits = (query.low_mark, query.high_mark)
opts = query.get_meta()
self.distinct = query.distinct
self.distinct_values = set()
self.distinct_on_field = None
self.distinct_field_convertor = None
self.queried_fields = []
self.model = query.model
self.pk_col = opts.pk.column
self.is_count = query.aggregates
self.extra_select = query.extra_select
self._set_db_table()
self._validate_query_is_possible(query)
if not query.default_ordering:
self.ordering = query.order_by
else:
self.ordering = query.order_by or opts.ordering
if self.ordering:
ordering = [ x for x in self.ordering if not (isinstance(x, basestring) and "__" in x) ]
if len(ordering) < len(self.ordering):
if not on_production() and not in_testing():
diff = set(self.ordering) - set(ordering)
log_once(DJANGAE_LOG.warning, "The following orderings were ignored as cross-table orderings are not supported on the datastore: %s", diff)
self.ordering = ordering
#If the query uses defer()/only() then we need to process deferred. We have to get all deferred columns
# for all (concrete) inherited models and then only include columns if they appear in that list
deferred_columns = {}
query.deferred_to_data(deferred_columns, query.deferred_to_columns_cb)
inherited_db_tables = [ x._meta.db_table for x in get_concrete_parents(self.model) ]
only_load = list(chain(*[ list(deferred_columns.get(x, [])) for x in inherited_db_tables ]))
if query.select:
for x in query.select:
if hasattr(x, "field"):
#In Django 1.6+ 'x' above is a SelectInfo (which is a tuple subclass), whereas in 1.5 it's a tuple
# in 1.6 x[1] == Field, but 1.5 x[1] == unicode (column name)
if x.field is None:
column = x.col.col[1] #This is the column we are getting
lookup_type = x.col.lookup_type
self.distinct_on_field = column
#This whole section of code is weird, and is probably better implemented as a custom Query type (like QueryByKeys)
# basically, appengine gives back dates as a time since the epoch, we convert it to a date, then floor it, then convert it back
# in our transform function. The transform is applied when the results are read back so that only distinct values are returned.
# this is very hacky...
if lookup_type in DATE_TRANSFORMS:
self.distinct_field_convertor = lambda value: DATE_TRANSFORMS[lookup_type](self.connection, value)
else:
raise CouldBeSupportedError("Unhandled lookup_type %s" % lookup_type)
else:
column = x.field.column
else:
column = x[1]
if only_load and column not in only_load:
continue
self.queried_fields.append(column)
else:
self.queried_fields = [ x.column for x in opts.fields if (not only_load) or (x.column in only_load) ]
self.keys_only = keys_only or self.queried_fields == [ opts.pk.column ]
assert self.queried_fields
#Projection queries don't return results unless all projected fields are
#indexed on the model. This means if you add a field, and all fields on the model
#are projectable, you will never get any results until you've resaved all of them.
#Because it's not possible to detect this situation, we only try a projection query if a
#subset of fields was specified (e.g. values_list('bananas')) which makes the behaviour a
#bit more predictable. It would be nice at some point to add some kind of force_projection()
#thing on a queryset that would do this whenever possible, but that's for the future, maybe.
try_projection = (self.keys_only is False) and bool(self.queried_fields)
if not self.queried_fields:
self.queried_fields = [ x.column for x in opts.fields ]
self.excluded_pks = set()
self.has_inequality_filter = False
self.all_filters = []
self.results = None
self.gae_query = None
projection_fields = []
if try_projection:
for field in self.queried_fields:
#We don't include the primary key in projection queries...
if field == self.pk_col:
continue
#Text and byte fields aren't indexed, so we can't do a
#projection query
f = get_field_from_column(self.model, field)
if not f:
raise CouldBeSupportedError("Attemping a cross-table select or dates query, or something?!")
assert f #If this happens, we have a cross-table select going on! #FIXME
db_type = f.db_type(connection)
if db_type in ("bytes", "text"):
projection_fields = []
break
projection_fields.append(field)
self.projection = list(set(projection_fields)) or None
if opts.parents:
self.projection = None
columns = set()
if isinstance(query.where, EmptyWhere):
#Empty where means return nothing!
raise EmptyResultSet()
else:
self.where = normalize_query(query.where, self.connection, filtered_columns=columns)
#DISABLE PROJECTION IF WE ARE FILTERING ON ONE OF THE PROJECTION_FIELDS
for field in self.projection or []:
if field in columns:
self.projection = None
break
try:
#If the PK was queried, we switch it in our queried
#fields store with __key__
pk_index = self.queried_fields.index(self.pk_col)
self.queried_fields[pk_index] = "__key__"
except ValueError:
pass
def __init__(self, connection, query, keys_only=False):
self.where = None
self.original_query = query
self.connection = connection
self.limits = (query.low_mark, query.high_mark)
self.results_returned = 0
opts = query.get_meta()
self.distinct = query.distinct
self.distinct_values = set()
self.distinct_on_field = None
self.distinct_field_convertor = None
self.queried_fields = []
self.model = query.model
self.pk_col = opts.pk.column
self.is_count = query.aggregates
self.extra_select = query.extra_select
self._set_db_table()
try:
self._validate_query_is_possible(query)
self.ordering = _convert_ordering(query)
except NotSupportedError as e:
# If we can detect here, or when parsing the WHERE tree that a query is unsupported
# we set this flag, and then throw NotSupportedError when execute is called.
# This will then wrap the exception in Django's NotSupportedError meaning users
# only need to catch that one, and not both Django's and ours
self.unsupported_query_message = str(e)
return
else:
self.unsupported_query_message = ""
# If the query uses defer()/only() then we need to process deferred. We have to get all deferred columns
# for all (concrete) inherited models and then only include columns if they appear in that list
deferred_columns = {}
query.deferred_to_data(deferred_columns, query.deferred_to_columns_cb)
inherited_db_tables = [x._meta.db_table for x in get_concrete_parents(self.model)]
only_load = list(chain(*[list(deferred_columns.get(x, [])) for x in inherited_db_tables]))
if query.select:
for x in query.select:
if hasattr(x, "field"):
# In Django 1.6+ 'x' above is a SelectInfo (which is a tuple subclass), whereas in 1.5 it's a tuple
# in 1.6 x[1] == Field, but 1.5 x[1] == unicode (column name)
if x.field is None:
column = x.col.col[1] # This is the column we are getting
lookup_type = x.col.lookup_type
self.distinct_on_field = column
# This whole section of code is weird, and is probably better implemented as a custom Query type (like QueryByKeys)
# basically, appengine gives back dates as a time since the epoch, we convert it to a date, then floor it, then convert it back
# in our transform function. The transform is applied when the results are read back so that only distinct values are returned.
# this is very hacky...
if lookup_type in DATE_TRANSFORMS:
self.distinct_field_convertor = lambda value: DATE_TRANSFORMS[lookup_type](self.connection, value)
else:
raise CouldBeSupportedError("Unhandled lookup_type %s" % lookup_type)
else:
column = x.field.column
else:
column = x[1]
if only_load and column not in only_load:
continue
self.queried_fields.append(column)
else:
# If no specific fields were specified, select all fields if the query is distinct (as App Engine only supports
# distinct on projection queries) or the ones specified by only_load
self.queried_fields = [x.column for x in opts.fields if (x.column in only_load) or self.distinct]
self.keys_only = keys_only or self.queried_fields == [opts.pk.column]
# Projection queries don't return results unless all projected fields are
# indexed on the model. This means if you add a field, and all fields on the model
# are projectable, you will never get any results until you've resaved all of them.
# Because it's not possible to detect this situation, we only try a projection query if a
# subset of fields was specified (e.g. values_list('bananas')) which makes the behaviour a
# bit more predictable. It would be nice at some point to add some kind of force_projection()
# thing on a queryset that would do this whenever possible, but that's for the future, maybe.
try_projection = (self.keys_only is False) and bool(self.queried_fields)
if not self.queried_fields:
# If we don't have any queried fields yet, it must have been an empty select and not a distinct
# and not an only/defer, so get all the fields
self.queried_fields = [ x.column for x in opts.fields ]
self.excluded_pks = set()
self.has_inequality_filter = False
self.all_filters = []
self.results = None
self.gae_query = None
projection_fields = []
if try_projection:
for field in self.queried_fields:
# We don't include the primary key in projection queries...
if field == self.pk_col:
order_fields = set([ x.strip("-") for x in self.ordering])
if self.pk_col in order_fields or "pk" in order_fields:
# If we were ordering on __key__ we can't do a projection at all
self.projection_fields = []
break
continue
# Text and byte fields aren't indexed, so we can't do a
# projection query
f = get_field_from_column(self.model, field)
if not f:
raise CouldBeSupportedError("Attempting a cross-table select or dates query, or something?!")
assert f # If this happens, we have a cross-table select going on! #FIXME
db_type = f.db_type(connection)
if db_type in ("bytes", "text", "list", "set"):
projection_fields = []
break
projection_fields.append(field)
self.projection = list(set(projection_fields)) or None
if opts.parents:
self.projection = None
if isinstance(query.where, EmptyWhere):
# Empty where means return nothing!
raise EmptyResultSet()
else:
try:
where_tables = _get_tables_from_where(query.where)
except TypeError:
# This exception is thrown by get_group_by_cols if one of the constraints is a SubQueryConstraint
# yeah, we can't do that.
self.unsupported_query_message = "Subquery WHERE constraints aren't supported"
return
if where_tables and where_tables != [ query.model._meta.db_table ]:
# Mark this query as unsupported and return
self.unsupported_query_message = "Cross-join WHERE constraints aren't supported: %s" % _cols_from_where_node(query.where)
return
from dnf import parse_dnf
try:
self.where, columns, self.excluded_pks = parse_dnf(query.where, self.connection, ordering=self.ordering)
except NotSupportedError as e:
# Mark this query as unsupported and return
self.unsupported_query_message = str(e)
return
# DISABLE PROJECTION IF WE ARE FILTERING ON ONE OF THE PROJECTION_FIELDS
for field in self.projection or []:
if field in columns:
self.projection = None
break
try:
# If the PK was queried, we switch it in our queried
# fields store with __key__
pk_index = self.queried_fields.index(self.pk_col)
self.queried_fields[pk_index] = "__key__"
except ValueError:
pass
def make_atom(self, child, qn, connection):
lvalue, lookup_type, value_annot, param = child
kwargs = {'connection': connection} if VERSION[:2] >= (1, 3) else {}
if lvalue and lvalue.field and hasattr(
lvalue.field,
'db_type') and lvalue.field.db_type(**kwargs) == 'hstore':
try:
lvalue, params = lvalue.process(lookup_type, param, connection)
except EmptyShortCircuit:
raise EmptyResultSet()
field = self.sql_for_columns(lvalue, qn, connection)
if lookup_type == 'exact':
if isinstance(param, dict):
return ('{0} = %s'.format(field), [param])
raise ValueError('invalid value')
elif lookup_type in ('gt', 'gte', 'lt', 'lte'):
if isinstance(param, dict) and len(param) == 1:
sign = (lookup_type[0] == 'g' and '>%s'
or '<%s') % (lookup_type[-1] == 'e' and '=' or '')
param_keys = list(param.keys())
return ('%s->\'%s\' %s %%s' % (field, param_keys[0], sign),
param.values())
raise ValueError('invalid value')
elif lookup_type in ['contains', 'icontains']:
if isinstance(param, dict):
values = list(param.values())
keys = list(param.keys())
if len(values) == 1 and isinstance(values[0],
(list, tuple)):
return ('%s->\'%s\' = ANY(%%s)' % (field, keys[0]),
[[str(x) for x in values[0]]])
return ('%s @> %%s' % field, [param])
elif isinstance(param, (list, tuple)):
if len(param) < 2:
return ('%s ? %%s' % field, [param[0]])
if param:
return ('%s ?& %%s' % field, [param])
raise ValueError('invalid value')
elif isinstance(param, six.string_types):
# if looking for a string perform the normal text lookup
# that is: look for occurence of string in all the keys
pass
else:
raise ValueError('invalid value')
elif lookup_type == 'isnull':
# do not perform any special format
return super(HStoreWhereNode,
self).make_atom(child, qn, connection)
else:
raise TypeError('invalid lookup type')
return super(HStoreWhereNode, self).make_atom(child, qn, connection)
def _process_where_node(self, parent, query):
"""
Process a single WHERE node, possibly recursing.
:param parent: The parent 'WHERE' clause
:return:
"""
if isinstance(parent, EmptyWhere):
raise EmptyResultSet()
where = []
for child in parent.children:
if isinstance(child, WhereNode):
where.append(self._process_where_node(child, query))
continue
# Field as represented in the query
query_field = child.lhs.output_field
# Field of the DB table for the model
real_field = child.lhs.target
# Origin field (if this is a foreign field)
origin_field = real_field.related_field if real_field.rel else real_field
was_list = isinstance(child.rhs, (list, tuple))
rhs_value = origin_field.get_db_prep_lookup(child.lookup_name,
child.rhs,
self.connection,
prepared=True)
if not was_list and rhs_value != []:
rhs_value = rhs_value[0]
if origin_field.primary_key:
if real_field.rel:
# If we're a related field, use the foreign table,
# but also don't use META(id), since it's actually embedded
lhs_table = real_field.related_model._meta.db_table
lhs = real_field.column
else:
lhs_table = query.alias_map[child.lhs.alias].table_name
lhs = 'META({}).id'.format(n1ql_escape(BUCKET_PLACEHOLDER))
if real_field.get_internal_type() in ('IntegerField',
'AutoField'):
# This could be cast as a string, so cast it back as an int
castfn = int
else:
castfn = lambda x_: x_
if isinstance(rhs_value, (list, tuple)):
rhs_value = [
DocID.encode(lhs_table, castfn(x)) for x in rhs_value
]
else:
rhs_value = DocID.encode(lhs_table, castfn(rhs_value))
else:
lhs = n1ql_escape(query_field.column)
real_field = query_field
placeholder = self.params.indexstr()
rhs_value, criteria = Operators.convert(rhs_value, lhs,
placeholder,
child.lookup_name,
real_field)
self.params.add(rhs_value)
where.append(' '.join(criteria))
if where:
connector = ' ' + parent.connector + ' '
neg = 'NOT ' if parent.negated else ''
return '(' + neg + (connector.join(where)) + ')'
else:
return None
def _prepare_for_transformation(self):
from django.db.models.sql.where import EmptyWhere
if isinstance(self.django_query.where, EmptyWhere):
# Empty where means return nothing!
raise EmptyResultSet()
def parse_constraint(child, connection, negated=False):
if isinstance(child, tuple):
# First, unpack the constraint
constraint, op, annotation, value = child
was_list = isinstance(value, (list, tuple))
if isinstance(value, query.Query):
value = value.get_compiler(connection.alias).as_sql()[0].execute()
else:
packed, value = constraint.process(op, value, connection)
alias, column, db_type = packed
field = constraint.field
else:
# Django 1.7+
field = child.lhs.target
column = child.lhs.target.column
op = child.lookup_name
value = child.rhs
annotation = value
was_list = isinstance(value, (list, tuple))
if isinstance(value, query.Query):
value = value.get_compiler(connection.alias).as_sql()[0].execute()
elif value != []:
value = child.lhs.output_field.get_db_prep_lookup(
child.lookup_name, child.rhs, connection, prepared=True)
is_pk = field and field.primary_key
if column == "id" and op == "iexact" and is_pk and isinstance(field, AutoField):
# When new instance is created, automatic primary key 'id' does not generate '_idx_iexact_id'.
# As the primary key 'id' (AutoField) is integer and is always case insensitive, we can deal with 'id_iexact=' query by using 'exact' rather than 'iexact'.
op = "exact"
if field and field.db_type(connection) in ("bytes", "text"):
raise NotSupportedError("Text and Blob fields are not indexed by the datastore, so you can't filter on them")
if op not in REQUIRES_SPECIAL_INDEXES:
# Don't convert if this op requires special indexes, it will be handled there
if field:
value = [ connection.ops.prep_lookup_value(field.model, x, field, column=column) for x in value]
# Don't ask me why, but on Django 1.6 constraint.process on isnull wipes out the value (it returns an empty list)
# so we have to special case this to use the annotation value instead
if op == "isnull":
if annotation is not None:
value = [ annotation ]
if is_pk and value[0]:
raise EmptyResultSet()
if not was_list:
value = value[0]
else:
if negated:
raise CouldBeSupportedError("Special indexing does not currently supported negated queries. See #80")
if not was_list:
value = value[0]
add_special_index(field.model, column, op) # Add the index if we can (e.g. on dev_appserver)
if op not in special_indexes_for_column(field.model, column):
raise RuntimeError("There is a missing index in your djangaeidx.yaml - \n\n{0}:\n\t{1}: [{2}]".format(
field.model, column, op)
)
indexer = REQUIRES_SPECIAL_INDEXES[op]
value = indexer.prep_value_for_query(value)
column = indexer.indexed_column_name(column, value=value)
op = indexer.prep_query_operator(op)
return column, op, value
def parse_where_and_check_projection(self, where, negated=False):
""" recursively parse the where tree and return a list of tuples of
(column, match_type, value), e.g. ('name', 'exact', 'John').
"""
result = []
if where.negated:
negated = not negated
if negated and where.connector != AND:
raise NotSupportedError("Only AND filters are supported for negated queries")
for child in where.children:
if isinstance(child, tuple):
constraint, op, annotation, value = child
if isinstance(value, (list, tuple)):
value = [ self.connection.ops.prep_lookup_value(self.model, x, constraint.field) for x in value]
else:
value = self.connection.ops.prep_lookup_value(self.model, value, constraint.field)
#Disable projection if it's not supported
if self.projection and constraint.col in self.projection:
if op in ("exact", "in", "isnull"):
#If we are projecting, but we are doing an
#equality filter on one of the columns, then we
#can't project
self.projection = None
if negated:
if op in ("exact", "in") and constraint.field.primary_key:
self.excluded_pks.append(value)
#else: FIXME when excluded_pks is handled, we can put the
#next section in an else block
if op == "exact":
if self.has_inequality_filter:
raise RuntimeError("You can only specify one inequality filter per query")
col = constraint.col
result.append((col, "gt_and_lt", value))
self.has_inequality_filter = True
else:
raise RuntimeError("Unsupported negated lookup: " + op)
else:
if constraint.field.primary_key:
if (value is None and op == "exact") or (op == "isnull" and value):
#If we are looking for a primary key that is None, then we always
#just return nothing
raise EmptyResultSet()
elif op in ("exact", "in"):
if isinstance(value, (list, tuple)):
self.included_pks.extend(list(value))
else:
self.included_pks.append(value)
else:
col = constraint.col
result.append((col, op, value))
else:
col = constraint.col
result.append((col, op, value))
else:
result.extend(self.parse_where_and_check_projection(child, negated))
return result
def _transform_query_17(connection, kind, query):
from django.db.models.sql.datastructures import Date, DateTime
if isinstance(query.where, EmptyWhere):
# Empty where means return nothing!
raise EmptyResultSet()
# Check for joins, we ignore select related tables as they aren't actually used (the connector marks select
# related as unsupported in its features)
tables = [k for k, v in query.alias_refcount.items() if v]
inherited_tables = set(
[x._meta.db_table for x in query.model._meta.parents])
select_related_tables = set([y[0][0] for y in query.related_select_cols])
tables = set(tables) - inherited_tables - select_related_tables
if len(tables) > 1:
raise NotSupportedError("""
The appengine database connector does not support JOINs. The requested join map follows\n
%s
""" % query.join_map)
ret = Query(query.model, kind)
ret.connection = connection
# Add the root concrete table as the source table
root_table = get_top_concrete_parent(query.model)._meta.db_table
ret.add_source_table(root_table)
# Extract the ordering of the query results
for order_col in _extract_ordering_from_query_17(query):
ret.add_order_by(order_col)
# Extract any projected columns (values/values_list/only/defer)
for projected_col in _extract_projected_columns_from_query_17(query):
ret.add_projected_column(projected_col)
for potential_annotation in query.select:
col = getattr(potential_annotation, "col", None)
if not col:
continue
if isinstance(col, (Date, DateTime)):
ret.add_annotation(col.col[-1], col)
# Add any extra selects
for col, select in query.extra_select.items():
ret.add_extra_select(col, select[0])
# This must happen after extracting projected cols
if query.distinct:
ret.set_distinct(list(query.distinct_fields))
# Extract any query offsets and limits
ret.low_mark = query.low_mark
ret.high_mark = query.high_mark
output = WhereNode()
output.connector = query.where.connector
_walk_django_where(query,
_django_17_query_walk_trunk,
_django_17_query_walk_leaf,
new_parent=output,
connection=connection,
model=query.model)
# If there no child nodes, just wipe out the where
if not output.children:
output = None
ret.where = output
return ret
def normalize_query(node, connection, negated=False, filtered_columns=None, _inequality_property=None):
"""
Converts a django_where_tree which is CNF to a DNF for use in the datastore.
This function does a lot of heavy lifting so optimization is welcome.
The connection is needed for calculating the values on the literals, it would
be nice if we can remove that so this only has one task, but it's fine for now
"""
_inequality_property = _inequality_property if _inequality_property is not None else []
def check_inequality_usage(_op, _column, _current_inequality):
if _op in INEQUALITY_OPERATORS:
if _current_inequality and _current_inequality[0] != _column:
raise NotSupportedError(
"You can only specify inequality filters ({}) on one property. There is already one on {}, you're attempting to use one on {}".format(
INEQUALITY_OPERATORS,
_current_inequality[0],
_column
)
)
else:
#We have to use a list, because Python 2.x doesn't have 'nonlocal' :/
_current_inequality.append(_column)
if isinstance(node, tuple) and isinstance(node[0], Constraint):
# This is where contraints are exploded
#
#
column, op, value = parse_constraint(node, connection)
if filtered_columns is not None:
assert isinstance(filtered_columns, set)
filtered_columns.add(column)
if op == 'in': # Explode INs into OR
if not isinstance(value, (list, tuple, set)):
raise ValueError("IN queries must be supplied a list of values")
if negated:
check_inequality_usage(">", column, _inequality_property)
return ('OR', [ ('OR', [(column, '>', x), (column, '<', x)]) for x in value ])
else:
if len(value) == 1:
return (column, '=', value[0])
return ('OR', [(column, '=', x) for x in value])
if op not in OPERATORS_MAP:
raise NotSupportedError("Unsupported operator %s" % op)
_op = OPERATORS_MAP[op]
check_inequality_usage(_op, column, _inequality_property) #Check we aren't doing an additional inequality
if negated and _op == '=': # Explode
check_inequality_usage('>', column, _inequality_property)
return ('OR', [(column, '>', value), (column, '<', value)])
elif op == "startswith":
#You can emulate starts with by adding the last unicode char
#to the value, then doing <=. Genius.
if value.endswith("%"):
value = value[:-1]
end_value = value[:]
if isinstance(end_value, str):
end_value = end_value.decode("utf-8")
end_value += u'\ufffd'
check_inequality_usage('>=', column, _inequality_property)
if not negated:
return ('AND', [(column, '>=', value), (column, '<', end_value)])
else:
return ('OR', [(column, '<', value), (column, '>=', end_value)])
elif op == "isnull":
if (value and not negated) or (not value and negated): #We're checking for isnull=True
#If we are checking that a primary key isnull, then don't do an impossible query!
if node[0].field.primary_key:
raise EmptyResultSet()
return (column, "=", None)
else: #We're checking for isnull=False
check_inequality_usage('>', column, _inequality_property)
return ('OR', [(column, '>', None), (column, '<', None)])
elif _op == None:
raise NotSupportedError("Unhandled lookup type %s" % op)
return (column, _op, value)
else:
# This is where the distribution is applied over the children
if not node.children:
return []
if node.negated:
negated = True
if len(node.children) > 1: # If there is more than one child then attempt to reduce
exploded = (node.connector, [
normalize_query(child, connection, negated=negated, filtered_columns=filtered_columns, _inequality_property=_inequality_property)
for child in node.children
])
if node.connector == 'AND':
if len(exploded[1]) > 1:
# We need to know if there are any ORs under this AND node
_ors = [x for x in exploded[1] if x[0] == 'OR' ]
def special_product(*args):
"""
Modified product to return the prods in an AND container
"""
pools = map(tuple, args)
result = [[]]
for pool in pools:
result = [x+[y] if y[0] != 'AND' else x+y[1] for x in result for y in pool]
for prod in result:
yield ('AND', list(prod))
if len(_ors) > 0:
# This is a bit complicated to explain in a comment
# But you can calculate the DNF by doing a clever product on the children of the AND node
# It's all to do with grouping the ORs in groups and the ANDs individualy
# This returns a DNF which is naturally wrapped in an OR
_literals = (x for x in exploded[1] if x[0] != 'OR')
flat_ors = (x[1] if x[0] == 'OR' else x for x in _ors)
flat_literals = ([y] if y[0] != 'AND' else ([q] for q in y[1]) for y in _literals)
return ('OR', [x for x in special_product(*chain(flat_ors, flat_literals))])
elif node.connector == 'OR':
if all(x[0] == 'OR' for x in exploded[1]):
return ('OR', list(chain.from_iterable((x[1] for x in exploded[1]))))
else:
exploded = normalize_query(
node.children[0],
connection,
negated=negated,
filtered_columns=filtered_columns,
_inequality_property=_inequality_property
)
if exploded[0] == exploded[1][0][0]: # crush any single child 'OR' or 'AND' hangover
return (exploded[0], [x for x in exploded[1][0][1]])
return exploded