def _validate_query_is_possible(self, query):
""" Given the *django* query, check the following:
- The query only has one inequality filter
- The query does no joins
- The query ordering is compatible with the filters
"""
#Check for joins
if query.count_active_tables() > 1:
raise NotSupportedError("""
The appengine database connector does not support JOINs. The requested join map follows\n
%s
""" % query.join_map)
if query.aggregates:
if query.aggregates.keys() == [ None ]:
if query.aggregates[None].col != "*":
raise NotSupportedError("Counting anything other than '*' is not supported")
else:
raise NotSupportedError("Unsupported aggregate query")
def check_inequality_usage(_op, _column, _current_inequality):
if _op in INEQUALITY_OPERATORS:
if _current_inequality and _current_inequality[0] != _column:
raise NotSupportedError(
"You can only specify inequality filters ({}) on one property. There is already one on {}, you're attempting to use one on {}".format(
INEQUALITY_OPERATORS,
_current_inequality[0],
_column
)
)
else:
#We have to use a list, because Python 2.x doesn't have 'nonlocal' :/
_current_inequality.append(_column)
def txn():
if key is not None:
if utils.key_exists(key):
raise IntegrityError("Tried to INSERT with existing key")
id_or_name = key.id_or_name()
if isinstance(id_or_name, basestring) and id_or_name.startswith("__"):
raise NotSupportedError("Datastore ids cannot start with __. Id was %s" % id_or_name)
markers = constraints.acquire(self.model, ent)
try:
results.append(datastore.Put(ent))
caching.add_entity_to_context_cache(self.model, ent)
except:
#Make sure we delete any created markers before we re-raise
constraints.release_markers(markers)
raise
def parse_constraint(child, connection):
#First, unpack the constraint
constraint, op, annotation, value = child
was_list = isinstance(value, (list, tuple))
packed, value = constraint.process(op, value, connection)
alias, column, db_type = packed
if constraint.field.db_type(connection) in ("bytes", "text"):
raise NotSupportedError("Text and Blob fields are not indexed by the datastore, so you can't filter on them")
if op not in REQUIRES_SPECIAL_INDEXES:
#Don't convert if this op requires special indexes, it will be handled there
value = [ connection.ops.prep_lookup_value(constraint.field.model, x, constraint.field, constraint=constraint) for x in value]
#Don't ask me why, but constraint.process on isnull wipes out the value (it returns an empty list)
# so we have to special case this to use the annotation value instead
if op == "isnull":
value = [ annotation ]
if not was_list:
value = value[0]
else:
if not was_list:
value = value[0]
add_special_index(constraint.field.model, column, op) #Add the index if we can (e.g. on dev_appserver)
if op not in special_indexes_for_column(constraint.field.model, column):
raise RuntimeError("There is a missing index in your djangaeidx.yaml - \n\n{0}:\n\t{1}: [{2}]".format(
constraint.field.model, column, op)
)
indexer = REQUIRES_SPECIAL_INDEXES[op]
column = indexer.indexed_column_name(column)
value = indexer.prep_value_for_query(value)
op = indexer.prep_query_operator(op)
return column, op, value
def parse_where_and_check_projection(self, where, negated=False):
""" recursively parse the where tree and return a list of tuples of
(column, match_type, value), e.g. ('name', 'exact', 'John').
"""
result = []
if where.negated:
negated = not negated
if negated and where.connector != AND:
raise NotSupportedError("Only AND filters are supported for negated queries")
for child in where.children:
if isinstance(child, tuple):
constraint, op, annotation, value = child
if isinstance(value, (list, tuple)):
value = [ self.connection.ops.prep_lookup_value(self.model, x, constraint.field) for x in value]
else:
value = self.connection.ops.prep_lookup_value(self.model, value, constraint.field)
#Disable projection if it's not supported
if self.projection and constraint.col in self.projection:
if op in ("exact", "in", "isnull"):
#If we are projecting, but we are doing an
#equality filter on one of the columns, then we
#can't project
self.projection = None
if negated:
if op in ("exact", "in") and constraint.field.primary_key:
self.excluded_pks.append(value)
#else: FIXME when excluded_pks is handled, we can put the
#next section in an else block
if op == "exact":
if self.has_inequality_filter:
raise RuntimeError("You can only specify one inequality filter per query")
col = constraint.col
result.append((col, "gt_and_lt", value))
self.has_inequality_filter = True
else:
raise RuntimeError("Unsupported negated lookup: " + op)
else:
if constraint.field.primary_key:
if (value is None and op == "exact") or (op == "isnull" and value):
#If we are looking for a primary key that is None, then we always
#just return nothing
raise EmptyResultSet()
elif op in ("exact", "in"):
if isinstance(value, (list, tuple)):
self.included_pks.extend(list(value))
else:
self.included_pks.append(value)
else:
col = constraint.col
result.append((col, op, value))
else:
col = constraint.col
result.append((col, op, value))
else:
result.extend(self.parse_where_and_check_projection(child, negated))
return result
def __init__(self, connection, query, keys_only=False, all_fields=False):
self.original_query = query
opts = query.get_meta()
if not query.default_ordering:
self.ordering = query.order_by
else:
self.ordering = query.order_by or opts.ordering
if self.ordering:
ordering = [ x for x in self.ordering if not (isinstance(x, basestring) and "__" in x) ]
if len(ordering) < len(self.ordering):
if not on_production() and not in_testing():
diff = set(self.ordering) - set(ordering)
log_once(DJANGAE_LOG.warning, "The following orderings were ignored as cross-table orderings are not supported on the datastore: %s", diff)
self.ordering = ordering
self.distinct_values = set()
self.distinct_on_field = None
self.distinct_field_convertor = None
self.queried_fields = []
if keys_only:
self.queried_fields = [ opts.pk.column ]
elif not all_fields:
for x in query.select:
if isinstance(x, tuple):
#Django < 1.6 compatibility
self.queried_fields.append(x[1])
else:
self.queried_fields.append(x.col[1])
if x.lookup_type == 'year':
assert self.distinct_on_field is None
self.distinct_on_field = x.col[1]
self.distinct_field_convertor = field_conv_year_only
elif x.lookup_type == 'month':
assert self.distinct_on_field is None
self.distinct_on_field = x.col[1]
self.distinct_field_convertor = field_conv_month_only
elif x.lookup_type == 'day':
assert self.distinct_on_field is None
self.distinct_on_field = x.col[1]
self.distinct_field_convertor = field_conv_day_only
else:
raise NotSupportedError("Unhandled lookup type: {0}".format(x.lookup_type))
#Projection queries don't return results unless all projected fields are
#indexed on the model. This means if you add a field, and all fields on the model
#are projectable, you will never get any results until you've resaved all of them.
#Because it's not possible to detect this situation, we only try a projection query if a
#subset of fields was specified (e.g. values_list('bananas')) which makes the behaviour a
#bit more predictable. It would be nice at some point to add some kind of force_projection()
#thing on a queryset that would do this whenever possible, but that's for the future, maybe.
try_projection = bool(self.queried_fields)
if not self.queried_fields:
self.queried_fields = [ x.column for x in opts.fields ]
self.connection = connection
self.pk_col = opts.pk.column
self.model = query.model
self.is_count = query.aggregates
self.keys_only = False #FIXME: This should be used where possible
self.included_pks = []
self.excluded_pks = []
self.has_inequality_filter = False
self.all_filters = []
self.results = None
self.extra_select = query.extra_select
self.gae_query = None
self._set_db_table()
self._validate_query_is_possible(query)
projection_fields = []
if try_projection:
for field in self.queried_fields:
#We don't include the primary key in projection queries...
if field == self.pk_col:
continue
#Text and byte fields aren't indexed, so we can't do a
#projection query
f = get_field_from_column(self.model, field)
if not f:
raise NotImplementedError("Attemping a cross-table select. Maybe? #FIXME")
assert f #If this happens, we have a cross-table select going on! #FIXME
db_type = f.db_type(connection)
if db_type in ("bytes", "text"):
projection_fields = []
break
projection_fields.append(field)
self.projection = list(set(projection_fields)) or None
if opts.parents:
self.projection = None
self.where = self.parse_where_and_check_projection(query.where)
try:
#If the PK was queried, we switch it in our queried
#fields store with __key__
pk_index = self.queried_fields.index(self.pk_col)
self.queried_fields[pk_index] = "__key__"
#If the only field queried was the key, then we can do a keys_only
#query
self.keys_only = len(self.queried_fields) == 1
except ValueError:
pass
def normalize_query(node, connection, negated=False, filtered_columns=None, _inequality_property=None):
"""
Converts a django_where_tree which is CNF to a DNF for use in the datastore.
This function does a lot of heavy lifting so optimization is welcome.
The connection is needed for calculating the values on the literals, it would
be nice if we can remove that so this only has one task, but it's fine for now
"""
_inequality_property = _inequality_property if _inequality_property is not None else []
def check_inequality_usage(_op, _column, _current_inequality):
if _op in INEQUALITY_OPERATORS:
if _current_inequality and _current_inequality[0] != _column:
raise NotSupportedError(
"You can only specify inequality filters ({}) on one property. There is already one on {}, you're attempting to use one on {}".format(
INEQUALITY_OPERATORS,
_current_inequality[0],
_column
)
)
else:
#We have to use a list, because Python 2.x doesn't have 'nonlocal' :/
_current_inequality.append(_column)
if isinstance(node, tuple) and isinstance(node[0], Constraint):
# This is where contraints are exploded
#
#
column, op, value = parse_constraint(node, connection)
if filtered_columns is not None:
assert isinstance(filtered_columns, set)
filtered_columns.add(column)
if op == 'in': # Explode INs into OR
if not isinstance(value, (list, tuple, set)):
raise ValueError("IN queries must be supplied a list of values")
if negated:
check_inequality_usage(">", column, _inequality_property)
return ('OR', [ ('OR', [(column, '>', x), (column, '<', x)]) for x in value ])
else:
if len(value) == 1:
return (column, '=', value[0])
return ('OR', [(column, '=', x) for x in value])
if op not in OPERATORS_MAP:
raise NotSupportedError("Unsupported operator %s" % op)
_op = OPERATORS_MAP[op]
check_inequality_usage(_op, column, _inequality_property) #Check we aren't doing an additional inequality
if negated and _op == '=': # Explode
check_inequality_usage('>', column, _inequality_property)
return ('OR', [(column, '>', value), (column, '<', value)])
elif op == "startswith":
#You can emulate starts with by adding the last unicode char
#to the value, then doing <=. Genius.
if value.endswith("%"):
value = value[:-1]
end_value = value[:]
if isinstance(end_value, str):
end_value = end_value.decode("utf-8")
end_value += u'\ufffd'
check_inequality_usage('>=', column, _inequality_property)
if not negated:
return ('AND', [(column, '>=', value), (column, '<', end_value)])
else:
return ('OR', [(column, '<', value), (column, '>=', end_value)])
elif op == "isnull":
if (value and not negated) or (not value and negated): #We're checking for isnull=True
#If we are checking that a primary key isnull, then don't do an impossible query!
if node[0].field.primary_key:
raise EmptyResultSet()
return (column, "=", None)
else: #We're checking for isnull=False
check_inequality_usage('>', column, _inequality_property)
return ('OR', [(column, '>', None), (column, '<', None)])
elif _op == None:
raise NotSupportedError("Unhandled lookup type %s" % op)
return (column, _op, value)
else:
# This is where the distribution is applied over the children
if not node.children:
return []
if node.negated:
negated = True
if len(node.children) > 1: # If there is more than one child then attempt to reduce
exploded = (node.connector, [
normalize_query(child, connection, negated=negated, filtered_columns=filtered_columns, _inequality_property=_inequality_property)
for child in node.children
])
if node.connector == 'AND':
if len(exploded[1]) > 1:
# We need to know if there are any ORs under this AND node
_ors = [x for x in exploded[1] if x[0] == 'OR' ]
def special_product(*args):
"""
Modified product to return the prods in an AND container
"""
pools = map(tuple, args)
result = [[]]
for pool in pools:
result = [x+[y] if y[0] != 'AND' else x+y[1] for x in result for y in pool]
for prod in result:
yield ('AND', list(prod))
if len(_ors) > 0:
# This is a bit complicated to explain in a comment
# But you can calculate the DNF by doing a clever product on the children of the AND node
# It's all to do with grouping the ORs in groups and the ANDs individualy
# This returns a DNF which is naturally wrapped in an OR
_literals = (x for x in exploded[1] if x[0] != 'OR')
flat_ors = (x[1] if x[0] == 'OR' else x for x in _ors)
flat_literals = ([y] if y[0] != 'AND' else ([q] for q in y[1]) for y in _literals)
return ('OR', [x for x in special_product(*chain(flat_ors, flat_literals))])
elif node.connector == 'OR':
if all(x[0] == 'OR' for x in exploded[1]):
return ('OR', list(chain.from_iterable((x[1] for x in exploded[1]))))
else:
exploded = normalize_query(
node.children[0],
connection,
negated=negated,
filtered_columns=filtered_columns,
_inequality_property=_inequality_property
)
if exploded[0] == exploded[1][0][0]: # crush any single child 'OR' or 'AND' hangover
return (exploded[0], [x for x in exploded[1][0][1]])
return exploded