def remove_empty_in(node): """ Once we are normalized, if any of the branches filters on an empty list, we can remove that entire branch from the query. If this leaves no branches, then the result set is empty """ # This is a bit ugly, but you try and do it more succinctly :) # We have the following possible situations for IN queries with an empty # value: # - Negated: One of the nodes in the and branch will always be true and is therefore # unnecessary, we leave it alone though # - Not negated: The entire AND branch will always be false, so that branch can be removed # if that was the last branch, then the queryset will be empty # Everything got wiped out! if node.connector == 'OR' and len(node.children) == 0: raise EmptyResultSet() for and_branch in node.children[:]: if and_branch.is_leaf and and_branch.operator == "IN" and not len(and_branch.value): node.children.remove(and_branch) if not node.children: raise EmptyResultSet()
def process_and_branch(query, and_branch): for child in and_branch[-1]: column, op, value = child[1] # for column, op, value in and_branch[-1]: if column == self.pk_col: column = "__key__" #FIXME: This EmptyResultSet check should happen during normalization so that Django doesn't count it as a query if op == "=" and "__key__ =" in query and query["__key__ ="] != value: # We've already done an exact lookup on a key, this query can't return anything! raise EmptyResultSet() if not isinstance(value, datastore.Key): value = get_datastore_key(self.model, value) key = "%s %s" % (column, op) try: if isinstance(value, basestring): value = coerce_unicode(value) if key in query: if type(query[key]) == list: if value not in query[key]: query[key].append(value) else: if query[key] != value: query[key] = [ query[key], value ] else: query[key] = value except datastore_errors.BadFilterError as e: raise NotSupportedError(str(e))
def set_leaf(self, column, operator, value, is_pk_field, negated, target_field=None): assert column assert operator assert isinstance(is_pk_field, bool) assert isinstance(negated, bool) if operator == "iexact" and isinstance(target_field, AutoField): # When new instance is created, automatic primary key 'id' does not generate '_idx_iexact_id'. # As the primary key 'id' (AutoField) is integer and is always case insensitive, # we can deal with 'id_iexact=' query by using 'exact' rather than 'iexact'. operator = "exact" value = int(value) if is_pk_field: # If this is a primary key, we need to make sure that the value # we pass to the query is a datastore Key. We have to deal with IN queries here # because they aren't flattened until the DNF stage model = get_top_concrete_parent(target_field.model) table = model._meta.db_table if isinstance(value, (list, tuple)): value = [ datastore.Key.from_path(table, x) for x in value if x ] else: if operator == "isnull" and value is True: # FIXME: Strictly, this isn't correct, this could be one of several branches # but id=None filters are silly anyway. This should be moved to after normalization.. # probably. This fixes a test in Django which does this in get_or_create for some reason raise EmptyResultSet() if not value: # Empty strings and 0 are forbidden as keys # so make this an impossible filter # FIXME: This is a hack! It screws with the ordering # because it's an inequality. Instead we should wipe this # filter out when preprocessing in the DNF (because it's impossible) value = datastore.Key.from_path('', 1) operator = '<' else: value = datastore.Key.from_path(table, value) column = "__key__" # Do any special index conversions necessary to perform this lookup if operator in REQUIRES_SPECIAL_INDEXES: add_special_index(target_field.model, column, operator, value) indexer = REQUIRES_SPECIAL_INDEXES[operator] index_type = indexer.prepare_index_type(operator, value) value = indexer.prep_value_for_query(value) if not indexer.validate_can_be_indexed(value, negated): raise NotSupportedError("Unsupported special index or value '%s %s'" % (column, operator)) column = indexer.indexed_column_name(column, value, index_type) operator = indexer.prep_query_operator(operator) self.column = column self.operator = convert_operator(operator) self.value = value
def _transform_query_18(connection, kind, query): from django.db.models.sql.where import EmptyWhere if isinstance(query.where, EmptyWhere): # Empty where means return nothing! raise EmptyResultSet() ret = Query(query.model, kind) ret.connection = connection # Add the root concrete table as the source table root_table = get_top_concrete_parent(query.model)._meta.db_table ret.add_source_table(root_table) # Extract the ordering of the query results for order_col in _extract_ordering_from_query_18(query): ret.add_order_by(order_col) # Extract any projected columns (values/values_list/only/defer) for projected_col in _extract_projected_columns_from_query_18(query): ret.add_projected_column(projected_col) # Add any extra selects for col, select in query.extra_select.items(): ret.add_extra_select(col, select[0]) if query.distinct: # This must happen after extracting projected cols ret.set_distinct(list(query.distinct_fields)) # Process annotations! if query.annotation_select: for k, v in query.annotation_select.items(): ret.add_annotation(k, v) # Extract any query offsets and limits ret.low_mark = query.low_mark ret.high_mark = query.high_mark output = WhereNode() output.connector = query.where.connector _walk_django_where( query, _django_18_query_walk_trunk, _django_18_query_walk_leaf, new_parent=output, connection=connection, negated=query.where.negated, model=query.model ) # If there no child nodes, just wipe out the where if not output.children: output = None ret.where = output return ret
def _prepare_for_transformation(self): from django.db.models.sql.where import NothingNode, WhereNode as DjangoWhereNode query = self.django_query # It could either be a NothingNode, or a WhereNode(AND NothingNode) if (isinstance(query.where, NothingNode) or (isinstance(query.where, DjangoWhereNode) and len(query.where.children) == 1 and isinstance(query.where.children[0], NothingNode))): # Empty where means return nothing! raise EmptyResultSet()
def parse_dnf(node, connection, ordering=None): should_in_memory_exclude = should_exclude_pks_in_memory(node, ordering) tree, filtered_columns, excluded_pks = parse_tree( node, connection, excluded_pks=set() if should_in_memory_exclude else None) if not should_exclude_pks_in_memory: assert excluded_pks is None if tree: tree = tripled(tree) if tree and tree[0] != 'OR': tree = ('OR', [tree]) # Filter out impossible branches of the where, if that then results in an empty tree then # raise an EmptyResultSet, otherwise replace the tree with the now simpler query if tree: final = [] for and_branch in tree[-1]: if and_branch[0] == 'LIT' and and_branch[-1] == IMPOSSIBLE_FILTER: continue elif and_branch[0] == 'AND' and IMPOSSIBLE_FILTER in [ x[-1] for x in and_branch[-1] ]: continue final.append(and_branch) if not final: raise EmptyResultSet() else: tree = (tree[0], final) # If there are more than 30 filters, and not all filters are PK filters if tree and len(tree[-1]) > 30: for and_branch in tree[-1]: if and_branch[0] == 'LIT': and_branch = [and_branch] for lit in and_branch[-1] if and_branch[ 0] == 'AND' else and_branch: # Go through each literal tuple if isinstance(lit[-1], datastore.Key) or isinstance( lit[-1][-1], datastore.Key ): # If the value is a key, then break the loop break else: # If we didn't find a literal with a datastore Key, then raise unsupported raise NotSupportedError( "The datastore doesn't support this query, more than 30 filters were needed" ) return tree, filtered_columns, excluded_pks or set()
def check_query(self): """ Checks if the current query is supported by the database. In general, we expect queries requiring JOINs (many-to-many relations, abstract model bases, or model spanning filtering), using DISTINCT (through `QuerySet.distinct()`, which is not required in most situations) or using the SQL-specific `QuerySet.extra()` to not work with nonrel back-ends. """ if hasattr(self.query, 'is_empty') and self.query.is_empty(): raise EmptyResultSet() if (len([a for a in self.query.alias_map if self.query.alias_refcount[a]]) > 1 or self.query.distinct or self.query.extra or self.query.having): raise DatabaseError("This query is not supported by the database.")
def detect_conflicting_key_filter(node): assert node.connector == "OR" for and_branch in node.children[:]: # If we have a Root OR with leaf elements, we don't need to worry if and_branch.is_leaf: break pk_equality_found = None for child in and_branch.children: if child.column == "__key__" and child.operator == "=": if pk_equality_found and pk_equality_found != child.value: # Remove this AND branch as it's impossible to return anything node.children.remove(and_branch) else: pk_equality_found = child.value if not node.children: raise EmptyResultSet()
def process_and_branch(query, and_branch): for column, op, value in and_branch[-1]: if column == self.pk_col: column = "__key__" #FIXME: This EmptyResultSet check should happen during normalization so that Django doesn't count it as a query if op == "=" and "__key__ =" in query: #We've already done an exact lookup on a key, this query can't return anything! raise EmptyResultSet() if not isinstance(value, datastore.Key): value = get_datastore_key(self.model, value) key = "%s %s" % (column, op) if key in query: query[key] = [ query[key], value ] else: query[key] = value
def check_query(self): """ Checks if the current query is supported by the database. In general, we expect queries requiring JOINs (many-to-many relations, abstract model bases, or model spanning filtering), using DISTINCT (through `QuerySet.distinct()`, which is not required in most situations) or using the SQL-specific `QuerySet.extra()` to not work with nonrel back-ends. """ if hasattr(self.query, 'is_empty') and self.query.is_empty(): raise EmptyResultSet() if (len([ a for a in self.query.alias_map if self.query.alias_refcount[a] ]) > 1 or self.query.distinct or self.query.extra): # or self.having -- Not quite working. # having is no longer part of the query as of 1.9; It moved to the compiler # https://github.com/django/django/commit/afe0bb7b13bb8dc4370f32225238012c873b0ee3 raise DatabaseError("This query is not supported by the database.")
def _prepare_for_transformation(self): from django.db.models.sql.where import NothingNode query = self.django_query def where_will_always_be_empty(where): if isinstance(where, NothingNode): return True if where.connector == 'AND' and any(isinstance(x, NothingNode) for x in where.children): return True if where.connector == 'OR' and len(where.children) == 1 and isinstance(where.children[0], NothingNode): return True return False # It could either be a NothingNode, or a WhereNode(AND NothingNode) if where_will_always_be_empty(query.where): # Empty where means return nothing! raise EmptyResultSet()
def remove_unnecessary_nodes(top_node): """ Sometimes you end up with a branch that has two nodes that have the same column and operator, but different values. When this happens we need to simplify, e.g.: AND:[username<A],[username<B] -> AND:[username<A] """ for and_branch in top_node.children[:]: seen = {} altered = False for node in and_branch.children: key = (node.column, node.operator) if key in seen: altered = True if node.operator in ('<', '<='): seen[key].value = min(seen[key].value, node.value) elif node.operator in ('>', '>='): seen[key].value = max(seen[key].value, node.value) elif node.operator == "=": # Impossible filter! remove the AND branch entirely if and_branch in top_node.children and seen[ key].value != node.value: top_node.children.remove(and_branch) break else: pass else: seen[key] = node if altered: and_branch.children = [x for x in seen.values()] # If all the OR clause are impossible filters we end up with no filters # at all, which is incorrect if not top_node.children: raise EmptyResultSet()
def _remove_impossible_branches(self): """ If we mark a child node as never returning results we either need to remove those nodes, or remove the branches of the tree they are on depending on the connector of the parent node. """ if not self._where: return def walk(node, negated): if node.negated: negated = not negated for child in node.children[:]: walk(child, negated) if child.will_never_return_results: if node.connector == 'AND': if child.negated: node.children.remove(child) else: node.will_never_return_results = True else: #OR if not child.negated: node.children.remove(child) if not node.children: node.will_never_return_results = True else: node.children[:] = [] walk(self._where, False) if self._where.will_never_return_results: # We removed all the children of the root where node, so no results raise EmptyResultSet()
def make_atom(self, child, qn, connection): lvalue, lookup_type, value_annot, param = child kwargs = {'connection': connection} if lvalue and lvalue.field and hasattr( lvalue.field, 'db_type') and lvalue.field.db_type(**kwargs) == 'hstore': try: lvalue, params = lvalue.process(lookup_type, param, connection) except EmptyShortCircuit: raise EmptyResultSet() field = self.sql_for_columns(lvalue, qn, connection) if lookup_type == 'exact': if isinstance(param, dict): return ('{0} = %s'.format(field), [param]) raise ValueError('invalid value') elif lookup_type in ('gt', 'gte', 'lt', 'lte'): if isinstance(param, dict): sign = (lookup_type[0] == 'g' and '>%s' or '<%s') % (lookup_type[-1] == 'e' and '=' or '') param_keys = list(param.keys()) conditions = [] for key in param_keys: cast = get_cast_for_param(value_annot, key) conditions.append('(%s->\'%s\')%s %s %%s' % (field, key, cast, sign)) return (" AND ".join(conditions), param.values()) raise ValueError('invalid value') elif lookup_type in ['contains', 'icontains']: if isinstance(param, dict): values = list(param.values()) keys = list(param.keys()) if len(values) == 1 and isinstance(values[0], (list, tuple)): # Can't cast here because the list could contain multiple types return ('%s->\'%s\' = ANY(%%s)' % (field, keys[0]), [[str(x) for x in values[0]]]) elif len(keys) == 1 and len(values) == 1: # Retrieve key and compare to param instead of using '@>' in order to cast hstore value cast = get_cast_for_param(value_annot, keys[0]) return ('(%s->\'%s\')%s = %%s' % (field, keys[0], cast), [values[0]]) return ('%s @> %%s' % field, [param]) elif isinstance(param, (list, tuple)): if len(param) == 0: raise ValueError('invalid value') if len(param) < 2: return ('%s ? %%s' % field, [param[0]]) if param: return ('%s ?& %%s' % field, [param]) raise ValueError('invalid value') elif isinstance(param, six.string_types): # if looking for a string perform the normal text lookup # that is: look for occurence of string in all the keys pass elif hasattr(child[0].field, 'serializer'): try: child[0].field._serialize_value(param) pass except Exception: raise ValueError('invalid value') else: raise ValueError('invalid value') elif lookup_type == 'isnull': if isinstance(param, dict): param_keys = list(param.keys()) conditions = [] for key in param_keys: op = 'IS NULL' if value_annot[key] else 'IS NOT NULL' conditions.append('(%s->\'%s\') %s' % (field, key, op)) return (" AND ".join(conditions), []) # do not perform any special format return super(HStoreWhereNode, self).make_atom(child, qn, connection) else: raise TypeError('invalid lookup type') return super(HStoreWhereNode, self).make_atom(child, qn, connection)
def __init__(self, connection, query, keys_only=False): self.original_query = query self.connection = connection self.limits = (query.low_mark, query.high_mark) opts = query.get_meta() self.distinct = query.distinct self.distinct_values = set() self.distinct_on_field = None self.distinct_field_convertor = None self.queried_fields = [] self.model = query.model self.pk_col = opts.pk.column self.is_count = query.aggregates self.extra_select = query.extra_select self._set_db_table() self._validate_query_is_possible(query) if not query.default_ordering: self.ordering = query.order_by else: self.ordering = query.order_by or opts.ordering if self.ordering: ordering = [ x for x in self.ordering if not (isinstance(x, basestring) and "__" in x) ] if len(ordering) < len(self.ordering): if not on_production() and not in_testing(): diff = set(self.ordering) - set(ordering) log_once(DJANGAE_LOG.warning, "The following orderings were ignored as cross-table orderings are not supported on the datastore: %s", diff) self.ordering = ordering #If the query uses defer()/only() then we need to process deferred. We have to get all deferred columns # for all (concrete) inherited models and then only include columns if they appear in that list deferred_columns = {} query.deferred_to_data(deferred_columns, query.deferred_to_columns_cb) inherited_db_tables = [ x._meta.db_table for x in get_concrete_parents(self.model) ] only_load = list(chain(*[ list(deferred_columns.get(x, [])) for x in inherited_db_tables ])) if query.select: for x in query.select: if hasattr(x, "field"): #In Django 1.6+ 'x' above is a SelectInfo (which is a tuple subclass), whereas in 1.5 it's a tuple # in 1.6 x[1] == Field, but 1.5 x[1] == unicode (column name) if x.field is None: column = x.col.col[1] #This is the column we are getting lookup_type = x.col.lookup_type self.distinct_on_field = column #This whole section of code is weird, and is probably better implemented as a custom Query type (like QueryByKeys) # basically, appengine gives back dates as a time since the epoch, we convert it to a date, then floor it, then convert it back # in our transform function. The transform is applied when the results are read back so that only distinct values are returned. # this is very hacky... if lookup_type in DATE_TRANSFORMS: self.distinct_field_convertor = lambda value: DATE_TRANSFORMS[lookup_type](self.connection, value) else: raise CouldBeSupportedError("Unhandled lookup_type %s" % lookup_type) else: column = x.field.column else: column = x[1] if only_load and column not in only_load: continue self.queried_fields.append(column) else: self.queried_fields = [ x.column for x in opts.fields if (not only_load) or (x.column in only_load) ] self.keys_only = keys_only or self.queried_fields == [ opts.pk.column ] assert self.queried_fields #Projection queries don't return results unless all projected fields are #indexed on the model. This means if you add a field, and all fields on the model #are projectable, you will never get any results until you've resaved all of them. #Because it's not possible to detect this situation, we only try a projection query if a #subset of fields was specified (e.g. values_list('bananas')) which makes the behaviour a #bit more predictable. It would be nice at some point to add some kind of force_projection() #thing on a queryset that would do this whenever possible, but that's for the future, maybe. try_projection = (self.keys_only is False) and bool(self.queried_fields) if not self.queried_fields: self.queried_fields = [ x.column for x in opts.fields ] self.excluded_pks = set() self.has_inequality_filter = False self.all_filters = [] self.results = None self.gae_query = None projection_fields = [] if try_projection: for field in self.queried_fields: #We don't include the primary key in projection queries... if field == self.pk_col: continue #Text and byte fields aren't indexed, so we can't do a #projection query f = get_field_from_column(self.model, field) if not f: raise CouldBeSupportedError("Attemping a cross-table select or dates query, or something?!") assert f #If this happens, we have a cross-table select going on! #FIXME db_type = f.db_type(connection) if db_type in ("bytes", "text"): projection_fields = [] break projection_fields.append(field) self.projection = list(set(projection_fields)) or None if opts.parents: self.projection = None columns = set() if isinstance(query.where, EmptyWhere): #Empty where means return nothing! raise EmptyResultSet() else: self.where = normalize_query(query.where, self.connection, filtered_columns=columns) #DISABLE PROJECTION IF WE ARE FILTERING ON ONE OF THE PROJECTION_FIELDS for field in self.projection or []: if field in columns: self.projection = None break try: #If the PK was queried, we switch it in our queried #fields store with __key__ pk_index = self.queried_fields.index(self.pk_col) self.queried_fields[pk_index] = "__key__" except ValueError: pass
def __init__(self, connection, query, keys_only=False): self.where = None self.original_query = query self.connection = connection self.limits = (query.low_mark, query.high_mark) self.results_returned = 0 opts = query.get_meta() self.distinct = query.distinct self.distinct_values = set() self.distinct_on_field = None self.distinct_field_convertor = None self.queried_fields = [] self.model = query.model self.pk_col = opts.pk.column self.is_count = query.aggregates self.extra_select = query.extra_select self._set_db_table() try: self._validate_query_is_possible(query) self.ordering = _convert_ordering(query) except NotSupportedError as e: # If we can detect here, or when parsing the WHERE tree that a query is unsupported # we set this flag, and then throw NotSupportedError when execute is called. # This will then wrap the exception in Django's NotSupportedError meaning users # only need to catch that one, and not both Django's and ours self.unsupported_query_message = str(e) return else: self.unsupported_query_message = "" # If the query uses defer()/only() then we need to process deferred. We have to get all deferred columns # for all (concrete) inherited models and then only include columns if they appear in that list deferred_columns = {} query.deferred_to_data(deferred_columns, query.deferred_to_columns_cb) inherited_db_tables = [x._meta.db_table for x in get_concrete_parents(self.model)] only_load = list(chain(*[list(deferred_columns.get(x, [])) for x in inherited_db_tables])) if query.select: for x in query.select: if hasattr(x, "field"): # In Django 1.6+ 'x' above is a SelectInfo (which is a tuple subclass), whereas in 1.5 it's a tuple # in 1.6 x[1] == Field, but 1.5 x[1] == unicode (column name) if x.field is None: column = x.col.col[1] # This is the column we are getting lookup_type = x.col.lookup_type self.distinct_on_field = column # This whole section of code is weird, and is probably better implemented as a custom Query type (like QueryByKeys) # basically, appengine gives back dates as a time since the epoch, we convert it to a date, then floor it, then convert it back # in our transform function. The transform is applied when the results are read back so that only distinct values are returned. # this is very hacky... if lookup_type in DATE_TRANSFORMS: self.distinct_field_convertor = lambda value: DATE_TRANSFORMS[lookup_type](self.connection, value) else: raise CouldBeSupportedError("Unhandled lookup_type %s" % lookup_type) else: column = x.field.column else: column = x[1] if only_load and column not in only_load: continue self.queried_fields.append(column) else: # If no specific fields were specified, select all fields if the query is distinct (as App Engine only supports # distinct on projection queries) or the ones specified by only_load self.queried_fields = [x.column for x in opts.fields if (x.column in only_load) or self.distinct] self.keys_only = keys_only or self.queried_fields == [opts.pk.column] # Projection queries don't return results unless all projected fields are # indexed on the model. This means if you add a field, and all fields on the model # are projectable, you will never get any results until you've resaved all of them. # Because it's not possible to detect this situation, we only try a projection query if a # subset of fields was specified (e.g. values_list('bananas')) which makes the behaviour a # bit more predictable. It would be nice at some point to add some kind of force_projection() # thing on a queryset that would do this whenever possible, but that's for the future, maybe. try_projection = (self.keys_only is False) and bool(self.queried_fields) if not self.queried_fields: # If we don't have any queried fields yet, it must have been an empty select and not a distinct # and not an only/defer, so get all the fields self.queried_fields = [ x.column for x in opts.fields ] self.excluded_pks = set() self.has_inequality_filter = False self.all_filters = [] self.results = None self.gae_query = None projection_fields = [] if try_projection: for field in self.queried_fields: # We don't include the primary key in projection queries... if field == self.pk_col: order_fields = set([ x.strip("-") for x in self.ordering]) if self.pk_col in order_fields or "pk" in order_fields: # If we were ordering on __key__ we can't do a projection at all self.projection_fields = [] break continue # Text and byte fields aren't indexed, so we can't do a # projection query f = get_field_from_column(self.model, field) if not f: raise CouldBeSupportedError("Attempting a cross-table select or dates query, or something?!") assert f # If this happens, we have a cross-table select going on! #FIXME db_type = f.db_type(connection) if db_type in ("bytes", "text", "list", "set"): projection_fields = [] break projection_fields.append(field) self.projection = list(set(projection_fields)) or None if opts.parents: self.projection = None if isinstance(query.where, EmptyWhere): # Empty where means return nothing! raise EmptyResultSet() else: try: where_tables = _get_tables_from_where(query.where) except TypeError: # This exception is thrown by get_group_by_cols if one of the constraints is a SubQueryConstraint # yeah, we can't do that. self.unsupported_query_message = "Subquery WHERE constraints aren't supported" return if where_tables and where_tables != [ query.model._meta.db_table ]: # Mark this query as unsupported and return self.unsupported_query_message = "Cross-join WHERE constraints aren't supported: %s" % _cols_from_where_node(query.where) return from dnf import parse_dnf try: self.where, columns, self.excluded_pks = parse_dnf(query.where, self.connection, ordering=self.ordering) except NotSupportedError as e: # Mark this query as unsupported and return self.unsupported_query_message = str(e) return # DISABLE PROJECTION IF WE ARE FILTERING ON ONE OF THE PROJECTION_FIELDS for field in self.projection or []: if field in columns: self.projection = None break try: # If the PK was queried, we switch it in our queried # fields store with __key__ pk_index = self.queried_fields.index(self.pk_col) self.queried_fields[pk_index] = "__key__" except ValueError: pass
def make_atom(self, child, qn, connection): lvalue, lookup_type, value_annot, param = child kwargs = {'connection': connection} if VERSION[:2] >= (1, 3) else {} if lvalue and lvalue.field and hasattr( lvalue.field, 'db_type') and lvalue.field.db_type(**kwargs) == 'hstore': try: lvalue, params = lvalue.process(lookup_type, param, connection) except EmptyShortCircuit: raise EmptyResultSet() field = self.sql_for_columns(lvalue, qn, connection) if lookup_type == 'exact': if isinstance(param, dict): return ('{0} = %s'.format(field), [param]) raise ValueError('invalid value') elif lookup_type in ('gt', 'gte', 'lt', 'lte'): if isinstance(param, dict) and len(param) == 1: sign = (lookup_type[0] == 'g' and '>%s' or '<%s') % (lookup_type[-1] == 'e' and '=' or '') param_keys = list(param.keys()) return ('%s->\'%s\' %s %%s' % (field, param_keys[0], sign), param.values()) raise ValueError('invalid value') elif lookup_type in ['contains', 'icontains']: if isinstance(param, dict): values = list(param.values()) keys = list(param.keys()) if len(values) == 1 and isinstance(values[0], (list, tuple)): return ('%s->\'%s\' = ANY(%%s)' % (field, keys[0]), [[str(x) for x in values[0]]]) return ('%s @> %%s' % field, [param]) elif isinstance(param, (list, tuple)): if len(param) < 2: return ('%s ? %%s' % field, [param[0]]) if param: return ('%s ?& %%s' % field, [param]) raise ValueError('invalid value') elif isinstance(param, six.string_types): # if looking for a string perform the normal text lookup # that is: look for occurence of string in all the keys pass else: raise ValueError('invalid value') elif lookup_type == 'isnull': # do not perform any special format return super(HStoreWhereNode, self).make_atom(child, qn, connection) else: raise TypeError('invalid lookup type') return super(HStoreWhereNode, self).make_atom(child, qn, connection)
def _process_where_node(self, parent, query): """ Process a single WHERE node, possibly recursing. :param parent: The parent 'WHERE' clause :return: """ if isinstance(parent, EmptyWhere): raise EmptyResultSet() where = [] for child in parent.children: if isinstance(child, WhereNode): where.append(self._process_where_node(child, query)) continue # Field as represented in the query query_field = child.lhs.output_field # Field of the DB table for the model real_field = child.lhs.target # Origin field (if this is a foreign field) origin_field = real_field.related_field if real_field.rel else real_field was_list = isinstance(child.rhs, (list, tuple)) rhs_value = origin_field.get_db_prep_lookup(child.lookup_name, child.rhs, self.connection, prepared=True) if not was_list and rhs_value != []: rhs_value = rhs_value[0] if origin_field.primary_key: if real_field.rel: # If we're a related field, use the foreign table, # but also don't use META(id), since it's actually embedded lhs_table = real_field.related_model._meta.db_table lhs = real_field.column else: lhs_table = query.alias_map[child.lhs.alias].table_name lhs = 'META({}).id'.format(n1ql_escape(BUCKET_PLACEHOLDER)) if real_field.get_internal_type() in ('IntegerField', 'AutoField'): # This could be cast as a string, so cast it back as an int castfn = int else: castfn = lambda x_: x_ if isinstance(rhs_value, (list, tuple)): rhs_value = [ DocID.encode(lhs_table, castfn(x)) for x in rhs_value ] else: rhs_value = DocID.encode(lhs_table, castfn(rhs_value)) else: lhs = n1ql_escape(query_field.column) real_field = query_field placeholder = self.params.indexstr() rhs_value, criteria = Operators.convert(rhs_value, lhs, placeholder, child.lookup_name, real_field) self.params.add(rhs_value) where.append(' '.join(criteria)) if where: connector = ' ' + parent.connector + ' ' neg = 'NOT ' if parent.negated else '' return '(' + neg + (connector.join(where)) + ')' else: return None
def _prepare_for_transformation(self): from django.db.models.sql.where import EmptyWhere if isinstance(self.django_query.where, EmptyWhere): # Empty where means return nothing! raise EmptyResultSet()
def parse_constraint(child, connection, negated=False): if isinstance(child, tuple): # First, unpack the constraint constraint, op, annotation, value = child was_list = isinstance(value, (list, tuple)) if isinstance(value, query.Query): value = value.get_compiler(connection.alias).as_sql()[0].execute() else: packed, value = constraint.process(op, value, connection) alias, column, db_type = packed field = constraint.field else: # Django 1.7+ field = child.lhs.target column = child.lhs.target.column op = child.lookup_name value = child.rhs annotation = value was_list = isinstance(value, (list, tuple)) if isinstance(value, query.Query): value = value.get_compiler(connection.alias).as_sql()[0].execute() elif value != []: value = child.lhs.output_field.get_db_prep_lookup( child.lookup_name, child.rhs, connection, prepared=True) is_pk = field and field.primary_key if column == "id" and op == "iexact" and is_pk and isinstance(field, AutoField): # When new instance is created, automatic primary key 'id' does not generate '_idx_iexact_id'. # As the primary key 'id' (AutoField) is integer and is always case insensitive, we can deal with 'id_iexact=' query by using 'exact' rather than 'iexact'. op = "exact" if field and field.db_type(connection) in ("bytes", "text"): raise NotSupportedError("Text and Blob fields are not indexed by the datastore, so you can't filter on them") if op not in REQUIRES_SPECIAL_INDEXES: # Don't convert if this op requires special indexes, it will be handled there if field: value = [ connection.ops.prep_lookup_value(field.model, x, field, column=column) for x in value] # Don't ask me why, but on Django 1.6 constraint.process on isnull wipes out the value (it returns an empty list) # so we have to special case this to use the annotation value instead if op == "isnull": if annotation is not None: value = [ annotation ] if is_pk and value[0]: raise EmptyResultSet() if not was_list: value = value[0] else: if negated: raise CouldBeSupportedError("Special indexing does not currently supported negated queries. See #80") if not was_list: value = value[0] add_special_index(field.model, column, op) # Add the index if we can (e.g. on dev_appserver) if op not in special_indexes_for_column(field.model, column): raise RuntimeError("There is a missing index in your djangaeidx.yaml - \n\n{0}:\n\t{1}: [{2}]".format( field.model, column, op) ) indexer = REQUIRES_SPECIAL_INDEXES[op] value = indexer.prep_value_for_query(value) column = indexer.indexed_column_name(column, value=value) op = indexer.prep_query_operator(op) return column, op, value
def parse_where_and_check_projection(self, where, negated=False): """ recursively parse the where tree and return a list of tuples of (column, match_type, value), e.g. ('name', 'exact', 'John'). """ result = [] if where.negated: negated = not negated if negated and where.connector != AND: raise NotSupportedError("Only AND filters are supported for negated queries") for child in where.children: if isinstance(child, tuple): constraint, op, annotation, value = child if isinstance(value, (list, tuple)): value = [ self.connection.ops.prep_lookup_value(self.model, x, constraint.field) for x in value] else: value = self.connection.ops.prep_lookup_value(self.model, value, constraint.field) #Disable projection if it's not supported if self.projection and constraint.col in self.projection: if op in ("exact", "in", "isnull"): #If we are projecting, but we are doing an #equality filter on one of the columns, then we #can't project self.projection = None if negated: if op in ("exact", "in") and constraint.field.primary_key: self.excluded_pks.append(value) #else: FIXME when excluded_pks is handled, we can put the #next section in an else block if op == "exact": if self.has_inequality_filter: raise RuntimeError("You can only specify one inequality filter per query") col = constraint.col result.append((col, "gt_and_lt", value)) self.has_inequality_filter = True else: raise RuntimeError("Unsupported negated lookup: " + op) else: if constraint.field.primary_key: if (value is None and op == "exact") or (op == "isnull" and value): #If we are looking for a primary key that is None, then we always #just return nothing raise EmptyResultSet() elif op in ("exact", "in"): if isinstance(value, (list, tuple)): self.included_pks.extend(list(value)) else: self.included_pks.append(value) else: col = constraint.col result.append((col, op, value)) else: col = constraint.col result.append((col, op, value)) else: result.extend(self.parse_where_and_check_projection(child, negated)) return result
def _transform_query_17(connection, kind, query): from django.db.models.sql.datastructures import Date, DateTime if isinstance(query.where, EmptyWhere): # Empty where means return nothing! raise EmptyResultSet() # Check for joins, we ignore select related tables as they aren't actually used (the connector marks select # related as unsupported in its features) tables = [k for k, v in query.alias_refcount.items() if v] inherited_tables = set( [x._meta.db_table for x in query.model._meta.parents]) select_related_tables = set([y[0][0] for y in query.related_select_cols]) tables = set(tables) - inherited_tables - select_related_tables if len(tables) > 1: raise NotSupportedError(""" The appengine database connector does not support JOINs. The requested join map follows\n %s """ % query.join_map) ret = Query(query.model, kind) ret.connection = connection # Add the root concrete table as the source table root_table = get_top_concrete_parent(query.model)._meta.db_table ret.add_source_table(root_table) # Extract the ordering of the query results for order_col in _extract_ordering_from_query_17(query): ret.add_order_by(order_col) # Extract any projected columns (values/values_list/only/defer) for projected_col in _extract_projected_columns_from_query_17(query): ret.add_projected_column(projected_col) for potential_annotation in query.select: col = getattr(potential_annotation, "col", None) if not col: continue if isinstance(col, (Date, DateTime)): ret.add_annotation(col.col[-1], col) # Add any extra selects for col, select in query.extra_select.items(): ret.add_extra_select(col, select[0]) # This must happen after extracting projected cols if query.distinct: ret.set_distinct(list(query.distinct_fields)) # Extract any query offsets and limits ret.low_mark = query.low_mark ret.high_mark = query.high_mark output = WhereNode() output.connector = query.where.connector _walk_django_where(query, _django_17_query_walk_trunk, _django_17_query_walk_leaf, new_parent=output, connection=connection, model=query.model) # If there no child nodes, just wipe out the where if not output.children: output = None ret.where = output return ret
def normalize_query(node, connection, negated=False, filtered_columns=None, _inequality_property=None): """ Converts a django_where_tree which is CNF to a DNF for use in the datastore. This function does a lot of heavy lifting so optimization is welcome. The connection is needed for calculating the values on the literals, it would be nice if we can remove that so this only has one task, but it's fine for now """ _inequality_property = _inequality_property if _inequality_property is not None else [] def check_inequality_usage(_op, _column, _current_inequality): if _op in INEQUALITY_OPERATORS: if _current_inequality and _current_inequality[0] != _column: raise NotSupportedError( "You can only specify inequality filters ({}) on one property. There is already one on {}, you're attempting to use one on {}".format( INEQUALITY_OPERATORS, _current_inequality[0], _column ) ) else: #We have to use a list, because Python 2.x doesn't have 'nonlocal' :/ _current_inequality.append(_column) if isinstance(node, tuple) and isinstance(node[0], Constraint): # This is where contraints are exploded # # column, op, value = parse_constraint(node, connection) if filtered_columns is not None: assert isinstance(filtered_columns, set) filtered_columns.add(column) if op == 'in': # Explode INs into OR if not isinstance(value, (list, tuple, set)): raise ValueError("IN queries must be supplied a list of values") if negated: check_inequality_usage(">", column, _inequality_property) return ('OR', [ ('OR', [(column, '>', x), (column, '<', x)]) for x in value ]) else: if len(value) == 1: return (column, '=', value[0]) return ('OR', [(column, '=', x) for x in value]) if op not in OPERATORS_MAP: raise NotSupportedError("Unsupported operator %s" % op) _op = OPERATORS_MAP[op] check_inequality_usage(_op, column, _inequality_property) #Check we aren't doing an additional inequality if negated and _op == '=': # Explode check_inequality_usage('>', column, _inequality_property) return ('OR', [(column, '>', value), (column, '<', value)]) elif op == "startswith": #You can emulate starts with by adding the last unicode char #to the value, then doing <=. Genius. if value.endswith("%"): value = value[:-1] end_value = value[:] if isinstance(end_value, str): end_value = end_value.decode("utf-8") end_value += u'\ufffd' check_inequality_usage('>=', column, _inequality_property) if not negated: return ('AND', [(column, '>=', value), (column, '<', end_value)]) else: return ('OR', [(column, '<', value), (column, '>=', end_value)]) elif op == "isnull": if (value and not negated) or (not value and negated): #We're checking for isnull=True #If we are checking that a primary key isnull, then don't do an impossible query! if node[0].field.primary_key: raise EmptyResultSet() return (column, "=", None) else: #We're checking for isnull=False check_inequality_usage('>', column, _inequality_property) return ('OR', [(column, '>', None), (column, '<', None)]) elif _op == None: raise NotSupportedError("Unhandled lookup type %s" % op) return (column, _op, value) else: # This is where the distribution is applied over the children if not node.children: return [] if node.negated: negated = True if len(node.children) > 1: # If there is more than one child then attempt to reduce exploded = (node.connector, [ normalize_query(child, connection, negated=negated, filtered_columns=filtered_columns, _inequality_property=_inequality_property) for child in node.children ]) if node.connector == 'AND': if len(exploded[1]) > 1: # We need to know if there are any ORs under this AND node _ors = [x for x in exploded[1] if x[0] == 'OR' ] def special_product(*args): """ Modified product to return the prods in an AND container """ pools = map(tuple, args) result = [[]] for pool in pools: result = [x+[y] if y[0] != 'AND' else x+y[1] for x in result for y in pool] for prod in result: yield ('AND', list(prod)) if len(_ors) > 0: # This is a bit complicated to explain in a comment # But you can calculate the DNF by doing a clever product on the children of the AND node # It's all to do with grouping the ORs in groups and the ANDs individualy # This returns a DNF which is naturally wrapped in an OR _literals = (x for x in exploded[1] if x[0] != 'OR') flat_ors = (x[1] if x[0] == 'OR' else x for x in _ors) flat_literals = ([y] if y[0] != 'AND' else ([q] for q in y[1]) for y in _literals) return ('OR', [x for x in special_product(*chain(flat_ors, flat_literals))]) elif node.connector == 'OR': if all(x[0] == 'OR' for x in exploded[1]): return ('OR', list(chain.from_iterable((x[1] for x in exploded[1])))) else: exploded = normalize_query( node.children[0], connection, negated=negated, filtered_columns=filtered_columns, _inequality_property=_inequality_property ) if exploded[0] == exploded[1][0][0]: # crush any single child 'OR' or 'AND' hangover return (exploded[0], [x for x in exploded[1][0][1]]) return exploded