def _validate_query_is_possible(self, query): """ Given the *django* query, check the following: - The query only has one inequality filter - The query does no joins - The query ordering is compatible with the filters """ #Check for joins if query.count_active_tables() > 1: raise NotSupportedError(""" The appengine database connector does not support JOINs. The requested join map follows\n %s """ % query.join_map) if query.aggregates: if query.aggregates.keys() == [ None ]: if query.aggregates[None].col != "*": raise NotSupportedError("Counting anything other than '*' is not supported") else: raise NotSupportedError("Unsupported aggregate query")
def check_inequality_usage(_op, _column, _current_inequality): if _op in INEQUALITY_OPERATORS: if _current_inequality and _current_inequality[0] != _column: raise NotSupportedError( "You can only specify inequality filters ({}) on one property. There is already one on {}, you're attempting to use one on {}".format( INEQUALITY_OPERATORS, _current_inequality[0], _column ) ) else: #We have to use a list, because Python 2.x doesn't have 'nonlocal' :/ _current_inequality.append(_column)
def txn(): if key is not None: if utils.key_exists(key): raise IntegrityError("Tried to INSERT with existing key") id_or_name = key.id_or_name() if isinstance(id_or_name, basestring) and id_or_name.startswith("__"): raise NotSupportedError("Datastore ids cannot start with __. Id was %s" % id_or_name) markers = constraints.acquire(self.model, ent) try: results.append(datastore.Put(ent)) caching.add_entity_to_context_cache(self.model, ent) except: #Make sure we delete any created markers before we re-raise constraints.release_markers(markers) raise
def parse_constraint(child, connection): #First, unpack the constraint constraint, op, annotation, value = child was_list = isinstance(value, (list, tuple)) packed, value = constraint.process(op, value, connection) alias, column, db_type = packed if constraint.field.db_type(connection) in ("bytes", "text"): raise NotSupportedError("Text and Blob fields are not indexed by the datastore, so you can't filter on them") if op not in REQUIRES_SPECIAL_INDEXES: #Don't convert if this op requires special indexes, it will be handled there value = [ connection.ops.prep_lookup_value(constraint.field.model, x, constraint.field, constraint=constraint) for x in value] #Don't ask me why, but constraint.process on isnull wipes out the value (it returns an empty list) # so we have to special case this to use the annotation value instead if op == "isnull": value = [ annotation ] if not was_list: value = value[0] else: if not was_list: value = value[0] add_special_index(constraint.field.model, column, op) #Add the index if we can (e.g. on dev_appserver) if op not in special_indexes_for_column(constraint.field.model, column): raise RuntimeError("There is a missing index in your djangaeidx.yaml - \n\n{0}:\n\t{1}: [{2}]".format( constraint.field.model, column, op) ) indexer = REQUIRES_SPECIAL_INDEXES[op] column = indexer.indexed_column_name(column) value = indexer.prep_value_for_query(value) op = indexer.prep_query_operator(op) return column, op, value
def parse_where_and_check_projection(self, where, negated=False): """ recursively parse the where tree and return a list of tuples of (column, match_type, value), e.g. ('name', 'exact', 'John'). """ result = [] if where.negated: negated = not negated if negated and where.connector != AND: raise NotSupportedError("Only AND filters are supported for negated queries") for child in where.children: if isinstance(child, tuple): constraint, op, annotation, value = child if isinstance(value, (list, tuple)): value = [ self.connection.ops.prep_lookup_value(self.model, x, constraint.field) for x in value] else: value = self.connection.ops.prep_lookup_value(self.model, value, constraint.field) #Disable projection if it's not supported if self.projection and constraint.col in self.projection: if op in ("exact", "in", "isnull"): #If we are projecting, but we are doing an #equality filter on one of the columns, then we #can't project self.projection = None if negated: if op in ("exact", "in") and constraint.field.primary_key: self.excluded_pks.append(value) #else: FIXME when excluded_pks is handled, we can put the #next section in an else block if op == "exact": if self.has_inequality_filter: raise RuntimeError("You can only specify one inequality filter per query") col = constraint.col result.append((col, "gt_and_lt", value)) self.has_inequality_filter = True else: raise RuntimeError("Unsupported negated lookup: " + op) else: if constraint.field.primary_key: if (value is None and op == "exact") or (op == "isnull" and value): #If we are looking for a primary key that is None, then we always #just return nothing raise EmptyResultSet() elif op in ("exact", "in"): if isinstance(value, (list, tuple)): self.included_pks.extend(list(value)) else: self.included_pks.append(value) else: col = constraint.col result.append((col, op, value)) else: col = constraint.col result.append((col, op, value)) else: result.extend(self.parse_where_and_check_projection(child, negated)) return result
def __init__(self, connection, query, keys_only=False, all_fields=False): self.original_query = query opts = query.get_meta() if not query.default_ordering: self.ordering = query.order_by else: self.ordering = query.order_by or opts.ordering if self.ordering: ordering = [ x for x in self.ordering if not (isinstance(x, basestring) and "__" in x) ] if len(ordering) < len(self.ordering): if not on_production() and not in_testing(): diff = set(self.ordering) - set(ordering) log_once(DJANGAE_LOG.warning, "The following orderings were ignored as cross-table orderings are not supported on the datastore: %s", diff) self.ordering = ordering self.distinct_values = set() self.distinct_on_field = None self.distinct_field_convertor = None self.queried_fields = [] if keys_only: self.queried_fields = [ opts.pk.column ] elif not all_fields: for x in query.select: if isinstance(x, tuple): #Django < 1.6 compatibility self.queried_fields.append(x[1]) else: self.queried_fields.append(x.col[1]) if x.lookup_type == 'year': assert self.distinct_on_field is None self.distinct_on_field = x.col[1] self.distinct_field_convertor = field_conv_year_only elif x.lookup_type == 'month': assert self.distinct_on_field is None self.distinct_on_field = x.col[1] self.distinct_field_convertor = field_conv_month_only elif x.lookup_type == 'day': assert self.distinct_on_field is None self.distinct_on_field = x.col[1] self.distinct_field_convertor = field_conv_day_only else: raise NotSupportedError("Unhandled lookup type: {0}".format(x.lookup_type)) #Projection queries don't return results unless all projected fields are #indexed on the model. This means if you add a field, and all fields on the model #are projectable, you will never get any results until you've resaved all of them. #Because it's not possible to detect this situation, we only try a projection query if a #subset of fields was specified (e.g. values_list('bananas')) which makes the behaviour a #bit more predictable. It would be nice at some point to add some kind of force_projection() #thing on a queryset that would do this whenever possible, but that's for the future, maybe. try_projection = bool(self.queried_fields) if not self.queried_fields: self.queried_fields = [ x.column for x in opts.fields ] self.connection = connection self.pk_col = opts.pk.column self.model = query.model self.is_count = query.aggregates self.keys_only = False #FIXME: This should be used where possible self.included_pks = [] self.excluded_pks = [] self.has_inequality_filter = False self.all_filters = [] self.results = None self.extra_select = query.extra_select self.gae_query = None self._set_db_table() self._validate_query_is_possible(query) projection_fields = [] if try_projection: for field in self.queried_fields: #We don't include the primary key in projection queries... if field == self.pk_col: continue #Text and byte fields aren't indexed, so we can't do a #projection query f = get_field_from_column(self.model, field) if not f: raise NotImplementedError("Attemping a cross-table select. Maybe? #FIXME") assert f #If this happens, we have a cross-table select going on! #FIXME db_type = f.db_type(connection) if db_type in ("bytes", "text"): projection_fields = [] break projection_fields.append(field) self.projection = list(set(projection_fields)) or None if opts.parents: self.projection = None self.where = self.parse_where_and_check_projection(query.where) try: #If the PK was queried, we switch it in our queried #fields store with __key__ pk_index = self.queried_fields.index(self.pk_col) self.queried_fields[pk_index] = "__key__" #If the only field queried was the key, then we can do a keys_only #query self.keys_only = len(self.queried_fields) == 1 except ValueError: pass
def normalize_query(node, connection, negated=False, filtered_columns=None, _inequality_property=None): """ Converts a django_where_tree which is CNF to a DNF for use in the datastore. This function does a lot of heavy lifting so optimization is welcome. The connection is needed for calculating the values on the literals, it would be nice if we can remove that so this only has one task, but it's fine for now """ _inequality_property = _inequality_property if _inequality_property is not None else [] def check_inequality_usage(_op, _column, _current_inequality): if _op in INEQUALITY_OPERATORS: if _current_inequality and _current_inequality[0] != _column: raise NotSupportedError( "You can only specify inequality filters ({}) on one property. There is already one on {}, you're attempting to use one on {}".format( INEQUALITY_OPERATORS, _current_inequality[0], _column ) ) else: #We have to use a list, because Python 2.x doesn't have 'nonlocal' :/ _current_inequality.append(_column) if isinstance(node, tuple) and isinstance(node[0], Constraint): # This is where contraints are exploded # # column, op, value = parse_constraint(node, connection) if filtered_columns is not None: assert isinstance(filtered_columns, set) filtered_columns.add(column) if op == 'in': # Explode INs into OR if not isinstance(value, (list, tuple, set)): raise ValueError("IN queries must be supplied a list of values") if negated: check_inequality_usage(">", column, _inequality_property) return ('OR', [ ('OR', [(column, '>', x), (column, '<', x)]) for x in value ]) else: if len(value) == 1: return (column, '=', value[0]) return ('OR', [(column, '=', x) for x in value]) if op not in OPERATORS_MAP: raise NotSupportedError("Unsupported operator %s" % op) _op = OPERATORS_MAP[op] check_inequality_usage(_op, column, _inequality_property) #Check we aren't doing an additional inequality if negated and _op == '=': # Explode check_inequality_usage('>', column, _inequality_property) return ('OR', [(column, '>', value), (column, '<', value)]) elif op == "startswith": #You can emulate starts with by adding the last unicode char #to the value, then doing <=. Genius. if value.endswith("%"): value = value[:-1] end_value = value[:] if isinstance(end_value, str): end_value = end_value.decode("utf-8") end_value += u'\ufffd' check_inequality_usage('>=', column, _inequality_property) if not negated: return ('AND', [(column, '>=', value), (column, '<', end_value)]) else: return ('OR', [(column, '<', value), (column, '>=', end_value)]) elif op == "isnull": if (value and not negated) or (not value and negated): #We're checking for isnull=True #If we are checking that a primary key isnull, then don't do an impossible query! if node[0].field.primary_key: raise EmptyResultSet() return (column, "=", None) else: #We're checking for isnull=False check_inequality_usage('>', column, _inequality_property) return ('OR', [(column, '>', None), (column, '<', None)]) elif _op == None: raise NotSupportedError("Unhandled lookup type %s" % op) return (column, _op, value) else: # This is where the distribution is applied over the children if not node.children: return [] if node.negated: negated = True if len(node.children) > 1: # If there is more than one child then attempt to reduce exploded = (node.connector, [ normalize_query(child, connection, negated=negated, filtered_columns=filtered_columns, _inequality_property=_inequality_property) for child in node.children ]) if node.connector == 'AND': if len(exploded[1]) > 1: # We need to know if there are any ORs under this AND node _ors = [x for x in exploded[1] if x[0] == 'OR' ] def special_product(*args): """ Modified product to return the prods in an AND container """ pools = map(tuple, args) result = [[]] for pool in pools: result = [x+[y] if y[0] != 'AND' else x+y[1] for x in result for y in pool] for prod in result: yield ('AND', list(prod)) if len(_ors) > 0: # This is a bit complicated to explain in a comment # But you can calculate the DNF by doing a clever product on the children of the AND node # It's all to do with grouping the ORs in groups and the ANDs individualy # This returns a DNF which is naturally wrapped in an OR _literals = (x for x in exploded[1] if x[0] != 'OR') flat_ors = (x[1] if x[0] == 'OR' else x for x in _ors) flat_literals = ([y] if y[0] != 'AND' else ([q] for q in y[1]) for y in _literals) return ('OR', [x for x in special_product(*chain(flat_ors, flat_literals))]) elif node.connector == 'OR': if all(x[0] == 'OR' for x in exploded[1]): return ('OR', list(chain.from_iterable((x[1] for x in exploded[1])))) else: exploded = normalize_query( node.children[0], connection, negated=negated, filtered_columns=filtered_columns, _inequality_property=_inequality_property ) if exploded[0] == exploded[1][0][0]: # crush any single child 'OR' or 'AND' hangover return (exploded[0], [x for x in exploded[1][0][1]]) return exploded