def _extract_projected_columns_from_query_17(query): result = [] if query.select: for x in query.select: if x.field is None: model = get_model_from_db_table(x.col.col[0]) if get_top_concrete_parent(model) != get_top_concrete_parent(query.model): raise NotSupportedError("Attempted a cross-join select which is not supported on the datastore") column = x.col.col[1] # This is the column we are getting else: column = x.field.column result.append(column) return result else: # If the query uses defer()/only() then we need to process deferred. We have to get all deferred columns # for all (concrete) inherited models and then only include columns if they appear in that list only_load = query.get_loaded_field_names() if only_load: for field, model in query.model._meta.get_concrete_fields_with_model(): model = model or query.model try: if field.name in only_load[model]: # Add a field that has been explicitly included result.append(field.column) except KeyError: # Model wasn't explicitly listed in the only_load table # Therefore, we need to load all fields from this model result.append(field.column) return result else: return []
def set_leaf(self, column, operator, value, is_pk_field, negated, namespace, target_field=None): assert column assert operator assert isinstance(is_pk_field, bool) assert isinstance(negated, bool) if operator == "iexact" and isinstance(target_field, AutoField): # When new instance is created, automatic primary key 'id' does not generate '_idx_iexact_id'. # As the primary key 'id' (AutoField) is integer and is always case insensitive, # we can deal with 'id_iexact=' query by using 'exact' rather than 'iexact'. operator = "exact" value = int(value) if is_pk_field: # If this is a primary key, we need to make sure that the value # we pass to the query is a datastore Key. We have to deal with IN queries here # because they aren't flattened until the DNF stage model = get_top_concrete_parent(target_field.model) table = model._meta.db_table if isinstance(value, (list, tuple)): value = [ datastore.Key.from_path(table, x, namespace=namespace) for x in value if x ] else: if operator == "isnull" and value is True: # FIXME: Strictly, this isn't correct, this could be one of several branches # but id=None filters are silly anyway. This should be moved to after normalization.. # probably. This fixes a test in Django which does this in get_or_create for some reason raise EmptyResultSet() if not value: # Empty strings and 0 are forbidden as keys # so make this an impossible filter # FIXME: This is a hack! It screws with the ordering # because it's an inequality. Instead we should wipe this # filter out when preprocessing in the DNF (because it's impossible) value = datastore.Key.from_path('', 1) operator = '<' else: value = datastore.Key.from_path(table, value, namespace=namespace) column = "__key__" # Do any special index conversions necessary to perform this lookup if operator in REQUIRES_SPECIAL_INDEXES: if is_pk_field: column = model._meta.pk.column value = unicode(value.id_or_name()) add_special_index(target_field.model, column, operator, value) indexer = REQUIRES_SPECIAL_INDEXES[operator] index_type = indexer.prepare_index_type(operator, value) value = indexer.prep_value_for_query(value) column = indexer.indexed_column_name(column, value, index_type) operator = indexer.prep_query_operator(operator) self.column = column self.operator = convert_operator(operator) self.value = value
def get_transformed_query(self): self._prepare_for_transformation() kind = self._determine_query_kind() ret = Query(self.model, kind) ret.connection = self.connection # Add the root concrete table as the source table root_table = get_top_concrete_parent(self.model)._meta.db_table ret.add_source_table(root_table) self._apply_ordering_to_query(ret) self._set_projected_columns_on_query(ret) self._apply_extra_selects_to_query(ret) self._apply_distinct_columns_to_query(ret) self._apply_annotations_to_query(ret) # Extract any query offsets and limits ret.low_mark = self.django_query.low_mark ret.high_mark = self.django_query.high_mark output = self._generate_where_node() # If there no child nodes, just wipe out the where if not output.children: output = None ret.where = output return ret
def __init__(self, model, kind): assert kind in VALID_QUERY_KINDS self.model = model self.concrete_model = get_top_concrete_parent(model) self.kind = kind self.projection_possible = True self.tables = [] self.columns = None # None means all fields self.init_list = [] self.distinct = False self.order_by = [] self.row_data = [] # For insert/updates self._where = None self.low_mark = self.high_mark = None self.annotations = [] self.per_entity_annotations = [] self.extra_selects = [] self.polymodel_filter_added = False # A list of PKs that should be excluded from the resultset self.excluded_pks = set()
def _set_db_table(self): """ Work out which Datastore kind we should actually be querying. This allows for poly models, i.e. non-abstract parent models which we support by storing all fields for both the parent model and its child models on the parent table. """ inheritance_root = get_top_concrete_parent(self.model) self.db_table = inheritance_root._meta.db_table
def _transform_query_18(connection, kind, query): from django.db.models.sql.where import EmptyWhere if isinstance(query.where, EmptyWhere): # Empty where means return nothing! raise EmptyResultSet() ret = Query(query.model, kind) ret.connection = connection # Add the root concrete table as the source table root_table = get_top_concrete_parent(query.model)._meta.db_table ret.add_source_table(root_table) # Extract the ordering of the query results for order_col in _extract_ordering_from_query_18(query): ret.add_order_by(order_col) # Extract any projected columns (values/values_list/only/defer) for projected_col in _extract_projected_columns_from_query_18(query): ret.add_projected_column(projected_col) # Add any extra selects for col, select in query.extra_select.items(): ret.add_extra_select(col, select[0]) if query.distinct: # This must happen after extracting projected cols ret.set_distinct(list(query.distinct_fields)) # Process annotations! if query.annotation_select: for k, v in query.annotation_select.items(): ret.add_annotation(k, v) # Extract any query offsets and limits ret.low_mark = query.low_mark ret.high_mark = query.high_mark output = WhereNode() output.connector = query.where.connector _walk_django_where( query, _django_18_query_walk_trunk, _django_18_query_walk_leaf, new_parent=output, connection=connection, negated=query.where.negated, model=query.model ) # If there no child nodes, just wipe out the where if not output.children: output = None ret.where = output return ret
def set_leaf(self, column, operator, value, is_pk_field, negated, lookup_name, namespace, target_field=None): assert column assert operator assert isinstance(is_pk_field, bool) assert isinstance(negated, bool) if operator == "iexact" and isinstance(target_field, AutoField): # When new instance is created, automatic primary key 'id' does not generate '_idx_iexact_id'. # As the primary key 'id' (AutoField) is integer and is always case insensitive, # we can deal with 'id_iexact=' query by using 'exact' rather than 'iexact'. operator = "exact" value = int(value) if is_pk_field: # If this is a primary key, we need to make sure that the value # we pass to the query is a datastore Key. We have to deal with IN queries here # because they aren't flattened until the DNF stage model = get_top_concrete_parent(target_field.model) table = model._meta.db_table if isinstance(value, (list, tuple)): value = [ datastore.Key.from_path(table, x, namespace=namespace) for x in value if x ] else: if (operator == "isnull" and value is True) or not value: # id=None will never return anything and # Empty strings and 0 are forbidden as keys self.will_never_return_results = True else: value = datastore.Key.from_path(table, value, namespace=namespace) column = "__key__" # Do any special index conversions necessary to perform this lookup special_indexer = get_indexer(target_field, operator) if special_indexer: if is_pk_field: column = model._meta.pk.column value = unicode(value.id_or_name()) add_special_index(target_field.model, column, special_indexer, operator, value) index_type = special_indexer.prepare_index_type(operator, value) value = special_indexer.prep_value_for_query(value) column = special_indexer.indexed_column_name(column, value, index_type) operator = special_indexer.prep_query_operator(operator) self.column = column self.operator = convert_operator(operator) self.value = value self.lookup_name = lookup_name
def __init__(self, connection, query): self.model = query.model self.select = SelectCommand(connection, query, keys_only=True) self.query = self.select.query self.namespace = connection.ops.connection.settings_dict.get("NAMESPACE") # It seems query.tables is populated in most cases, but I have seen cases (albeit in testing) # where this isn't the case (particularly when not filtering on anything). In that case # fallback to the model table (perhaps we should do self.table_to_delete = ( query.tables[0] if query.tables else utils.get_top_concrete_parent(query.model)._meta.db_table )
def unique_identifiers_from_entity(model, entity, ignore_pk=False, ignore_null_values=True): """ Given an instance, this function returns a list of identifier strings that represent unique field/value combinations. """ from djangae.db.utils import get_top_concrete_parent unique_combinations = _unique_combinations(model, ignore_pk) meta = model._meta identifiers = [] for combination in unique_combinations: combo_identifiers = [[]] include_combination = True for field_name in combination: field = meta.get_field(field_name) if field.primary_key: value = entity.key().id_or_name() else: value = entity.get(field.column) # Get the value from the entity # If ignore_null_values is True, then we don't include combinations where the value is None # or if the field is a multivalue field where None means no value (you can't store None in a list) if (value is None and ignore_null_values) or (not value and isinstance(value, (list, set))): include_combination = False break if not isinstance(value, (list, set)): value = [value] new_combo_identifers = [] for existing in combo_identifiers: for v in value: identifier = "{}:{}".format(field.column, _format_value_for_identifier(v)) new_combo_identifers.append(existing + [identifier]) combo_identifiers = new_combo_identifers if include_combination: for ident in combo_identifiers: identifiers.append(get_top_concrete_parent(model)._meta.db_table + "|" + "|".join(ident)) return identifiers
def _django_18_query_walk_leaf(node, negated, new_parent, connection, model): new_node = WhereNode() if not hasattr(node, "lhs"): raise NotSupportedError("Attempted probable subquery, these aren't supported on the datastore") # Leaf if hasattr(node.lhs, 'target'): # from Django 1.9, some node.lhs might not have a target attribute # as they might be wrapping date fields field = node.lhs.target operator = node.lookup_name else: field = node.lhs.lhs.target operator = node.lhs.lookup_name # This deals with things like datefield__month__gt=X which means from this point # on, operator will have two parts in that particular case and will probably need to # be dealt with by a special indexer if node.lookup_name != operator: operator = "{}__{}".format(operator, node.lookup_name) if get_top_concrete_parent(field.model) != get_top_concrete_parent(model): raise NotSupportedError("Cross-join where filters are not supported on the datastore") # Make sure we don't let people try to filter on a text field, otherwise they just won't # get any results! if field.db_type(connection) in ("bytes", "text"): raise NotSupportedError("You can't filter on text or blob fields on the datastore") if operator == "isnull" and field.model._meta.parents.values(): raise NotSupportedError("isnull lookups on inherited relations aren't supported on the datastore") lhs = field.column try: if hasattr(node.rhs, "get_compiler"): # This is a subquery raise NotSupportedError("Attempted to run a subquery on the datastore") elif isinstance(node.rhs, ValuesListQuerySet): # We explicitly handle ValuesListQuerySet because of the # common case of pk__in=Something.objects.values_list("pk", flat=True) # this WILL execute another query, but that is to be expected on a # non-relational datastore. node.rhs = [ x for x in node.rhs ] # Evaluate the queryset rhs = node.process_rhs(None, connection) # Process the RHS as if it was a list elif isinstance(node.rhs, QuerySet): # In Django 1.9, ValuesListQuerySet doesn't exist anymore, and instead # values_list returns a QuerySet if node.rhs._iterable_class == FlatValuesListIterable: # if the queryset has FlatValuesListIterable as iterable class # then it's a flat list, and we just need to evaluate the # queryset converting it into a list node.rhs = [ x for x in node.rhs ] else: # otherwise, we try to get the PK from the queryset node.rhs = [ x.pk for x in node.rhs ] rhs = node.process_rhs(None, connection) # Process the RHS as if it was a list else: rhs = node.process_rhs(None, connection) except EmptyResultSet: if operator == 'in': # Deal with this later rhs = [ [] ] else: raise if operator in ('in', 'range'): rhs = rhs[-1] elif operator == 'isnull': rhs = node.rhs else: rhs = rhs[-1][0] new_node.set_leaf( lhs, operator, rhs, is_pk_field=field==model._meta.pk, negated=negated, namespace=connection.ops.connection.settings_dict.get("NAMESPACE"), target_field=field, ) # For some reason, this test: # test_update_with_related_manager (get_or_create.tests.UpdateOrCreateTests) # ends up with duplicate nodes in the where tree. I don't know why. But this # weirdly causes the datastore query to return nothing. # so here we don't add duplicate nodes, I can't think of a case where that would # change the query if it's under the same parent. if new_node in new_parent.children: return new_parent.children.append(new_node)
def _django_17_query_walk_leaf(node, negated, new_parent, connection, model): new_node = WhereNode() if not hasattr(node, "lhs"): raise NotSupportedError( "Attempted probable subquery, these aren't supported on the datastore" ) # Leaf if get_top_concrete_parent( node.lhs.target.model) != get_top_concrete_parent(model): raise NotSupportedError( "Cross-join where filters are not supported on the datastore") field = node.lhs.target # Make sure we don't let people try to filter on a text field, otherwise they just won't # get any results! if field.db_type(connection) in ("bytes", "text"): raise NotSupportedError( "You can't filter on text or blob fields on the datastore") if node.lookup_name == "isnull" and field.model._meta.parents.values(): raise NotSupportedError( "isnull lookups on inherited relations aren't supported on the datastore" ) lhs = field.column try: if hasattr(node.rhs, "get_compiler"): # This is a subquery raise NotSupportedError( "Attempted to run a subquery on the datastore") elif isinstance(node.rhs, ValuesListQuerySet): # We explicitly handle ValuesListQuerySet because of the # common case of pk__in=Something.objects.values_list("pk", flat=True) # this WILL execute another query, but that is to be expected on a # non-relational datastore. node.rhs = [x for x in node.rhs] # Evaluate the queryset rhs = node.process_rhs( None, connection) # Process the RHS as if it was a list else: rhs = node.process_rhs(None, connection) except EmptyResultSet: if node.lookup_name == 'in': # Deal with this later rhs = [[]] else: raise if node.lookup_name in ('in', 'range'): rhs = rhs[-1] elif node.lookup_name == 'isnull': rhs = node.rhs else: rhs = rhs[-1][0] new_node.set_leaf( lhs, node.lookup_name, rhs, is_pk_field=field == model._meta.pk, negated=negated, namespace=connection.ops.connection.settings_dict.get("NAMESPACE"), target_field=node.lhs.target, ) # For some reason, this test: # test_update_with_related_manager (get_or_create.tests.UpdateOrCreateTests) # ends up with duplicate nodes in the where tree. I don't know why. But this # weirdly causes the datastore query to return nothing. # so here we don't add duplicate nodes, I can't think of a case where that would # change the query if it's under the same parent. if new_node in new_parent.children: return new_parent.children.append(new_node)
def _generate_kind_name(self, model, column): return "_djangae_idx_{}_{}".format( get_top_concrete_parent(model)._meta.db_table, column )
def _where_node_leaf_callback(self, node, negated, new_parent, connection, model, compiler): new_node = WhereNode() def convert_rhs_op(node): db_rhs = getattr(node.rhs, '_db', None) if db_rhs is not None and db_rhs != connection.alias: raise ValueError( "Subqueries aren't allowed across different databases. Force " "the inner query to be evaluated using `list(inner_query)`." ) value = node.get_rhs_op(connection, node.rhs) operator = value.split()[0].lower().strip() if operator == 'between': operator = 'range' return operator if not hasattr(node, "lhs"): raise NotSupportedError("Attempted probable subquery, these aren't supported on the datastore") # Although we do nothing with this. We need to call it as many lookups # perform validation etc. if not hasattr(node.rhs, "_as_sql"): # Don't call on querysets try: node.process_rhs(compiler, connection) except EmptyResultSet: if node.lookup_name == 'in': node.rhs = [] else: raise # Leaf if hasattr(node.lhs, 'target'): # from Django 1.9, some node.lhs might not have a target attribute # as they might be wrapping date fields field = node.lhs.target operator = convert_rhs_op(node) else: field = node.lhs.lhs.target operator = convert_rhs_op(node) # This deals with things like datefield__month__gt=X which means from this point # on, operator will have two parts in that particular case and will probably need to # be dealt with by a special indexer if node.lookup_name != node.lhs.lookup_name: operator = "{}__{}".format(node.lhs.lookup_name, node.lookup_name) if get_top_concrete_parent(field.model) != get_top_concrete_parent(model): raise NotSupportedError("Cross-join where filters are not supported on the datastore") # Make sure we don't let people try to filter on a text field, otherwise they just won't # get any results! if field.db_type(connection) in ("bytes", "text"): raise NotSupportedError("You can't filter on text or blob fields on the datastore") if operator == "isnull" and field.model._meta.parents.values(): raise NotSupportedError("isnull lookups on inherited relations aren't supported on the datastore") lhs = field.column if hasattr(node.rhs, "get_compiler"): # This is a subquery raise NotSupportedError("Attempted to run a subquery on the datastore") elif isinstance(node.rhs, ValuesListQuerySet): # We explicitly handle ValuesListQuerySet because of the # common case of pk__in=Something.objects.values_list("pk", flat=True) # this WILL execute another query, but that is to be expected on a # non-relational datastore. rhs = [x for x in node.rhs] # Evaluate the queryset elif isinstance(node.rhs, QuerySet): # In Django 1.9, ValuesListQuerySet doesn't exist anymore, and instead # values_list returns a QuerySet if node.rhs._iterable_class == FlatValuesListIterable: # if the queryset has FlatValuesListIterable as iterable class # then it's a flat list, and we just need to evaluate the # queryset converting it into a list rhs = [x for x in node.rhs] else: # otherwise, we try to get the PK from the queryset rhs = list(node.rhs.values_list('pk', flat=True)) else: rhs = node.rhs was_iter = hasattr(node.rhs, "__iter__") rhs = node.get_db_prep_lookup(rhs, connection)[-1] if rhs and not was_iter and hasattr(rhs, "__iter__"): rhs = rhs[0] new_node.set_leaf( lhs, operator, rhs, is_pk_field=field==model._meta.pk, negated=negated, lookup_name=node.lookup_name, namespace=connection.ops.connection.settings_dict.get("NAMESPACE"), target_field=field, ) # For some reason, this test: # test_update_with_related_manager (get_or_create.tests.UpdateOrCreateTests) # ends up with duplicate nodes in the where tree. I don't know why. But this # weirdly causes the datastore query to return nothing. # so here we don't add duplicate nodes, I can't think of a case where that would # change the query if it's under the same parent. if new_node in new_parent.children: return new_parent.children.append(new_node)
def _django_17_query_walk_leaf(node, negated, new_parent, connection, model): new_node = WhereNode() if not hasattr(node, "lhs"): raise NotSupportedError("Attempted probable subquery, these aren't supported on the datastore") # Leaf if get_top_concrete_parent(node.lhs.target.model) != get_top_concrete_parent(model): raise NotSupportedError("Cross-join where filters are not supported on the datastore") field = node.lhs.target # Make sure we don't let people try to filter on a text field, otherwise they just won't # get any results! if field.db_type(connection) in ("bytes", "text"): raise NotSupportedError("You can't filter on text or blob fields on the datastore") if node.lookup_name == "isnull" and field.model._meta.parents.values(): raise NotSupportedError("isnull lookups on inherited relations aren't supported on the datastore") lhs = field.column try: if hasattr(node.rhs, "get_compiler"): # This is a subquery raise NotSupportedError("Attempted to run a subquery on the datastore") else: rhs = node.process_rhs(None, connection) except EmptyResultSet: if node.lookup_name == 'in': # Deal with this later rhs = [ [] ] else: raise if node.lookup_name in ('in', 'range'): rhs = rhs[-1] elif node.lookup_name == 'isnull': rhs = node.rhs else: rhs = rhs[-1][0] new_node.set_leaf( lhs, node.lookup_name, rhs, is_pk_field=field==model._meta.pk, negated=negated, target_field=node.lhs.target, ) # For some reason, this test: # test_update_with_related_manager (get_or_create.tests.UpdateOrCreateTests) # ends up with duplicate nodes in the where tree. I don't know why. But this # weirdly causes the datastore query to return nothing. # so here we don't add duplicate nodes, I can't think of a case where that would # change the query if it's under the same parent. if new_node in new_parent.children: return new_parent.children.append(new_node)
def _django_17_query_walk_leaf(node, negated, new_parent, connection, model): new_node = WhereNode() if not hasattr(node, "lhs"): raise NotSupportedError("Attempted probable subquery, these aren't supported on the datastore") # Leaf if get_top_concrete_parent(node.lhs.target.model) != get_top_concrete_parent(model): raise NotSupportedError("Cross-join where filters are not supported on the datastore") field = node.lhs.target # Make sure we don't let people try to filter on a text field, otherwise they just won't # get any results! if field.db_type(connection) in ("bytes", "text"): raise NotSupportedError("You can't filter on text or blob fields on the datastore") if node.lookup_name == "isnull" and field.model._meta.parents.values(): raise NotSupportedError("isnull lookups on inherited relations aren't supported on the datastore") lhs = field.column try: if hasattr(node.rhs, "get_compiler"): # This is a subquery raise NotSupportedError("Attempted to run a subquery on the datastore") elif isinstance(node.rhs, ValuesListQuerySet): # We explicitly handle ValuesListQuerySet because of the # common case of pk__in=Something.objects.values_list("pk", flat=True) # this WILL execute another query, but that is to be expected on a # non-relational datastore. node.rhs = [ x for x in node.rhs ] # Evaluate the queryset rhs = node.process_rhs(None, connection) # Process the RHS as if it was a list else: rhs = node.process_rhs(None, connection) except EmptyResultSet: if node.lookup_name == 'in': # Deal with this later rhs = [ [] ] else: raise if node.lookup_name in ('in', 'range'): rhs = rhs[-1] elif node.lookup_name == 'isnull': rhs = node.rhs else: rhs = rhs[-1][0] new_node.set_leaf( lhs, node.lookup_name, rhs, is_pk_field=field==model._meta.pk, negated=negated, namespace=connection.ops.connection.settings_dict.get("NAMESPACE"), target_field=node.lhs.target, ) # For some reason, this test: # test_update_with_related_manager (get_or_create.tests.UpdateOrCreateTests) # ends up with duplicate nodes in the where tree. I don't know why. But this # weirdly causes the datastore query to return nothing. # so here we don't add duplicate nodes, I can't think of a case where that would # change the query if it's under the same parent. if new_node in new_parent.children: return new_parent.children.append(new_node)
def _transform_query_17(connection, kind, query): from django.db.models.sql.datastructures import Date, DateTime if isinstance(query.where, EmptyWhere): # Empty where means return nothing! raise EmptyResultSet() # Check for joins, we ignore select related tables as they aren't actually used (the connector marks select # related as unsupported in its features) tables = [k for k, v in query.alias_refcount.items() if v] inherited_tables = set( [x._meta.db_table for x in query.model._meta.parents]) select_related_tables = set([y[0][0] for y in query.related_select_cols]) tables = set(tables) - inherited_tables - select_related_tables if len(tables) > 1: raise NotSupportedError(""" The appengine database connector does not support JOINs. The requested join map follows\n %s """ % query.join_map) ret = Query(query.model, kind) ret.connection = connection # Add the root concrete table as the source table root_table = get_top_concrete_parent(query.model)._meta.db_table ret.add_source_table(root_table) # Extract the ordering of the query results for order_col in _extract_ordering_from_query_17(query): ret.add_order_by(order_col) # Extract any projected columns (values/values_list/only/defer) for projected_col in _extract_projected_columns_from_query_17(query): ret.add_projected_column(projected_col) for potential_annotation in query.select: col = getattr(potential_annotation, "col", None) if not col: continue if isinstance(col, (Date, DateTime)): ret.add_annotation(col.col[-1], col) # Add any extra selects for col, select in query.extra_select.items(): ret.add_extra_select(col, select[0]) # This must happen after extracting projected cols if query.distinct: ret.set_distinct(list(query.distinct_fields)) # Extract any query offsets and limits ret.low_mark = query.low_mark ret.high_mark = query.high_mark output = WhereNode() output.connector = query.where.connector _walk_django_where(query, _django_17_query_walk_trunk, _django_17_query_walk_leaf, new_parent=output, connection=connection, model=query.model) # If there no child nodes, just wipe out the where if not output.children: output = None ret.where = output return ret
def _generate_kind_name(self, model, column): return "_djangae_idx_{}_{}".format( get_top_concrete_parent(model)._meta.db_table, column)
def set_leaf( self, column, operator, value, is_pk_field, negated, lookup_name, namespace, target_field=None): assert column assert operator assert isinstance(is_pk_field, bool) assert isinstance(negated, bool) if operator == "iexact" and isinstance(target_field, AutoField): # When new instance is created, automatic primary key 'id' # does not generate '_idx_iexact_id'. # As the primary key 'id' (AutoField) is integer and is always case insensitive, # we can deal with 'id_iexact=' query by using 'exact' rather than 'iexact'. operator = "exact" value = int(value) if is_pk_field: # If this is a primary key, we need to make sure that the value # we pass to the query is a datastore Key. We have to deal with IN queries here # because they aren't flattened until the DNF stage model = get_top_concrete_parent(target_field.model) table = model._meta.db_table if isinstance(value, (list, tuple)): value = [ rpc.Key.from_path(table, x, namespace=namespace) for x in value if x ] else: # Django 1.11 has operators as symbols, earlier versions use "exact" etc. if (operator == "isnull" and value is True) or (operator in ("exact", "lt", "lte", "<", "<=", "=") and not value): # id=None will never return anything and # Empty strings and 0 are forbidden as keys self.will_never_return_results = True elif operator in ("gt", "gte", ">", ">=") and not value: # If the value is 0 or "", then we need to manipulate the value and operator here to # get the right result (given that both are invalid keys) so for both we return # >= 1 or >= "\0" for strings if isinstance(value, six.integer_types): value = 1 else: value = "\0" value = rpc.Key.from_path(table, value, namespace=namespace) operator = "gte" else: value = rpc.Key.from_path(table, value, namespace=namespace) column = "__key__" # Do any special index conversions necessary to perform this lookup special_indexer = get_indexer(target_field, operator) if special_indexer: if is_pk_field: column = model._meta.pk.column value = unicode(value.id_or_name()) add_special_index(target_field.model, column, special_indexer, operator, value) index_type = special_indexer.prepare_index_type(operator, value) value = special_indexer.prep_value_for_query( value, model=target_field.model, column=column, connection=connections[self.using] ) column = special_indexer.indexed_column_name(column, value, index_type) operator = special_indexer.prep_query_operator(operator) self.column = column self.operator = convert_operator(operator) self.value = value self.lookup_name = lookup_name
def _where_node_leaf_callback(self, node, negated, new_parent, connection, model, compiler): new_node = WhereNode(new_parent.using) def convert_rhs_op(node): db_rhs = getattr(node.rhs, '_db', None) if db_rhs is not None and db_rhs != connection.alias: raise ValueError( "Subqueries aren't allowed across different databases. Force " "the inner query to be evaluated using `list(inner_query)`." ) value = node.get_rhs_op(connection, node.rhs) operator = value.split()[0].lower().strip() if operator == 'between': operator = 'range' return operator if not hasattr(node, "lhs"): raise NotSupportedError("Attempted probable subquery, these aren't supported on the Datastore") # Don't call on querysets if not hasattr(node.rhs, "_as_sql") and not isinstance(node.rhs, DjangoQuery): try: # Although we do nothing with this. We need to call it as many lookups # perform validation etc. node.process_rhs(compiler, connection) except EmptyResultSet: if node.lookup_name == 'in': node.rhs = [] else: raise # Leaf if hasattr(node.lhs, 'target'): # from Django 1.9, some node.lhs might not have a target attribute # as they might be wrapping date fields field = node.lhs.target operator = convert_rhs_op(node) elif isinstance(node.lhs, Aggregate): raise NotSupportedError("Aggregate filters are not supported on the Datastore") else: field = node.lhs.lhs.target operator = convert_rhs_op(node) # This deals with things like datefield__month__gt=X which means from this point # on, operator will have two parts in that particular case and will probably need to # be dealt with by a special indexer if node.lookup_name != node.lhs.lookup_name: operator = "{}__{}".format(node.lhs.lookup_name, node.lookup_name) if get_top_concrete_parent(field.model) != get_top_concrete_parent(model): raise NotSupportedError("Cross-join where filters are not supported on the Datastore") # Make sure we don't let people try to filter on a text field, otherwise they just won't # get any results! lookup_supports_text = getattr(node, "lookup_supports_text", False) if field.db_type(connection) in ("bytes", "text") and not lookup_supports_text: raise NotSupportedError("You can't filter on text or blob fields on the Datastore") if operator == "isnull" and field.model._meta.parents.values(): raise NotSupportedError("isnull lookups on inherited relations aren't supported on the Datastore") lhs = field.column if hasattr(node.rhs, "get_compiler"): if len(node.rhs.select) == 1: # In Django >= 1.11 this is a values list type query, which we explicitly handle # because of the common case of pk__in=Something.objects.values_list("pk", flat=True) qs = QuerySet(query=node.rhs, using=self.connection.alias) # We make the query for the values, but wrap in a list to trick the # was_iter code below. This whole set of if/elif statements needs rethinking! rhs = [list(qs.values_list("pk", flat=True))] else: # This is a subquery raise NotSupportedError("Attempted to run a subquery on the Datastore") elif isinstance(node.rhs, ValuesListQuerySet): # We explicitly handle ValuesListQuerySet because of the # common case of pk__in=Something.objects.values_list("pk", flat=True) # this WILL execute another query, but that is to be expected on a # non-relational database. rhs = [x for x in node.rhs] # Evaluate the queryset elif isinstance(node.rhs, QuerySet): # In Django 1.9, ValuesListQuerySet doesn't exist anymore, and instead # values_list returns a QuerySet if node.rhs._iterable_class == FlatValuesListIterable: # if the queryset has FlatValuesListIterable as iterable class # then it's a flat list, and we just need to evaluate the # queryset converting it into a list rhs = [x for x in node.rhs] else: # otherwise, we try to get the PK from the queryset rhs = list(node.rhs.values_list('pk', flat=True)) else: rhs = node.rhs was_iter = hasattr(node.rhs, "__iter__") rhs = node.get_db_prep_lookup(rhs, connection)[-1] if rhs and not was_iter and hasattr(rhs, "__iter__"): rhs = rhs[0] new_node.set_leaf( lhs, operator, rhs, is_pk_field=field==model._meta.pk, negated=negated, lookup_name=node.lookup_name, namespace=connection.ops.connection.settings_dict.get("NAMESPACE"), target_field=field, ) # For some reason, this test: # test_update_with_related_manager (get_or_create.tests.UpdateOrCreateTests) # ends up with duplicate nodes in the where tree. I don't know why. But this # weirdly causes the Datastore query to return nothing. # so here we don't add duplicate nodes, I can't think of a case where that would # change the query if it's under the same parent. if new_node in new_parent.children: return new_parent.children.append(new_node)
def _django_18_query_walk_leaf(node, negated, new_parent, connection, model): new_node = WhereNode() if not hasattr(node, "lhs"): raise NotSupportedError( "Attempted probable subquery, these aren't supported on the datastore" ) # Leaf if hasattr(node.lhs, 'target'): # from Django 1.9, some node.lhs might not have a target attribute # as they might be wrapping date fields field = node.lhs.target operator = node.lookup_name else: field = node.lhs.lhs.target operator = node.lhs.lookup_name # This deals with things like datefield__month__gt=X which means from this point # on, operator will have two parts in that particular case and will probably need to # be dealt with by a special indexer if node.lookup_name != operator: operator = "{}__{}".format(operator, node.lookup_name) if get_top_concrete_parent(field.model) != get_top_concrete_parent(model): raise NotSupportedError( "Cross-join where filters are not supported on the datastore") # Make sure we don't let people try to filter on a text field, otherwise they just won't # get any results! if field.db_type(connection) in ("bytes", "text"): raise NotSupportedError( "You can't filter on text or blob fields on the datastore") if operator == "isnull" and field.model._meta.parents.values(): raise NotSupportedError( "isnull lookups on inherited relations aren't supported on the datastore" ) lhs = field.column try: if hasattr(node.rhs, "get_compiler"): # This is a subquery raise NotSupportedError( "Attempted to run a subquery on the datastore") elif isinstance(node.rhs, ValuesListQuerySet): # We explicitly handle ValuesListQuerySet because of the # common case of pk__in=Something.objects.values_list("pk", flat=True) # this WILL execute another query, but that is to be expected on a # non-relational datastore. node.rhs = [x for x in node.rhs] # Evaluate the queryset rhs = node.process_rhs( None, connection) # Process the RHS as if it was a list elif isinstance(node.rhs, QuerySet): # In Django 1.9, ValuesListQuerySet doesn't exist anymore, and instead # values_list returns a QuerySet if node.rhs._iterable_class == FlatValuesListIterable: # if the queryset has FlatValuesListIterable as iterable class # then it's a flat list, and we just need to evaluate the # queryset converting it into a list node.rhs = [x for x in node.rhs] else: # otherwise, we try to get the PK from the queryset node.rhs = [x.pk for x in node.rhs] rhs = node.process_rhs( None, connection) # Process the RHS as if it was a list else: rhs = node.process_rhs(None, connection) except EmptyResultSet: if operator == 'in': # Deal with this later rhs = [[]] else: raise if operator in ('in', 'range'): rhs = rhs[-1] elif operator == 'isnull': rhs = node.rhs else: rhs = rhs[-1][0] new_node.set_leaf( lhs, operator, rhs, is_pk_field=field == model._meta.pk, negated=negated, namespace=connection.ops.connection.settings_dict.get("NAMESPACE"), target_field=field, ) # For some reason, this test: # test_update_with_related_manager (get_or_create.tests.UpdateOrCreateTests) # ends up with duplicate nodes in the where tree. I don't know why. But this # weirdly causes the datastore query to return nothing. # so here we don't add duplicate nodes, I can't think of a case where that would # change the query if it's under the same parent. if new_node in new_parent.children: return new_parent.children.append(new_node)
def _transform_query_19(connection, kind, query): from django.db.models.sql.where import NothingNode, WhereNode as DjangoWhereNode # It could either be a NothingNode, or a WhereNode(AND NothingNode) if (isinstance(query.where, NothingNode) or (isinstance(query.where, DjangoWhereNode) and len(query.where.children) == 1 and isinstance(query.where.children[0], NothingNode))): # Empty where means return nothing! raise EmptyResultSet() ret = Query(query.model, kind) ret.connection = connection # Add the root concrete table as the source table root_table = get_top_concrete_parent(query.model)._meta.db_table ret.add_source_table(root_table) # Extract the ordering of the query results for order_col in _extract_ordering_from_query_18(query): ret.add_order_by(order_col) # Extract any projected columns (values/values_list/only/defer) for projected_col in _extract_projected_columns_from_query_18(query): ret.add_projected_column(projected_col) # Add any extra selects for col, select in query.extra_select.items(): ret.add_extra_select(col, select[0]) if query.distinct: # This must happen after extracting projected cols ret.set_distinct(list(query.distinct_fields)) # Process annotations! if query.annotation_select: for k, v in query.annotation_select.items(): ret.add_annotation(k, v) # Extract any query offsets and limits ret.low_mark = query.low_mark ret.high_mark = query.high_mark output = WhereNode() output.connector = query.where.connector _walk_django_where(query, _django_18_query_walk_trunk, _django_18_query_walk_leaf, new_parent=output, connection=connection, negated=query.where.negated, model=query.model) # If there no child nodes, just wipe out the where if not output.children: output = None ret.where = output return ret
def set_leaf(self, column, operator, value, is_pk_field, negated, namespace, target_field=None): assert column assert operator assert isinstance(is_pk_field, bool) assert isinstance(negated, bool) if operator == "iexact" and isinstance(target_field, AutoField): # When new instance is created, automatic primary key 'id' does not generate '_idx_iexact_id'. # As the primary key 'id' (AutoField) is integer and is always case insensitive, # we can deal with 'id_iexact=' query by using 'exact' rather than 'iexact'. operator = "exact" value = int(value) if is_pk_field: # If this is a primary key, we need to make sure that the value # we pass to the query is a datastore Key. We have to deal with IN queries here # because they aren't flattened until the DNF stage model = get_top_concrete_parent(target_field.model) table = model._meta.db_table if isinstance(value, (list, tuple)): value = [ datastore.Key.from_path(table, x, namespace=namespace) for x in value if x ] else: if operator == "isnull" and value is True: # FIXME: Strictly, this isn't correct, this could be one of several branches # but id=None filters are silly anyway. This should be moved to after normalization.. # probably. This fixes a test in Django which does this in get_or_create for some reason raise EmptyResultSet() if not value: # Empty strings and 0 are forbidden as keys # so make this an impossible filter # FIXME: This is a hack! It screws with the ordering # because it's an inequality. Instead we should wipe this # filter out when preprocessing in the DNF (because it's impossible) value = datastore.Key.from_path('', 1) operator = '<' else: value = datastore.Key.from_path(table, value, namespace=namespace) column = "__key__" # Do any special index conversions necessary to perform this lookup primary_operation = operator.split("__")[0] special_indexer = REQUIRES_SPECIAL_INDEXES.get(primary_operation) if special_indexer: if is_pk_field: column = model._meta.pk.column value = unicode(value.id_or_name()) add_special_index(target_field.model, column, primary_operation, value) index_type = special_indexer.prepare_index_type(operator, value) value = special_indexer.prep_value_for_query(value) column = special_indexer.indexed_column_name( column, value, index_type) operator = special_indexer.prep_query_operator(operator) self.column = column self.operator = convert_operator(operator) self.value = value
def _transform_query_17(connection, kind, query): from django.db.models.sql.datastructures import Date, DateTime if isinstance(query.where, EmptyWhere): # Empty where means return nothing! raise EmptyResultSet() # Check for joins, we ignore select related tables as they aren't actually used (the connector marks select # related as unsupported in its features) tables = [ k for k, v in query.alias_refcount.items() if v ] inherited_tables = set([x._meta.db_table for x in query.model._meta.parents ]) select_related_tables = set([y[0][0] for y in query.related_select_cols ]) tables = set(tables) - inherited_tables - select_related_tables if len(tables) > 1: raise NotSupportedError(""" The appengine database connector does not support JOINs. The requested join map follows\n %s """ % query.join_map) ret = Query(query.model, kind) ret.connection = connection # Add the root concrete table as the source table root_table = get_top_concrete_parent(query.model)._meta.db_table ret.add_source_table(root_table) # Extract the ordering of the query results for order_col in _extract_ordering_from_query_17(query): ret.add_order_by(order_col) # Extract any projected columns (values/values_list/only/defer) for projected_col in _extract_projected_columns_from_query_17(query): ret.add_projected_column(projected_col) for potential_annotation in query.select: col = getattr(potential_annotation, "col", None) if not col: continue if isinstance(col, (Date, DateTime)): ret.add_annotation(col.col[-1], col) # Add any extra selects for col, select in query.extra_select.items(): ret.add_extra_select(col, select[0]) # This must happen after extracting projected cols if query.distinct: ret.set_distinct(list(query.distinct_fields)) # Extract any query offsets and limits ret.low_mark = query.low_mark ret.high_mark = query.high_mark output = WhereNode() output.connector = query.where.connector _walk_django_where( query, _django_17_query_walk_trunk, _django_17_query_walk_leaf, new_parent=output, connection=connection, model=query.model ) # If there no child nodes, just wipe out the where if not output.children: output = None ret.where = output return ret