def test_query_is_unique(self): qry = datastore.Query(UniqueModel._meta.db_table) qry["unique_field ="] = "test" self.assertTrue(query_is_unique(UniqueModel, qry)) del qry["unique_field ="] qry["unique_field >"] = "test" self.assertFalse(query_is_unique(UniqueModel, qry)) del qry["unique_field >"] qry["unique_combo_one ="] = "one" self.assertFalse(query_is_unique(UniqueModel, qry)) qry["unique_combo_two ="] = "two" self.assertTrue(query_is_unique(UniqueModel, qry))
def _build_gae_query(self): """ Build and return the Datastore Query object. """ query_kwargs = { "kind": str(self.db_table) } if self.distinct: if self.projection: query_kwargs["distinct"] = True else: logging.warning("Ignoring distinct on a query where a projection wasn't possible") if self.keys_only: query_kwargs["keys_only"] = self.keys_only elif self.projection: query_kwargs["projection"] = self.projection query = Query( **query_kwargs ) if has_concrete_parents(self.model) and not self.model._meta.proxy: query["class ="] = self.model._meta.db_table ordering = [] for order in self.ordering: if isinstance(order, (long, int)): direction = datastore.Query.ASCENDING if order == 1 else datastore.Query.DESCENDING order = self.queried_fields[0] else: direction = datastore.Query.DESCENDING if order.startswith("-") else datastore.Query.ASCENDING order = order.lstrip("-") if order == self.model._meta.pk.column or order == "pk": order = "__key__" #Flip the ordering if someone called reverse() on the queryset if not self.original_query.standard_ordering: direction = datastore.Query.DESCENDING if direction == datastore.Query.ASCENDING else datastore.Query.ASCENDING ordering.append((order, direction)) def process_and_branch(query, and_branch): for child in and_branch[-1]: column, op, value = child[1] # for column, op, value in and_branch[-1]: if column == self.pk_col: column = "__key__" #FIXME: This EmptyResultSet check should happen during normalization so that Django doesn't count it as a query if op == "=" and "__key__ =" in query and query["__key__ ="] != value: # We've already done an exact lookup on a key, this query can't return anything! raise EmptyResultSet() if not isinstance(value, datastore.Key): value = get_datastore_key(self.model, value) key = "%s %s" % (column, op) try: if isinstance(value, basestring): value = coerce_unicode(value) if key in query: if type(query[key]) == list: if value not in query[key]: query[key].append(value) else: if query[key] != value: query[key] = [ query[key], value ] else: query[key] = value except datastore_errors.BadFilterError as e: raise NotSupportedError(str(e)) if self.where: queries = [] # print query._Query__kind, self.where for and_branch in self.where[1]: # Duplicate the query for all the "OR"s queries.append(Query(**query_kwargs)) queries[-1].update(query) # Make sure we copy across filters (e.g. class =) try: if and_branch[0] == "LIT": and_branch = ("AND", [and_branch]) process_and_branch(queries[-1], and_branch) except EmptyResultSet: # This is a little hacky but basically if there is only one branch in the or, and it raises # and EmptyResultSet, then we just bail, however if there is more than one branch the query the # query might still return something. This logic needs cleaning up and moving to the DNF phase if len(self.where[1]) == 1: return NoOpQuery() else: queries.pop() if not queries: return NoOpQuery() included_pks = [ qry["__key__ ="] for qry in queries if "__key__ =" in qry ] if len(included_pks) == len(queries): # If all queries have a key, we can perform a Get return QueryByKeys(self.model, queries, ordering) # Just use whatever query to determine the matches else: if len(queries) > 1: # Disable keys only queries for MultiQuery new_queries = [] for i, query in enumerate(queries): if i > 30: raise NotSupportedError("Too many subqueries (max: 30, got {}). Probably cause too many IN/!= filters".format( len(queries) )) qry = Query(query._Query__kind, projection=query._Query__query_options.projection) qry.update(query) try: qry.Order(*ordering) except datastore_errors.BadArgumentError as e: raise NotSupportedError(e) new_queries.append(qry) query = datastore.MultiQuery(new_queries, ordering) else: query = queries[0] try: query.Order(*ordering) except datastore_errors.BadArgumentError as e: raise NotSupportedError(e) else: try: query.Order(*ordering) except datastore_errors.BadArgumentError as e: raise NotSupportedError(e) # If the resulting query was unique, then wrap as a unique query which # will hit the cache first unique_identifier = query_is_unique(self.model, query) if unique_identifier: return UniqueQuery(unique_identifier, query, self.model) DJANGAE_LOG.debug("Select query: {0}, {1}".format(self.model.__name__, self.where)) return query
def _build_query(self): self._sanity_check() queries = [] projection = self._exclude_pk(self.query.columns) or None query_kwargs = { "kind": self.query.concrete_model._meta.db_table, "distinct": self.query.distinct or None, "keys_only": self.keys_only or None, "projection": projection, } ordering = convert_django_ordering_to_gae(self.query.order_by) if self.query.distinct and not ordering: # If we specified we wanted a distinct query, but we didn't specify # an ordering, we must set the ordering to the distinct columns, otherwise # App Engine shouts at us. Nastily. And without remorse. ordering = self.query.columns[:] # Deal with the no filters case if self.query.where is None: query = Query(**query_kwargs) try: query.Order(*ordering) except datastore_errors.BadArgumentError as e: raise NotSupportedError(e) return query assert self.query.where # Go through the normalized query tree for and_branch in self.query.where.children: query = Query(**query_kwargs) # This deals with the oddity that the root of the tree may well be a leaf filters = [and_branch] if and_branch.is_leaf else and_branch.children for filter_node in filters: lookup = "{} {}".format(filter_node.column, filter_node.operator) value = filter_node.value # This is a special case. Annoyingly Django's decimal field doesn't # ever call ops.get_prep_save or lookup or whatever when you are filtering # on a query. It *does* do it on a save, so we basically need to do a # conversion here, when really it should be handled elsewhere if isinstance(value, decimal.Decimal): field = get_field_from_column(self.query.model, filter_node.column) value = self.connection.ops.value_to_db_decimal(value, field.max_digits, field.decimal_places) elif isinstance(value, basestring): value = coerce_unicode(value) # If there is already a value for this lookup, we need to make the # value a list and append the new entry if lookup in query and not isinstance(query[lookup], (list, tuple)) and query[lookup] != value: query[lookup] = [query[lookup]] + [value] else: # If the value is a list, we can't just assign it to the query # which will treat each element as its own value. So in this # case we nest it. This has the side effect of throwing a BadValueError # which we could throw ourselves, but the datastore might start supporting # list values in lookups.. you never know! if isinstance(value, (list, tuple)): query[lookup] = [value] else: # Common case: just add the raw where constraint query[lookup] = value if ordering: try: query.Order(*ordering) except datastore_errors.BadArgumentError as e: # This is the easiest way to detect unsupported orderings # ideally we'd detect this at the query normalization stage # but it's a lot of hassle, this is much easier and seems to work OK raise NotSupportedError(e) queries.append(query) if can_perform_datastore_get(self.query): # Yay for optimizations! return QueryByKeys(self.query.model, queries, ordering) if len(queries) == 1: identifier = query_is_unique(self.query.model, queries[0]) if identifier: # Yay for optimizations! return UniqueQuery(identifier, queries[0], self.query.model) return queries[0] else: return datastore.MultiQuery(queries, ordering)
def _build_query(self): self._sanity_check() queries = [] projection = self._exclude_pk(self.query.columns) or None query_kwargs = { "kind": self.query.concrete_model._meta.db_table, "distinct": self.query.distinct or None, "keys_only": self.keys_only or None, "projection": projection } ordering = convert_django_ordering_to_gae(self.query.order_by) if self.query.distinct and not ordering: # If we specified we wanted a distinct query, but we didn't specify # an ordering, we must set the ordering to the distinct columns, otherwise # App Engine shouts at us. Nastily. And without remorse. ordering = self.query.columns[:] # Deal with the no filters case if self.query.where is None: query = Query(**query_kwargs) try: query.Order(*ordering) except datastore_errors.BadArgumentError as e: raise NotSupportedError(e) return query assert self.query.where # Go through the normalized query tree for and_branch in self.query.where.children: query = Query(**query_kwargs) # This deals with the oddity that the root of the tree may well be a leaf filters = [and_branch ] if and_branch.is_leaf else and_branch.children for filter_node in filters: lookup = "{} {}".format(filter_node.column, filter_node.operator) value = filter_node.value # This is a special case. Annoyingly Django's decimal field doesn't # ever call ops.get_prep_save or lookup or whatever when you are filtering # on a query. It *does* do it on a save, so we basically need to do a # conversion here, when really it should be handled elsewhere if isinstance(value, decimal.Decimal): field = get_field_from_column(self.query.model, filter_node.column) value = self.connection.ops.value_to_db_decimal( value, field.max_digits, field.decimal_places) elif isinstance(value, basestring): value = unicode(value) # If there is already a value for this lookup, we need to make the # value a list and append the new entry if lookup in query and not isinstance( query[lookup], (list, tuple)) and query[lookup] != value: query[lookup] = [query[lookup]] + [value] else: # If the value is a list, we can't just assign it to the query # which will treat each element as its own value. So in this # case we nest it. This has the side effect of throwing a BadValueError # which we could throw ourselves, but the datastore might start supporting # list values in lookups.. you never know! if isinstance(value, (list, tuple)): query[lookup] = [value] else: # Common case: just add the raw where constraint query[lookup] = value if ordering: try: query.Order(*ordering) except datastore_errors.BadArgumentError as e: # This is the easiest way to detect unsupported orderings # ideally we'd detect this at the query normalization stage # but it's a lot of hassle, this is much easier and seems to work OK raise NotSupportedError(e) queries.append(query) if can_perform_datastore_get(self.query): # Yay for optimizations! return QueryByKeys(self.query.model, queries, ordering) if len(queries) == 1: identifier = query_is_unique(self.query.model, queries[0]) if identifier: # Yay for optimizations! return UniqueQuery(identifier, queries[0], self.query.model) return queries[0] else: return datastore.MultiQuery(queries, ordering)
def _build_gae_query(self): """ Build and return the Datstore Query object. """ query_kwargs = { "kind": str(self.db_table) } if self.distinct: query_kwargs["distinct"] = True if self.keys_only: query_kwargs["keys_only"] = self.keys_only elif self.projection: query_kwargs["projection"] = self.projection query = Query( **query_kwargs ) if has_concrete_parents(self.model) and not self.model._meta.proxy: query["class ="] = self.model._meta.db_table ordering = [] for order in self.ordering: if isinstance(order, int): direction = datastore.Query.ASCENDING if order == 1 else datastore.Query.DESCENDING order = self.queried_fields[0] else: direction = datastore.Query.DESCENDING if order.startswith("-") else datastore.Query.ASCENDING order = order.lstrip("-") if order == self.model._meta.pk.column or order == "pk": order = "__key__" ordering.append((order, direction)) def process_and_branch(query, and_branch): for column, op, value in and_branch[-1]: if column == self.pk_col: column = "__key__" #FIXME: This EmptyResultSet check should happen during normalization so that Django doesn't count it as a query if op == "=" and "__key__ =" in query: #We've already done an exact lookup on a key, this query can't return anything! raise EmptyResultSet() if not isinstance(value, datastore.Key): value = get_datastore_key(self.model, value) key = "%s %s" % (column, op) if key in query: query[key] = [ query[key], value ] else: query[key] = value if self.where: queries = [] #If there is a single filter, we make it out it's an OR with only one branch #just so that the code below is simpler if isinstance(self.where, tuple) and len(self.where) == 3: self.where = ('OR', [(u'AND', [ self.where ])]) elif isinstance(self.where, tuple) and self.where[0] == 'AND': self.where = ('OR', [self.where]) elif isinstance(self.where, tuple) and self.where[0] == 'OR' and isinstance(self.where[1][0], tuple) and self.where[1][0][0] != 'AND': self.where = ('OR', [ ('AND', [x]) for x in self.where[-1] ]) operator = self.where[0] assert operator == 'OR' #print query._Query__kind, self.where for and_branch in self.where[1]: #Duplicate the query for all the "OR"s queries.append(Query(**query_kwargs)) queries[-1].update(query) #Make sure we copy across filters (e.g. class =) try: process_and_branch(queries[-1], and_branch) except EmptyResultSet: return NoOpQuery() def all_queries_same_except_key(_queries): """ Returns True if all queries in the list of queries filter on the same thing except for "__key__ =". Determine if we can do a Get basically. """ test = _queries[0] for qry in _queries: if "__key__ =" not in qry.keys(): return False if qry._Query__kind != test._Query__kind: return False if qry.keys() != test.keys(): return False for k, v in qry.items(): if k.startswith("__key__"): continue if v != test[k]: return False return True if all_queries_same_except_key(queries): included_pks = [ qry["__key__ ="] for qry in queries ] return QueryByKeys(queries[0], included_pks, ordering) #Just use whatever query to determine the matches else: if len(queries) > 1: #Disable keys only queries for MultiQuery new_queries = [] for query in queries: qry = Query(query._Query__kind, projection=query._Query__query_options.projection) qry.update(query) new_queries.append(qry) query = datastore.MultiQuery(new_queries, ordering) else: query = queries[0] query.Order(*ordering) else: query.Order(*ordering) #If the resulting query was unique, then wrap as a unique query which #will hit the cache first unique_identifier = query_is_unique(self.model, query) if unique_identifier: return UniqueQuery(unique_identifier, query, self.model) DJANGAE_LOG.debug("Select query: {0}, {1}".format(self.model.__name__, self.where)) return query