def test_unhashable(self):
class Unhashable:
__hash__ = None
with self.assertRaisesMessage(TypeError,
"unhashable type: 'Unhashable'"):
make_hashable(Unhashable())
def resolve_field(self, name, value):
field = self.cache_object_type()._meta.get_field(name)
# Field is Foreign Key
if isinstance(field, models.ForeignKey):
try:
obj = field.related_model.objects.get(
**{field.remote_field.field_name: value})
return str(obj)
except models.ObjectDoesNotExist:
pass # Reference no longer exists (deleted?)
if isinstance(field, models.ManyToManyField):
try:
qs = [
models.Q(**{field.related_model._meta.pk.name: v})
for v in value
]
query = qs.pop()
for q in qs:
query |= q
return ', '.join(
str(obj)
for obj in field.related_model.objects.filter(query))
except models.ObjectDoesNotExist:
pass # Reference no longer exists (deleted?)
# Other fields (code copied straight from django/db/models/base.py :: _get_FIELD_display)
choices_dict = dict(make_hashable(field.flatchoices))
return force_str(choices_dict.get(make_hashable(value), value),
strings_only=True)
def get_data_collection_techniques_display(self):
choices_dict = dict(
make_hashable(QuestionBase.DATA_COLLECTION_TECHNIQUE_OPTIONS))
return [
force_str(choices_dict.get(make_hashable(value), value),
strings_only=True)
for value in self.data_collection_techniques or []
]
def __hash__(self):
# Ignore params and messages ordering.
if hasattr(self, 'message'):
return hash((
self.message,
self.code,
tuple(sorted(make_hashable(self.params))) if self.params else None,
))
if hasattr(self, 'error_dict'):
return hash(tuple(sorted(make_hashable(self.error_dict))))
return hash(tuple(sorted(self.error_list, key=operator.attrgetter('message'))))
def _get_FIELD_display(self, field):
"""
Closure to replace default model method of the same name.
Cargo-culted from `django.db.models.base.Model._get_FIELD_display`
"""
choices = field.get_choices()
value = getattr(self, field.attname)
choices_dict = dict(make_hashable(choices))
# force_str() to coerce lazy strings.
return force_str(choices_dict.get(make_hashable(value), value), strings_only=True)
def __hash__(self):
if hasattr(self, "message"):
return hash((
self.message,
self.code,
make_hashable(self.params),
))
if hasattr(self, "error_dict"):
return hash(make_hashable(self.error_dict))
return hash(
tuple(sorted(self.error_list, key=operator.attrgetter("message"))))
def __hash__(self):
return hash((
self.__class__,
self.connector,
self.negated,
*make_hashable(self.children),
))
def test_count_equal(self):
tests = (
({"a": 1, "b": ["a", 1]}, (("a", 1), ("b", ("a", 1)))),
({"a": 1, "b": ("a", [1, 2])}, (("a", 1), ("b", ("a", (1, 2))))),
)
for value, expected in tests:
with self.subTest(value=value):
self.assertCountEqual(make_hashable(value), expected)
def test_count_equal(self):
tests = (
({'a': 1, 'b': ['a', 1]}, (('a', 1), ('b', ('a', 1)))),
({'a': 1, 'b': ('a', [1, 2])}, (('a', 1), ('b', ('a', (1, 2))))),
)
for value, expected in tests:
with self.subTest(value=value):
self.assertCountEqual(make_hashable(value), expected)
def identity(self):
return (
self.field,
self.model,
self.related_name,
self.related_query_name,
make_hashable(self.limit_choices_to),
self.parent_link,
self.on_delete,
self.symmetrical,
self.multiple,
)
def test_count_equal(self):
tests = (
({
'a': 1,
'b': ['a', 1]
}, (('a', 1), ('b', ('a', 1)))),
({
'a': 1,
'b': ('a', [1, 2])
}, (('a', 1), ('b', ('a', (1, 2))))),
)
for value, expected in tests:
with self.subTest(value=value):
self.assertCountEqual(make_hashable(value), expected)
def identity(self):
constructor_signature = inspect.signature(self.__init__)
args, kwargs = self._constructor_args
signature = constructor_signature.bind_partial(*args, **kwargs)
signature.apply_defaults()
arguments = signature.arguments.items()
identity = [self.__class__]
for arg, value in arguments:
if isinstance(value, fields.Field):
value = type(value)
else:
value = make_hashable(value)
identity.append((arg, value))
return tuple(identity)
def test_equal(self):
tests = (
([], ()),
(['a', 1], ('a', 1)),
({}, ()),
({'a'}, ('a',)),
(frozenset({'a'}), {'a'}),
({'a': 1}, (('a', 1),)),
(('a', ['b', 1]), ('a', ('b', 1))),
(('a', {'b': 1}), ('a', (('b', 1),))),
)
for value, expected in tests:
with self.subTest(value=value):
self.assertEqual(make_hashable(value), expected)
def identity(self):
constructor_signature = inspect.signature(self.__init__)
args, kwargs = self._constructor_args
signature = constructor_signature.bind_partial(*args, **kwargs)
signature.apply_defaults()
arguments = signature.arguments.items()
identity = [self.__class__]
for arg, value in arguments:
if isinstance(value, fields.Field):
value = type(value)
else:
value = make_hashable(value)
identity.append((arg, value))
return tuple(identity)
def test_equal(self):
tests = (
([], ()),
(["a", 1], ("a", 1)),
({}, ()),
({"a"}, ("a",)),
(frozenset({"a"}), {"a"}),
({"a": 1, "b": 2}, (("a", 1), ("b", 2))),
({"b": 2, "a": 1}, (("a", 1), ("b", 2))),
(("a", ["b", 1]), ("a", ("b", 1))),
(("a", {"b": 1}), ("a", (("b", 1),))),
)
for value, expected in tests:
with self.subTest(value=value):
self.assertEqual(make_hashable(value), expected)
def get_group_by(self, select, order_by):
"""
Original SQLCompiler.get_group_by produces type error with tuple(params) when checking
with seen in the last if statement. This is fixed in the newer version (checking on 3.0).
"""
if self.query.group_by is None:
return []
expressions = []
if self.query.group_by is not True:
# If the group by is set to a list (by .values() call most likely),
# then we need to add everything in it to the GROUP BY clause.
# Backwards compatibility hack for setting query.group_by. Remove
# when we have public API way of forcing the GROUP BY clause.
# Converts string references to expressions.
for expr in self.query.group_by:
if not hasattr(expr, 'as_sql'):
expressions.append(self.query.resolve_ref(expr))
else:
expressions.append(expr)
# Note that even if the group_by is set, it is only the minimal
# set to group by. So, we need to add cols in select, order_by, and
# having into the select in any case.
for expr, _, _ in select:
cols = expr.get_group_by_cols()
for col in cols:
expressions.append(col)
for expr, (sql, params, is_ref) in order_by:
# Skip References to the select clause, as all expressions in the
# select clause are already part of the group by.
if not expr.contains_aggregate and not is_ref:
expressions.extend(expr.get_source_expressions())
having_group_by = self.having.get_group_by_cols() if self.having else (
)
for expr in having_group_by:
expressions.append(expr)
result = []
seen = set()
expressions = self.collapse_group_by(expressions, having_group_by)
for expr in expressions:
sql, params = self.compile(expr)
params_hash = make_hashable(params)
if (sql, params_hash) not in seen:
result.append((sql, params))
seen.add((sql, params_hash))
return result
def test_equal(self):
tests = (
([], ()),
(['a', 1], ('a', 1)),
({}, ()),
({'a'}, ('a', )),
(frozenset({'a'}), {'a'}),
({
'a': 1
}, (('a', 1), )),
(('a', ['b', 1]), ('a', ('b', 1))),
(('a', {
'b': 1
}), ('a', (('b', 1), ))),
)
for value, expected in tests:
with self.subTest(value=value):
self.assertEqual(make_hashable(value), expected)
def get_secondary_sectors_display(self):
choices_dict = dict(make_hashable(SectorTags.choices()))
return [
force_str(choices_dict.get(make_hashable(value), value), strings_only=True)
for value in self.secondary_sectors or []
]
def get_order_by(self):
"""
Return a list of 2-tuples of form (expr, (sql, params, is_ref)) for the
ORDER BY clause.
The order_by clause can alter the select clause (for example it
can add aliases to clauses that do not yet have one, or it can
add totally new select clauses).
"""
if self.query.extra_order_by:
ordering = self.query.extra_order_by
elif not self.query.default_ordering:
ordering = self.query.order_by
elif self.query.order_by:
ordering = self.query.order_by
elif self.query.get_meta().ordering:
ordering = self.query.get_meta().ordering
self._meta_ordering = ordering
else:
ordering = []
if self.query.standard_ordering:
asc, desc = ORDER_DIR['ASC']
else:
asc, desc = ORDER_DIR['DESC']
order_by = []
for field in ordering:
if hasattr(field, 'resolve_expression'):
if isinstance(field, Value):
# output_field must be resolved for constants.
field = Cast(field, field.output_field)
if not isinstance(field, OrderBy):
field = field.asc()
if not self.query.standard_ordering:
field = field.copy()
field.reverse_ordering()
order_by.append((field, True))
else:
order_by.append((field, False))
continue
if field == '?': # random
order_by.append((OrderBy(Random()), False))
continue
col, order = get_order_dir(field, asc)
descending = order == 'DESC'
if col in self.query.annotation_select:
# Reference to expression in SELECT clause
order_by.append(
(OrderBy(Ref(col, self.query.annotation_select[col]),
descending=descending), True))
continue
if col in self.query.annotations:
# References to an expression which is masked out of the SELECT
# clause.
if self.query.combinator and self.select:
# Don't use the resolved annotation because other
# combined queries might define it differently.
expr = F(col)
else:
expr = self.query.annotations[col]
if isinstance(expr, Value):
# output_field must be resolved for constants.
expr = Cast(expr, expr.output_field)
order_by.append((OrderBy(expr, descending=descending), False))
continue
if '.' in field:
# This came in through an extra(order_by=...) addition. Pass it
# on verbatim.
table, col = col.split('.', 1)
order_by.append((OrderBy(RawSQL(
'%s.%s' % (self.quote_name_unless_alias(table), col), []),
descending=descending), False))
continue
if not self.query.extra or col not in self.query.extra:
if self.query.combinator and self.select:
# Don't use the first model's field because other
# combined queries might define it differently.
order_by.append((OrderBy(F(col),
descending=descending), False))
else:
# 'col' is of the form 'field' or 'field1__field2' or
# '-field1__field2__field', etc.
order_by.extend(
self.find_ordering_name(field,
self.query.get_meta(),
default_order=asc))
else:
if col not in self.query.extra_select:
order_by.append((OrderBy(RawSQL(*self.query.extra[col]),
descending=descending), False))
else:
order_by.append(
(OrderBy(Ref(col, RawSQL(*self.query.extra[col])),
descending=descending), True))
result = []
seen = set()
for expr, is_ref in order_by:
resolved = expr.resolve_expression(self.query,
allow_joins=True,
reuse=None)
if self.query.combinator and self.select:
src = resolved.get_source_expressions()[0]
expr_src = expr.get_source_expressions()[0]
# Relabel order by columns to raw numbers if this is a combined
# query; necessary since the columns can't be referenced by the
# fully qualified name and the simple column names may collide.
for idx, (sel_expr, _, col_alias) in enumerate(self.select):
if is_ref and col_alias == src.refs:
src = src.source
elif col_alias and not (isinstance(expr_src, F)
and col_alias == expr_src.name):
continue
if src == sel_expr:
resolved.set_source_expressions(
[RawSQL('%d' % (idx + 1), ())])
break
else:
if col_alias:
raise DatabaseError(
'ORDER BY term does not match any column in the result set.'
)
# Add column used in ORDER BY clause without an alias to
# the selected columns.
order_by_idx = len(self.query.select) + 1
col_name = f'__orderbycol{order_by_idx}'
for q in self.query.combined_queries:
q.add_annotation(expr_src, col_name)
self.query.add_select_col(src, col_name)
resolved.set_source_expressions(
[RawSQL(f'{order_by_idx}', ())])
sql, params = self.compile(resolved)
# Don't add the same column twice, but the order direction is
# not taken into account so we strip it. When this entire method
# is refactored into expressions, then we can check each part as we
# generate it.
without_ordering = self.ordering_parts.search(sql)[1]
params_hash = make_hashable(params)
if (without_ordering, params_hash) in seen:
continue
seen.add((without_ordering, params_hash))
result.append((resolved, (sql, params, is_ref)))
return result
def identity(self):
return super().identity + (
self.through,
make_hashable(self.through_fields),
self.db_constraint,
)
def test_unhashable(self):
class Unhashable:
__hash__ = None
with self.assertRaisesMessage(TypeError, "unhashable type: 'Unhashable'"):
make_hashable(Unhashable())
def get_group_by(self, select, order_by):
"""
Return a list of 2-tuples of form (sql, params).
The logic of what exactly the GROUP BY clause contains is hard
to describe in other words than "if it passes the test suite,
then it is correct".
"""
# Some examples:
# SomeModel.objects.annotate(Count('somecol'))
# GROUP BY: all fields of the model
#
# SomeModel.objects.values('name').annotate(Count('somecol'))
# GROUP BY: name
#
# SomeModel.objects.annotate(Count('somecol')).values('name')
# GROUP BY: all cols of the model
#
# SomeModel.objects.values('name', 'pk').annotate(Count('somecol')).values('pk')
# GROUP BY: name, pk
#
# SomeModel.objects.values('name').annotate(Count('somecol')).values('pk')
# GROUP BY: name, pk
#
# In fact, the self.query.group_by is the minimal set to GROUP BY. It
# can't be ever restricted to a smaller set, but additional columns in
# HAVING, ORDER BY, and SELECT clauses are added to it. Unfortunately
# the end result is that it is impossible to force the query to have
# a chosen GROUP BY clause - you can almost do this by using the form:
# .values(*wanted_cols).annotate(AnAggregate())
# but any later annotations, extra selects, values calls that
# refer some column outside of the wanted_cols, order_by, or even
# filter calls can alter the GROUP BY clause.
# The query.group_by is either None (no GROUP BY at all), True
# (group by select fields), or a list of expressions to be added
# to the group by.
if self.query.group_by is None:
return []
expressions = []
if self.query.group_by is not True:
# If the group by is set to a list (by .values() call most likely),
# then we need to add everything in it to the GROUP BY clause.
# Backwards compatibility hack for setting query.group_by. Remove
# when we have public API way of forcing the GROUP BY clause.
# Converts string references to expressions.
for expr in self.query.group_by:
if not hasattr(expr, 'as_sql'):
expressions.append(self.query.resolve_ref(expr))
else:
expressions.append(expr)
# Note that even if the group_by is set, it is only the minimal
# set to group by. So, we need to add cols in select, order_by, and
# having into the select in any case.
if django.VERSION >= (3, 0, 0):
ref_sources = {
expr.source
for expr in expressions if isinstance(expr, Ref)
}
for expr, _, _ in select:
cols = expr.get_group_by_cols()
if django.VERSION >= (3, 0, 0):
# Skip members of the select clause that are already included
# by reference.
if expr in ref_sources:
continue
for col in cols:
expressions.append(col)
# for MSSQL, the stored procedure used by pyodbc doesn't allow
# queries in the format:
# SELECT [AGGREGATE_FUNCTION](*) FROM (SELECT id, F(a) FROM ... GROUP BY id, F(a)) subquery
# the accepted format is:
# SELECT [AGGREGATE_FUNCTION](*) FROM (SELECT id, F(a) FROM ... GROUP BY id, F(a), a) subquery
# Therefore we add the referenced columns in the get_group_by function
for sub_expr in _flatten_expressions_only(expr):
if isinstance(sub_expr, Col):
expr_cols = sub_expr.get_group_by_cols()
for expr_col in expr_cols:
expressions.append(expr_col)
for expr, (sql, params, is_ref) in order_by:
if django.VERSION >= (3, 0, 0):
# Skip References to the select clause, as all expressions in the
# select clause are already part of the group by.
if not is_ref:
expressions.extend(expr.get_group_by_cols())
else:
# Skip References to the select clause, as all expressions in the
# select clause are already part of the group by.
if not expr.contains_aggregate and not is_ref:
expressions.extend(expr.get_source_expressions())
having_group_by = self.having.get_group_by_cols() if self.having else (
)
for expr in having_group_by:
expressions.append(expr)
result = []
seen = set()
expressions = self.collapse_group_by(expressions, having_group_by)
for expr in expressions:
sql, params = self.compile(expr)
if django.VERSION >= (3, 0, 0):
sql, params = expr.select_format(self, sql, params)
else:
if isinstance(expr, Subquery) and not sql.startswith('('):
# Subquery expression from HAVING clause may not contain
# wrapping () because they could be removed when a subquery is
# the "rhs" in an expression (see Subquery._prepare()).
sql = '(%s)' % sql
params_hash = make_hashable(params)
if (sql, params_hash) not in seen:
result.append((sql, params))
seen.add((sql, params_hash))
return result
def _get_FIELD_display(self, field):
value = getattr(self, field.attribute_name)
choices_dict = dict(make_hashable(field.choices))
# force_str() to coerce lazy strings.
return force_str(choices_dict.get(make_hashable(value), value),
strings_only=True)
def get_enum_display(Enum, value):
choices_dict = dict(make_hashable(Enum.choices))
# force_str() to coerce lazy strings.
return force_str(choices_dict.get(make_hashable(value), value),
strings_only=True)
def __hash__(self):
return hash(make_hashable(self.identity))
def __hash__(self):
return hash((self.__class__, self.connector, self.negated, *make_hashable(self.children)))
