def _get_join_paths(self, table_alias, accessor_path):
try:
model_class = self.queryset._get_model_class_from_table(table_alias)
except CacheBotException:
# this is a many to many field
try:
model_class = [f.rel.to for m in get_models() for f in m._meta.local_many_to_many if f.m2m_db_table() == table_alias][0]
except:
import ipdb; ipdb.set_trace()
accessor_path = model_class._meta.pk.attname
yield model_class, accessor_path
join_from_tables = ifilter(lambda x: x[1] == table_alias, self.queryset.query.join_map.keys())
for join_tuple in join_from_tables:
if join_tuple[0]:
for model_class, join_accessor_path in self._get_join_paths(join_tuple[0], join_tuple[2]):
if join_accessor_path == model_class._meta.pk.attname:
for attname, related in self.queryset._get_reverse_relations(model_class):
join_accessor_path = attname
yield model_class, LOOKUP_SEP.join((join_accessor_path, accessor_path))
elif join_accessor_path.split(LOOKUP_SEP)[-1] == 'id':
accessor_path_split = join_accessor_path.split(LOOKUP_SEP)
join_accessor_path = LOOKUP_SEP.join(accessor_path_split[:-1])
yield model_class, LOOKUP_SEP.join((join_accessor_path, accessor_path))
elif join_accessor_path.endswith('_id'):
join_accessor_path = join_accessor_path[:-3]
yield model_class, LOOKUP_SEP.join((join_accessor_path, accessor_path))
else:
yield model_class, LOOKUP_SEP.join((join_accessor_path, accessor_path))
def _get_join_paths(self, table_alias, accessor_path):
model_class, m2m = self.queryset._get_model_class_from_table(
table_alias)
if m2m:
accessor_path = model_class._meta.pk.attname
yield model_class, accessor_path
for join_tuple in self.queryset.query.join_map.keys():
if join_tuple[0] and join_tuple[1] == table_alias:
for model_class, join_accessor_path in self._get_join_paths(
join_tuple[0], join_tuple[2]):
if join_accessor_path == model_class._meta.pk.attname:
for attname, related in self.queryset._get_reverse_relations(
model_class):
join_accessor_path = attname
yield model_class, LOOKUP_SEP.join(
(join_accessor_path, accessor_path))
elif join_accessor_path.split(LOOKUP_SEP)[-1] == 'id':
accessor_path_split = join_accessor_path.split(
LOOKUP_SEP)
join_accessor_path = LOOKUP_SEP.join(
accessor_path_split[:-1])
yield model_class, LOOKUP_SEP.join(
(join_accessor_path, accessor_path))
elif join_accessor_path.endswith('_id'):
join_accessor_path = join_accessor_path[:-3]
yield model_class, LOOKUP_SEP.join(
(join_accessor_path, accessor_path))
else:
yield model_class, LOOKUP_SEP.join(
(join_accessor_path, accessor_path))
def _handle_filter(self, query, item, used_aliases):
# The generic strategy here is:
# 1. Let Django's SQL system handle all the joins etc
# by adding the rangestart and rangeend filters with
# dummy data.
# 2. Iterate over the where tree to pickup the spurious
# where clauses generated in (1), and replace them with
# appropriate SubWhereNode objects that have their own SQL.
filter, value = item
qtype = filter.rsplit(LOOKUP_SEP, 1)
if len(qtype) == 1:
qtype = qtype[0]
query_first = ''
else:
query_first = qtype[0] + '__'
qtype = qtype[1]
if qtype[-3:] == '_id':
qtype = qtype[:-3]
if self.negated:
qtype = REVERSE[qtype]
new_q = Q(
**{
LOOKUP_SEP.join((query_first + 'rangestart', direction_map[qtype][0])):
TOKENS[0],
LOOKUP_SEP.join((query_first + 'rangeend', direction_map[qtype][1])):
TOKENS[1],
})
query.add_q(new_q)
self._update_where(query, query.where, value)
def _chemicalite_filter(self, args, original_kwargs):
chemicalite = connections[self.db].ops
opts = self.query.get_meta()
args = list(args)
kwargs = {}
for lookup, value in original_kwargs.items():
parts = lookup.split(LOOKUP_SEP)
if len(parts) == 1 or parts[-1] not in self.query.query_terms:
lookup_type = 'exact'
else:
lookup_type = parts.pop()
chemfield = _ChemWhereNode._check_chem_field(opts,
LOOKUP_SEP.join(parts))
if (lookup_type in chemicalite.structure_operators and
chemfield and isinstance(chemfield, _MoleculeField)):
stridx_attname = chemfield.stridx_attname
stridx_lookup_type = chemicalite.stridx_lookup[lookup_type]
stridx_parts = (parts[:-1] +
[stridx_attname, stridx_lookup_type])
stridx_lookup = LOOKUP_SEP.join(stridx_parts)
args.extend((Q(**{lookup: value}),
Q(**{stridx_lookup: value})))
else:
kwargs[lookup] = value
return tuple(args), kwargs
def patch_child(self, parts, lookup, value):
inherited_flag = ["is_%s_inherited" % parts[-1]]
inherited_value = ["%s_value" % parts[-1]]
lookup = [lookup] if lookup else []
is_inherited = LOOKUP_SEP.join(parts[:-1] + inherited_flag)
field = LOOKUP_SEP.join(parts[:-1] + inherited_value + lookup)
print is_inherited
print field
return Q(**{is_inherited: False, field: value})
def lookup_allowed(self, lookup, value):
# overriden to allow filter on cell_filter fields
# import django.contrib.admin.options
# django.contrib.admin.options.QUERY_TERMS.update({'not':'not'})
original = DjangoModelAdmin.lookup_allowed(self, lookup, value)
if original:
return True
model = self.model
parts = lookup.split(LOOKUP_SEP)
if len(parts) > 1 and parts[-1] in QUERY_TERMS:
parts.pop()
pk_attr_name = None
for part in parts[:-1]:
field, _, _, _ = model._meta.get_field_by_name(part)
if hasattr(field, 'rel'):
model = field.rel.to
pk_attr_name = model._meta.pk.name
elif isinstance(field, RelatedObject):
model = field.model
pk_attr_name = model._meta.pk.name
else:
pk_attr_name = None
if pk_attr_name and len(parts) > 1 and parts[-1] == pk_attr_name:
parts.pop()
clean_lookup = LOOKUP_SEP.join(parts)
flat_filter = [isinstance(v, tuple) and v[0] or v for v in self.list_filter]
flat_filter.extend([isinstance(v, tuple) and v[0] or v for v in self.cell_filter])
return clean_lookup in self.extra_allowed_filter or clean_lookup in flat_filter
def reverse_field_path(model, path):
""" Create a reversed field path.
E.g. Given (Order, "user__groups"),
return (Group, "user__order").
Final field must be a related model, not a data field.
"""
reversed_path = []
parent = model
pieces = path.split(LOOKUP_SEP)
for piece in pieces:
field, model, direct, m2m = parent._meta.get_field_by_name(piece)
# skip trailing data field if extant:
if len(reversed_path) == len(pieces) - 1: # final iteration
try:
get_model_from_relation(field)
except NotRelationField:
break
if direct:
related_name = field.related_query_name()
parent = field.rel.to
else:
related_name = field.field.name
parent = field.model
reversed_path.insert(0, related_name)
return (parent, LOOKUP_SEP.join(reversed_path))
def reverse_field_path(model, path):
""" Create a reversed field path.
E.g. Given (Order, "user__groups"),
return (Group, "user__order").
Final field must be a related model, not a data field.
"""
reversed_path = []
parent = model
pieces = path.split(LOOKUP_SEP)
for piece in pieces:
field, model, direct, m2m = parent._meta.get_field_by_name(piece)
# skip trailing data field if extant:
if len(reversed_path) == len(pieces)-1: # final iteration
try:
get_model_from_relation(field)
except NotRelationField:
break
if direct:
related_name = field.related_query_name()
parent = field.rel.to
else:
related_name = field.field.name
parent = field.model
reversed_path.insert(0, related_name)
return (parent, LOOKUP_SEP.join(reversed_path))
def get_queryset(self, request, **resources):
if self.queryset is None:
return None
# Make filters from URL variables or resources
filters = dict((k, v) for k, v in resources.iteritems() if k in self.meta.model_fields)
qs = self.queryset.filter(**filters)
# Make filters from GET variables
for field in request.GET.iterkeys():
tokens = field.split(LOOKUP_SEP)
field_name = tokens[0]
if not field_name in self.meta.model_fields or filters.has_key(field_name):
continue
converter = self.model._meta.get_field(field).to_python if len(tokens) == 1 else lambda v: v
value = map(converter, request.GET.getlist(field))
if len(value) > 1:
tokens.append('in')
else:
value = value.pop()
try:
qs = qs.filter(**{LOOKUP_SEP.join(tokens): value})
except FieldError, e:
logger.warning(e)
def _get_reverse_relations(self, model_class):
for related in chain(model_class._meta.get_all_related_objects(), model_class._meta.get_all_related_many_to_many_objects()):
if related.opts.db_table in self.query.tables:
related_name = related.get_accessor_name()
yield related_name, related
for attname, join_related in self._get_reverse_relations(related.model):
yield LOOKUP_SEP.join((related_name + '_cache', attname)), join_related
def get_filters(self, request, **resources):
" Make filters from GET variables. "
filters = dict()
for field in request.GET.iterkeys():
tokens = field.split(LOOKUP_SEP)
field_name = tokens[0]
exclude = False
if tokens[-1] == 'not':
exclude = True
tokens.pop()
if not field_name in self.meta.model_fields:
continue
converter = self.model._meta.get_field(
field_name).to_python if len(tokens) == 1 else lambda v: v
value = map(converter, request.GET.getlist(field))
if len(value) > 1:
tokens.append('in')
else:
value = value.pop()
filters[LOOKUP_SEP.join(tokens)] = (value, exclude)
return filters
def _get_related_models(self, parent_model):
"""
A recursive function that looks at what tables this query spans, and
finds that table's primary key accessor name and model class.
"""
related_models = set()
rev_reversemapping = dict([
(v, k) for k, v in self._reversemapping.iteritems()
])
if rev_reversemapping:
for attname, related in self._get_reverse_relations(parent_model):
if attname in rev_reversemapping:
related_models.add(
(rev_reversemapping[attname], related.model))
for field in parent_model._meta.fields:
if field.rel and field.rel.to._meta.db_table in self.query.tables and field.rel.to != parent_model:
related_models.add((field.attname, field.rel.to))
for attname, model_class in related_models:
yield attname, model_class
if attname.endswith("_id"):
attname = attname[:-3]
for join_attname, model_klass in self._get_related_models(
model_class):
yield LOOKUP_SEP.join((attname, join_attname)), model_klass
def apply_sorting(self, obj_list, options=None):
"""
Given a dictionary of options, apply some ORM-level sorting to the
provided ``QuerySet``.
Looks for the ``order_by`` key and handles either ascending (just the
field name) or descending (the field name with a ``-`` in front).
The field name should be the resource field, **NOT** model field.
"""
if options is None:
options = {}
parameter_name = 'order_by'
if not 'order_by' in options:
if not 'sort_by' in options:
# Nothing to alter the order. Return what we've got.
return obj_list
else:
warnings.warn("'sort_by' is a deprecated parameter. Please use 'order_by' instead.")
parameter_name = 'sort_by'
order_by_args = []
if hasattr(options, 'getlist'):
order_bits = options.getlist(parameter_name)
else:
order_bits = options.get(parameter_name)
if not isinstance(order_bits, (list, tuple)):
order_bits = [order_bits]
for order_by in order_bits:
order_by_bits = order_by.split(LOOKUP_SEP)
field_name = order_by_bits[0]
order = ''
if order_by_bits[0].startswith('-'):
field_name = order_by_bits[0][1:]
order = '-'
if not field_name in self.fields:
# It's not a field we know about. Move along citizen.
raise InvalidSortError("No matching '%s' field for ordering on." % field_name)
if not field_name in self._meta.ordering:
raise InvalidSortError("The '%s' field does not allow ordering." % field_name)
if self.fields[field_name].attribute is None:
raise InvalidSortError("The '%s' field has no 'attribute' for ordering with." % field_name)
order_by_args.append("%s%s" % (order, LOOKUP_SEP.join([self.fields[field_name].attribute] + order_by_bits[1:])))
reverse = False
if order == '-': reverse = True
ordered_list = sorted(obj_list, key= attrgetter(field_name), reverse=reverse)
return ordered_list
def _get_reverse_relations(self, model_class):
for related in chain(model_class._meta.get_all_related_objects(), model_class._meta.get_all_related_many_to_many_objects()):
if related.opts.db_table in self.query.tables and related.model != model_class:
related_name = related.get_accessor_name()
yield related_name, related
if related.model != related.parent_model:
for attname, join_related in self._get_reverse_relations(related.model):
yield LOOKUP_SEP.join((related_name + '_cache', attname)), join_related
def patch_parent(self, parts, lookup, value):
_parts = parts[:]
last_field_name, model = self.traverse_models(_parts, self.model)
lookup = [lookup] if lookup else []
parents_name, parents_field_name = model.FIELD_INHERITANCE_MAP[last_field_name]
parent_lookup = LOOKUP_SEP.join(parts[:-1] + [parents_name] + [parents_field_name] + lookup)
print parent_lookup
return Q(**{parent_lookup: value})
def build_filters(self, filters=None):
"""
Given a dictionary of filters, create the necessary ORM-level filters.
Keys should be resource fields, **NOT** model fields.
Valid values are either a list of Django filter types (i.e.
``['startswith', 'exact', 'lte']``), the ``ALL`` constant or the
``ALL_WITH_RELATIONS`` constant.
"""
# At the declarative level:
# filtering = {
# 'resource_field_name': ['exact', 'gt', 'gte', 'lt', 'lte', 'range'],
# 'resource_field_name_3': ALL,
# 'resource_field_name_4': ALL_WITH_RELATIONS,
# ...
# }
# Accepts the filters as a dict. None by default, meaning no filters.
if filters is None:
filters = {}
qs_filters = {}
for filter_expr, value in filters.items():
filter_bits = filter_expr.split(LOOKUP_SEP)
field_name = filter_bits.pop(0)
filter_type = 'exact'
if not field_name in self.fields:
# It's not a field we know about. Move along citizen.
continue
if len(filter_bits) and filter_bits[-1] in QUERY_TERMS.keys():
filter_type = filter_bits.pop()
lookup_bits = self.check_filtering(field_name, filter_type,
filter_bits)
if value in ['true', 'True', True]:
value = True
elif value in ['false', 'False', False]:
value = False
elif value in ('nil', 'none', 'None', None):
value = None
# Split on ',' if not empty string and either an in or range filter.
if filter_type in ('in', 'range') and len(value):
if hasattr(filters, 'getlist'):
value = filters.getlist(filter_expr)
else:
value = value.split(',')
redis_model_field_name = LOOKUP_SEP.join(lookup_bits)
qs_filter = "%s%s%s" % (redis_model_field_name, LOOKUP_SEP,
filter_type)
qs_filters[qs_filter] = value
return dict_strip_unicode_keys(qs_filters)
def build_filters(self, filters=None):
"""
Given a dictionary of filters, create the necessary ORM-level filters.
Keys should be resource fields, **NOT** model fields.
Valid values are either a list of Django filter types (i.e.
``['startswith', 'exact', 'lte']``), the ``ALL`` constant or the
``ALL_WITH_RELATIONS`` constant.
"""
# At the declarative level:
# filtering = {
# 'resource_field_name': ['exact', 'startswith', 'endswith', 'contains'],
# 'resource_field_name_2': ['exact', 'gt', 'gte', 'lt', 'lte', 'range'],
# 'resource_field_name_3': ALL,
# 'resource_field_name_4': ALL_WITH_RELATIONS,
# ...
# }
# Accepts the filters as a dict. None by default, meaning no filters.
if filters is None:
filters = {}
qs_filters = {}
for filter_expr, value in filters.items():
filter_bits = filter_expr.split(LOOKUP_SEP)
field_name = filter_bits.pop(0)
filter_type = 'exact'
if not field_name in self.fields:
# It's not a field we know about. Move along citizen.
continue
if len(filter_bits) and filter_bits[-1] in QUERY_TERMS.keys():
filter_type = filter_bits.pop()
lookup_bits = self.check_filtering(field_name, filter_type, filter_bits)
if value in ['true', 'True', True]:
value = True
elif value in ['false', 'False', False]:
value = False
elif value in ('nil', 'none', 'None', None):
value = None
# Split on ',' if not empty string and either an in or range filter.
if filter_type in ('in', 'range') and len(value):
if hasattr(filters, 'getlist'):
value = filters.getlist(filter_expr)
else:
value = value.split(',')
db_field_name = LOOKUP_SEP.join(lookup_bits)
qs_filter = "%s%s%s" % (db_field_name, LOOKUP_SEP, filter_type)
qs_filters[qs_filter] = value
return dict_strip_unicode_keys(qs_filters)
def lookup_allowed(self, lookup, value):
# overriden to allow filter on cell_filter fields
# import django.contrib.admin.options
# django.contrib.admin.options.QUERY_TERMS.update({'not':'not'})
original = super(IModelAdmin, self).lookup_allowed(lookup, value)
if original:
return True
parts = lookup.split(LOOKUP_SEP)
if len(parts) > 1 and parts[-1] in QUERY_TERMS:
parts.pop()
clean_lookup = LOOKUP_SEP.join(parts)
return clean_lookup in self.extra_allowed_filter
def _get_join_paths(self, table_alias, accessor_path):
model_class, m2m = self.queryset._get_model_class_from_table(table_alias)
if m2m:
accessor_path = model_class._meta.pk.attname
yield model_class, accessor_path
for join_tuple in self.queryset.query.join_map.keys():
if join_tuple[0] and join_tuple[1] == table_alias:
for model_class, join_accessor_path in self._get_join_paths(join_tuple[0], join_tuple[2]):
if join_accessor_path == model_class._meta.pk.attname:
for attname, related in self.queryset._get_reverse_relations(model_class):
join_accessor_path = attname
yield model_class, LOOKUP_SEP.join((join_accessor_path, accessor_path))
elif join_accessor_path.split(LOOKUP_SEP)[-1] == 'id':
accessor_path_split = join_accessor_path.split(LOOKUP_SEP)
join_accessor_path = LOOKUP_SEP.join(accessor_path_split[:-1])
yield model_class, LOOKUP_SEP.join((join_accessor_path, accessor_path))
elif join_accessor_path.endswith('_id'):
join_accessor_path = join_accessor_path[:-3]
yield model_class, LOOKUP_SEP.join((join_accessor_path, accessor_path))
else:
yield model_class, LOOKUP_SEP.join((join_accessor_path, accessor_path))
def passes_through(base_model, path, target, addition=''):
if not path:
return False
if base_model == target and addition:
return addition
elif base_model == target and not addition:
return True
steps = path.split(LOOKUP_SEP)
model = base_model
new_path = ()
for step in steps:
model = next_model(step, model)
new_path += (step, )
if model == target:
return LOOKUP_SEP.join(new_path + (addition, ))
return ''
def _expand_money_params(kwargs):
from moneyed import Money
from django.db.models.sql.constants import LOOKUP_SEP, QUERY_TERMS
to_append = {}
for name, value in kwargs.items():
if isinstance(value, Money):
# Get rid of __lt, __gt etc for the currency lookup
path = name.split(LOOKUP_SEP)
if QUERY_TERMS.has_key(path[-1]):
clean_name = LOOKUP_SEP.join(path[:-1])
else:
clean_name = name
to_append[name] = value.amount
to_append[currency_field_name(clean_name)] = smart_unicode(value.currency)
kwargs.update(to_append)
return kwargs
def lookup_allowed(self, lookup, value): model = self.model # Check FKey lookups that are allowed, so that popups produced by # ForeignKeyRawIdWidget, on the basis of ForeignKey.limit_choices_to, # are allowed to work. for l in model._meta.related_fkey_lookups: for k, v in widgets.url_params_from_lookup_dict(l).items(): if k == lookup and v == value: return True parts = lookup.split(LOOKUP_SEP) # Last term in lookup is a query term (__exact, __startswith etc) # This term can be ignored. if len(parts) > 1 and parts[-1] in QUERY_TERMS: parts.pop() # Special case -- foo__id__exact and foo__id queries are implied # if foo has been specificially included in the lookup list; so # drop __id if it is the last part. However, first we need to find # the pk attribute name. pk_attr_name = None for part in parts[:-1]: field, _, _, _ = model._meta.get_field_by_name(part) if hasattr(field, 'rel'): model = field.rel.to pk_attr_name = model._meta.pk.name elif isinstance(field, RelatedObject): model = field.model pk_attr_name = model._meta.pk.name else: pk_attr_name = None if pk_attr_name and len(parts) > 1 and parts[-1] == pk_attr_name: parts.pop() try: self.model._meta.get_field_by_name(parts[0]) except FieldDoesNotExist: # Lookups on non-existants fields are ok, since they're ignored # later. return True else: if len(parts) == 1: return True clean_lookup = LOOKUP_SEP.join(parts) return clean_lookup in self.list_filter or clean_lookup == self.date_hierarchy
def build_filters(self, filters=None):
# Override the filters so we can stop Tastypie silently ignoring
# invalid filters. That will cause an invalid filtering just to return
# lots of results.
if filters is None:
filters = {}
qs_filters = {}
for filter_expr, value in filters.items():
filter_bits = filter_expr.split(LOOKUP_SEP)
field_name = filter_bits.pop(0)
filter_type = 'exact'
if not field_name in self.fields:
# Don't just ignore this. Tell the world. Shame I have to
# override all this, just to do this.
raise InvalidFilterError('Not a valid filtering field: %s'
% field_name)
if len(filter_bits) and filter_bits[-1] in QUERY_TERMS.keys():
filter_type = filter_bits.pop()
lookup_bits = self.check_filtering(field_name, filter_type,
filter_bits)
if value in ['true', 'True', True]:
value = True
elif value in ['false', 'False', False]:
value = False
elif value in ('nil', 'none', 'None', None):
value = None
# Split on ',' if not empty string and either an in or range
# filter.
if filter_type in ('in', 'range') and len(value):
if hasattr(filters, 'getlist'):
value = filters.getlist(filter_expr)
else:
value = value.split(',')
db_field_name = LOOKUP_SEP.join(lookup_bits)
qs_filter = "%s%s%s" % (db_field_name, LOOKUP_SEP, filter_type)
qs_filters[qs_filter] = value
return dict_strip_unicode_keys(qs_filters)
def build_filters(self, filters=None):
# Override the filters so we can stop Tastypie silently ignoring
# invalid filters. That will cause an invalid filtering just to return
# lots of results.
if filters is None:
filters = {}
qs_filters = {}
for filter_expr, value in filters.items():
filter_bits = filter_expr.split(LOOKUP_SEP)
field_name = filter_bits.pop(0)
filter_type = 'exact'
if not field_name in self.fields:
# Don't just ignore this. Tell the world. Shame I have to
# override all this, just to do this.
raise InvalidFilterError('Not a valid filtering field: %s' %
field_name)
if len(filter_bits) and filter_bits[-1] in QUERY_TERMS:
filter_type = filter_bits.pop()
lookup_bits = self.check_filtering(field_name, filter_type,
filter_bits)
if value in ['true', 'True', True]:
value = True
elif value in ['false', 'False', False]:
value = False
elif value in ('nil', 'none', 'None', None):
value = None
# Split on ',' if not empty string and either an in or range
# filter.
if filter_type in ('in', 'range') and len(value):
if hasattr(filters, 'getlist'):
value = filters.getlist(filter_expr)
else:
value = value.split(',')
db_field_name = LOOKUP_SEP.join(lookup_bits)
qs_filter = '%s%s%s' % (db_field_name, LOOKUP_SEP, filter_type)
qs_filters[qs_filter] = value
return dict_strip_unicode_keys(qs_filters)
def _expand_money_params(kwargs):
from moneyed import Money
from django.db.models.sql.constants import LOOKUP_SEP, QUERY_TERMS
to_append = {}
for name, value in kwargs.items():
if isinstance(value, Money):
# Get rid of __lt, __gt etc for the currency lookup
path = name.split(LOOKUP_SEP)
if QUERY_TERMS.has_key(path[-1]):
clean_name = LOOKUP_SEP.join(path[:-1])
else:
clean_name = name
to_append[name] = value.amount
to_append[currency_field_name(clean_name)] = smart_unicode(
value.currency)
kwargs.update(to_append)
return kwargs
def path_v2(model, *conditions):
'''
Returns a filter of all objects of type model, subject to the list of conditions.
The first argument, model, must be a model. It cannot be an instance of a model, or any other object.
The list of conditions can be arbitrarily long, but must consist entirely of instances of models.
The function uses path_v1() to find the forward path from model to conditions[i] (for all i), and then applies the appropriate filter.
If there is more than one path from model to conditions[i], the function asks the user (via the raw_input() function) which path(s) to filter on.
'''
if not (isclass(model) and issubclass(model, Model)):
raise TypeError("path_v2 argument 1 must be a model")
if not len(conditions):
return model.objects.all()
models = [None for i in range(len(conditions))]
for i in range(len(conditions)):
if not isinstance(conditions[i], Model):
raise TypeError(
"path_v2 argument 2 must be a list of instances of models")
elif isinstance(conditions[i], model):
raise TypeError(
"path_v2 argument 2 must not contain any instances of argument 1"
)
models[i] = type(conditions[i])
kwargs = {}
paths = None
repeat = None
use = None
for i in range(len(conditions)):
paths = path_v1(model, models[i])
for (path, _, _) in paths:
repeat = True
path = LOOKUP_SEP.join(path)
while repeat:
use = raw_input("Include the path " + path +
" as a condition ([y]/n)? ")
repeat = False
if use.lower() == 'y' or use.lower() == 'yes' or not use:
kwargs[path] = conditions[i]
elif not (use.lower() == 'n' or use.lower() == 'no'):
repeat = True
return model.objects.filter(**kwargs)
def filters_for_model(model, fields=None, exclude=None, filter_for_field=None):
field_dict = SortedDict()
opts = model._meta
if fields is None:
fields = [f.name for f in sorted(opts.fields + opts.many_to_many)]
for f in fields:
if exclude is not None and f in exclude:
continue
if f.split(LOOKUP_SEP)[-1] in QUERY_TERMS.keys():
lookup_type = f.split(LOOKUP_SEP)[-1]
f = LOOKUP_SEP.join(f.split(LOOKUP_SEP)[:-1])
else:
lookup_type = None
field = get_model_field(model, f)
if field is None:
field_dict[f] = None
continue
filter_ = filter_for_field(field, f, lookup_type)
if filter_:
field_dict[f] = filter_
return field_dict
def _get_related_models(self, parent_model):
"""
A recursive function that looks at what tables this query spans, and
finds that table's primary key accessor name and model class.
"""
related_models = set()
rev_reversemapping = dict([(v,k) for k,v in self._reversemapping.iteritems()])
if rev_reversemapping:
for attname, related in self._get_reverse_relations(parent_model):
related_models.add((rev_reversemapping[attname], related.model))
for field in parent_model._meta.fields:
if field.rel and field.rel.to._meta.db_table in self.query.tables:
related_models.add((field.attname, field.rel.to))
for attname, model_class in related_models:
yield attname, model_class
if attname.endswith("_id"):
attname = attname[:-3]
for join_attname, model_class in self._get_related_models(model_class):
yield LOOKUP_SEP.join((attname,join_attname)), model_class
def filters_for_model(model, fields=None, exclude=None, filter_for_field=None):
field_dict = SortedDict()
opts = model._meta
if fields is None:
fields = [f.name for f in sorted(opts.fields + opts.many_to_many)]
for f in fields:
if exclude is not None and f in exclude:
continue
if f.split(LOOKUP_SEP)[-1] in QUERY_TERMS.keys():
lookup_type = f.split(LOOKUP_SEP)[-1]
f = LOOKUP_SEP.join(f.split(LOOKUP_SEP)[:-1])
else:
lookup_type = None
field = get_model_field(model, f)
if field is None:
field_dict[f] = None
continue
filter_ = filter_for_field(field, f, lookup_type)
if filter_:
field_dict[f] = filter_
return field_dict
def apply_sorting(self, obj_list, options=None):
"""
Given a dictionary of options, apply some ORM-level sorting to the
provided ``QuerySet``.
Looks for the ``sort_by`` key and handles either ascending (just the
field name) or descending (the field name with a ``-`` in front).
The field name should be the resource field, **NOT** model field.
"""
if options is None:
options = {}
if not 'sort_by' in options:
# Nothing to alter the sort order. Return what we've got.
return obj_list
sort_by_bits = options['sort_by'].split(LOOKUP_SEP)
field_name = sort_by_bits[0]
order = ''
if sort_by_bits[0].startswith('-'):
field_name = sort_by_bits[0][1:]
order = '-'
if not field_name in self.fields:
# It's not a field we know about. Move along citizen.
raise InvalidSortError("No matching '%s' field for ordering on." % field_name)
if not field_name in self._meta.ordering:
raise InvalidSortError("The '%s' field does not allow ordering." % field_name)
if self.fields[field_name].attribute is None:
raise InvalidSortError("The '%s' field has no 'attribute' for ordering with." % field_name)
sort_expr = "%s%s" % (order, LOOKUP_SEP.join([self.fields[field_name].attribute] + sort_by_bits[1:]))
return obj_list.order_by(sort_expr)
def process_queryset(queryset,display_fields=None):
"""
This is used in the custom_view below, but its de-coupled so it can be used
programatically as well. Simply pass in a queryset and a list of relations to display
and viola.
Relations look like: ['address__zip','contact__date']
"""
display_fields = display_fields or []
extra_select_kwargs = {}
select_related = []
used_routes = []
distinct = True
for i in display_fields:
if i in queryset.query.aggregates or i in queryset.query.extra:
continue # Want below check to work only for relations, excluding aggregates.
select_related_token = i
if LOOKUP_SEP in i:
"""
Since select_related() is rather flexible about what it receives
(ignoring things it doesn't like), we'll just haphazardly pass
all filter and display fields in for now.
"""
select_related_token = i.split(LOOKUP_SEP)
select_related_token.pop() # get rid of the field name
select_related_token = LOOKUP_SEP.join(select_related_token)
"""
Here we remove distinct status for queries which have reverse relations
and possibly numerous results per original record
"""
if distinct and is_reverse_related(i,queryset.model):
distinct = False
primary_model = queryset.model
join_route = i
if len(i.split(LOOKUP_SEP)) > 2:
second_to_last = LOOKUP_SEP.join(i.split(LOOKUP_SEP)[0:-1])
join_route = LOOKUP_SEP.join(i.split(LOOKUP_SEP)[-2:])
primary_model = get_closest_relation(queryset.model,second_to_last)[0]
primary_table = primary_model._meta.db_table
if primary_table in queryset.query.table_map:
primary_table = queryset.query.table_map[primary_table][-1] # Will the last one always work?
join_model, join_field, join_name = get_closest_relation(primary_model,join_route)
join_table = join_model._meta.db_table
try:
join_table = queryset.query.table_map[join_table][-1]
queryset = queryset.extra(select={i: '%s.%s' % (join_table,join_field.column)})
except KeyError:
"""
Design decision. This will work fine if the ModelAdmin does the displaying of
related objects for us. At this time, Django doesn't. We have a patch in place
but aren't using it.
for now we just need the join column between the primary table and the join table.
"""
join_table = join_model._meta.db_table
join_column = "id" # Always this for now.
for field_name in primary_model._meta.get_all_field_names():
from django.db import models
try:
field = primary_model._meta.get_field(field_name)
if (isinstance(field,models.OneToOneField) or isinstance(field,models.ForeignKey)) and \
field.rel.to == join_model:
# 2 foreignkeys on the same model issue
# https://code.djangoproject.com/ticket/12890
if field_name not in select_related_token:
continue
whereclause = '%s.%s=%s.%s' % (join_table,join_column,primary_table,field.column)
if not (select_related_token,join_table) in used_routes:
queryset = queryset.extra(select={i: '%s.%s' % (join_table,join_field.column)},\
tables=[join_table],where=[whereclause])
else:
queryset = queryset.extra(select={i: '%s.%s' % (join_table,join_field.column)},where=[whereclause])
except models.FieldDoesNotExist:
pass
if not (select_related_token,join_table) in used_routes:
used_routes.append((select_related_token, join_table))
if not select_related_token in select_related:
select_related.append(select_related_token)
if select_related:
queryset = queryset.select_related(*select_related)
if distinct:
queryset = queryset.distinct()
return queryset
def display_list_redux(query_class,_model_class=None,inclusions=None,
model_exclusions=None,_relation_list=None):
"""
User Args:
inclusions: A list of string-module relationships we want to be allowed to
choose from, if not passed in, it means we want all the relations
query_class: The class at the top of the tree hierarchy, essentially what
we are reporting on.
Function Args:
_model_class: As we progress through the tree, we need to keep track of what model we are on.
_relation_list: What our function returns, but we need to pass it through so it can find
it's way to the top... may be able to change this though
function return: A list of tuples, where each tuple represents
(string-module relationship, human-readable-value)
ex. [
('first_name', 'Consumer :: First Name'),
('address__zip', 'Consmer :: Address :: Zip'),
('pwi__refer_date', 'Consumer :: PWI :: Referral Date')
]
"""
_relation_list = _relation_list or []
model_exclusions = model_exclusions or []
inclusions = inclusions or []
# if no model class is passed, then we are at the beginning or "base_class"
query_aggregates = None
if query_class.__class__ == query.QuerySet:
query_aggregates = query_class.query.aggregates
query_class = query_class.model
_model_class = _model_class or query_class
model_exclusions.append(_model_class._meta.module_name)
current_inclusions = [r.split(LOOKUP_SEP,1)[0] for r in inclusions] # these are the ONLY fields and relations to be returned
# Non-relational fields are easy and just get appended to the list as is pretty much
non_relation_fields = [f for f in _model_class._meta.fields]
# Now handle the relations
# Get the forward ones
relations = [(f.rel.to, f.name, f.verbose_name.lower()) for f in _model_class._meta.fields if \
hasattr(f.rel, "to") and \
f.rel.to._meta.module_name not in model_exclusions]
# and grab our backward ones
relations.extend([(r.model, r.field.related_query_name(), r.model._meta.verbose_name) for r in _model_class._meta.get_all_related_objects() if \
r.model._meta.module_name not in model_exclusions])
# We have to handle the inclusion list separately because if there isn't one, we don't want to filter over nothing
if current_inclusions:
non_relation_fields = [f for f in non_relation_fields if f.name in current_inclusions]
relations = [r for r in relations if r[0]._meta.module_name in current_inclusions] # r == (model, model.verbose_name)
# At this point we are finally adding our fields to the tuple list
if query_aggregates:
[_relation_list.append(( q, q )) for q in query_aggregates.keys()]
for field in non_relation_fields:
if _model_class != query_class:
_relation_list.append((
field.name,
field.verbose_name.lower()
#' :: '.join([_model_class._meta.verbose_name.lower(), field.verbose_name.lower()])
))
else: _relation_list.append(( field.name, ' :: '.join([_model_class._meta.module_name, field.verbose_name.lower()]) ))
## Recursion happens at this point, we are basically going down one tree / relations at a time before we do another one
# so taking consumer... it will do consumer->address->zip and then it will do consumer->emergency_contact and
# then whatever backward relations
for relation in relations:
# prepare the inclusion for the next recursive call by chopping off all relations that match the one in our loop
relation_inclusions = [name.split(LOOKUP_SEP, 1)[1] for name in inclusions if LOOKUP_SEP in name and name.split(LOOKUP_SEP,1)[0] == relation[1]]
if not current_inclusions:
return _relation_list
# recurse
###
# we use copy.deepcopy on model_exclusions because we don't want a global list of exclusions everytime it adds a new one,
# just the ones down this tree
relation_pair_list = display_list_redux(query_class,_model_class=relation[0],\
inclusions=relation_inclusions,model_exclusions=copy.deepcopy(model_exclusions),\
)
# build the module-relation and human-readable string
###
# We check if _model_class != query_class because we have a case here where once we hit the top of the tree,
# then we don't want to append the query_class to the module-relation
if _model_class != query_class:
_relation_list.extend([
(LOOKUP_SEP.join([relation[1], relation_pair[0]]),
' :: '.join([relation[2], relation_pair[1]]))
for relation_pair in relation_pair_list
])
else:
_relation_list.extend([
(LOOKUP_SEP.join([relation[1], relation_pair[0]]),
' :: '.join([_model_class._meta.module_name, relation[2], relation_pair[1]])) \
for relation_pair in relation_pair_list
])
# go back up the recursion tree now
return _relation_list
def display_list(query_class,_model_class=None,inclusions=None,exclusions=None,depth=None,\
model_exclusions=None,_max_depth=None,_relation_list=None):
"""
User Args:
depth: how far our relation follows, we want to make sure to include forward then backward relations, as well.
ex. depth=1, consumer->field, consumer->disability_primary->field, consumer->address->field, consumer<-goal<-field, consumer<-pwi<-field
inclusions: A list of string-module relationships we want to be allowed to choose from, if not passed in, it means we want all the relations
exclusions: A list of string-module relationships to not follow, it could be a field or whole relation.
query_class: The class at the top of the tree hierarchy, essentially what we are reporting on.
Function Args:
_model_class: As we progress through the tree, we need to keep track of what model we are on.
_max_depth: Takes the depth, and depth starts as a counter from 0, just easier to read this way
_relation_list: What our function returns, but we need to pass it through so it can find it's way to the top... may be able to change this though
function return: A list of tuples, where each tuple represents (string-module relationship, human-readable-value)
ex. [
('first_name', 'Consumer :: First Name'),
('address__zip', 'Consmer :: Address :: Zip'),
('pwi__refer_date', 'Consumer :: PWI :: Referral Date')
]
"""
_relation_list = _relation_list or []
exclusions = exclusions or []
model_exclusions = model_exclusions or []
inclusions = inclusions or []
# if no model class is passed, then we are at the beginning or "base_class"
query_aggregates = None
if query_class.__class__ == query.QuerySet:
query_aggregates = query_class.query.aggregates
query_class = query_class.model
_model_class = _model_class or query_class
## less typing when calling the function, we use depth to set _max_depth from the first call, and use _max_depth henceforth.
# thus depth just keeps track of what level we are on
if not _max_depth:
_max_depth = depth
depth = 0
exclusions.extend(['logentry', 'message', 'id']) # these are always excluded fields
## We dont want our backward relations in the next recursive step, so we add it to our exclusions.
# as well as the base class.
exclusions.append(_model_class._meta.module_name)
model_exclusions.append(_model_class._meta.module_name)
current_inclusions = [r.split(LOOKUP_SEP,1)[0] for r in inclusions] # these are the ONLY fields and relations to be returned
current_exclusions = [r for r in exclusions if LOOKUP_SEP not in r] # these are the fields / relations we don't want to show up
# Non-relational fields are easy and just get appended to the list as is pretty much
non_relation_fields = [f for f in _model_class._meta.fields if \
f.name not in current_exclusions and
f.name not in model_exclusions]
# Now handle the relations
# Get the forward ones
relations = [(f.rel.to, f.name, f.verbose_name.lower()) for f in _model_class._meta.fields if \
hasattr(f.rel, "to") and \
f.name not in current_exclusions and
f.rel.to._meta.module_name not in model_exclusions]
# and grab our backward ones
relations.extend([(r.model, r.field.related_query_name(), r.model._meta.verbose_name) for r in _model_class._meta.get_all_related_objects() if \
r.model._meta.module_name not in current_exclusions and
r.model._meta.module_name not in model_exclusions])
# We have to handle the inclusion list separately because if there isn't one, we don't want to filter over nothing
if current_inclusions:
non_relation_fields = [f for f in non_relation_fields if f.name in current_inclusions]
relations = [r for r in relations if r[0]._meta.module_name in current_inclusions] # r == (model, model.verbose_name)
# At this point we are finally adding our fields to the tuple list
if query_aggregates:
[_relation_list.append(( q, q )) for q in query_aggregates.keys()]
for field in non_relation_fields:
if _model_class != query_class:
_relation_list.append((
field.name,
field.verbose_name.lower()
#' :: '.join([_model_class._meta.verbose_name.lower(), field.verbose_name.lower()])
))
else: _relation_list.append(( field.name, ' :: '.join([_model_class._meta.module_name, field.verbose_name.lower()]) ))
## Recursion happens at this point, we are basically going down one tree / relations at a time before we do another one
# so taking consumer... it will do consumer->address->zip and then it will do consumer->emergency_contact and
# then whatever backward relations
for relation in relations:
# prepare the inclusion/exclusion for the next recursive call by chopping off all relations that match the one in our loop
relation_inclusions = [name.split(LOOKUP_SEP, 1)[1] for name in inclusions if LOOKUP_SEP in name and name.split(LOOKUP_SEP,1)[0] == relation[1]]
relation_exclusions = [name.split(LOOKUP_SEP, 1)[1] for name in exclusions if LOOKUP_SEP in name and name.split(LOOKUP_SEP,1)[0] == relation[1]]
if current_inclusions: pass # if we have inclusions we want to continue with, don't return this tree yet
elif depth >= _max_depth: return _relation_list # return this tree
# if we have reached a star in the exclusions, then skip the rest of this relation
if [True for r in relation_exclusions if '*' in r.split(LOOKUP_SEP, 1)[0]]:
continue
# recurse
###
# we use copy.deepcopy on model_exclusions because we don't want a global list of exclusions everytime it adds a new one,
# just the ones down this tree
relation_pair_list = display_list(query_class,_model_class=relation[0],\
inclusions=relation_inclusions,exclusions=relation_exclusions,model_exclusions=copy.deepcopy(model_exclusions),\
depth=depth + 1,_max_depth=_max_depth)
# build the module-relation and human-readable string
###
# We check if _model_class != query_class because we have a case here where once we hit the top of the tree,
# then we don't want to append the query_class to the module-relation
if _model_class != query_class:
_relation_list.extend([
(LOOKUP_SEP.join([relation[1], relation_pair[0]]),
' :: '.join([relation[2], relation_pair[1]]))
for relation_pair in relation_pair_list
])
else:
_relation_list.extend([
(LOOKUP_SEP.join([relation[1], relation_pair[0]]),
' :: '.join([_model_class._meta.module_name, relation[2], relation_pair[1]])) \
for relation_pair in relation_pair_list
])
# go back up the recursion tree now
return _relation_list
def clip_lookup(lookup):
return LOOKUP_SEP.join(lookup.split(LOOKUP_SEP)[1:]) or None
def deep_prefetch_related_objects(objects, lookups):
"""
Helper function for prefetch_related functionality.
Populates prefetched objects caches for a list of results
from a QuerySet.
Differs from :meth:`django.db.models.query.prefetch_related_objects`
in that that it can prefetch "non-strict" lookups through GFK.
:param objects: result cache of base queryset.
:param lookups: list with lookups.
"""
#How it works
#------------
# - Since data is not of same type - "unit" of data is a model instance
# (object), not QuerySet.
# - Goals that I was tried to achieve:
# - Prefetching 'non-strict' lookups.
# - Absence of doubled queries to DB, other redundancy in processing
# - and infinite recursions.
# - Simplicity of design in comparison with original
# `django.db.models.query.prefetch_related_objects`
# - Sets are used to avoid duplicates.
# - Python's sets distinguish objects by value. But task require
# to distinguish objects by id, not by value, because two objects
# that are same by value, might be included in two different
# querysets, and both of them must be populated with cache.
# - BUT fetched cache data for object are distinguished by value.
# Rationale is to distinguish unique queries and prevent their
# repeated execution. We don't care about "id" of queries, we care
# about their value. Composite identifier for query is
# object and lookup for which query being constructed.
# Only one query exists for one object (by value) and lookup part.
# In function there slightly more complex structure than
# `obj -> lookup` used for storing data for traversed lookups
# (``seen``) - that's done to reduce redundancy in data, but
# "primary key" for stroed data is `obj -> lookup`.
# - Data flows through `buffer`, during processing.
# Objects discovered while traversing DB structure are being added
# and processed objects are removed.
#todo beauty and refactoring
if len(objects) == 0:
return # nothing to do
#lookup -> model -> {(id(obj), obj), ...}
#id(obj) - because objects must be distinguished by id.
#DefaultOrederedDict because order of lookups matter,
#see tests.LookupOrderingTest.test_order of Django test suite.
buffer = DefaultOrderedDict(lambda: defaultdict(set))
update_buffer(buffer, objects, reversed(lookups))
seen = tree() # model -> attr ->
# single -> bool
# cache_name -> str
# cache ->
# obj -> [cache]
while True:
try:
lookup = last(buffer.keys())
except TypeError:
break
try:
model, current = buffer[lookup].popitem()
except KeyError:
del buffer[lookup]
continue
attr = lookup.split(LOOKUP_SEP)[0]
sample = current.pop()
current.add(sample)
_, object = sample
prefetcher, _, attr_found, _ = get_prefetcher(object, attr)
#Lookup is not valid for that object, it must be skipped.
#No exception, because data it is that data is of diffrerent types,
#so - such situation is normal.
if not attr_found:
continue
if LOOKUP_SEP not in lookup and prefetcher is None:
raise ValueError("'%s' does not resolve to a item that supports "
"prefetching - this is an invalid parameter to "
"prefetch_related()." % lookup)
if prefetcher is None:
clipped = clip_lookup(lookup)
if clipped:
update_buffer(
buffer,
filter(is_not_none,
[getattr(o, attr) for _, o in current]), [clipped])
continue
to_discard = set()
for e in current: # no need to query for already prefetched data
obj = e[1]
obj_model = obj.__class__
p, d, _, is_fetched = get_prefetcher(obj, attr)
cache = None
if is_fetched: # case of Django internal cache
single, cache_name = get_info(p, d)
cache = get_cache(obj, single, cache_name, attr)
single, cache_name = get_info(p, d)
update_seen(seen, model, attr, single, cache_name, obj, cache)
to_discard.add(e)
elif (model in seen and # case of `seen`
attr in seen[obj_model] and obj
in seen[obj_model][attr]['cache']):
single = seen[obj_model][attr]['single']
cache_name = seen[obj_model][attr]['cache_name']
cache = seen[obj_model][attr]['cache'][obj]
to_discard.add(e)
set_cache(obj, single, cache, cache_name, attr)
if cache is not None and len(cache) != 0: # if data was cached
clipped = clip_lookup(lookup) # it still must get
if clipped: # into `buffer`.
update_buffer(buffer, cache, [clipped])
current -= to_discard
if current:
prefetch_qs, rel_attr_fn, cur_attr_fn, single, cache_name = \
prefetcher.get_prefetch_query_set(
map(itemgetter(1), current)
)
#prefetch lookups from prefetch queries are merged into processing.
additional_lookups = getattr(prefetch_qs,
'_prefetch_related_lookups', [])
if additional_lookups:
setattr(prefetch_qs, '_prefetch_related_lookups', [])
discovered = list(prefetch_qs)
lookups_for_discovered = additional_lookups
clipped_lookup = LOOKUP_SEP.join(lookup.split(LOOKUP_SEP)[1:])
if len(clipped_lookup) > 0:
lookups_for_discovered = chain([clipped_lookup],
additional_lookups)
if lookups_for_discovered:
update_buffer(buffer, discovered,
reversed(list(lookups_for_discovered)))
rel_to_cur = defaultdict(list)
for obj in discovered:
val = rel_attr_fn(obj)
rel_to_cur[val].append(obj)
for pair in current: # queried data is set up to objects
obj = pair[1]
val = cur_attr_fn(obj)
cache = rel_to_cur.get(val, [])
update_seen(seen, model, attr, single, cache_name, obj, cache)
set_cache(obj, single, cache, cache_name, attr)
def process_queryset(queryset, display_fields=None):
"""
This is used in the custom_view below, but its de-coupled so it can be used
programatically as well. Simply pass in a queryset and a list of relations to display
and viola.
Relations look like: ['address__zip','contact__date']
"""
display_fields = display_fields or []
extra_select_kwargs = {}
select_related = []
used_routes = []
distinct = True
for i in display_fields:
if i in queryset.query.aggregates or i in queryset.query.extra:
continue # Want below check to work only for relations, excluding aggregates.
select_related_token = i
if LOOKUP_SEP in i:
"""
Since select_related() is rather flexible about what it receives
(ignoring things it doesn't like), we'll just haphazardly pass
all filter and display fields in for now.
"""
select_related_token = i.split(LOOKUP_SEP)
select_related_token.pop() # get rid of the field name
select_related_token = LOOKUP_SEP.join(select_related_token)
"""
Here we remove distinct status for queries which have reverse relations
and possibly numerous results per original record
"""
if distinct and is_reverse_related(i, queryset.model):
distinct = False
primary_model = queryset.model
join_route = i
if len(i.split(LOOKUP_SEP)) > 2:
second_to_last = LOOKUP_SEP.join(i.split(LOOKUP_SEP)[0:-1])
join_route = LOOKUP_SEP.join(i.split(LOOKUP_SEP)[-2:])
primary_model = get_closest_relation(queryset.model,
second_to_last)[0]
primary_table = primary_model._meta.db_table
if primary_table in queryset.query.table_map:
primary_table = queryset.query.table_map[primary_table][
-1] # Will the last one always work?
join_model, join_field, join_name = get_closest_relation(
primary_model, join_route)
join_table = join_model._meta.db_table
try:
join_table = queryset.query.table_map[join_table][-1]
queryset = queryset.extra(
select={i: '%s.%s' % (join_table, join_field.column)})
except KeyError:
"""
Design decision. This will work fine if the ModelAdmin does the displaying of
related objects for us. At this time, Django doesn't. We have a patch in place
but aren't using it.
for now we just need the join column between the primary table and the join table.
"""
join_table = join_model._meta.db_table
join_column = "id" # Always this for now.
for field_name in primary_model._meta.get_all_field_names():
from django.db import models
try:
field = primary_model._meta.get_field(field_name)
if (isinstance(field,models.OneToOneField) or isinstance(field,models.ForeignKey)) and \
field.rel.to == join_model:
# 2 foreignkeys on the same model issue
# https://code.djangoproject.com/ticket/12890
if field_name not in select_related_token:
continue
whereclause = '%s.%s=%s.%s' % (
join_table, join_column, primary_table,
field.column)
if not (select_related_token,
join_table) in used_routes:
queryset = queryset.extra(select={i: '%s.%s' % (join_table,join_field.column)},\
tables=[join_table],where=[whereclause])
else:
queryset = queryset.extra(select={
i:
'%s.%s' % (join_table, join_field.column)
},
where=[whereclause])
except models.FieldDoesNotExist:
pass
if not (select_related_token, join_table) in used_routes:
used_routes.append((select_related_token, join_table))
if not select_related_token in select_related:
select_related.append(select_related_token)
if select_related:
queryset = queryset.select_related(*select_related)
if distinct:
queryset = queryset.distinct()
return queryset
def display_list(query_class,_model_class=None,inclusions=None,exclusions=None,depth=None,\
model_exclusions=None,_max_depth=None,_relation_list=None):
"""
User Args:
depth: how far our relation follows, we want to make sure to include forward then backward relations, as well.
ex. depth=1, consumer->field, consumer->disability_primary->field, consumer->address->field, consumer<-goal<-field, consumer<-pwi<-field
inclusions: A list of string-module relationships we want to be allowed to choose from, if not passed in, it means we want all the relations
exclusions: A list of string-module relationships to not follow, it could be a field or whole relation.
query_class: The class at the top of the tree hierarchy, essentially what we are reporting on.
Function Args:
_model_class: As we progress through the tree, we need to keep track of what model we are on.
_max_depth: Takes the depth, and depth starts as a counter from 0, just easier to read this way
_relation_list: What our function returns, but we need to pass it through so it can find it's way to the top... may be able to change this though
function return: A list of tuples, where each tuple represents (string-module relationship, human-readable-value)
ex. [
('first_name', 'Consumer :: First Name'),
('address__zip', 'Consmer :: Address :: Zip'),
('pwi__refer_date', 'Consumer :: PWI :: Referral Date')
]
"""
_relation_list = _relation_list or []
exclusions = exclusions or []
model_exclusions = model_exclusions or []
inclusions = inclusions or []
# if no model class is passed, then we are at the beginning or "base_class"
query_aggregates = None
if query_class.__class__ == query.QuerySet:
query_aggregates = query_class.query.aggregates
query_class = query_class.model
_model_class = _model_class or query_class
## less typing when calling the function, we use depth to set _max_depth from the first call, and use _max_depth henceforth.
# thus depth just keeps track of what level we are on
if not _max_depth:
_max_depth = depth
depth = 0
exclusions.extend(['logentry', 'message',
'id']) # these are always excluded fields
## We dont want our backward relations in the next recursive step, so we add it to our exclusions.
# as well as the base class.
exclusions.append(_model_class._meta.module_name)
model_exclusions.append(_model_class._meta.module_name)
current_inclusions = [
r.split(LOOKUP_SEP, 1)[0] for r in inclusions
] # these are the ONLY fields and relations to be returned
current_exclusions = [
r for r in exclusions if LOOKUP_SEP not in r
] # these are the fields / relations we don't want to show up
# Non-relational fields are easy and just get appended to the list as is pretty much
non_relation_fields = [f for f in _model_class._meta.fields if \
f.name not in current_exclusions and
f.name not in model_exclusions]
# Now handle the relations
# Get the forward ones
relations = [(f.rel.to, f.name, f.verbose_name.lower()) for f in _model_class._meta.fields if \
hasattr(f.rel, "to") and \
f.name not in current_exclusions and
f.rel.to._meta.module_name not in model_exclusions]
# and grab our backward ones
relations.extend([(r.model, r.field.related_query_name(), r.model._meta.verbose_name) for r in _model_class._meta.get_all_related_objects() if \
r.model._meta.module_name not in current_exclusions and
r.model._meta.module_name not in model_exclusions])
# We have to handle the inclusion list separately because if there isn't one, we don't want to filter over nothing
if current_inclusions:
non_relation_fields = [
f for f in non_relation_fields if f.name in current_inclusions
]
relations = [
r for r in relations
if r[0]._meta.module_name in current_inclusions
] # r == (model, model.verbose_name)
# At this point we are finally adding our fields to the tuple list
if query_aggregates:
[_relation_list.append((q, q)) for q in query_aggregates.keys()]
for field in non_relation_fields:
if _model_class != query_class:
_relation_list.append((
field.name, field.verbose_name.lower()
#' :: '.join([_model_class._meta.verbose_name.lower(), field.verbose_name.lower()])
))
else:
_relation_list.append((field.name, ' :: '.join(
[_model_class._meta.module_name,
field.verbose_name.lower()])))
## Recursion happens at this point, we are basically going down one tree / relations at a time before we do another one
# so taking consumer... it will do consumer->address->zip and then it will do consumer->emergency_contact and
# then whatever backward relations
for relation in relations:
# prepare the inclusion/exclusion for the next recursive call by chopping off all relations that match the one in our loop
relation_inclusions = [
name.split(LOOKUP_SEP, 1)[1] for name in inclusions if
LOOKUP_SEP in name and name.split(LOOKUP_SEP, 1)[0] == relation[1]
]
relation_exclusions = [
name.split(LOOKUP_SEP, 1)[1] for name in exclusions if
LOOKUP_SEP in name and name.split(LOOKUP_SEP, 1)[0] == relation[1]
]
if current_inclusions:
pass # if we have inclusions we want to continue with, don't return this tree yet
elif depth >= _max_depth:
return _relation_list # return this tree
# if we have reached a star in the exclusions, then skip the rest of this relation
if [
True for r in relation_exclusions
if '*' in r.split(LOOKUP_SEP, 1)[0]
]:
continue
# recurse
###
# we use copy.deepcopy on model_exclusions because we don't want a global list of exclusions everytime it adds a new one,
# just the ones down this tree
relation_pair_list = display_list(query_class,_model_class=relation[0],\
inclusions=relation_inclusions,exclusions=relation_exclusions,model_exclusions=copy.deepcopy(model_exclusions),\
depth=depth + 1,_max_depth=_max_depth)
# build the module-relation and human-readable string
###
# We check if _model_class != query_class because we have a case here where once we hit the top of the tree,
# then we don't want to append the query_class to the module-relation
if _model_class != query_class:
_relation_list.extend([
(LOOKUP_SEP.join([relation[1], relation_pair[0]]),
' :: '.join([relation[2], relation_pair[1]]))
for relation_pair in relation_pair_list
])
else:
_relation_list.extend([
(LOOKUP_SEP.join([relation[1], relation_pair[0]]),
' :: '.join([_model_class._meta.module_name, relation[2], relation_pair[1]])) \
for relation_pair in relation_pair_list
])
# go back up the recursion tree now
return _relation_list
def clip_lookup(lookup):
return LOOKUP_SEP.join(lookup.split(LOOKUP_SEP)[1:]) or None
def display_list_redux(query_class,
_model_class=None,
inclusions=None,
model_exclusions=None,
_relation_list=None):
"""
User Args:
inclusions: A list of string-module relationships we want to be allowed to
choose from, if not passed in, it means we want all the relations
query_class: The class at the top of the tree hierarchy, essentially what
we are reporting on.
Function Args:
_model_class: As we progress through the tree, we need to keep track of what model we are on.
_relation_list: What our function returns, but we need to pass it through so it can find
it's way to the top... may be able to change this though
function return: A list of tuples, where each tuple represents
(string-module relationship, human-readable-value)
ex. [
('first_name', 'Consumer :: First Name'),
('address__zip', 'Consmer :: Address :: Zip'),
('pwi__refer_date', 'Consumer :: PWI :: Referral Date')
]
"""
_relation_list = _relation_list or []
model_exclusions = model_exclusions or []
inclusions = inclusions or []
# if no model class is passed, then we are at the beginning or "base_class"
query_aggregates = None
if query_class.__class__ == query.QuerySet:
query_aggregates = query_class.query.aggregates
query_class = query_class.model
_model_class = _model_class or query_class
model_exclusions.append(_model_class._meta.module_name)
current_inclusions = [
r.split(LOOKUP_SEP, 1)[0] for r in inclusions
] # these are the ONLY fields and relations to be returned
# Non-relational fields are easy and just get appended to the list as is pretty much
non_relation_fields = [f for f in _model_class._meta.fields]
# Now handle the relations
# Get the forward ones
relations = [(f.rel.to, f.name, f.verbose_name.lower()) for f in _model_class._meta.fields if \
hasattr(f.rel, "to") and \
f.rel.to._meta.module_name not in model_exclusions]
# and grab our backward ones
relations.extend([(r.model, r.field.related_query_name(), r.model._meta.verbose_name) for r in _model_class._meta.get_all_related_objects() if \
r.model._meta.module_name not in model_exclusions])
# We have to handle the inclusion list separately because if there isn't one, we don't want to filter over nothing
if current_inclusions:
non_relation_fields = [
f for f in non_relation_fields if f.name in current_inclusions
]
relations = [
r for r in relations
if r[0]._meta.module_name in current_inclusions
] # r == (model, model.verbose_name)
# At this point we are finally adding our fields to the tuple list
if query_aggregates:
[_relation_list.append((q, q)) for q in query_aggregates.keys()]
for field in non_relation_fields:
if _model_class != query_class:
_relation_list.append((
field.name, field.verbose_name.lower()
#' :: '.join([_model_class._meta.verbose_name.lower(), field.verbose_name.lower()])
))
else:
_relation_list.append((field.name, ' :: '.join(
[_model_class._meta.module_name,
field.verbose_name.lower()])))
## Recursion happens at this point, we are basically going down one tree / relations at a time before we do another one
# so taking consumer... it will do consumer->address->zip and then it will do consumer->emergency_contact and
# then whatever backward relations
for relation in relations:
# prepare the inclusion for the next recursive call by chopping off all relations that match the one in our loop
relation_inclusions = [
name.split(LOOKUP_SEP, 1)[1] for name in inclusions if
LOOKUP_SEP in name and name.split(LOOKUP_SEP, 1)[0] == relation[1]
]
if not current_inclusions:
return _relation_list
# recurse
###
# we use copy.deepcopy on model_exclusions because we don't want a global list of exclusions everytime it adds a new one,
# just the ones down this tree
relation_pair_list = display_list_redux(query_class,_model_class=relation[0],\
inclusions=relation_inclusions,model_exclusions=copy.deepcopy(model_exclusions),\
)
# build the module-relation and human-readable string
###
# We check if _model_class != query_class because we have a case here where once we hit the top of the tree,
# then we don't want to append the query_class to the module-relation
if _model_class != query_class:
_relation_list.extend([
(LOOKUP_SEP.join([relation[1], relation_pair[0]]),
' :: '.join([relation[2], relation_pair[1]]))
for relation_pair in relation_pair_list
])
else:
_relation_list.extend([
(LOOKUP_SEP.join([relation[1], relation_pair[0]]),
' :: '.join([_model_class._meta.module_name, relation[2], relation_pair[1]])) \
for relation_pair in relation_pair_list
])
# go back up the recursion tree now
return _relation_list
def __validate_definition(self, *args, **kwargs):
def error(message):
raise FieldError("%s: %s: %s" % (self.related.model._meta, self.name, message))
original_choices_callback = self._choices_callback
# The choices we're defined by a string
# therefore it should be a cls method
if isinstance(self._choices_callback, basestring):
callback = getattr(self.related.model, self._choices_callback, None)
if not callable(callback):
error('Cannot find method specified by choices.')
args_length = 2 # Since the callback is a method we must emulate the 'self'
self._choices_callback = callback
else:
args_length = 1 # It's a callable, it needs no reference to model instance
spec = inspect.getargspec(self._choices_callback)
# Make sure the callback has the correct number or arg
if spec.defaults is not None:
spec_defaults_len = len(spec.defaults)
args_length += spec_defaults_len
self._choices_relationships = spec.args[-spec_defaults_len:]
else:
self._choices_relationships = []
if len(spec.args) != args_length:
error('Specified choices callback must accept only a single arg')
self._choices_callback_field_descriptors = {}
# We make sure field descriptors are valid
for descriptor in self._choices_relationships:
lookups = descriptor.split(LOOKUP_SEP)
meta = self.related.model._meta
depth = len(lookups)
step = 1
fields = []
for lookup in lookups:
try:
field = meta.get_field(lookup)
# The field is a foreign key to another model
if isinstance(field, ForeignKey):
try:
meta = field.rel.to._meta
except AttributeError:
# The model hasn't been loaded yet
# so we must stop here and start over
# when it is loaded.
if isinstance(field.rel.to, basestring):
self._choices_callback = original_choices_callback
return add_lazy_relation(field.model, field,
field.rel.to,
self.__validate_definition)
else:
raise
step += 1
# We cannot go deeper if it's not a model
elif step != depth:
error('Invalid descriptor "%s", "%s" is not a ForeignKey to a model' % (
LOOKUP_SEP.join(lookups), LOOKUP_SEP.join(lookups[:step])))
fields.append(field)
except FieldDoesNotExist:
# Lookup failed, suggest alternatives
depth_descriptor = LOOKUP_SEP.join(descriptor[:step - 1])
if depth_descriptor:
depth_descriptor += LOOKUP_SEP
choice_descriptors = [(depth_descriptor + name) for name in meta.get_all_field_names()]
error('Invalid descriptor "%s", choices are %s' % (
LOOKUP_SEP.join(descriptor), ', '.join(choice_descriptors)))
self._choices_callback_field_descriptors[descriptor] = fields
def path_v4(model, *conditions):
'''
Returns a filter of all objects of type model, subject to the list of conditions.
The first argument, model, must be a model. It cannot be an instance of a model, or any other object.
The list of conditions can be arbitrarily long, but each condition must be one of the following:
* an instance of a model
* a 3-tuple (model, field, value), where field is a string which is the name of a field of model, and value is a valid value of that field
* a 4-tuple (model, field, query_term, value), where field is a string which is the name of a field of model, query_term is a string which is the name of a Django query term (defined in django.db.models.sql.constants.QUERY_TERMS), and value is a valid value of that field
The function uses path_v1() to find the forward path from model to conditions[i] or conditions[i][0] (for all i), and then applies the appropriate filter.
The function asks the user (via the raw_input() function) which path(s) to filter on.
'''
conditions = list(conditions)
if not (isclass(model) and issubclass(model, Model)):
raise TypeError("path_v4 argument 1 must be a model")
iter_conditions = range(len(conditions))
models = [None for i in iter_conditions]
fields = ['' for i in iter_conditions]
lookup_type = ['' for i in iter_conditions]
values = [None for i in iter_conditions]
for i in iter_conditions:
if isinstance(conditions[i], tuple) and isinstance(
conditions[i][-1], Model):
conditions[i] = conditions[i][-1]
if isinstance(conditions[i], Model):
if isinstance(conditions[i], model):
raise TypeError(
"path_v4 argument 2 must not contain any instances of argument 1"
)
values[i] = conditions[i]
models[i] = type(conditions[i])
elif isinstance(conditions[i], tuple) and iscorrecttuple(
conditions[i]):
fields[i] = conditions[i][1]
models[i] = conditions[i][0]
if len(conditions[i]) == 3:
values[i] = conditions[i][2]
else:
values[i] = conditions[i][3]
lookup_type[i] = conditions[i][2]
else:
raise TypeError(
"path_v4 argument 2 must be a list, with each item being either an instance of a model or valid (model, field, value) 3-tuple"
)
repeat = False
use = ''
kwargs = {}
paths = []
num_paths = 0
for i in iter_conditions:
for path in [
LOOKUP_SEP.join(path + (fields[i], )).strip(LOOKUP_SEP)
for (path, _, _) in path_v1(model, models[i])
]:
repeat = True
while repeat:
use = raw_input("Include the path " + path +
" as a condition ([y]/n)? ")
repeat = False
if use.lower() == 'y' or use.lower() == 'yes' or not use:
kwargs[path] = values[i]
elif not (use.lower() == 'n' or use.lower() == 'no'):
repeat = True
return model.objects.filter(**kwargs)
def build_filters(self, filters=None):
"""
An enhanced tastypie method that will first check to see if a value
passed inside of a filter can be decoded as JSON. If so, it passes those
values to ``::meth::<get_query_bits_from_dict> get_query_bits_from_dict``.
"""
if filters is None:
filters = {}
qs_filters = {}
if hasattr(self._meta, 'queryset'):
# Get the possible query terms from the current QuerySet.
if hasattr(self._meta.queryset.query.query_terms, 'keys'):
# Django 1.4 & below compatibility.
query_terms = self._meta.queryset.query.query_terms.keys()
else:
# Django 1.5+.
query_terms = self._meta.queryset.query.query_terms
else:
query_terms = QUERY_TERMS.keys()
for filter_expr, value in filters.items():
filter_type = 'exact'
# Check first to see if a value can be decoded as json
try:
value_dict = json.loads(value)
# Because the python json module is not as strict as JSON standards
# We must check to make sure that the value loaded is a dict
if not isinstance(value_dict, dict):
value_dict = None
except ValueError:
value_dict = None
# If a value is decoded, it passes that dict into a special method
if value_dict:
# Get the value and filter_bits from the method.
filter_bits, value = self.get_query_bits_from_dict(value_dict,
keys_list=[], value=None)
# Because the field_name and the filter_expr are backward,
# we need to set the field name = to filter expr
field_name = filter_expr
# If not, stick with standard tastypie stuff
else:
filter_bits = filter_expr.split(LOOKUP_SEP)
field_name = filter_bits.pop(0)
if not field_name in self.fields:
# It's not a field we know about. Move along citizen.
continue
if len(filter_bits) and filter_bits[-1] in query_terms:
filter_type = filter_bits.pop()
lookup_bits = self.check_filtering(field_name, filter_type,
filter_bits)
value = self.filter_value_to_python(value, field_name, filters,
filter_expr, filter_type)
db_field_name = LOOKUP_SEP.join(lookup_bits)
qs_filter = "%s%s%s" % (db_field_name, LOOKUP_SEP, filter_type)
qs_filters[qs_filter] = value
return dict_strip_unicode_keys(qs_filters)
def build_filters(self, filters=None):
"""
Given a dictionary of filters, create the necessary ORM-level filters.
Keys should be resource fields, **NOT** model fields.
Valid values are either a list of Django filter types (i.e.
``['startswith', 'exact', 'lte']``), the ``ALL`` constant or the
``ALL_WITH_RELATIONS`` constant.
"""
# At the declarative level:
# filtering = {
# 'resource_field_name': ['exact', 'startswith', 'endswith', 'contains'],
# 'resource_field_name_2': ['exact', 'gt', 'gte', 'lt', 'lte', 'range'],
# 'resource_field_name_3': ALL,
# 'resource_field_name_4': ALL_WITH_RELATIONS,
# ...
# }
# Accepts the filters as a dict. None by default, meaning no filters.
if filters is None:
filters = {}
qs_filters = {}
for filter_expr, value in filters.items():
filter_bits = filter_expr.split(LOOKUP_SEP)
if not filter_bits[0] in self.fields:
# It's not a field we know about. Move along citizen.
continue
if not filter_bits[0] in self._meta.filtering:
raise InvalidFilterError("The '%s' field does not allow filtering." % filter_bits[0])
if filter_bits[-1] in QUERY_TERMS.keys():
filter_type = filter_bits.pop()
else:
filter_type = 'exact'
# Check to see if it's allowed lookup type.
if not self._meta.filtering[filter_bits[0]] in (ALL, ALL_WITH_RELATIONS):
# Must be an explicit whitelist.
if not filter_type in self._meta.filtering[filter_bits[0]]:
raise InvalidFilterError("'%s' is not an allowed filter on the '%s' field." % (filter_expr, filter_bits[0]))
# Check to see if it's a relational lookup and if that's allowed.
if len(filter_bits) > 1:
if not self._meta.filtering[filter_bits[0]] == ALL_WITH_RELATIONS:
raise InvalidFilterError("Lookups are not allowed more than one level deep on the '%s' field." % filter_bits[0])
if self.fields[filter_bits[0]].attribute is None:
raise InvalidFilterError("The '%s' field has no 'attribute' for searching with." % filter_bits[0])
if value == 'true':
value = True
elif value == 'false':
value = False
elif value in ('nil', 'none', 'None'):
value = None
db_field_name = LOOKUP_SEP.join([self.fields[filter_bits[0]].attribute] + filter_bits[1:])
qs_filter = "%s%s%s" % (db_field_name, LOOKUP_SEP, filter_type)
qs_filters[qs_filter] = value
return dict_strip_unicode_keys(qs_filters)
def deep_prefetch_related_objects(objects, lookups):
"""
Helper function for prefetch_related functionality.
Populates prefetched objects caches for a list of results
from a QuerySet.
Differs from :meth:`django.db.models.query.prefetch_related_objects`
in that that it can prefetch "non-strict" lookups through GFK.
:param objects: result cache of base queryset.
:param lookups: list with lookups.
"""
#How it works
#------------
# - Since data is not of same type - "unit" of data is a model instance
# (object), not QuerySet.
# - Goals that I was tried to achieve:
# - Prefetching 'non-strict' lookups.
# - Absence of doubled queries to DB, other redundancy in processing
# - and infinite recursions.
# - Simplicity of design in comparison with original
# `django.db.models.query.prefetch_related_objects`
# - Sets are used to avoid duplicates.
# - Python's sets distinguish objects by value. But task require
# to distinguish objects by id, not by value, because two objects
# that are same by value, might be included in two different
# querysets, and both of them must be populated with cache.
# - BUT fetched cache data for object are distinguished by value.
# Rationale is to distinguish unique queries and prevent their
# repeated execution. We don't care about "id" of queries, we care
# about their value. Composite identifier for query is
# object and lookup for which query being constructed.
# Only one query exists for one object (by value) and lookup part.
# In function there slightly more complex structure than
# `obj -> lookup` used for storing data for traversed lookups
# (``seen``) - that's done to reduce redundancy in data, but
# "primary key" for stroed data is `obj -> lookup`.
# - Data flows through `buffer`, during processing.
# Objects discovered while traversing DB structure are being added
# and processed objects are removed.
#todo beauty and refactoring
if len(objects) == 0:
return # nothing to do
#lookup -> model -> {(id(obj), obj), ...}
#id(obj) - because objects must be distinguished by id.
#DefaultOrederedDict because order of lookups matter,
#see tests.LookupOrderingTest.test_order of Django test suite.
buffer = DefaultOrderedDict(lambda: defaultdict(set))
update_buffer(buffer, objects, reversed(lookups))
seen = tree() # model -> attr ->
# single -> bool
# cache_name -> str
# cache ->
# obj -> [cache]
while True:
try:
lookup = last(buffer.keys())
except TypeError:
break
try:
model, current = buffer[lookup].popitem()
except KeyError:
del buffer[lookup]
continue
attr = lookup.split(LOOKUP_SEP)[0]
sample = current.pop()
current.add(sample)
_, object = sample
prefetcher, _, attr_found, _ = get_prefetcher(object, attr)
#Lookup is not valid for that object, it must be skipped.
#No exception, because data it is that data is of diffrerent types,
#so - such situation is normal.
if not attr_found:
continue
if LOOKUP_SEP not in lookup and prefetcher is None:
raise ValueError("'%s' does not resolve to a item that supports "
"prefetching - this is an invalid parameter to "
"prefetch_related()." % lookup)
if prefetcher is None:
clipped = clip_lookup(lookup)
if clipped:
update_buffer(
buffer,
filter(is_not_none, [getattr(o, attr)
for _, o in current]),
[clipped])
continue
to_discard = set()
for e in current: # no need to query for already prefetched data
obj = e[1]
obj_model = obj.__class__
p, d, _, is_fetched = get_prefetcher(obj, attr)
cache = None
if is_fetched: # case of Django internal cache
single, cache_name = get_info(p, d)
cache = get_cache(obj, single, cache_name, attr)
single, cache_name = get_info(p, d)
update_seen(seen, model, attr, single, cache_name, obj, cache)
to_discard.add(e)
elif (model in seen and # case of `seen`
attr in seen[obj_model] and
obj in seen[obj_model][attr]['cache']):
single = seen[obj_model][attr]['single']
cache_name = seen[obj_model][attr]['cache_name']
cache = seen[obj_model][attr]['cache'][obj]
to_discard.add(e)
set_cache(obj, single, cache, cache_name, attr)
if cache is not None and len(cache) != 0: # if data was cached
clipped = clip_lookup(lookup) # it still must get
if clipped: # into `buffer`.
update_buffer(buffer, cache, [clipped])
current -= to_discard
if current:
prefetch_qs, rel_attr_fn, cur_attr_fn, single, cache_name = \
prefetcher.get_prefetch_query_set(
map(itemgetter(1), current)
)
#prefetch lookups from prefetch queries are merged into processing.
additional_lookups = getattr(prefetch_qs,
'_prefetch_related_lookups', [])
if additional_lookups:
setattr(prefetch_qs, '_prefetch_related_lookups', [])
discovered = list(prefetch_qs)
lookups_for_discovered = additional_lookups
clipped_lookup = LOOKUP_SEP.join(lookup.split(LOOKUP_SEP)[1:])
if len(clipped_lookup) > 0:
lookups_for_discovered = chain([clipped_lookup],
additional_lookups)
if lookups_for_discovered:
update_buffer(buffer, discovered, reversed(list(lookups_for_discovered)))
rel_to_cur = defaultdict(list)
for obj in discovered:
val = rel_attr_fn(obj)
rel_to_cur[val].append(obj)
for pair in current: # queried data is set up to objects
obj = pair[1]
val = cur_attr_fn(obj)
cache = rel_to_cur.get(val, [])
update_seen(seen, model, attr, single, cache_name, obj, cache)
set_cache(obj, single, cache, cache_name, attr)
def _perform_search(self):
"""
Generates the query using the validated constraint formset. Also compiles a natural
language string (``self.natural_string``) in the format of
"N [Model]s with [field] [operator] '[term]'".
"""
natural_string = []
# 2-tuples of an operator and Q instance, sent through reduce() after it's built
query_list = []
for i, constraint_form in enumerate(self.constraint_formset):
type_operator = constraint_form.cleaned_data['type']
field_list = constraint_form.cleaned_data['field']
constraint_operator = constraint_form.cleaned_data['operator']
term = constraint_form.cleaned_data['term']
end_term = constraint_form.cleaned_data['end_term']
verbose_name = self.model_config._fields[field_list]
# Build this field's query.
# Search fields bound together in a tuple are considered OR conditions for a single
# virtual field name.
query = None
# Iterate multiple fields defined in a compound column
for field in field_list:
value = term
target_field = resolve_orm_path(self.model, field)
# Prep an inverted lookup
negative = constraint_operator.startswith('!')
if negative:
constraint_operator = constraint_operator[1:]
if constraint_operator == "isnull":
value = not negative
negative = False
# Bake the queryset language string
field_query = LOOKUP_SEP.join((field, constraint_operator))
q = Q(**{
field_query: value,
})
log.debug("Querying %s [%d]: %s%s=%r", self.model.__name__, i, field_query,
"!" if negative else "", value)
# Negate if necessary
if negative:
q = ~q
if query is None:
query = q
else:
query |= q
query_list.append((type_operator, query))
# Do some natural processing
if isinstance(value, (tuple, list)):
value = ' - '.join(map(unicode, value))
else:
value = str(value)
bits = [verbose_name, constraint_form['operator'].value(), value]
if i != 0: # Skip the leading "and" on the first constraint
bits.insert(0, constraint_form['type'].value())
natural_string.append(bits)
# The first query's "type" should be ignored for the sake of the reduce line below.
query = reduce(lambda q1, (op, q2): op(q1, q2), query_list[1:], query_list[0][1])
queryset = self.build_queryset(self.model, query)
data_rows = self.process_results(queryset)
self.results = {
'count': len(queryset),
'list': data_rows,
'fields': self._get_display_fields(self.model, self.model_config),
'natural_string': "where " + ', '.join(map(' '.join, natural_string)),
}
def path_v5(model, paths, *conditions):
'''
Returns a filter of all objects of type model, subject to the list of conditions.
The first argument, model, must be a model. It cannot be an instance of a model, or any other object.
The second argument, paths, is a dictionary. paths[condition_model] is a list of selected paths from model to condition_model that should be queried.
The list of conditions can be arbitrarily long, but each condition must be one of the following:
* an instance of a model
* a 3-tuple (model, field, value), where field is a string which is the name of a field of model, and value is a valid value of that field
* a 4-tuple (model, field, query_term, value), where field is a string which is the name of a field of model, query_term is a string which is the name of a Django query term (defined in django.db.models.sql.constants.QUERY_TERMS), and value is a valid value of that field
path_v1() is used outside of the scope of this function, to determine paths. Then path_v5 applies the appropriate filter.
'''
conditions = list(conditions)
if not is_useful_model(model):
raise TypeError("path_v5 argument 1 must be a useful model")
iter_conditions = range(len(conditions))
models = [None for i in iter_conditions]
fields = ['' for i in iter_conditions]
lookup_type = ['' for i in iter_conditions]
values = [None for i in iter_conditions]
for i in iter_conditions:
if isinstance(conditions[i], tuple) and isinstance(
conditions[i][-1], Model):
conditions[i] = conditions[i][-1]
if isinstance(conditions[i], Model):
if isinstance(conditions[i], model):
raise TypeError(
"path_v5 argument 3 must not contain any instances of argument 1"
)
values[i] = conditions[i]
models[i] = type(conditions[i])
elif isinstance(conditions[i], tuple) and iscorrecttuple(
conditions[i]):
fields[i] = conditions[i][1]
models[i] = conditions[i][0]
if len(conditions[i]) == 3:
values[i] = conditions[i][2]
else:
values[i] = conditions[i][3]
lookup_type[i] = conditions[i][2]
else:
raise TypeError(
"path_v5 argument 3 must be a list, with each item being either an instance of a model or valid (model, field, value) 3-tuple"
)
repeat = False
use = ''
kwargs = {}
all_paths = []
for i in iter_conditions:
for path in paths[models[i]]:
kwargs[LOOKUP_SEP.join(
(path, fields[i],
lookup_type[i])).strip(LOOKUP_SEP)] = values[i]
all_paths.append(path)
automatic_conditions_for_registrations = []
for path in all_paths:
new_path = passes_through(model, path, StudentRegistration, 'end_date')
if new_path and isinstance(new_path, basestring):
automatic_conditions_for_registrations.append(new_path)
for condition in automatic_conditions_for_registrations:
condition_being_used = False
for path in all_paths:
if condition in path:
condition_being_used = True
break
if not condition_being_used:
kwargs[condition + LOOKUP_SEP + 'gte'] = datetime.now()
return model.objects.filter(**kwargs).distinct()