def _get_implicit_requirements(self, fields, requirements):
"""Extract internal prefetch requirements from serializer fields."""
for name, field in six.iteritems(fields):
source = field.source
# Requires may be manually set on the field -- if not,
# assume the field requires only its source.
requires = getattr(field, 'requires', None) or [source]
for require in requires:
if not require:
# ignore fields with empty source
continue
requirement = require.split('.')
if requirement[-1] == '':
# Change 'a.b.' -> 'a.b.*',
# supporting 'a.b.' for backwards compatibility.
requirement[-1] = '*'
requirements.insert(requirement, TreeMap(), update=True)
def _build_queryset(
self,
serializer=None,
filters=None,
queryset=None,
requirements=None,
extra_filters=None,
disable_prefetches=False,
):
"""Build a queryset that pulls in all data required by this request.
Handles nested prefetching of related data and deferring fields
at the queryset level.
Arguments:
serializer: An optional serializer to use a base for the queryset.
If no serializer is passed, the `get_serializer` method will
be used to initialize the base serializer for the viewset.
filters: An optional TreeMap of nested filters.
queryset: An optional base queryset.
requirements: An optional TreeMap of nested requirements.
"""
is_root_level = False
if not serializer:
serializer = self.view.get_serializer()
is_root_level = True
queryset = self._get_queryset(queryset=queryset, serializer=serializer)
model = getattr(serializer.Meta, 'model', None)
if not model:
return queryset
prefetches = {}
# build a nested Prefetch queryset
# based on request parameters and serializer fields
fields = serializer.fields
if requirements is None:
requirements = TreeMap()
self._get_implicit_requirements(fields, requirements)
# Implicit requirements (i.e. via `requires`) can potentially
# include fields that haven't been explicitly included.
# Such fields would not be in `fields`, so they need to be added.
implicitly_included = set(requirements.keys()) - set(fields.keys())
if implicitly_included:
all_fields = serializer.get_all_fields()
fields.update({
field: all_fields[field]
for field in implicitly_included if field in all_fields
})
if filters is None:
filters = self._get_requested_filters()
# build nested Prefetch queryset
self._build_requested_prefetches(prefetches, requirements, model,
fields, filters)
# build remaining prefetches out of internal requirements
# that are not already covered by request requirements
self._build_implicit_prefetches(model, prefetches, requirements)
# use requirements at this level to limit fields selected
# only do this for GET requests where we are not requesting the
# entire fieldset
if ('*' not in requirements and not self.view.is_update()
and not self.view.is_delete()):
id_fields = getattr(serializer, 'get_id_fields', lambda: [])()
# only include local model fields
only = [
field for field in set(id_fields + list(requirements.keys()))
if is_model_field(model, field)
and not is_field_remote(model, field)
]
queryset = queryset.only(*only)
# add request filters
query = self._filters_to_query(includes=filters.get('_include'),
excludes=filters.get('_exclude'),
serializer=serializer)
# add additional filters specified by calling view
if extra_filters:
query = extra_filters if not query else extra_filters & query
if query:
# Convert internal django ValidationError to
# APIException-based one in order to resolve validation error
# from 500 status code to 400.
try:
queryset = queryset.filter(query)
except InternalValidationError as e:
raise ValidationError(
dict(e) if hasattr(e, 'error_dict') else list(e))
except Exception as e:
# Some other Django error in parsing the filter.
# Very likely a bad query, so throw a ValidationError.
err_msg = getattr(e, 'message', '')
raise ValidationError(err_msg)
# A serializer can have this optional function
# to dynamically apply additional filters on
# any queries that will use that serializer
# You could use this to have (for example) different
# serializers for different subsets of a model or to
# implement permissions which work even in sideloads
if hasattr(serializer, 'filter_queryset'):
queryset = self._serializer_filter(serializer=serializer,
queryset=queryset)
# add prefetches and remove duplicates if necessary
prefetch = prefetches.values()
if prefetch and not disable_prefetches:
queryset = queryset.prefetch_related(*prefetch)
elif isinstance(queryset, Manager):
queryset = queryset.all()
if has_joins(queryset) or not is_root_level:
queryset = queryset.distinct()
if self.DEBUG:
queryset._using_prefetches = prefetches
return queryset
def _get_requested_filters(self, **kwargs):
"""
Convert 'filters' query params into a dict that can be passed
to Q. Returns a dict with two fields, 'include' and 'exclude',
which can be used like:
result = self._get_requested_filters()
q = Q(**result['include'] & ~Q(**result['exclude'])
"""
filters_map = (kwargs.get('filters_map')
or self.view.get_request_feature(self.view.FILTER))
out = TreeMap()
for spec, value in six.iteritems(filters_map):
# Inclusion or exclusion?
if spec[0] == '-':
spec = spec[1:]
inex = '_exclude'
else:
inex = '_include'
# for relational filters, separate out relation path part
if '|' in spec:
rel, spec = spec.split('|')
rel = rel.split('.')
else:
rel = None
parts = spec.split('.')
# Last part could be operator, e.g. "events.capacity.gte"
if len(parts) > 1 and parts[-1] in self.VALID_FILTER_OPERATORS:
operator = parts.pop()
else:
operator = None
# All operators except 'range' and 'in' should have one value
if operator == 'range':
value = value[:2]
elif operator == 'in':
# no-op: i.e. accept `value` as an arbitrarily long list
pass
elif operator in self.VALID_FILTER_OPERATORS:
value = value[0]
if (operator == 'isnull'
and isinstance(value, six.string_types)):
value = is_truthy(value)
elif operator == 'eq':
operator = None
node = FilterNode(parts, operator, value)
# insert into output tree
path = rel if rel else []
path += [inex, node.key]
out.insert(path, node)
return out
def _filter_queryset(self,
serializer=None,
filters=None,
queryset=None,
requirements=None):
"""Recursive queryset builder.
Handles nested prefetching of related data and deferring fields
at the queryset level.
Arguments:
serializer: An optional serializer to use a base for the queryset.
If no serializer is passed, the `get_serializer` method will
be used to initialize the base serializer for the viewset.
filters: An optional TreeMap of nested filters.
queryset: An optional base queryset.
requirements: An optional TreeMap of nested requirements.
"""
is_root_level = False
if serializer:
if queryset is None:
queryset = serializer.Meta.model.objects
else:
serializer = self.view.get_serializer()
is_root_level = True
model = getattr(serializer.Meta, 'model', None)
if not model:
return queryset
prefetches = {}
fields = serializer.fields
if requirements is None:
requirements = TreeMap()
self._extract_requirements(fields, requirements)
if filters is None:
filters = self._extract_filters()
# build nested Prefetch queryset
self._add_request_prefetches(prefetches, requirements, model, fields,
filters)
# add any remaining requirements as prefetches
self._add_internal_prefetches(prefetches, requirements)
# use requirements at this level to limit fields selected
# only do this for GET requests where we are not requesting the
# entire fieldset
if ('*' not in requirements and not self.view.is_update()
and not self.view.is_delete()):
id_fields = getattr(serializer, 'get_id_fields', lambda: [])()
# only include local model fields
only = [
field for field in set(id_fields + list(requirements.keys()))
if is_model_field(model, field)
and not is_field_remote(model, field)
]
queryset = queryset.only(*only)
# add filters
query = self._filters_to_query(includes=filters.get('_include'),
excludes=filters.get('_exclude'),
serializer=serializer)
if query:
# Convert internal django ValidationError to APIException-based one
# in order to resolve validation errors from 500 status code to
# 400.
try:
queryset = queryset.filter(query)
except InternalValidationError as e:
raise ValidationError(
dict(e) if hasattr(e, 'error_dict') else list(e))
except Exception as e:
# Some other Django error in parsing the filter. Very likely
# a bad query, so throw a ValidationError.
err_msg = getattr(e, 'message', '')
raise ValidationError(err_msg)
# A serializer can have this optional function
# to dynamically apply additional filters on
# any queries that will use that serializer
# You could use this to have (for example) different
# serializers for different subsets of a model or to
# implement permissions which work even in sideloads
if hasattr(serializer, 'filter_queryset'):
queryset = serializer.filter_queryset(queryset)
prefetch = prefetches.values()
queryset = queryset.prefetch_related(*prefetch)
if has_joins(queryset) or not is_root_level:
queryset = queryset.distinct()
return queryset
def _build_queryset(self,
serializer=None,
filters=None,
queryset=None,
requirements=None):
"""Build a queryset that pulls in all data required by this request.
Handles nested prefetching of related data and deferring fields
at the queryset level.
Arguments:
serializer: An optional serializer to use a base for the queryset.
If no serializer is passed, the `get_serializer` method will
be used to initialize the base serializer for the viewset.
filters: An optional TreeMap of nested filters.
queryset: An optional base queryset.
requirements: An optional TreeMap of nested requirements.
"""
is_root_level = False
if serializer:
if queryset is None:
queryset = serializer.Meta.model.objects
else:
serializer = self.view.get_serializer()
is_root_level = True
model = serializer.get_model()
if not model:
return queryset
meta = Meta(model)
prefetches = {}
# build a nested Prefetch queryset
# based on request parameters and serializer fields
fields = serializer.fields
if requirements is None:
requirements = TreeMap()
self._get_implicit_requirements(fields, requirements)
if filters is None:
filters = self._get_requested_filters()
# build nested Prefetch queryset
self._build_requested_prefetches(prefetches, requirements, model,
fields, filters, is_root_level)
# build remaining prefetches out of internal requirements
# that are not already covered by request requirements
self._build_implicit_prefetches(model, prefetches, requirements)
# use requirements at this level to limit fields selected
# only do this for GET requests where we are not requesting the
# entire fieldset
is_gui = self.view.get_format() == 'admin'
if ('*' not in requirements and not self.view.is_update()
and not self.view.is_delete() and not is_gui):
id_fields = getattr(serializer, 'get_id_fields', lambda: [])()
# only include local model fields
only = [
field for field in set(id_fields + list(requirements.keys()))
if meta.is_field(field) and not meta.is_field_remote(field)
]
queryset = queryset.only(*only)
# add request filters
query = self._filters_to_query(filters)
if query:
# Convert internal django ValidationError to
# APIException-based one in order to resolve validation error
# from 500 status code to 400.
try:
queryset = queryset.filter(query)
except InternalValidationError as e:
raise ValidationError(
dict(e) if hasattr(e, 'error_dict') else list(e))
except Exception as e:
# Some other Django error in parsing the filter.
# Very likely a bad query, so throw a ValidationError.
err_msg = getattr(e, 'message', '')
raise ValidationError(err_msg)
# A serializer can have this optional function
# to dynamically apply additional filters on
# any queries that will use that serializer
# You could use this to have (for example) different
# serializers for different subsets of a model or to
# implement permissions which work even in sideloads
if hasattr(serializer, 'filter_queryset'):
queryset = serializer.filter_queryset(queryset)
# add prefetches and remove duplicates if necessary
prefetch = prefetches.values()
queryset = queryset.prefetch_related(*prefetch)
if has_joins(queryset) or not is_root_level:
queryset = queryset.distinct()
if self.DEBUG:
queryset._using_prefetches = prefetches
return queryset
def _get_requested_filters(self, **kwargs):
"""
Convert 'filters' query params into a dict that can be passed
to Q. Returns a dict with two fields, 'include' and 'exclude',
which can be used like:
result = self._get_requested_filters()
q = Q(**result['_include'] & ~Q(**result['_exclude'])
"""
filters_map = kwargs.get('filters_map')
view = getattr(self, 'view', None)
if view:
serializer_class = view.get_serializer_class()
serializer = serializer_class()
if not filters_map:
filters_map = view.get_request_feature(view.FILTER)
else:
serializer = None
out = TreeMap()
for key, value in six.iteritems(filters_map):
# Inclusion or exclusion?
if key[0] == '-':
key = key[1:]
category = '_exclude'
else:
category = '_include'
# for relational filters, separate out relation path part
if '|' in key:
rel, key = key.split('|')
rel = rel.split('.')
else:
rel = None
terms = key.split('.')
# Last part could be operator, e.g. "events.capacity.gte"
if len(terms) > 1 and terms[-1] in self.VALID_FILTER_OPERATORS:
operator = terms.pop()
else:
operator = None
# All operators except 'range' and 'in' should have one value
if operator == 'range':
value = value[:2]
if value[0] == '':
operator = 'lte'
value = value[1]
elif value[1] == '':
operator = 'gte'
value = value[0]
elif operator == 'in':
# no-op: i.e. accept `value` as an arbitrarily long list
pass
elif operator in self.VALID_FILTER_OPERATORS:
value = value[0]
if (operator == 'isnull'
and isinstance(value, six.string_types)):
value = is_truthy(value)
elif operator == 'eq':
operator = None
if serializer:
s = serializer
if rel:
# get related serializer
model_fields, serializer_fields = serializer.resolve(rel)
s = serializer_fields[-1]
s = getattr(s, 'serializer', s)
rel = [Meta.get_query_name(f) for f in model_fields]
# perform model-field resolution
model_fields, serializer_fields = s.resolve(terms)
field = serializer_fields[-1] if serializer_fields else None
# if the field is a boolean,
# coerce the value
if field and isinstance(
field,
(serializers.BooleanField, serializers.NullBooleanField)):
value = is_truthy(value)
key = '__'.join([Meta.get_query_name(f) for f in model_fields])
else:
key = '__'.join(terms)
if operator:
key += '__%s' % operator
# insert into output tree
path = rel if rel else []
path += [category, key]
out.insert(path, value)
return out
def _filter_queryset(
self,
serializer=None,
filters=None,
queryset=None,
requirements=None
):
"""Recursive queryset builder.
Handles nested prefetching of related data and deferring fields
at the queryset level.
Arguments:
serializer: An optional serializer to use a base for the queryset.
If no serializer is passed, the `get_serializer` method will
be used to initialize the base serializer for the viewset.
filters: An optional TreeMap of nested filters.
queryset: An optional base queryset.
requirements: An optional TreeMap of nested requirements.
"""
is_root_level = False
if serializer:
if queryset is None:
queryset = serializer.Meta.model.objects
else:
serializer = self.view.get_serializer()
is_root_level = True
model = getattr(serializer.Meta, 'model', None)
if not model:
return queryset
prefetches = {}
fields = serializer.fields
if requirements is None:
requirements = TreeMap()
self._extract_requirements(
fields,
requirements
)
if filters is None:
filters = self._extract_filters()
# build nested Prefetch queryset
self._add_request_prefetches(
prefetches,
requirements,
model,
fields,
filters
)
# add any remaining requirements as prefetches
self._add_internal_prefetches(
prefetches,
requirements
)
# use requirements at this level to limit fields selected
# only do this for GET requests where we are not requesting the
# entire fieldset
if '*' not in requirements and not self.view.is_update():
id_fields = getattr(serializer, 'get_id_fields', lambda: [])()
# only include local model fields
only = [
field for field in set(id_fields + list(requirements.keys()))
if is_model_field(model, field) and
not is_field_remote(model, field)
]
queryset = queryset.only(*only)
# add filters
query = self._filters_to_query(
includes=filters.get('_include'),
excludes=filters.get('_exclude'),
serializer=serializer
)
if query:
# Convert internal django ValidationError to APIException-based one
# in order to resolve validation errors from 500 status code to
# 400.
try:
queryset = queryset.filter(query)
except InternalValidationError as e:
raise ValidationError(
dict(e) if hasattr(e, 'error_dict') else list(e)
)
# A serializer can have this optional function
# to dynamically apply additional filters on
# any queries that will use that serializer
# You could use this to have (for example) different
# serializers for different subsets of a model or to
# implement permissions which work even in sideloads
if hasattr(serializer, 'filter_queryset'):
queryset = serializer.filter_queryset(queryset)
prefetch = prefetches.values()
queryset = queryset.prefetch_related(*prefetch)
if has_joins(queryset) or not is_root_level:
queryset = queryset.distinct()
return queryset
def _extract_filters(self, **kwargs):
"""
Convert 'filters' query params into a dict that can be passed
to Q. Returns a dict with two fields, 'include' and 'exclude',
which can be used like:
result = self._extract_filters()
q = Q(**result['include'] & ~Q(**result['exclude'])
"""
filters_map = (
kwargs.get('filters_map') or
self.view.get_request_feature(self.view.FILTER)
)
out = TreeMap()
for spec, value in six.iteritems(filters_map):
# Inclusion or exclusion?
if spec[0] == '-':
spec = spec[1:]
inex = '_exclude'
else:
inex = '_include'
# for relational filters, separate out relation path part
if '|' in spec:
rel, spec = spec.split('|')
rel = rel.split('.')
else:
rel = None
parts = spec.split('.')
# Last part could be operator, e.g. "events.capacity.gte"
if len(parts) > 1 and parts[-1] in self.VALID_FILTER_OPERATORS:
operator = parts.pop()
else:
operator = None
# All operators except 'range' and 'in' should have one value
if operator == 'range':
value = value[:2]
elif operator == 'in':
# no-op: i.e. accept `value` as an arbitrarily long list
pass
elif operator in self.VALID_FILTER_OPERATORS:
value = value[0]
if (
operator == 'isnull' and
isinstance(value, six.string_types)
):
value = value.lower() not in self.FALSEY_STRINGS
elif operator == 'eq':
operator = None
node = FilterNode(parts, operator, value)
# insert into output tree
path = rel if rel else []
path += [inex, node.key]
out.insert(path, node)
return out