def get_skip_and_limit(self, params=None):
if params is None:
params = request.args
if self.paginate:
# _limit and _skip validation
if not isint(params.get('_limit', 1)):
raise ValidationError({
'error':
'_limit must be an integer (got "%s" instead).' %
params['_limit']
})
if not isint(params.get('_skip', 1)):
raise ValidationError({
'error':
'_skip must be an integer (got "%s" instead).' %
params['_skip']
})
if params.get('_limit') and int(params['_limit']) > self.max_limit:
raise ValidationError({
'error':
"The limit you set is larger than the maximum limit for this resource (max_limit = %d)."
% self.max_limit
})
limit = min(int(params.get('_limit', self.default_limit)),
self.max_limit)
# Fetch one more so we know if there are more results.
return int(params.get('_skip', 0)), limit
else:
return 0, self.max_limit
def raw_data(self):
if not hasattr(self, '_raw_data'):
if request.method in ('PUT', 'POST') or request.data:
if request.mimetype and 'json' not in request.mimetype:
raise ValidationError({
'error':
"Please send valid JSON with a 'Content-Type: application/json' header."
})
if request.headers.get('Transfer-Encoding') == 'chunked':
raise ValidationError({
'error':
"Chunked Transfer-Encoding is not supported."
})
try:
self._raw_data = json.loads(
request.data, parse_constant=self._enforce_strict_json)
except ValueError:
raise ValidationError(
{'error': 'The request contains invalid JSON.'})
if not isinstance(self._raw_data, dict):
raise ValidationError(
{'error': 'JSON data must be a dict.'})
else:
self._raw_data = {}
return self._raw_data
def validate_request(self, obj=None):
# Don't work on original raw data, we may reuse the resource for bulk updates.
self.data = self.raw_data.copy()
# Do renaming in two passes to prevent potential multiple renames depending on dict traversal order.
# E.g. if a -> b, b -> c, then a should never be renamed to c.
fields_to_delete = []
fields_to_update = {}
for k, v in self._rename_fields.items():
if v in self.data:
fields_to_update[k] = self.data[v]
fields_to_delete.append(v)
for k in fields_to_delete:
del self.data[k]
for k, v in fields_to_update.items():
self.data[k] = v
if self.schema:
if request.method == 'PUT' and obj != None:
obj_data = dict([(key, getattr(obj, key)) for key in list(obj._fields.keys())])
else:
obj_data = None
schema = self.schema(self.data, obj_data)
try:
self.data = schema.full_clean()
except SchemaValidationError:
raise ValidationError({'field-errors': schema.field_errors, 'errors': schema.errors })
def raw_data(self):
if not hasattr(self, '_raw_data'):
if request.method in ('PUT', 'POST') or request.data:
if request.mimetype and 'json' not in request.mimetype:
raise ValidationError({'error': "Please send valid JSON with a 'Content-Type: application/json' header."})
try:
self._raw_data = json.loads(request.data)
except ValueError:
raise ValidationError({'error': 'The request contains invalid JSON.'})
if not isinstance(self._raw_data, dict):
raise ValidationError({'error': 'JSON data must be a dict.'})
else:
self._raw_data = {}
return self._raw_data
def handle_validation_error(self, e):
if isinstance(e, ValidationError):
raise
elif isinstance(e, mongoengine.ValidationError):
raise ValidationError(serialize_mongoengine_validation_error(e))
else:
raise
def raw_data(self):
if not hasattr(self, '_raw_data'):
if request.method in ('PUT', 'POST'):
try:
self._raw_data = json.loads(request.data)
except ValueError, e:
raise ValidationError(
{'error': 'The request contains invalid JSON.'})
else:
self._raw_data = {}
def _save(self, obj):
try:
self.save_object(obj)
except mongoengine.ValidationError, e:
def serialize_errors(errors):
if hasattr(errors, 'iteritems'):
return dict((k, serialize_errors(v)) for (k, v) in errors.iteritems())
else:
return unicode(errors)
raise ValidationError({'field-errors': serialize_errors(e.errors)})
def validate_request(self, obj=None):
# Don't work on original raw data, we may reuse the resource for bulk updates.
self.data = self.raw_data.copy()
if not self.schema and self.form:
from werkzeug.datastructures import MultiDict
if request.method == 'PUT' and obj != None:
# We treat 'PUT' like 'PATCH', i.e. when fields are not
# specified, existing values are used.
# TODO: This is not implemented properly for nested objects yet.
obj_data = self.serialize(obj)
obj_data.update(self.data)
self.data = obj_data
# @TODO this should rename form fields otherwise in a resource you could say "model_id" and in a form still have to use "model".
# Do renaming in two passes to prevent potential multiple renames depending on dict traversal order.
# E.g. if a -> b, b -> c, then a should never be renamed to c.
fields_to_delete = []
fields_to_update = {}
for k, v in self._rename_fields.iteritems():
if self.data.has_key(v):
fields_to_update[k] = self.data[v]
fields_to_delete.append(v)
for k in fields_to_delete:
del self.data[k]
for k, v in fields_to_update.iteritems():
self.data[k] = v
if self.schema:
from cleancat import ValidationError as SchemaValidationError
if request.method == 'PUT' and obj != None:
obj_data = dict([(key, getattr(obj, key))
for key in obj._fields.keys()])
else:
obj_data = None
schema = self.schema(self.data, obj_data)
try:
self.data = schema.full_clean()
except SchemaValidationError, e:
raise ValidationError({
'field-errors': schema.field_errors,
'errors': schema.errors
})
def validate_request(self, obj=None):
# @TODO this should rename form fields otherwise in a resource you could say "model_id" and in a form still have to use "model".
for k, v in self.rename_fields.iteritems(
): # _rename_fields => rename_fields
if self.data.has_key(v):
self.data[k] = self.data[v]
del self.data[v]
if self.form:
from werkzeug.datastructures import MultiDict
if request.method == 'PUT' and obj != None:
# We treat 'PUT' like 'PATCH', i.e. when fields are not
# specified, existing values are used.
# TODO: This is not implemented properly for nested objects yet.
obj_data = self.serialize(obj)
obj_data.update(self.data)
self.data = obj_data
# We need to convert JSON data into form data.
# e.g. { "people": [ { "name": "A" } ] } into { "people-0-name": "A" }
def json_to_form_data(prefix, json_data):
form_data = {}
for k, v in json_data.iteritems():
if isinstance(v, list): # FieldList
for n, el in enumerate(v):
form_data.update(
json_to_form_data('%s%s-%d-' % (prefix, k, n),
el))
else:
if isinstance(v, dict): # DictField
v = json.dumps(v)
form_data['%s%s' % (prefix, k)] = v
return form_data
json_data = json_to_form_data('', self.data)
data = MultiDict(json_data)
form = self.form(data, csrf_enabled=False)
if not form.validate():
raise ValidationError({'field-errors': form.errors})
self.data = form.data
def __init__(self):
doc_fields = self.document._fields.keys()
if self.fields == None:
self.fields = doc_fields
self._related_resources = self.get_related_resources()
self._rename_fields = self.get_rename_fields()
self._reverse_rename_fields = {}
for k, v in self._rename_fields.iteritems():
self._reverse_rename_fields[v] = k
assert len(self._rename_fields) == len(self._reverse_rename_fields), \
'Cannot rename multiple fields to the same name'
self._filters = self.get_filters()
self._child_document_resources = self.get_child_document_resources()
self.data = None
self._dirty_fields = None
if request.method in ('PUT', 'POST'):
if request.mimetype and 'json' not in request.mimetype:
raise ValidationError({
'error':
"Please send valid JSON with a 'Content-Type: application/json' header."
})
def __init__(self):
doc_fields = self.document._fields.keys()
if self.fields == None:
self.fields = doc_fields
self._related_resources = self.get_related_resources()
self._rename_fields = self.get_rename_fields()
self._reverse_rename_fields = {}
for k, v in self._rename_fields.iteritems():
self._reverse_rename_fields[v] = k
assert len(self._rename_fields) == len(self._reverse_rename_fields), \
'Cannot rename multiple fields to the same name'
self._filters = self.get_filters()
self._child_document_resources = self.get_child_document_resources()
self.data = None
self.raw_data = {}
self._dirty_fields = None
if request.method in ('PUT', 'POST'):
try:
self.raw_data = json.loads(request.data)
except ValueError, e:
raise ValidationError({'error': 'invalid json.'})
def get_objects(self, all=False, qs=None, qfilter=None):
params = request.args
custom_qs = True
if qs is None:
custom_qs = False
qs = self.get_queryset()
# If a queryset filter was provided, pass our current
# queryset in and get a new one out
if qfilter:
qs = qfilter(qs)
for key, value in params.iteritems():
# If this is a resource identified by a URI, we need
# to extract the object id at this point since
# MongoEngine only understands the object id
if self.uri_prefix:
url = urlparse(value)
uri = url.path
value = uri.lstrip(self.uri_prefix)
negate = False
op_name = ''
parts = key.split('__')
for i in range(len(parts) + 1, 0, -1):
field = '__'.join(parts[:i])
allowed_operators = self._filters.get(field)
if allowed_operators:
parts = parts[i:]
break
if allowed_operators is None:
continue
if parts:
# either an operator or a query lookup! See what's allowed.
op_name = parts[-1]
if op_name in allowed_operators:
# operator; drop it
parts.pop()
else:
# assume it's part of a lookup
op_name = ''
if parts and parts[-1] == 'not':
negate = True
parts.pop()
operator = allowed_operators.get(op_name, None)
if operator is None:
continue
if parts:
field = '%s__%s' % (field, '__'.join(parts))
field = self._reverse_rename_fields.get(field, field)
if self.allow_operator_value_none and value == 'None':
value = None
qs = operator().apply(qs, field, value, negate)
limit = None
if params.get('_order_by'):
param = params.get('_order_by')
if not self.allowed_ordering:
raise ValidationError({
"field-errors": {
"_order_by":
"Ordering is not allowed for this resource"
}
})
elif param not in self.allowed_ordering:
raise ValidationError({
"field-errors": {
"_order_by": "Ordering is not allowed for %s" % param
}
})
qs = qs.order_by(*params['_order_by'].split(','))
if not custom_qs and not all:
if self.paginate:
try:
lim = int(params.get('_limit', 100))
except ValueError:
raise ValidationError(
{"field-errors": {
'_limit': 'should be a number',
}})
limit = min(lim, self.max_limit) + 1
if not limit > 1:
raise ValidationError({
"field-errors": {
"_limit": "Limit must be a positive number"
}
})
try:
qs = qs.skip(int(params.get('_skip', 0))).limit(limit)
except ValueError:
raise ValidationError(
{"field-errors": {
'_skip': 'should be a number'
}})
# Fetch one more so we know if there are more results.
else:
qs = qs.limit(self.max_limit + 1)
# Needs to be at the end as it returns a list.
if self.select_related:
qs = qs.select_related()
if limit:
# It is OK to evaluate the queryset as we will do so anyway.
qs = [
o for o in qs
] # don't use list() because mongoengine will do a count query
has_more = len(qs) == limit
if has_more:
qs = qs[:-1]
else:
has_more = None
# bulk-fetch related resources for moar speed
def cmp_fields(ordering):
# Takes a list of fields and directions and returns a
# comparison function for sorted() to perform client-side
# sorting.
# Example: sorted(objs, cmp_fields([('date_created', -1)]))
def _cmp(x, y):
for field, direction in ordering:
result = cmp(x, y) * direction
if result:
return result
return 0
return _cmp
if self.related_resources_hints:
if params and '_fields' in params:
only_fields = set(params['_fields'].split(','))
else:
only_fields = None
document_queryset = {}
for obj in qs:
for field_name in self.related_resources_hints.keys():
if only_fields and field_name not in only_fields:
continue
resource = self.get_related_resources()[field_name]
method = getattr(obj, field_name)
if callable(method):
q = method()
if field_name in document_queryset.keys():
document_queryset[field_name] = (
document_queryset[field_name] | q._query_obj)
else:
document_queryset[field_name] = q._query_obj
hints = {}
for k, v in document_queryset.iteritems():
doc = self.get_related_resources()[k].document
query = doc.objects.filter(v)
# Don't let MongoDB do the sorting as it won't use the index.
# Store the ordering so we can do client sorting afterwards.
ordering = query._ordering
query._ordering = []
# NO I REALLY DONT WANT YOUR ORDERING
document_ordering = query._document._meta['ordering']
query._document._meta['ordering'] = []
results = [
o for o in query
] # don't use list() because mongoengine will do a count query
query._document._meta['ordering'] = document_ordering
document_queryset[k] = sorted(results, cmp_fields(ordering))
hint_index = {}
if k in self.related_resources_hints.keys():
hint_field = self.related_resources_hints[k]
for obj in document_queryset[k]:
hinted = str(obj._data[hint_field].id)
if hinted not in hint_index:
hint_index[hinted] = [obj]
else:
hint_index[hinted].append(obj)
hints[k] = hint_index
for obj in qs:
for field, hint_index in hints.iteritems():
obj_id = obj.id
if isinstance(obj_id, DBRef):
obj_id = obj_id.id
elif isinstance(obj_id, ObjectId):
obj_id = str(obj_id)
if obj_id not in hint_index.keys():
setattr(obj, field, [])
continue
setattr(obj, field, hint_index[obj_id])
return qs, has_more
v = []
if isinstance(v, datetime.datetime): # DateTimeField
v = v.strftime('%Y-%m-%d %H:%M:%S')
if isinstance(v, DBRef): # ReferenceField
v = v.id
if v is None:
v = ''
form_data['%s%s' % (prefix, k)] = v
return form_data
json_data = json_to_form_data('', self.data)
data = MultiDict(json_data)
form = self.form(data, csrf_enabled=False)
if not form.validate():
raise ValidationError({'field-errors': form.errors})
self.data = form.data
def get_queryset(self):
return self.document.objects
def get_object(self, pk, qfilter=None):
qs = self.get_queryset()
# If a queryset filter was provided, pass our current
# queryset in and get a new one out
if qfilter:
qs = qfilter(qs)
return qs.get(pk=pk)
def get_objects(self, all=False, qs=None, qfilter=None):
def validate_request(self, obj=None):
# Don't work on original raw data, we may reuse the resource for bulk updates.
self.data = self.raw_data.copy()
if not self.schema and self.form:
if request.method == 'PUT' and obj != None:
# We treat 'PUT' like 'PATCH', i.e. when fields are not
# specified, existing values are used.
# TODO: This is not implemented properly for nested objects yet.
obj_data = self.serialize(obj)
obj_data.update(self.data)
self.data = obj_data
# @TODO this should rename form fields otherwise in a resource you could say "model_id" and in a form still have to use "model".
# Do renaming in two passes to prevent potential multiple renames depending on dict traversal order.
# E.g. if a -> b, b -> c, then a should never be renamed to c.
fields_to_delete = []
fields_to_update = {}
for k, v in self._rename_fields.iteritems():
if self.data.has_key(v):
fields_to_update[k] = self.data[v]
fields_to_delete.append(v)
for k in fields_to_delete:
del self.data[k]
for k, v in fields_to_update.iteritems():
self.data[k] = v
if self.schema:
if request.method == 'PUT' and obj != None:
obj_data = dict([(key, getattr(obj, key)) for key in obj._fields.keys()])
else:
obj_data = None
schema = self.schema(self.data, obj_data)
try:
self.data = schema.full_clean()
except SchemaValidationError:
raise ValidationError({'field-errors': schema.field_errors, 'errors': schema.errors })
elif self.form:
# We need to convert JSON data into form data.
# e.g. { "people": [ { "name": "A" } ] } into { "people-0-name": "A" }
def json_to_form_data(prefix, json_data):
form_data = {}
for k, v in json_data.iteritems():
if isinstance(v, list): # FieldList
for n, el in enumerate(v):
if isinstance(el, dict): # only dict type is supported
form_data.update(json_to_form_data('%s%s-%d-' % (prefix, k, n), el))
else:
if isinstance(v, dict): # DictField
v = json.dumps(v, cls=MongoEncoder)
if isinstance(v, bool) and v == False: # BooleanField
v = []
if isinstance(v, datetime.datetime): # DateTimeField
v = v.strftime('%Y-%m-%d %H:%M:%S')
if isinstance(v, DBRef): # ReferenceField
v = v.id
if v is None:
v = ''
form_data['%s%s' % (prefix, k)] = v
return form_data
json_data = json_to_form_data('', self.data)
data = MultiDict(json_data)
form = self.form(data, csrf_enabled=False)
if not form.validate():
raise ValidationError({'field-errors': form.errors})
self.data = form.data
def get_objects(self, all=False, qs=None, qfilter=None):
params = request.args
custom_qs = True
if qs is None:
custom_qs = False
qs = self.get_queryset()
# If a queryset filter was provided, pass our current
# queryset in and get a new one out
if qfilter:
qs = qfilter(qs)
for key, value in params.iteritems():
# If this is a resource identified by a URI, we need
# to extract the object id at this point since
# MongoEngine only understands the object id
if self.uri_prefix:
url = urlparse(value)
uri = url.path
value = uri.lstrip(self.uri_prefix)
negate = False
op_name = ''
parts = key.split('__')
for i in range(len(parts) + 1, 0, -1):
field = '__'.join(parts[:i])
allowed_operators = self._filters.get(field)
if allowed_operators:
parts = parts[i:]
break
if allowed_operators is None:
continue
if parts:
# either an operator or a query lookup! See what's allowed.
op_name = parts[-1]
if op_name in allowed_operators:
# operator; drop it
parts.pop()
else:
# assume it's part of a lookup
op_name = ''
if parts and parts[-1] == 'not':
negate = True
parts.pop()
operator = allowed_operators.get(op_name, None)
if operator is None:
continue
if parts:
field = '%s__%s' % (field, '__'.join(parts))
field = self._reverse_rename_fields.get(field, field)
qs = operator().apply(qs, field, value, negate)
limit = None
if self.allowed_ordering and params.get(
'_order_by', '').strip('-') in self.allowed_ordering:
qs = qs.order_by(*params['_order_by'].split(','))
if not custom_qs and not all:
if self.paginate:
# _limit and _skip validation
if not isint(params.get('_limit', 1)):
raise ValidationError({
'error':
'_limit must be an integer (got "%s" instead).' %
params['_limit']
})
if not isint(params.get('_skip', 1)):
raise ValidationError({
'error':
'_skip must be an integer (got "%s" instead).' %
params['_skip']
})
if params.get('_limit') and int(
params['_limit']) > self.max_limit:
raise ValidationError({
'error':
"The limit you set is larger than the maximum limit for this resource (max_limit = %d)."
% self.max_limit
})
limit = min(int(params.get('_limit', 100)), self.max_limit) + 1
# Fetch one more so we know if there are more results.
qs = qs.skip(int(params.get('_skip', 0))).limit(limit)
else:
qs = qs.limit(self.max_limit + 1)
# Needs to be at the end as it returns a list.
if self.select_related:
qs = qs.select_related()
if limit:
# It is OK to evaluate the queryset as we will do so anyway.
qs = [
o for o in qs
] # don't use list() because mongoengine will do a count query
has_more = len(qs) == limit
if has_more:
qs = qs[:-1]
else:
has_more = None
# bulk-fetch related resources for moar speed
if self.related_resources_hints:
if params and '_fields' in params:
only_fields = set(params['_fields'].split(','))
else:
only_fields = None
self.fetch_related_resources(qs, only_fields)
return qs, has_more