def get_skip_and_limit(self, params=None): if params is None: params = request.args if self.paginate: # _limit and _skip validation if not isint(params.get('_limit', 1)): raise ValidationError({ 'error': '_limit must be an integer (got "%s" instead).' % params['_limit'] }) if not isint(params.get('_skip', 1)): raise ValidationError({ 'error': '_skip must be an integer (got "%s" instead).' % params['_skip'] }) if params.get('_limit') and int(params['_limit']) > self.max_limit: raise ValidationError({ 'error': "The limit you set is larger than the maximum limit for this resource (max_limit = %d)." % self.max_limit }) limit = min(int(params.get('_limit', self.default_limit)), self.max_limit) # Fetch one more so we know if there are more results. return int(params.get('_skip', 0)), limit else: return 0, self.max_limit
def raw_data(self): if not hasattr(self, '_raw_data'): if request.method in ('PUT', 'POST') or request.data: if request.mimetype and 'json' not in request.mimetype: raise ValidationError({ 'error': "Please send valid JSON with a 'Content-Type: application/json' header." }) if request.headers.get('Transfer-Encoding') == 'chunked': raise ValidationError({ 'error': "Chunked Transfer-Encoding is not supported." }) try: self._raw_data = json.loads( request.data, parse_constant=self._enforce_strict_json) except ValueError: raise ValidationError( {'error': 'The request contains invalid JSON.'}) if not isinstance(self._raw_data, dict): raise ValidationError( {'error': 'JSON data must be a dict.'}) else: self._raw_data = {} return self._raw_data
def validate_request(self, obj=None): # Don't work on original raw data, we may reuse the resource for bulk updates. self.data = self.raw_data.copy() # Do renaming in two passes to prevent potential multiple renames depending on dict traversal order. # E.g. if a -> b, b -> c, then a should never be renamed to c. fields_to_delete = [] fields_to_update = {} for k, v in self._rename_fields.items(): if v in self.data: fields_to_update[k] = self.data[v] fields_to_delete.append(v) for k in fields_to_delete: del self.data[k] for k, v in fields_to_update.items(): self.data[k] = v if self.schema: if request.method == 'PUT' and obj != None: obj_data = dict([(key, getattr(obj, key)) for key in list(obj._fields.keys())]) else: obj_data = None schema = self.schema(self.data, obj_data) try: self.data = schema.full_clean() except SchemaValidationError: raise ValidationError({'field-errors': schema.field_errors, 'errors': schema.errors })
def raw_data(self): if not hasattr(self, '_raw_data'): if request.method in ('PUT', 'POST') or request.data: if request.mimetype and 'json' not in request.mimetype: raise ValidationError({'error': "Please send valid JSON with a 'Content-Type: application/json' header."}) try: self._raw_data = json.loads(request.data) except ValueError: raise ValidationError({'error': 'The request contains invalid JSON.'}) if not isinstance(self._raw_data, dict): raise ValidationError({'error': 'JSON data must be a dict.'}) else: self._raw_data = {} return self._raw_data
def handle_validation_error(self, e): if isinstance(e, ValidationError): raise elif isinstance(e, mongoengine.ValidationError): raise ValidationError(serialize_mongoengine_validation_error(e)) else: raise
def raw_data(self): if not hasattr(self, '_raw_data'): if request.method in ('PUT', 'POST'): try: self._raw_data = json.loads(request.data) except ValueError, e: raise ValidationError( {'error': 'The request contains invalid JSON.'}) else: self._raw_data = {}
def _save(self, obj): try: self.save_object(obj) except mongoengine.ValidationError, e: def serialize_errors(errors): if hasattr(errors, 'iteritems'): return dict((k, serialize_errors(v)) for (k, v) in errors.iteritems()) else: return unicode(errors) raise ValidationError({'field-errors': serialize_errors(e.errors)})
def validate_request(self, obj=None): # Don't work on original raw data, we may reuse the resource for bulk updates. self.data = self.raw_data.copy() if not self.schema and self.form: from werkzeug.datastructures import MultiDict if request.method == 'PUT' and obj != None: # We treat 'PUT' like 'PATCH', i.e. when fields are not # specified, existing values are used. # TODO: This is not implemented properly for nested objects yet. obj_data = self.serialize(obj) obj_data.update(self.data) self.data = obj_data # @TODO this should rename form fields otherwise in a resource you could say "model_id" and in a form still have to use "model". # Do renaming in two passes to prevent potential multiple renames depending on dict traversal order. # E.g. if a -> b, b -> c, then a should never be renamed to c. fields_to_delete = [] fields_to_update = {} for k, v in self._rename_fields.iteritems(): if self.data.has_key(v): fields_to_update[k] = self.data[v] fields_to_delete.append(v) for k in fields_to_delete: del self.data[k] for k, v in fields_to_update.iteritems(): self.data[k] = v if self.schema: from cleancat import ValidationError as SchemaValidationError if request.method == 'PUT' and obj != None: obj_data = dict([(key, getattr(obj, key)) for key in obj._fields.keys()]) else: obj_data = None schema = self.schema(self.data, obj_data) try: self.data = schema.full_clean() except SchemaValidationError, e: raise ValidationError({ 'field-errors': schema.field_errors, 'errors': schema.errors })
def validate_request(self, obj=None): # @TODO this should rename form fields otherwise in a resource you could say "model_id" and in a form still have to use "model". for k, v in self.rename_fields.iteritems( ): # _rename_fields => rename_fields if self.data.has_key(v): self.data[k] = self.data[v] del self.data[v] if self.form: from werkzeug.datastructures import MultiDict if request.method == 'PUT' and obj != None: # We treat 'PUT' like 'PATCH', i.e. when fields are not # specified, existing values are used. # TODO: This is not implemented properly for nested objects yet. obj_data = self.serialize(obj) obj_data.update(self.data) self.data = obj_data # We need to convert JSON data into form data. # e.g. { "people": [ { "name": "A" } ] } into { "people-0-name": "A" } def json_to_form_data(prefix, json_data): form_data = {} for k, v in json_data.iteritems(): if isinstance(v, list): # FieldList for n, el in enumerate(v): form_data.update( json_to_form_data('%s%s-%d-' % (prefix, k, n), el)) else: if isinstance(v, dict): # DictField v = json.dumps(v) form_data['%s%s' % (prefix, k)] = v return form_data json_data = json_to_form_data('', self.data) data = MultiDict(json_data) form = self.form(data, csrf_enabled=False) if not form.validate(): raise ValidationError({'field-errors': form.errors}) self.data = form.data
def __init__(self): doc_fields = self.document._fields.keys() if self.fields == None: self.fields = doc_fields self._related_resources = self.get_related_resources() self._rename_fields = self.get_rename_fields() self._reverse_rename_fields = {} for k, v in self._rename_fields.iteritems(): self._reverse_rename_fields[v] = k assert len(self._rename_fields) == len(self._reverse_rename_fields), \ 'Cannot rename multiple fields to the same name' self._filters = self.get_filters() self._child_document_resources = self.get_child_document_resources() self.data = None self._dirty_fields = None if request.method in ('PUT', 'POST'): if request.mimetype and 'json' not in request.mimetype: raise ValidationError({ 'error': "Please send valid JSON with a 'Content-Type: application/json' header." })
def __init__(self): doc_fields = self.document._fields.keys() if self.fields == None: self.fields = doc_fields self._related_resources = self.get_related_resources() self._rename_fields = self.get_rename_fields() self._reverse_rename_fields = {} for k, v in self._rename_fields.iteritems(): self._reverse_rename_fields[v] = k assert len(self._rename_fields) == len(self._reverse_rename_fields), \ 'Cannot rename multiple fields to the same name' self._filters = self.get_filters() self._child_document_resources = self.get_child_document_resources() self.data = None self.raw_data = {} self._dirty_fields = None if request.method in ('PUT', 'POST'): try: self.raw_data = json.loads(request.data) except ValueError, e: raise ValidationError({'error': 'invalid json.'})
def get_objects(self, all=False, qs=None, qfilter=None): params = request.args custom_qs = True if qs is None: custom_qs = False qs = self.get_queryset() # If a queryset filter was provided, pass our current # queryset in and get a new one out if qfilter: qs = qfilter(qs) for key, value in params.iteritems(): # If this is a resource identified by a URI, we need # to extract the object id at this point since # MongoEngine only understands the object id if self.uri_prefix: url = urlparse(value) uri = url.path value = uri.lstrip(self.uri_prefix) negate = False op_name = '' parts = key.split('__') for i in range(len(parts) + 1, 0, -1): field = '__'.join(parts[:i]) allowed_operators = self._filters.get(field) if allowed_operators: parts = parts[i:] break if allowed_operators is None: continue if parts: # either an operator or a query lookup! See what's allowed. op_name = parts[-1] if op_name in allowed_operators: # operator; drop it parts.pop() else: # assume it's part of a lookup op_name = '' if parts and parts[-1] == 'not': negate = True parts.pop() operator = allowed_operators.get(op_name, None) if operator is None: continue if parts: field = '%s__%s' % (field, '__'.join(parts)) field = self._reverse_rename_fields.get(field, field) if self.allow_operator_value_none and value == 'None': value = None qs = operator().apply(qs, field, value, negate) limit = None if params.get('_order_by'): param = params.get('_order_by') if not self.allowed_ordering: raise ValidationError({ "field-errors": { "_order_by": "Ordering is not allowed for this resource" } }) elif param not in self.allowed_ordering: raise ValidationError({ "field-errors": { "_order_by": "Ordering is not allowed for %s" % param } }) qs = qs.order_by(*params['_order_by'].split(',')) if not custom_qs and not all: if self.paginate: try: lim = int(params.get('_limit', 100)) except ValueError: raise ValidationError( {"field-errors": { '_limit': 'should be a number', }}) limit = min(lim, self.max_limit) + 1 if not limit > 1: raise ValidationError({ "field-errors": { "_limit": "Limit must be a positive number" } }) try: qs = qs.skip(int(params.get('_skip', 0))).limit(limit) except ValueError: raise ValidationError( {"field-errors": { '_skip': 'should be a number' }}) # Fetch one more so we know if there are more results. else: qs = qs.limit(self.max_limit + 1) # Needs to be at the end as it returns a list. if self.select_related: qs = qs.select_related() if limit: # It is OK to evaluate the queryset as we will do so anyway. qs = [ o for o in qs ] # don't use list() because mongoengine will do a count query has_more = len(qs) == limit if has_more: qs = qs[:-1] else: has_more = None # bulk-fetch related resources for moar speed def cmp_fields(ordering): # Takes a list of fields and directions and returns a # comparison function for sorted() to perform client-side # sorting. # Example: sorted(objs, cmp_fields([('date_created', -1)])) def _cmp(x, y): for field, direction in ordering: result = cmp(x, y) * direction if result: return result return 0 return _cmp if self.related_resources_hints: if params and '_fields' in params: only_fields = set(params['_fields'].split(',')) else: only_fields = None document_queryset = {} for obj in qs: for field_name in self.related_resources_hints.keys(): if only_fields and field_name not in only_fields: continue resource = self.get_related_resources()[field_name] method = getattr(obj, field_name) if callable(method): q = method() if field_name in document_queryset.keys(): document_queryset[field_name] = ( document_queryset[field_name] | q._query_obj) else: document_queryset[field_name] = q._query_obj hints = {} for k, v in document_queryset.iteritems(): doc = self.get_related_resources()[k].document query = doc.objects.filter(v) # Don't let MongoDB do the sorting as it won't use the index. # Store the ordering so we can do client sorting afterwards. ordering = query._ordering query._ordering = [] # NO I REALLY DONT WANT YOUR ORDERING document_ordering = query._document._meta['ordering'] query._document._meta['ordering'] = [] results = [ o for o in query ] # don't use list() because mongoengine will do a count query query._document._meta['ordering'] = document_ordering document_queryset[k] = sorted(results, cmp_fields(ordering)) hint_index = {} if k in self.related_resources_hints.keys(): hint_field = self.related_resources_hints[k] for obj in document_queryset[k]: hinted = str(obj._data[hint_field].id) if hinted not in hint_index: hint_index[hinted] = [obj] else: hint_index[hinted].append(obj) hints[k] = hint_index for obj in qs: for field, hint_index in hints.iteritems(): obj_id = obj.id if isinstance(obj_id, DBRef): obj_id = obj_id.id elif isinstance(obj_id, ObjectId): obj_id = str(obj_id) if obj_id not in hint_index.keys(): setattr(obj, field, []) continue setattr(obj, field, hint_index[obj_id]) return qs, has_more
v = [] if isinstance(v, datetime.datetime): # DateTimeField v = v.strftime('%Y-%m-%d %H:%M:%S') if isinstance(v, DBRef): # ReferenceField v = v.id if v is None: v = '' form_data['%s%s' % (prefix, k)] = v return form_data json_data = json_to_form_data('', self.data) data = MultiDict(json_data) form = self.form(data, csrf_enabled=False) if not form.validate(): raise ValidationError({'field-errors': form.errors}) self.data = form.data def get_queryset(self): return self.document.objects def get_object(self, pk, qfilter=None): qs = self.get_queryset() # If a queryset filter was provided, pass our current # queryset in and get a new one out if qfilter: qs = qfilter(qs) return qs.get(pk=pk) def get_objects(self, all=False, qs=None, qfilter=None):
def validate_request(self, obj=None): # Don't work on original raw data, we may reuse the resource for bulk updates. self.data = self.raw_data.copy() if not self.schema and self.form: if request.method == 'PUT' and obj != None: # We treat 'PUT' like 'PATCH', i.e. when fields are not # specified, existing values are used. # TODO: This is not implemented properly for nested objects yet. obj_data = self.serialize(obj) obj_data.update(self.data) self.data = obj_data # @TODO this should rename form fields otherwise in a resource you could say "model_id" and in a form still have to use "model". # Do renaming in two passes to prevent potential multiple renames depending on dict traversal order. # E.g. if a -> b, b -> c, then a should never be renamed to c. fields_to_delete = [] fields_to_update = {} for k, v in self._rename_fields.iteritems(): if self.data.has_key(v): fields_to_update[k] = self.data[v] fields_to_delete.append(v) for k in fields_to_delete: del self.data[k] for k, v in fields_to_update.iteritems(): self.data[k] = v if self.schema: if request.method == 'PUT' and obj != None: obj_data = dict([(key, getattr(obj, key)) for key in obj._fields.keys()]) else: obj_data = None schema = self.schema(self.data, obj_data) try: self.data = schema.full_clean() except SchemaValidationError: raise ValidationError({'field-errors': schema.field_errors, 'errors': schema.errors }) elif self.form: # We need to convert JSON data into form data. # e.g. { "people": [ { "name": "A" } ] } into { "people-0-name": "A" } def json_to_form_data(prefix, json_data): form_data = {} for k, v in json_data.iteritems(): if isinstance(v, list): # FieldList for n, el in enumerate(v): if isinstance(el, dict): # only dict type is supported form_data.update(json_to_form_data('%s%s-%d-' % (prefix, k, n), el)) else: if isinstance(v, dict): # DictField v = json.dumps(v, cls=MongoEncoder) if isinstance(v, bool) and v == False: # BooleanField v = [] if isinstance(v, datetime.datetime): # DateTimeField v = v.strftime('%Y-%m-%d %H:%M:%S') if isinstance(v, DBRef): # ReferenceField v = v.id if v is None: v = '' form_data['%s%s' % (prefix, k)] = v return form_data json_data = json_to_form_data('', self.data) data = MultiDict(json_data) form = self.form(data, csrf_enabled=False) if not form.validate(): raise ValidationError({'field-errors': form.errors}) self.data = form.data
def get_objects(self, all=False, qs=None, qfilter=None): params = request.args custom_qs = True if qs is None: custom_qs = False qs = self.get_queryset() # If a queryset filter was provided, pass our current # queryset in and get a new one out if qfilter: qs = qfilter(qs) for key, value in params.iteritems(): # If this is a resource identified by a URI, we need # to extract the object id at this point since # MongoEngine only understands the object id if self.uri_prefix: url = urlparse(value) uri = url.path value = uri.lstrip(self.uri_prefix) negate = False op_name = '' parts = key.split('__') for i in range(len(parts) + 1, 0, -1): field = '__'.join(parts[:i]) allowed_operators = self._filters.get(field) if allowed_operators: parts = parts[i:] break if allowed_operators is None: continue if parts: # either an operator or a query lookup! See what's allowed. op_name = parts[-1] if op_name in allowed_operators: # operator; drop it parts.pop() else: # assume it's part of a lookup op_name = '' if parts and parts[-1] == 'not': negate = True parts.pop() operator = allowed_operators.get(op_name, None) if operator is None: continue if parts: field = '%s__%s' % (field, '__'.join(parts)) field = self._reverse_rename_fields.get(field, field) qs = operator().apply(qs, field, value, negate) limit = None if self.allowed_ordering and params.get( '_order_by', '').strip('-') in self.allowed_ordering: qs = qs.order_by(*params['_order_by'].split(',')) if not custom_qs and not all: if self.paginate: # _limit and _skip validation if not isint(params.get('_limit', 1)): raise ValidationError({ 'error': '_limit must be an integer (got "%s" instead).' % params['_limit'] }) if not isint(params.get('_skip', 1)): raise ValidationError({ 'error': '_skip must be an integer (got "%s" instead).' % params['_skip'] }) if params.get('_limit') and int( params['_limit']) > self.max_limit: raise ValidationError({ 'error': "The limit you set is larger than the maximum limit for this resource (max_limit = %d)." % self.max_limit }) limit = min(int(params.get('_limit', 100)), self.max_limit) + 1 # Fetch one more so we know if there are more results. qs = qs.skip(int(params.get('_skip', 0))).limit(limit) else: qs = qs.limit(self.max_limit + 1) # Needs to be at the end as it returns a list. if self.select_related: qs = qs.select_related() if limit: # It is OK to evaluate the queryset as we will do so anyway. qs = [ o for o in qs ] # don't use list() because mongoengine will do a count query has_more = len(qs) == limit if has_more: qs = qs[:-1] else: has_more = None # bulk-fetch related resources for moar speed if self.related_resources_hints: if params and '_fields' in params: only_fields = set(params['_fields'].split(',')) else: only_fields = None self.fetch_related_resources(qs, only_fields) return qs, has_more