def raw_data(self):
"""Validate and return parsed JSON payload."""
if not has_request_context():
# `raw_data` doesn't make sense if we don't have a request
raise AttributeError
if not hasattr(self, '_raw_data'):
if request.method in ('PUT', 'POST') or request.data:
if request.mimetype and 'json' not in request.mimetype:
raise ValidationError({
'error':
"Please send valid JSON with a 'Content-Type: application/json' header."
})
if request.headers.get('Transfer-Encoding') == 'chunked':
raise ValidationError({
'error':
"Chunked Transfer-Encoding is not supported."
})
try:
self._raw_data = json.loads(
request.data.decode('utf-8'),
parse_constant=self._enforce_strict_json)
except ValueError:
raise ValidationError(
{'error': 'The request contains invalid JSON.'})
if not isinstance(self._raw_data, dict):
raise ValidationError(
{'error': 'JSON data must be a dict.'})
else:
self._raw_data = {}
return self._raw_data
def validate_data(doc, sender=None, project=None):
d = fdict(doc, delimiter=delimiter)
for key in list(d.keys()):
key = key.strip()
nodes = key.split(delimiter)
is_quantity_key = int(nodes[-1] in quantity_keys)
if len(nodes) > max_depth + is_quantity_key:
raise ValidationError(
{"error": f"max nesting ({max_depth}) exceeded for {key}"})
if is_quantity_key:
continue
for node in nodes:
for char in node:
if char in invalidChars:
raise ValidationError({
"error":
f"invalid character '{char}' in {node} ({key})"
})
value = str(d[key])
words = value.split()
try_quantity = bool(len(words) == 2 and is_float(words[0]))
if try_quantity or isinstance(d[key], (int, float)):
try:
q = Q_(value).to_compact()
if not q.check(0):
q.ito_reduced_units()
if sender:
_key = key.replace(".", "__")
query = {"project": project, f"data__{_key}__exists": True}
sample = (sender.objects.only(f"data.{key}.unit").filter(
**query).first())
if sample:
sample_fdct = fdict(sample["data"],
delimiter=delimiter)
q.ito(sample_fdct[f"{key}.unit"])
v = Decimal(str(q.magnitude))
vt = v.as_tuple()
if vt.exponent < 0:
dgts = len(vt.digits)
dgts = max_dgts if dgts > max_dgts else dgts
v = f"{v:.{dgts}g}"
if try_quantity:
q = Q_(f"{v} {q.units}")
except Exception as ex:
raise ValidationError({"error": str(ex)})
d[key] = {
"display": str(q),
"value": q.magnitude,
"unit": str(q.units)
}
return d.to_dict_nested()
def handle_validation_error(self, e):
if isinstance(e, ValidationError):
raise
elif isinstance(e, mongoengine.ValidationError):
raise ValidationError(serialize_mongoengine_validation_error(e))
else:
raise
def pre_save_post_validation(cls, sender, document, **kwargs):
document.other = validate_data(document.other)
if len(document.urls) > 5:
raise ValidationError(
{"error": f"too many URL references (max. 5)"})
for label in document.urls.keys():
len_label = len(label)
if len_label < 3 or len_label > 8:
raise ValidationError({
"error":
f"length of URL label {label} should be 3-8 characters"
})
for char in label:
if char in invalidChars:
raise ValidationError(
{"error": f"invalid character '{char}' in {label}"})
def date_prep(field, value, op):
try:
value = isoparse(value)
except ValueError:
raise ValidationError("Invalid date format - use ISO 8601")
return {f"{field}__{op}": value}
def post(self, **kwargs):
if kwargs.pop("pk"):
raise NotFound("Did you mean to use PUT?")
# Set the view_method on a resource instance
raw_data = self._resource.raw_data
if isinstance(raw_data, dict):
# create single object
self._resource.view_method = methods.Create
return self.create_object()
elif isinstance(raw_data, list):
limit = self._resource.bulk_update_limit
if len(raw_data) > limit:
raise ValidationError(f"Can only create {limit} documents at once")
raw_data_deque = deque(raw_data)
self._resource.view_method = methods.BulkCreate
count = 0
tic = time.perf_counter()
while len(raw_data_deque):
self._resource._raw_data = raw_data_deque.popleft()
skip = bool(raw_data_deque)
kwargs = {"skip_post_save": skip}
if not skip:
kwargs["remaining_time"] = TIMEOUT - (time.perf_counter() - tic)
self.create_object(**kwargs)
count += 1
dt = time.perf_counter() - tic
avg = dt / count
if dt + avg > TIMEOUT:
break
msg = f"Created {count} objects in {dt:0.1f}s ({avg:0.1f}s per object)."
print(msg)
ret = {"count": count}
if raw_data_deque:
remain = len(raw_data_deque)
msg += f" Remaining {remain} objects skipped to avoid Server Timeout."
ret["warning"] = msg
return ret, "201 Created"
else:
raise ValidationError("wrong payload type")
def get_skip_and_limit(self, params=None):
"""
Perform validation and return sanitized values for _skip and _limit
params of the request that's currently being processed.
"""
max_limit = self.get_max_limit()
if params is None:
params = self.params
if self.paginate:
# _limit and _skip validation
if not isint(params.get('_limit', 1)):
raise ValidationError({
'error':
'_limit must be an integer (got "%s" instead).' %
params['_limit']
})
if not isint(params.get('_skip', 1)):
raise ValidationError({
'error':
'_skip must be an integer (got "%s" instead).' %
params['_skip']
})
if params.get('_limit') and int(params['_limit']) > max_limit:
raise ValidationError({
'error':
"The limit you set is larger than the maximum limit for this resource (max_limit = %d)."
% max_limit
})
if params.get('_skip') and int(params['_skip']) < 0:
raise ValidationError({
'error':
'_skip must be a non-negative integer (got "%s" instead).'
% params['_skip']
})
limit = min(int(params.get('_limit', self.default_limit)),
max_limit)
# Fetch one more so we know if there are more results.
return int(params.get('_skip', 0)), limit
else:
return 0, max_limit
def validate_request(self, obj=None):
"""
Validate the request that's currently being processed and fill in
the self.data dict that'll later be used to save/update an object.
`obj` points to the object that's being updated, or is empty if a new
object is being created.
"""
# When creating or updating a single object, delegate the validation
# to a more specific subresource, if it exists
if (request.method == 'PUT' and obj) or request.method == 'POST':
subresource = self._subresource(obj)
if subresource:
subresource._raw_data = self._raw_data
subresource.validate_request(obj=obj)
self.data = subresource.data
return
# Don't work on original raw data, we may reuse the resource for bulk
# updates.
self.data = self.raw_data.copy()
# Do renaming in two passes to prevent potential multiple renames
# depending on dict traversal order.
# E.g. if a -> b, b -> c, then a should never be renamed to c.
fields_to_delete = []
fields_to_update = {}
for k, v in self._rename_fields.items():
if v in self.data:
fields_to_update[k] = self.data[v]
fields_to_delete.append(v)
for k in fields_to_delete:
del self.data[k]
for k, v in fields_to_update.items():
self.data[k] = v
# If CleanCat schema exists on this resource, use it to perform the
# validation
if self.schema:
if request.method == 'PUT' and obj is not None:
obj_data = dict([(key, getattr(obj, key))
for key in obj._fields.keys()])
else:
obj_data = None
schema = self.schema(self.data, obj_data)
try:
self.data = schema.full_clean()
except SchemaValidationError:
raise ValidationError({
'field-errors': schema.field_errors,
'errors': schema.errors
})
def get_objects(self, qs=None, qfilter=None):
"""
Return objects fetched from the database based on all the parameters
of the request that's currently being processed.
Params:
- Custom queryset can be passed via `qs`. Otherwise `self.get_queryset`
is used.
- Pass `qfilter` function to modify the queryset.
"""
params = self.params
custom_qs = True
if qs is None:
custom_qs = False
qs = self.get_queryset()
# If a queryset filter was provided, pass our current queryset in and
# get a new one out
if qfilter:
qs = qfilter(qs)
# Apply filters and ordering, based on the params supplied by the
# request
qs = self.apply_filters(qs, params)
qs = self.apply_ordering(qs, params)
# Apply limit and skip to the queryset
limit = None
if self.view_method == methods.BulkUpdate:
# limit the number of objects that can be bulk-updated at a time
qs = qs.limit(self.bulk_update_limit)
elif not custom_qs:
# no need to skip/limit if a custom `qs` was provided
skip, limit = self.get_skip_and_limit(params)
qs = qs.skip(skip).limit(limit+1)
# Needs to be at the end as it returns a list, not a queryset
if self.select_related:
qs = qs.select_related()
# Evaluate the queryset
objs = list(qs)
# Raise a validation error if bulk update would result in more than
# bulk_update_limit updates
if self.view_method == methods.BulkUpdate and len(objs) >= self.bulk_update_limit:
raise ValidationError({
'errors': ["It's not allowed to update more than %d objects at once" % self.bulk_update_limit]
})
# Determine the value of has_more
if self.view_method != methods.BulkUpdate and self.paginate:
has_more = len(objs) > limit
if has_more:
objs = objs[:-1]
else:
has_more = None
# bulk-fetch related resources for moar speed
if self.related_resources_hints:
self.fetch_related_resources(objs, self.get_requested_fields(params=params))
return objs, has_more
def send_email(to, subject, template):
try:
sns_client.publish(TopicArn=to, Message=template, Subject=subject)
except Exception as ex:
raise ValidationError({"error": str(ex)})