def deserialize(self, node, cstruct):
"""
In Colander speak: Converts a serialized value (a cstruct) into a
Python data structure (a appstruct).
Or: Converts a GeoJSON string into a `WKBElement`.
"""
if cstruct is null or cstruct == '':
return null
try:
data = geojson.loads(cstruct)
except Invalid as exc:
raise exc
except:
raise Invalid(node, 'Invalid geometry: %s' % cstruct)
if not isinstance(data, geojson.GeoJSON):
raise Invalid(node, 'Invalid geometry: %s' % cstruct)
geom_type = data['type'].upper()
allowed_types = self.geometry_types
if geom_type in allowed_types:
if not is_valid_geometry(data):
raise Invalid(node, 'Invalid geometry: %s' % cstruct)
else:
return wkbelement_from_geojson(data, self.srid)
else:
raise Invalid(
node, 'Invalid geometry type. Expected: %s. Got: %s.' %
(allowed_types, geom_type))
def deserialize(self, field, pstruct):
result = []
error = None
if pstruct is null:
pstruct = []
field.sequence_fields = []
item_field = field.children[0]
for num, substruct in enumerate(pstruct):
subfield = item_field.clone()
try:
subval = subfield.deserialize(substruct)
except Invalid as e:
subval = e.value
if error is None:
error = Invalid(field.schema, value=result)
error.add(e, num)
result.append(subval)
field.sequence_fields.append(subfield)
if error is not None:
raise error
return result
def deserialize(self, field, pstruct):
error = None
result = {}
if pstruct is null:
pstruct = {}
checkbox_schema = pstruct.get(self.checkbox_element, null)
if checkbox_schema is null:
return null
result[self.checkbox_element] = True
for num, subfield in enumerate(field.children):
name = subfield.name
subval = pstruct.get(name, null)
try:
result[name] = subfield.deserialize(subval)
except Invalid as e:
result[name] = e.value
if error is None:
error = Invalid(field.schema, value=result)
error.add(e, num)
if error is not None:
raise error
result[self.checkbox_element] = True
return result
def serialize(self, appstruct: Request):
if appstruct is colander.null:
return colander.null
if not IRequest.providedBy(appstruct):
msg = '{:s} does not implement IRequest interface'.format(
str(appstruct))
raise Invalid(self, msg)
content_type = getattr(appstruct,
"content_type",
"application/octet-stream")
if content_type != "application/json":
msg = "{:s} is not a json request.".format((str(appstruct)))
raise Invalid(self, msg)
try:
json_body = appstruct.json_body or {}
except (JSONDecodeError, AttributeError):
json_body = {}
appstruct = {
"body": json_body,
"queryset": appstruct.GET,
}
return super(RequestSchema, self).serialize(appstruct)
def validator(node, value):
user = request.user
if user is None or user != value['subject']:
error = Invalid(node, msg='')
error.add(Invalid(node['subject'],
msg='Must be the currently logged-in user'))
raise error
def deserialize(self, field, pstruct):
if pstruct is null:
return null
response = pstruct.get('g-recaptcha-response') or ''
if not response:
raise Invalid(field.schema, 'No input')
settings = self.request.registry.settings
privatekey = settings['pyramid_recaptcha.private_key']
remoteip = self.request.remote_addr
request = requests.post(self.url,
data={
'secret': privatekey,
'response': response,
'remoteip': remoteip
})
if request.status_code != 200:
raise Invalid(
field.schema,
"There was an error talking to the recaptcha \
server ({0})".format(request.status_code))
try:
response = request.json()
except ValueError as e:
raise Invalid(
field.schema,
"There was an error talking to the recaptcha \
server ({0})".format(e))
valid = response['success']
if valid is not True:
# TODO: Should provide more meaningful error messages
raise Invalid(field.schema, "Incorrect solution")
return 'Verified'
def deserialize(self, field, pstruct):
result = []
error = None
if pstruct is null:
pstruct = []
field.sequence_fields = []
item_field = field.children[0]
for num, substruct in enumerate(pstruct):
subfield = item_field.clone()
if item_field.parent is not None:
subfield._parent = weakref.ref(item_field.parent)
try:
subval = subfield.deserialize(substruct)
except Invalid as e:
subval = e.value
if error is None:
error = Invalid(field.schema, value=result)
error.add(e, num)
subfield.cstruct = subval
result.append(subval)
field.sequence_fields.append(subfield)
if error is not None:
raise error
return result
def convert_types(mapping, row):
""" Translate a row of input data (e.g. from a CSV file) into the
structure understood by the dataset loader, i.e. where all
dimensions are dicts and all types have been converted.
This will validate the incoming data and emit a colander.Invalid
exception if validation was unsuccessful."""
out = {}
errors = Invalid(SchemaNode(Mapping(unknown="preserve")))
for dimension, meta in mapping.items():
meta["dimension"] = dimension
# handle AttributeDimensions, Measures and DateDimensions.
# this is clever, but possibly not always true.
if "column" in meta:
try:
out[dimension] = _cast(row, meta, dimension)
except Invalid, i:
errors.add(i)
# handle CompoundDimensions.
else:
out[dimension] = {}
for attribute, ameta in meta.get("attributes", {}).items():
try:
out[dimension][attribute] = _cast(row, ameta, dimension + "." + attribute)
except Invalid, i:
errors.add(i)
def validator(node, value):
rate_validator = registry.getAdapter(value['object'], IRateValidator)
if not rate_validator.validate(value['rate']):
error = Invalid(node, msg='')
msg = rate_validator.helpful_error_message()
error.add(Invalid(node['rate'], msg=msg))
raise error
def validator(node, value):
rate_validator = registry.getAdapter(value['object'], IRateValidator)
if not rate_validator.validate(value['rate']):
error = Invalid(node, msg='')
msg = rate_validator.helpful_error_message()
error.add(Invalid(node['rate'], msg=msg))
raise error
def validator(self, node: SchemaNode, value: List[str]) -> None:
assert isinstance(value, list)
if len(value) < self.minimum_number:
raise Invalid(
node,
"You must specify at least {}".format(self.minimum_number))
if len(value) != len(set(value)):
raise Invalid(node, "You have specified duplicate diagnoses")
def deserialize(self, node, cstruct):
id = cstruct.get('id') if isinstance(cstruct, dict) else cstruct
if id is null or id is None:
raise Invalid(node, "No entity specified")
obj = self.context.network.Entity.current_by_id(id)
if obj is None:
raise Invalid(node, "No such entity: %r")
return obj
def repo_name_validator(node, value):
chars = set(value)
if not chars <= REPO_NAME_CHARS:
raise Invalid(
node, '%r contains invalid characters %r' %
(value, chars - REPO_NAME_CHARS))
if value in ('.', '..'):
raise Invalid(node, '%r is not a valid repo_name' % (value, ))
def validator(node, value):
user = request.user
if user is None or user != value['subject']:
error = Invalid(node, msg='')
error.add(
Invalid(node['subject'],
msg='Must be the currently logged-in user'))
raise error
def validate_rate_is_unique(node, value):
existing = _get_rates_user_non_anonymized(context, request, value)
existing += _get_rates_user_anonymized(context, request, value)
existing = _remove_following_versions(existing, context, request)
if existing:
error = Invalid(node, msg='')
msg = 'Another rate by the same user already exists'
error.add(Invalid(node['object'], msg=msg))
raise error
def validator(self, node: SchemaNode, value: dict):
"""Extra validation depending on the status of the topic."""
topics = value.get('topic', [])
if 'other' in topics:
if not value.get('topic_other', None):
raise Invalid(node['topic_other'],
msg='Required if "other" in topic')
if _has_duplicates(topics):
raise Invalid(node['topic'], msg='Duplicates are not allowed')
def __call__(self, node, value):
provided_keys = six.iterkeys(value)
match = set(self.keys).intersection(set(provided_keys))
if len(match) > 1:
msg = u"Choose between {0}".format(",".join(match))
raise Invalid(node, msg)
elif len(match) == 0:
msg = u"Provide at least one of {0}".format(",".join(self.keys))
raise Invalid(node, msg)
def validate_rate_is_unique(node, value):
existing = _get_rates_user_non_anonymized(context, request, value)
existing += _get_rates_user_anonymized(context, request, value)
existing = _remove_following_versions(existing, context, request)
if existing:
error = Invalid(node, msg='')
msg = 'Another rate by the same user already exists'
error.add(Invalid(node['object'], msg=msg))
raise error
def validate_user_is_active(node: SchemaNode, value: dict):
user = request.validated.get('user', None)
if user is None:
return
elif not user.active:
error = Invalid(node)
error.add(Invalid(node[child_node_name],
msg='User account not yet activated'))
raise error
def validate_user_is_active(node: SchemaNode, value: dict):
user = request.validated.get('user', None)
if user is None:
return
elif not user.active:
error = Invalid(node)
error.add(
Invalid(node[child_node_name],
msg='User account not yet activated'))
raise error
def validate_login_password(node: SchemaNode, value: dict):
password = value['password']
user = request.validated.get('user', None)
if user is None:
return
valid = user.is_password_valid(registry, password)
if not valid:
error = Invalid(node)
error.add(Invalid(node['password'], msg=error_msg_wrong_login))
raise error
def validate_login_password(node: SchemaNode, value: dict):
password = value['password']
user = request.validated.get('user', None)
if user is None:
return
valid = user.is_password_valid(registry, password)
if not valid:
error = Invalid(node)
error.add(Invalid(node['password'], msg=error_msg_wrong_login))
raise error
def validator(self, node: SchemaNode, value: dict):
"""Extra validation depending on the status of the heard froms."""
heard_froms = value.get('heard_froms', [])
if 'other' in heard_froms:
if not value.get('heard_from_other', None):
raise Invalid(node['heard_from_other'],
msg='Required if "other" in heard_froms')
if _has_duplicates(heard_froms):
raise Invalid(node['heard_froms'],
msg='Duplicates are not allowed')
def validator(self, node: SchemaNode, value: List[str]) -> None:
assert isinstance(value, list)
_ = self.gettext
if len(value) < self.minimum_number:
raise Invalid(
node,
_("You must specify at least") + f" {self.minimum_number}",
)
if len(value) != len(set(value)):
raise Invalid(node, _("You have specified duplicate diagnoses"))
def validate_login_password(node: SchemaNode, value: dict):
password = value['password']
user = request.validated.get('user', None)
if user is None:
return
sheet = registry.content.get_sheet(user, IPasswordAuthentication)
valid = sheet.check_plaintext_password(password)
if not valid:
error = Invalid(node)
error.add(Invalid(node['password'], msg=error_msg_wrong_login))
raise error
def __call__(self, node, value):
if self.min is not None:
if len(value) < self.min:
min_err = self.translationStringFactory(
self.min_message.format(min=self.min))
raise Invalid(node, min_err)
if self.max is not None:
if len(value) > self.max:
max_err = self.translationStringFactory(
self.max_message.format(max=self.max))
raise Invalid(node, max_err)
def exists(node, value):
if request.registry.content.istype(context, content_type):
if value != context.__name__:
try:
context.__parent__.check_name(value)
except Exception as e:
raise Invalid(node, e.args[0], value)
else:
try:
context.check_name(value)
except Exception as e:
raise Invalid(node, e.args[0], value)
def deserialize(self, node, cstruct):
if cstruct is null or cstruct is None:
return null
if not isinstance(cstruct, basestring):
raise Invalid(node, '%r is not a string' % cstruct)
try:
appstruct = [int(value) for value in cstruct.split(' ')]
except ValueError:
raise Invalid(node, '%r has invalid integer values' % cstruct)
return appstruct
def validate_email(node, value):
locator = request.registry.getMultiAdapter((context, request),
IUserLocator)
user = locator.get_user_by_email(value)
if user is None:
msg = 'No user exists with this email: {0}'.format(value)
raise Invalid(node, msg)
if not IPasswordAuthentication.providedBy(user):
msg = 'This user has no password to reset: {0}'.format(value)
raise Invalid(node, msg)
if not user.active:
user.activate()
request.validated['user'] = user
def validate_login(node: SchemaNode, value: dict):
login = value[child_node_name]
locator = registry.getMultiAdapter((context, request), IUserLocator)
if child_node_name == 'email':
login = login.lower().strip()
user = locator.get_user_by_email(login)
else:
user = locator.get_user_by_login(login)
if user is None:
error = Invalid(node)
error.add(Invalid(node['password'], msg=error_msg_wrong_login))
raise error
else:
request.validated['user'] = user
def test_validate_fails_widgeterror_and_schemaerror(self):
from colander import Invalid
controls = [("name", "Name"), ("title", "Title")]
widget_invalid = Invalid(None, None, dict(controls))
schema_invalid = Invalid(None, None)
schema = DummySchema(schema_invalid)
field = self._makeOne(schema)
field.widget = DummyWidget(exc=widget_invalid)
e = validation_failure_exc(field.validate, controls)
self.assertEqual(field.widget.error, schema_invalid)
self.assertEqual(e.cstruct, dict(controls))
self.assertEqual(e.field, field)
self.assertEqual(e.error, schema_invalid)
def validator(self, node, value):
"""Make `status_other` required if `status` == `other`."""
status = value.get('status', None)
if status == 'other':
if not value.get('status_other', None):
status_other = node['status_other']
raise Invalid(status_other, msg='Required iff status == other')
else:
# TODO: Allow multiple errors at the same time
name = node['name']
if not value.get('name', None):
raise Invalid(name, msg='Required iff status != other')
country = node['country']
if not value.get('country', None):
raise Invalid(country, msg='Required iff status != other')
def deserialize(self, node, cstruct):
if cstruct is null:
return null
if type(cstruct) is not dict:
raise Invalid(node, "Invalid types: it must be dict")
for k, v in cstruct.items():
if type(v) is not list:
raise Invalid(node, "Invalid types of '{}': it must be a list".format(k))
for i in v:
if type(i) is not str:
raise Invalid(node, "Invalid types of '{}[{}]': it must be a str".format(k, v.index(i)))
return cstruct
def validator(node, value):
if isinstance(value, string_types):
expected = [str(et.name) for et in list(enum_type)]
if value not in expected:
err_msg_fmt = "%r is not a valid %s"
raise Invalid(err_msg_fmt % (value, class_name))
value = enum_type[value].value
return value
expected = [et.value for et in list(enum_type)]
if value not in expected:
err_msg_fmt = "%r is not a valid value for %s"
raise Invalid(err_msg_fmt % (value, class_name))
return value
def test_validate_fails_widgeterror_and_schemaerror(self):
from colander import Invalid
fields = [
('name', 'Name'),
('title', 'Title'),
]
widget_invalid = Invalid(None, None, dict(fields))
schema_invalid = Invalid(None, None)
schema = DummySchema(schema_invalid)
field = self._makeOne(schema)
field.widget = DummyWidget(exc=widget_invalid)
e = validation_failure_exc(field.validate, fields)
self.assertEqual(field.widget.error, schema_invalid)
self.assertEqual(e.cstruct, dict(fields))
self.assertEqual(e.field, field)
self.assertEqual(e.error, schema_invalid)
def _ser(self, node, value, options=None):
dict_ = None
try:
if hasattr(value, 'to_dict'):
dict_ = value.to_dict('webapi')
# for k in dict_.keys():
# if dict_[k] is None:
# dict_[k] = null
else:
raise TypeError('Object does not have a to_dict function')
except Exception as e:
raise e
raise Invalid(node, '{0}" does not implement GnomeObj functionality: {1}'.format(value, e))
# if options is not None:
# if not options.get('raw_paths', True):
# datafiles = node.get_nodes_by_attr('isdatafile')
# for d in datafiles:
# if d in dict_:
# if dict_[d] is None:
# continue
# elif isinstance(dict_[d], six.string_types):
# dict_[d] = os.path.split(dict_[d])[1]
# elif isinstance(dict_[d], collections.Iterable):
# #List, tuple, etc
# for i, filename in enumerate(dict_[d]):
# dict_[d][i] = os.path.split(filename)[1]
return dict_
def deserialize(self, node, cstruct):
if cstruct is null:
return null
value = self.decode(cstruct)
if value is None:
raise Invalid(node, 'Missing')
return value
def translate_error(node, value):
min_err = _(_('${val} is less than minimum value ${min}'),
mapping={
'val': 'value',
'min': 'min'
})
raise Invalid(node, min_err)
def __call__(self, node, value):
if value not in (Authenticated, Everyone):
try:
self.db.get_token(value)
except CredentialsNotFound:
msg = u"Credentials id '%s' could not be found." % value
raise Invalid(node, msg, value)
def __call__(self, node, value):
if self.db_session is not None:
query = self.db_session.query(UserProfile)
query = query.filter(UserProfile.username == value)
exists = bool(query.count())
if exists != self.should_exist:
raise Invalid(node, self.msg)
def validate_entity(data, schema, context, deep=True):
schema = entity_schema(schema)
inv = Invalid(schema)
try:
data = schema.deserialize(data)
except Invalid as inv_real:
inv = inv_real
if deep:
for direction, attribute in (('incoming', 'source'), ('outgoing', 'target')):
data[direction], i = validate_deep(data, context, direction, attribute)
if len(i.children):
inv.add(i)
if len(inv.children):
raise inv
return data
def _raise_if_unknown_field_value(self, field_name: str,
err: Invalid,
json_object: dict):
"""Raise an 'unknown_xxx' WebSocketError error if appropriate."""
errdict = err.asdict()
if (self._is_only_key(errdict, field_name) and
field_name in json_object):
field_value = json_object[field_name]
raise WebSocketError('unknown_' + field_name, field_value)
def deserialize(self, field, pstruct):
error = None
result = {}
if pstruct is null:
pstruct = {}
for num, subfield in enumerate(field.children):
name = subfield.name
subval = pstruct.get(name, null)
try:
result[name] = subfield.deserialize(subval)
except Invalid, e:
result[name] = e.value
if error is None:
error = Invalid(field.schema, value=result)
error.add(e, num)
def validator(node, value):
catalogs = find_service(context, 'catalogs')
query = search_query._replace(
references=(Reference(None, IRate, 'subject', value['subject']),
Reference(None, IRate, 'object', value['object'])),
resolve=True,
)
same_rates = catalogs.search(query).elements
if not same_rates:
return
item = find_interface(context, IRateItem)
old_versions = registry.content.get_sheet_field(item,
IVersions,
'elements')
for rate in same_rates:
if rate not in old_versions:
error = Invalid(node, msg='')
msg = 'Another rate by the same user already exists'
error.add(Invalid(node['object'], msg=msg))
raise error
def validate_deep(data, context, direction, attribute):
relations = []
node = sequence(direction)
inv = Invalid(node)
for i, relation_data in enumerate(data.get(direction, [])):
try:
other = relation_data.get(attribute, {})
schema = context.network.get_relation_schema(relation_data.get('type'))
if schema is None:
raise Invalid(node, "Invalid relation type: %e" % relation_data.get('type'))
relation = validate_relation(relation_data, schema, context, ignore_entities=True)
schema = context.network.get_entity_schema(other.get('type'))
if schema is None:
raise Invalid(node, "Invalid entity type: %e" % other.get('type'))
relation[attribute] = validate_entity(other, schema, context, deep=False)
relations.append(relation)
except Invalid as sub:
inv.add(sub, i)
return relations, inv
def deserialize(self, field, pstruct):
error = None
result = {}
if pstruct is null:
pstruct = {}
# Convert the list of selected parent id's into a list of sqlalchemy parent objects in appstruct form.
if 'parents' in pstruct:
parents = []
for parent_id in pstruct['parents']:
parents.append(field.view.get_schema(parent_id).dictify())
pstruct['parents'] = parents
# Remove parent fields from the list of custom fields as they will be inherited through the parent 'link'
if 'custom_field' in pstruct:
for custom_field in pstruct['custom_fields']:
if custom_field.parent_field:
pstruct.pop(custom_field)
for num, subfield in enumerate(field.children):
name = subfield.name
subval = pstruct.get(name, null)
try:
result[name] = subfield.deserialize(subval)
except Invalid as e:
result[name] = e.value
if error is None:
error = Invalid(field.schema, value=result)
error.add(e, num)
if error is not None:
raise error
return result
def sequence_impl(self, node, value, callback, accept_scalar):
if accept_scalar is None:
accept_scalar = self.accept_scalar
value = self._validate(node, value, accept_scalar)
error = None
result = []
for num, subval in enumerate(value):
try:
result.append(callback(node.children[0], subval))
except Invalid as e:
if error is None:
error = Invalid(node)
error.add(e, num)
if error is not None:
raise error
if not result:
return null
return result
def _impl(self, node, value, callback):
value = self._validate(node, value)
error = None
result = {}
for index, (k, v) in enumerate(value.iteritems()):
key_node = node["key"]
value_node = node["value"].clone()
value_node.name = k
try:
name = callback(key_node, k)
result[name] = callback(value_node, v)
except Invalid as e:
if error is None:
error = Invalid(node)
error.add(e, index)
if error is not None:
raise error
return result
def deserialize(self, field, pstruct):
error = None
result = {}
if pstruct is null:
pstruct = {}
for num, subfield in enumerate(field.children):
name = subfield.name
subval = get_dict_item(pstruct, name, null)
try:
result[name] = subfield.deserialize(subval)
except Invalid as e:
result[name] = e.value
if error is None:
error = Invalid(field.schema, value=result)
error.add(e, num)
if error is not None:
raise error
return result
def _add_colander_invalid_error_to_request(error: Invalid, request: Request, location: str):
for name, msg in error.asdict().items():
request.errors.append(error_entry(location, name, msg))
def _raise_invalid_json_from_colander_invalid(self, err: Invalid):
errdict = err.asdict()
errlist = ['{}: {}'.format(k, errdict[k]) for k in errdict.keys()]
details = ' / '.join(sorted(errlist))
raise WebSocketError('invalid_json', details)
