def validate(self, context, data_dict, schema, action):
thing, action_type = action.split('_')
t = data_dict.get('type')
if not t or t not in self._schemas: # pragma: no cover
return data_dict, {'type': "Unsupported {thing} type: {t}".format(
thing=thing, t=t)}
scheming_schema = self._schemas[t]
scheming_fields = scheming_schema['fields']
for f in scheming_fields:
if action_type == 'show':
if f['field_name'] not in schema:
validators = [convert_from_extras, ignore_missing]
else:
validators = [ignore_missing]
if 'output_validators' in f:
validators += validators_from_string(f['output_validators'])
else:
if 'validators' in f:
validators = validators_from_string(f['validators'])
else:
validators = [ignore_missing, unicode]
if f['field_name'] not in schema:
validators = validators + [convert_to_extras]
schema[f['field_name']] = validators
return navl_validate(data_dict, schema, context)
def _field_create_validators(f, schema, convert_extras):
"""
Return the validators to use when creating for scheming field f,
normally the same as the validators used for updating
"""
if 'create_validators' not in f:
return _field_validators(f, schema, convert_extras)
validators = validation.validators_from_string(
f['create_validators'],
f,
schema
)
if convert_extras:
validators.append(convert_to_extras)
# If this field contains children, we need a special validator to handle
# them.
if 'repeating_subfields' in f:
validators = {
sf['field_name']: _field_create_validators(sf, schema, False)
for sf in f['repeating_subfields']
}
return validators
def _field_validators(f, schema, convert_extras):
"""
Return the validators for a scheming field f
"""
if 'validators' in f:
validators = validation.validators_from_string(
f['validators'],
f,
schema
)
elif helpers.scheming_field_required(f):
validators = [not_empty]
else:
validators = [ignore_missing]
if convert_extras:
validators.append(convert_to_extras)
# If this field contains children, we need a special validator to handle
# them.
if 'repeating_subfields' in f:
validators = {
sf['field_name']: _field_validators(sf, schema, False)
for sf in f['repeating_subfields']
}
return validators
def validate(self, context, data_dict, schema, action):
"""
Validate and convert for package_create, package_update and
package_show actions.
"""
thing, action_type = action.split('_')
t = data_dict.get('type')
if not t or t not in self._schemas: # pragma: no cover
return data_dict, {'type': [
"Unsupported dataset type: {t}".format(t=t)]}
scheming_schema = self._schemas[t]
for f in scheming_schema['dataset_fields']:
if action_type == 'show':
if f['field_name'] not in schema:
validators = [convert_from_extras, ignore_missing]
else:
validators = [ignore_missing]
if 'output_validators' in f:
validators += validators_from_string(f['output_validators'])
else:
if 'validators' in f:
validators = validators_from_string(f['validators'])
else:
validators = [ignore_missing, unicode]
if f['field_name'] not in schema:
validators = validators + [convert_to_extras]
schema[f['field_name']] = validators
resource_schema = schema['resources']
for f in scheming_schema['resource_fields']:
if action_type == 'show':
validators = [ignore_missing]
if 'output_validators' in f:
validators += validators_from_string(f['output_validators'])
else:
if 'validators' in f:
validators = validators_from_string(f['validators'])
else:
validators = [ignore_missing, unicode]
resource_schema[f['field_name']] = validators
return navl_validate(data_dict, schema, context)
def _field_output_validators(f, convert_extras):
"""
Return the output validators for a scheming field f
"""
if convert_extras:
validators = [convert_from_extras, ignore_missing]
else:
validators = [ignore_missing]
if 'output_validators' in f:
validators += validators_from_string(f['output_validators'], f)
return validators
def _field_output_validators(f, schema, convert_extras):
"""
Return the output validators for a scheming field f
"""
if convert_extras:
validators = [convert_from_extras, ignore_missing]
else:
validators = [ignore_missing]
if 'output_validators' in f:
validators += validators_from_string(f['output_validators'], f, schema)
return validators
def _field_create_validators(f, schema, convert_extras):
"""
Return the validators to use when creating for scheming field f,
normally the same as the validators used for updating
"""
if 'create_validators' not in f:
return _field_validators(f, schema, convert_extras)
validators = validators_from_string(f['create_validators'], f, schema)
if convert_extras:
validators = validators + [convert_to_extras]
return validators
def _field_create_validators(f, schema, convert_extras):
"""
Return the validators to use when creating for scheming field f,
normally the same as the validators used for updating
"""
if 'create_validators' not in f:
return _field_validators(f, schema, convert_extras)
validators = validators_from_string(f['create_validators'], f, schema)
if convert_extras:
validators = validators + [convert_to_extras]
return validators
def _field_validators(f, schema, convert_extras):
"""
Return the validators for a scheming field f
"""
validators = []
if 'validators' in f:
validators = validators_from_string(f['validators'], f, schema)
elif helpers.scheming_field_required(f):
validators = [not_empty, unicode]
else:
validators = [ignore_missing, unicode]
if convert_extras:
validators = validators + [convert_to_extras]
return validators
def _field_validators(f, schema, convert_extras):
"""
Return the validators for a scheming field f
"""
validators = []
if 'validators' in f:
validators = validators_from_string(f['validators'], f, schema)
elif helpers.scheming_field_required(f):
validators = [not_empty, unicode]
else:
validators = [ignore_missing, unicode]
if convert_extras:
validators = validators + [convert_to_extras]
return validators
def _field_output_validators(f, schema, convert_extras,
convert_from_extras_type=convert_from_extras):
"""
Return the output validators for a scheming field f
"""
if 'repeating_subfields' in f:
validators = {
sf['field_name']: _field_output_validators(sf, schema, False)
for sf in f['repeating_subfields']
}
elif convert_extras:
validators = [convert_from_extras_type, ignore_missing]
else:
validators = [ignore_missing]
if 'output_validators' in f:
validators += validation.validators_from_string(
f['output_validators'], f, schema)
return validators
def _field_create_validators(f, schema, convert_extras):
"""
Return the validators to use when creating for scheming field f,
normally the same as the validators used for updating
"""
if 'create_validators' not in f:
return _field_validators(f, schema, convert_extras)
validators = validation.validators_from_string(f['create_validators'], f,
schema)
if convert_extras:
validators.append(convert_to_extras)
# If this field contains children, we need a special validator to handle
# them.
if 'subfields' in f:
validators = [validation.composite_form(f, schema)] + validators
return validators
def _field_validators(f, schema, convert_extras):
"""
Return the validators for a scheming field f
"""
if 'validators' in f:
validators = validation.validators_from_string(f['validators'], f,
schema)
elif helpers.scheming_field_required(f):
validators = [not_empty]
else:
validators = [ignore_missing]
if convert_extras:
validators.append(convert_to_extras)
# If this field contains children, we need a special validator to handle
# them.
if 'subfields' in f:
validators = [validation.composite_form(f, schema)] + validators
return validators
def fluent_tags(field, schema):
"""
Accept multilingual lists of tags in the following forms
and convert to a json string for storage.
1. a multilingual dict of lists of tag strings, eg.
{"en": ["big", "large"], "fr": ["grande"]}
2. separate fields per language with comma-separated values
(for form submissions)
fieldname-en = "big,large"
fieldname-fr = "grande"
Validation of each tag is performed with validators
tag_length_validator and tag_name_validator. When using
ckanext-scheming these may be overridden with the
"tag_validators" field value
"""
# XXX this validator is too long and should be broken up
required_langs = []
if field and field.get('required'):
required_langs = fluent_form_languages(field, schema=schema)
tag_validators = [tag_length_validator, tag_name_validator]
if field and 'tag_validators' in field:
tag_validators = validators_from_string(
field['tag_validators'], field, schema)
def validator(key, data, errors, context):
if errors[key]:
return
value = data[key]
# 1. dict of lists of tag strings
if value is not missing:
if not isinstance(value, dict):
errors[key].append(_('expecting JSON object'))
return
for lang, keys in value.items():
try:
m = re.match(ISO_639_LANGUAGE, lang)
except TypeError:
errors[key].append(_('invalid type for language code: %r')
% lang)
continue
if not m:
errors[key].append(_('invalid language code: "%s"') % lang)
continue
if not isinstance(keys, list):
errors[key].append(_('invalid type for "%s" value') % lang)
continue
out = []
for i, v in enumerate(keys):
if not isinstance(v, basestring):
errors[key].append(
_('invalid type for "{lang}" value item {num}').format(
lang=lang, num=i))
continue
if isinstance(v, str):
try:
out.append(v.decode('utf-8'))
except UnicodeDecodeError:
errors[key]. append(_(
'expected UTF-8 encoding for '
'"{lang}" value item {num}').format(
lang=lang, num=i))
else:
out.append(v)
tags = []
errs = []
for tag in out:
newtag, tagerrs = _validate_single_tag(tag, tag_validators)
errs.extend(tagerrs)
tags.append(newtag)
if errs:
errors[key].extend(errs)
value[lang] = tags
for lang in required_langs:
if value.get(lang):
continue
errors[key].append(_('Required language "%s" missing') % lang)
if not errors[key]:
data[key] = json.dumps(value)
return
# 2. separate fields
output = {}
prefix = key[-1] + '-'
extras = data.get(key[:-1] + ('__extras',), {})
for name, text in extras.iteritems():
if not name.startswith(prefix):
continue
lang = name.split('-', 1)[1]
m = re.match(ISO_639_LANGUAGE, lang)
if not m:
errors[name] = [_('invalid language code: "%s"') % lang]
output = None
continue
if not isinstance(text, basestring):
errors[name].append(_('invalid type'))
continue
if isinstance(text, str):
try:
text = text.decode('utf-8')
except UnicodeDecodeError:
errors[name].append(_('expected UTF-8 encoding'))
continue
if output is not None and text:
tags = []
errs = []
for tag in text.split(','):
newtag, tagerrs = _validate_single_tag(tag, tag_validators)
errs.extend(tagerrs)
tags.append(newtag)
output[lang] = tags
if errs:
errors[key[:-1] + (name,)] = errs
for lang in required_langs:
if extras.get(prefix + lang):
continue
errors[key[:-1] + (key[-1] + '-' + lang,)] = [_('Missing value')]
output = None
if output is None:
return
for lang in output:
del extras[prefix + lang]
data[key] = json.dumps(output)
return validator
def fluent_tags(field, schema):
"""
Accept multilingual lists of tags in the following forms
and convert to a json string for storage.
1. a multilingual dict of lists of tag strings, eg.
{"en": ["big", "large"], "fr": ["grande"]}
2. separate fields per language with comma-separated values
(for form submissions)
fieldname-en = "big,large"
fieldname-fr = "grande"
Validation of each tag is performed with validators
tag_length_validator and tag_name_validator. When using
ckanext-scheming these may be overridden with the
"tag_validators" field value
"""
# XXX this validator is too long and should be broken up
required_langs = []
alternate_langs = {}
if field and field.get('required'):
required_langs = fluent_form_languages(field, schema=schema)
alternate_langs = fluent_alternate_languages(field, schema=schema)
tag_validators = [tag_length_validator, tag_name_validator]
if field and 'tag_validators' in field:
tag_validators = validators_from_string(
field['tag_validators'], field, schema)
def validator(key, data, errors, context):
if errors[key]:
return
value = data[key]
# 1. dict of lists of tag strings
if value is not missing:
if not isinstance(value, dict):
errors[key].append(_('expecting JSON object'))
return
for lang, keys in value.items():
try:
m = re.match(BCP_47_LANGUAGE, lang)
except TypeError:
errors[key].append(_('invalid type for language code: %r')
% lang)
continue
if not m:
errors[key].append(_('invalid language code: "%s"') % lang)
continue
if not isinstance(keys, list):
errors[key].append(_('invalid type for "%s" value') % lang)
continue
out = []
for i, v in enumerate(keys):
if not isinstance(v, basestring):
errors[key].append(
_('invalid type for "{lang}" value item {num}').format(
lang=lang, num=i))
continue
if isinstance(v, str):
try:
out.append(v.decode('utf-8'))
except UnicodeDecodeError:
errors[key]. append(_(
'expected UTF-8 encoding for '
'"{lang}" value item {num}').format(
lang=lang, num=i))
else:
out.append(v)
tags = []
errs = []
for tag in out:
newtag, tagerrs = _validate_single_tag(tag, tag_validators)
errs.extend(tagerrs)
tags.append(newtag)
if errs:
errors[key].extend(errs)
value[lang] = tags
for lang in required_langs:
if value.get(lang) or any(
value.get(l) for l in alternate_langs.get(lang, [])):
continue
errors[key].append(_('Required language "%s" missing') % lang)
if not errors[key]:
data[key] = json.dumps(value)
return
# 2. separate fields
output = {}
prefix = key[-1] + '-'
extras = data.get(key[:-1] + ('__extras',), {})
for name, text in extras.iteritems():
if not name.startswith(prefix):
continue
lang = name.split('-', 1)[1]
m = re.match(BCP_47_LANGUAGE, lang)
if not m:
errors[name] = [_('invalid language code: "%s"') % lang]
output = None
continue
if not isinstance(text, basestring):
errors[name].append(_('invalid type'))
continue
if isinstance(text, str):
try:
text = text.decode('utf-8')
except UnicodeDecodeError:
errors[name].append(_('expected UTF-8 encoding'))
continue
if output is not None and text:
tags = []
errs = []
for tag in text.split(','):
newtag, tagerrs = _validate_single_tag(tag, tag_validators)
errs.extend(tagerrs)
tags.append(newtag)
output[lang] = tags
if errs:
errors[key[:-1] + (name,)] = errs
for lang in required_langs:
if extras.get(prefix + lang) or any(
extras.get(prefix + l) for l in alternate_langs.get(lang, [])):
continue
errors[key[:-1] + (key[-1] + '-' + lang,)] = [_('Missing value')]
output = None
if output is None:
return
for lang in output:
del extras[prefix + lang]
data[key] = json.dumps(output)
return validator
def validate(self, context, data_dict, schema, action):
"""
Validate and convert for package_create, package_update and
package_show actions.
"""
thing, action_type = action.split('_')
t = data_dict.get('type')
if not t or t not in self._schemas:
return data_dict, {
'type': ["Unsupported dataset type: {t}".format(t=t)]
}
scheming_schema = self._expanded_schemas[t]
before = scheming_schema.get('before_validators')
after = scheming_schema.get('after_validators')
if action_type == 'show':
get_validators = _field_output_validators
before = after = None
elif action_type == 'create':
get_validators = _field_create_validators
else:
get_validators = _field_validators
if before:
schema['__before'] = validation.validators_from_string(
before, None, scheming_schema)
if after:
schema['__after'] = validation.validators_from_string(
after, None, scheming_schema)
fg = ((scheming_schema['dataset_fields'], schema, True),
(scheming_schema['resource_fields'], schema['resources'], False))
composite_convert_fields = []
for field_list, destination, convert_extras in fg:
for f in field_list:
convert_this = convert_extras and f['field_name'] not in schema
destination[f['field_name']] = get_validators(
f, scheming_schema, convert_this)
if convert_this and 'repeating_subfields' in f:
composite_convert_fields.append(f['field_name'])
def composite_convert_to(key, data, errors, context):
unflat = unflatten(data)
for f in composite_convert_fields:
if f not in unflat:
continue
data[(f, )] = json.dumps(unflat[f],
default=lambda x: None
if x == missing else x)
convert_to_extras((f, ), data, errors, context)
del data[(f, )]
if action_type == 'show':
if composite_convert_fields:
for ex in data_dict['extras']:
if ex['key'] in composite_convert_fields:
data_dict[ex['key']] = json.loads(ex['value'])
data_dict['extras'] = [
ex for ex in data_dict['extras']
if ex['key'] not in composite_convert_fields
]
else:
dataset_composite = {
f['field_name']
for f in scheming_schema['dataset_fields']
if 'repeating_subfields' in f
}
if dataset_composite:
expand_form_composite(data_dict, dataset_composite)
resource_composite = {
f['field_name']
for f in scheming_schema['resource_fields']
if 'repeating_subfields' in f
}
if resource_composite and 'resources' in data_dict:
for res in data_dict['resources']:
expand_form_composite(res, resource_composite)
# convert composite package fields to extras so they are stored
if composite_convert_fields:
schema = dict(schema,
__after=schema.get('__after', []) +
[composite_convert_to])
return navl_validate(data_dict, schema, context)