def validate(self, context, data_dict, schema, action): thing, action_type = action.split('_') t = data_dict.get('type') if not t or t not in self._schemas: # pragma: no cover return data_dict, {'type': "Unsupported {thing} type: {t}".format( thing=thing, t=t)} scheming_schema = self._schemas[t] scheming_fields = scheming_schema['fields'] for f in scheming_fields: if action_type == 'show': if f['field_name'] not in schema: validators = [convert_from_extras, ignore_missing] else: validators = [ignore_missing] if 'output_validators' in f: validators += validators_from_string(f['output_validators']) else: if 'validators' in f: validators = validators_from_string(f['validators']) else: validators = [ignore_missing, unicode] if f['field_name'] not in schema: validators = validators + [convert_to_extras] schema[f['field_name']] = validators return navl_validate(data_dict, schema, context)
def _field_create_validators(f, schema, convert_extras): """ Return the validators to use when creating for scheming field f, normally the same as the validators used for updating """ if 'create_validators' not in f: return _field_validators(f, schema, convert_extras) validators = validation.validators_from_string( f['create_validators'], f, schema ) if convert_extras: validators.append(convert_to_extras) # If this field contains children, we need a special validator to handle # them. if 'repeating_subfields' in f: validators = { sf['field_name']: _field_create_validators(sf, schema, False) for sf in f['repeating_subfields'] } return validators
def _field_validators(f, schema, convert_extras): """ Return the validators for a scheming field f """ if 'validators' in f: validators = validation.validators_from_string( f['validators'], f, schema ) elif helpers.scheming_field_required(f): validators = [not_empty] else: validators = [ignore_missing] if convert_extras: validators.append(convert_to_extras) # If this field contains children, we need a special validator to handle # them. if 'repeating_subfields' in f: validators = { sf['field_name']: _field_validators(sf, schema, False) for sf in f['repeating_subfields'] } return validators
def validate(self, context, data_dict, schema, action): """ Validate and convert for package_create, package_update and package_show actions. """ thing, action_type = action.split('_') t = data_dict.get('type') if not t or t not in self._schemas: # pragma: no cover return data_dict, {'type': [ "Unsupported dataset type: {t}".format(t=t)]} scheming_schema = self._schemas[t] for f in scheming_schema['dataset_fields']: if action_type == 'show': if f['field_name'] not in schema: validators = [convert_from_extras, ignore_missing] else: validators = [ignore_missing] if 'output_validators' in f: validators += validators_from_string(f['output_validators']) else: if 'validators' in f: validators = validators_from_string(f['validators']) else: validators = [ignore_missing, unicode] if f['field_name'] not in schema: validators = validators + [convert_to_extras] schema[f['field_name']] = validators resource_schema = schema['resources'] for f in scheming_schema['resource_fields']: if action_type == 'show': validators = [ignore_missing] if 'output_validators' in f: validators += validators_from_string(f['output_validators']) else: if 'validators' in f: validators = validators_from_string(f['validators']) else: validators = [ignore_missing, unicode] resource_schema[f['field_name']] = validators return navl_validate(data_dict, schema, context)
def _field_output_validators(f, convert_extras): """ Return the output validators for a scheming field f """ if convert_extras: validators = [convert_from_extras, ignore_missing] else: validators = [ignore_missing] if 'output_validators' in f: validators += validators_from_string(f['output_validators'], f) return validators
def _field_output_validators(f, schema, convert_extras): """ Return the output validators for a scheming field f """ if convert_extras: validators = [convert_from_extras, ignore_missing] else: validators = [ignore_missing] if 'output_validators' in f: validators += validators_from_string(f['output_validators'], f, schema) return validators
def _field_create_validators(f, schema, convert_extras): """ Return the validators to use when creating for scheming field f, normally the same as the validators used for updating """ if 'create_validators' not in f: return _field_validators(f, schema, convert_extras) validators = validators_from_string(f['create_validators'], f, schema) if convert_extras: validators = validators + [convert_to_extras] return validators
def _field_validators(f, schema, convert_extras): """ Return the validators for a scheming field f """ validators = [] if 'validators' in f: validators = validators_from_string(f['validators'], f, schema) elif helpers.scheming_field_required(f): validators = [not_empty, unicode] else: validators = [ignore_missing, unicode] if convert_extras: validators = validators + [convert_to_extras] return validators
def _field_output_validators(f, schema, convert_extras, convert_from_extras_type=convert_from_extras): """ Return the output validators for a scheming field f """ if 'repeating_subfields' in f: validators = { sf['field_name']: _field_output_validators(sf, schema, False) for sf in f['repeating_subfields'] } elif convert_extras: validators = [convert_from_extras_type, ignore_missing] else: validators = [ignore_missing] if 'output_validators' in f: validators += validation.validators_from_string( f['output_validators'], f, schema) return validators
def _field_create_validators(f, schema, convert_extras): """ Return the validators to use when creating for scheming field f, normally the same as the validators used for updating """ if 'create_validators' not in f: return _field_validators(f, schema, convert_extras) validators = validation.validators_from_string(f['create_validators'], f, schema) if convert_extras: validators.append(convert_to_extras) # If this field contains children, we need a special validator to handle # them. if 'subfields' in f: validators = [validation.composite_form(f, schema)] + validators return validators
def _field_validators(f, schema, convert_extras): """ Return the validators for a scheming field f """ if 'validators' in f: validators = validation.validators_from_string(f['validators'], f, schema) elif helpers.scheming_field_required(f): validators = [not_empty] else: validators = [ignore_missing] if convert_extras: validators.append(convert_to_extras) # If this field contains children, we need a special validator to handle # them. if 'subfields' in f: validators = [validation.composite_form(f, schema)] + validators return validators
def fluent_tags(field, schema): """ Accept multilingual lists of tags in the following forms and convert to a json string for storage. 1. a multilingual dict of lists of tag strings, eg. {"en": ["big", "large"], "fr": ["grande"]} 2. separate fields per language with comma-separated values (for form submissions) fieldname-en = "big,large" fieldname-fr = "grande" Validation of each tag is performed with validators tag_length_validator and tag_name_validator. When using ckanext-scheming these may be overridden with the "tag_validators" field value """ # XXX this validator is too long and should be broken up required_langs = [] if field and field.get('required'): required_langs = fluent_form_languages(field, schema=schema) tag_validators = [tag_length_validator, tag_name_validator] if field and 'tag_validators' in field: tag_validators = validators_from_string( field['tag_validators'], field, schema) def validator(key, data, errors, context): if errors[key]: return value = data[key] # 1. dict of lists of tag strings if value is not missing: if not isinstance(value, dict): errors[key].append(_('expecting JSON object')) return for lang, keys in value.items(): try: m = re.match(ISO_639_LANGUAGE, lang) except TypeError: errors[key].append(_('invalid type for language code: %r') % lang) continue if not m: errors[key].append(_('invalid language code: "%s"') % lang) continue if not isinstance(keys, list): errors[key].append(_('invalid type for "%s" value') % lang) continue out = [] for i, v in enumerate(keys): if not isinstance(v, basestring): errors[key].append( _('invalid type for "{lang}" value item {num}').format( lang=lang, num=i)) continue if isinstance(v, str): try: out.append(v.decode('utf-8')) except UnicodeDecodeError: errors[key]. append(_( 'expected UTF-8 encoding for ' '"{lang}" value item {num}').format( lang=lang, num=i)) else: out.append(v) tags = [] errs = [] for tag in out: newtag, tagerrs = _validate_single_tag(tag, tag_validators) errs.extend(tagerrs) tags.append(newtag) if errs: errors[key].extend(errs) value[lang] = tags for lang in required_langs: if value.get(lang): continue errors[key].append(_('Required language "%s" missing') % lang) if not errors[key]: data[key] = json.dumps(value) return # 2. separate fields output = {} prefix = key[-1] + '-' extras = data.get(key[:-1] + ('__extras',), {}) for name, text in extras.iteritems(): if not name.startswith(prefix): continue lang = name.split('-', 1)[1] m = re.match(ISO_639_LANGUAGE, lang) if not m: errors[name] = [_('invalid language code: "%s"') % lang] output = None continue if not isinstance(text, basestring): errors[name].append(_('invalid type')) continue if isinstance(text, str): try: text = text.decode('utf-8') except UnicodeDecodeError: errors[name].append(_('expected UTF-8 encoding')) continue if output is not None and text: tags = [] errs = [] for tag in text.split(','): newtag, tagerrs = _validate_single_tag(tag, tag_validators) errs.extend(tagerrs) tags.append(newtag) output[lang] = tags if errs: errors[key[:-1] + (name,)] = errs for lang in required_langs: if extras.get(prefix + lang): continue errors[key[:-1] + (key[-1] + '-' + lang,)] = [_('Missing value')] output = None if output is None: return for lang in output: del extras[prefix + lang] data[key] = json.dumps(output) return validator
def fluent_tags(field, schema): """ Accept multilingual lists of tags in the following forms and convert to a json string for storage. 1. a multilingual dict of lists of tag strings, eg. {"en": ["big", "large"], "fr": ["grande"]} 2. separate fields per language with comma-separated values (for form submissions) fieldname-en = "big,large" fieldname-fr = "grande" Validation of each tag is performed with validators tag_length_validator and tag_name_validator. When using ckanext-scheming these may be overridden with the "tag_validators" field value """ # XXX this validator is too long and should be broken up required_langs = [] alternate_langs = {} if field and field.get('required'): required_langs = fluent_form_languages(field, schema=schema) alternate_langs = fluent_alternate_languages(field, schema=schema) tag_validators = [tag_length_validator, tag_name_validator] if field and 'tag_validators' in field: tag_validators = validators_from_string( field['tag_validators'], field, schema) def validator(key, data, errors, context): if errors[key]: return value = data[key] # 1. dict of lists of tag strings if value is not missing: if not isinstance(value, dict): errors[key].append(_('expecting JSON object')) return for lang, keys in value.items(): try: m = re.match(BCP_47_LANGUAGE, lang) except TypeError: errors[key].append(_('invalid type for language code: %r') % lang) continue if not m: errors[key].append(_('invalid language code: "%s"') % lang) continue if not isinstance(keys, list): errors[key].append(_('invalid type for "%s" value') % lang) continue out = [] for i, v in enumerate(keys): if not isinstance(v, basestring): errors[key].append( _('invalid type for "{lang}" value item {num}').format( lang=lang, num=i)) continue if isinstance(v, str): try: out.append(v.decode('utf-8')) except UnicodeDecodeError: errors[key]. append(_( 'expected UTF-8 encoding for ' '"{lang}" value item {num}').format( lang=lang, num=i)) else: out.append(v) tags = [] errs = [] for tag in out: newtag, tagerrs = _validate_single_tag(tag, tag_validators) errs.extend(tagerrs) tags.append(newtag) if errs: errors[key].extend(errs) value[lang] = tags for lang in required_langs: if value.get(lang) or any( value.get(l) for l in alternate_langs.get(lang, [])): continue errors[key].append(_('Required language "%s" missing') % lang) if not errors[key]: data[key] = json.dumps(value) return # 2. separate fields output = {} prefix = key[-1] + '-' extras = data.get(key[:-1] + ('__extras',), {}) for name, text in extras.iteritems(): if not name.startswith(prefix): continue lang = name.split('-', 1)[1] m = re.match(BCP_47_LANGUAGE, lang) if not m: errors[name] = [_('invalid language code: "%s"') % lang] output = None continue if not isinstance(text, basestring): errors[name].append(_('invalid type')) continue if isinstance(text, str): try: text = text.decode('utf-8') except UnicodeDecodeError: errors[name].append(_('expected UTF-8 encoding')) continue if output is not None and text: tags = [] errs = [] for tag in text.split(','): newtag, tagerrs = _validate_single_tag(tag, tag_validators) errs.extend(tagerrs) tags.append(newtag) output[lang] = tags if errs: errors[key[:-1] + (name,)] = errs for lang in required_langs: if extras.get(prefix + lang) or any( extras.get(prefix + l) for l in alternate_langs.get(lang, [])): continue errors[key[:-1] + (key[-1] + '-' + lang,)] = [_('Missing value')] output = None if output is None: return for lang in output: del extras[prefix + lang] data[key] = json.dumps(output) return validator
def validate(self, context, data_dict, schema, action): """ Validate and convert for package_create, package_update and package_show actions. """ thing, action_type = action.split('_') t = data_dict.get('type') if not t or t not in self._schemas: return data_dict, { 'type': ["Unsupported dataset type: {t}".format(t=t)] } scheming_schema = self._expanded_schemas[t] before = scheming_schema.get('before_validators') after = scheming_schema.get('after_validators') if action_type == 'show': get_validators = _field_output_validators before = after = None elif action_type == 'create': get_validators = _field_create_validators else: get_validators = _field_validators if before: schema['__before'] = validation.validators_from_string( before, None, scheming_schema) if after: schema['__after'] = validation.validators_from_string( after, None, scheming_schema) fg = ((scheming_schema['dataset_fields'], schema, True), (scheming_schema['resource_fields'], schema['resources'], False)) composite_convert_fields = [] for field_list, destination, convert_extras in fg: for f in field_list: convert_this = convert_extras and f['field_name'] not in schema destination[f['field_name']] = get_validators( f, scheming_schema, convert_this) if convert_this and 'repeating_subfields' in f: composite_convert_fields.append(f['field_name']) def composite_convert_to(key, data, errors, context): unflat = unflatten(data) for f in composite_convert_fields: if f not in unflat: continue data[(f, )] = json.dumps(unflat[f], default=lambda x: None if x == missing else x) convert_to_extras((f, ), data, errors, context) del data[(f, )] if action_type == 'show': if composite_convert_fields: for ex in data_dict['extras']: if ex['key'] in composite_convert_fields: data_dict[ex['key']] = json.loads(ex['value']) data_dict['extras'] = [ ex for ex in data_dict['extras'] if ex['key'] not in composite_convert_fields ] else: dataset_composite = { f['field_name'] for f in scheming_schema['dataset_fields'] if 'repeating_subfields' in f } if dataset_composite: expand_form_composite(data_dict, dataset_composite) resource_composite = { f['field_name'] for f in scheming_schema['resource_fields'] if 'repeating_subfields' in f } if resource_composite and 'resources' in data_dict: for res in data_dict['resources']: expand_form_composite(res, resource_composite) # convert composite package fields to extras so they are stored if composite_convert_fields: schema = dict(schema, __after=schema.get('__after', []) + [composite_convert_to]) return navl_validate(data_dict, schema, context)