def test_ignore_missing(self):
schema = {
"__junk": [ignore],
"__extras": [ignore, default("weee")],
"0": [default("default")],
"1": [ignore_missing, default("default")],
}
converted_data, errors = validate_flattened(data, schema)
assert not errors
assert converted_data == {("0",): "0 value"}, converted_data
def test_ignore_missing():
schema = {
"__junk": [ignore],
"__extras": [ignore, default("weee")],
"0": [default("default")],
"1": [ignore_missing, default("default")],
}
converted_data, errors = validate_flattened(data, schema)
assert not errors
assert converted_data == {('0', ): '0 value'}, converted_data
def test_default():
schema = {
"__junk": [ignore],
"__extras": [ignore, default("weee")],
"__before": [ignore],
"__after": [ignore],
"0": [default("default")],
"1": [default("default")],
}
converted_data, errors = validate_flattened(data, schema)
assert not errors
assert converted_data == {('1',): 'default', ('0',): '0 value'}, converted_data
def test_ignore_missing():
log.debug('Starting test_ignore_missing()')
schema = {
"__junk": [ignore],
"__extras": [ignore, default("weee")],
"0": [default("default")],
"1": [ignore_missing, default("default")],
}
converted_data, errors = validate_flattened(data, schema)
log.debug('converted_data: {0}'.format(converted_data))
assert not errors
assert converted_data == {('0', ): '0 value'}, converted_data
def test_ignore_missing():
log.debug('Starting test_ignore_missing()')
schema = {
"__junk": [ignore],
"__extras": [ignore, default("weee")],
"0": [default("default")],
"1": [ignore_missing, default("default")],
}
converted_data, errors = validate_flattened(data, schema)
log.debug('converted_data: {0}'.format(converted_data))
assert not errors
assert converted_data == {('0',): '0 value'}, converted_data
def form_to_db_schema_options(self, package_type=None, options=None):
schema = form_to_db_package_schema()
for key in self.kata_fields_required:
schema[key] = [not_empty, self.convert_to_extras_kata, unicode]
for key in self.kata_fields_recommended:
schema[key] = [ignore_missing, self.convert_to_extras_kata, unicode]
schema['temporal_coverage_begin'].append(validate_lastmod)
schema['temporal_coverage_end'].append(validate_lastmod)
schema['language'] = [validate_language, self.convert_to_extras_kata, unicode]
schema['phone'].append(validate_phonenum)
schema['maintainer_email'].append(validate_email)
schema['tag_string'].append(not_empty)
schema.update({
'version': [not_empty, unicode, validate_lastmod, check_last_and_update_pid],
'versionPID': [self.update_pid, unicode, self.pid_to_extras],
'author': {'value': [not_empty, unicode, org_auth_to_extras]},
'organization': {'value': [not_empty, unicode, org_auth_to_extras]},
'access': [not_missing, self.convert_to_extras_kata, validate_access],
'accessRights': [ignore_missing, self.convert_to_extras_kata, unicode],
'langdis': [default(False), unicode],
'__extras': [check_author_org],
'projdis': [default(False), unicode, check_project],
'__junk': [check_junk],
'name': [unicode, ignore_missing, self.update_name],
'accessRights': [check_accessrights, self.convert_to_extras_kata, unicode],
'accessrequestURL': [check_accessrequesturl, self.convert_to_extras_kata, unicode],
'project_name': [check_project_dis, unicode, self.convert_to_extras_kata],
'funder': [check_project_dis, unicode, self.convert_to_extras_kata],
'project_funding': [check_project_dis, unicode, self.convert_to_extras_kata],
'project_homepage': [check_project_dis, unicode, self.convert_to_extras_kata],
'resources': default_resource_schema(),
'discipline': [add_to_group],
})
schema['title'] = {'value': [not_missing, ltitle_to_extras],
'lang': [not_missing]}
schema['evtype'] = {'value': [ignore_missing, unicode, event_to_extras]}
schema['evwho'] = {'value': [ignore_missing, unicode, event_to_extras]}
schema['evwhen'] = {'value': [ignore_missing, unicode, event_to_extras]}
schema['evdescr'] = {'value': [ignore_missing, unicode, event_to_extras]}
schema['groups'] = {
'id': [ignore_missing, unicode],
'name': [ignore_missing, unicode],
'title': [ignore_missing, unicode],
'__extras': [ignore],
}
return schema
def default_user_schema(schema):
schema.update(
{
"name": [navl_validators.not_empty, core_validators.user_name_validator, unicode],
"email": [navl_validators.default(u""), unicode],
}
)
return schema
def test_default(self):
schema = {
"__junk": [ignore],
"__extras": [ignore, default("weee")],
"__before": [ignore],
"__after": [ignore],
"0": [default("default")],
"1": [default("default")],
}
converted_data, errors = validate_flattened(data, schema)
assert not errors
assert converted_data == {
('1', ): 'default',
('0', ): '0 value'
}, converted_data
def db_to_form_schema_options(self, options = None):
schema = db_to_form_package_schema()
context = options['context']
for key in self.kata_field:
schema[key] = [self.convert_from_extras_kata, ignore_missing, unicode]
schema['versionPID'] = [pid_from_extras, ignore_missing, unicode]
schema['author'] = [org_auth_from_extras, ignore_missing, unicode]
schema['organization'] = [org_auth_from_extras, ignore_missing, unicode]
schema['langdis'] = [default(False), unicode]
schema['projdis'] = [default(False), unicode]
schema['title'] = [ltitle_from_extras, ignore_missing]
schema['evtype'] = [event_from_extras, ignore_missing, unicode]
schema['evwho'] = [event_from_extras, ignore_missing, unicode]
schema['evwhen'] = [event_from_extras, ignore_missing, unicode]
schema['evdescr'] = [event_from_extras, ignore_missing, unicode]
return schema
def default_user_schema(schema):
schema.update({
'name': [
navl_validators.not_empty, core_validators.user_name_validator,
unicode
],
'email': [navl_validators.default(u''), unicode]
})
return schema
def test_simple():
schema = {
"name": [not_empty],
"age": [ignore_missing, convert_int],
"gender": [default("female")],
}
data = {"name": "fred", "age": "32"}
converted_data, errors = validate(data, schema)
assert not errors
assert converted_data == {
"gender": "female",
"age": 32,
"name": "fred",
}, converted_data
data = {"name": "", "age": "dsa32", "extra": "extra"}
converted_data, errors = validate(data, schema)
assert errors == {
"age": [u"Please enter an integer value"],
"name": [u"Missing value"],
}, errors
assert converted_data == {
"gender": "female",
"age": "dsa32",
"name": "",
"__extras": {"extra": "extra"},
}
data = {
"name": "fred",
"numbers": [
{"number": "13221312"},
{"number": "432423432", "code": "+44"},
],
}
schema = {
"name": [not_empty],
"numbers": {
"number": [convert_int],
"code": [not_empty],
"__extras": [ignore],
},
}
converted_data, errors = validate(data, schema)
assert errors == {"numbers": [{"code": [u"Missing value"]}]}
def default_user_schema():
# changes from core:
# - username can be uppercase
# - email is not required
schema = core_schema.default_user_schema()
schema['name'] = [navl_validators.not_empty,
core_validators.user_name_validator,
unicode]
schema['email'] = [navl_validators.default(u''),
unicode]
return schema
def form_to_db_schema(self, package_type=None):
schema = logic.schema.package_form_schema()
schema.update({
'name': [not_empty, unicode, ecportal_name_validator,
package_name_validator],
'license_id': [ignore_missing, reduce_list, map_licenses, unicode],
'keyword_string': [ignore_missing, keyword_string_convert],
'alternative_title': [ignore_missing, unicode, convert_to_extras],
'description': [not_empty, unicode],
'status': [not_empty, convert_to_tags(STATUS_VOCAB_NAME)],
'identifier': [ignore_missing, unicode, convert_to_extras],
'interoperability_level': [ignore_missing, convert_to_tags(INTEROP_VOCAB_NAME)],
'type_of_dataset': [ignore_missing, convert_to_tags(DATASET_TYPE_VOCAB_NAME)],
'published_by': [not_empty, unicode, publisher_exists,
convert_to_groups('name')],
'capacity': [ignore_missing, unicode, default(u'private'),
convert_to_groups('capacity')],
'release_date': [ignore_missing, ecportal_date_to_db, convert_to_extras],
'modified_date': [ignore_missing, ecportal_date_to_db, convert_to_extras],
'accrual_periodicity': [ignore_missing, unicode, convert_to_extras],
'temporal_coverage_from': [ignore_missing, ecportal_date_to_db,
convert_to_extras],
'temporal_coverage_to': [ignore_missing, ecportal_date_to_db,
convert_to_extras],
'temporal_granularity': [ignore_missing, convert_to_tags(TEMPORAL_VOCAB_NAME)],
'geographical_coverage': [ignore_missing, convert_to_tags(GEO_VOCAB_NAME)],
'language': [ignore_missing, convert_to_tags(LANGUAGE_VOCAB_NAME)],
'metadata_language': [ignore_missing, member_of_vocab(LANGUAGE_VOCAB_NAME), convert_to_extras],
'version_description': [ignore_missing, unicode, convert_to_extras],
'rdf': [ignore_missing, unicode, update_rdf, convert_to_extras],
'contact_name': [ignore_missing, unicode, convert_to_extras],
'contact_email': [ignore_missing, requires_field('contact_name'),
unicode, convert_to_extras],
'contact_address': [ignore_missing, requires_field('contact_name'),
unicode, convert_to_extras],
'contact_telephone': [ignore_missing, requires_field('contact_name'),
unicode, convert_to_extras],
'contact_webpage': [ignore_missing, requires_field('contact_name'),
unicode, convert_to_extras],
'__after': [duplicate_extras_key,
rename('keywords', 'tags'),
rename('description', 'notes')],
})
schema['groups'].update({
'capacity': [ignore_missing, unicode]
})
schema['resources'].update({
'type': [ignore_missing, unicode, convert_resource_type]
})
return schema
def test_simple():
schema = {
"name": [not_empty],
"age": [ignore_missing, convert_int],
"gender": [default("female")],
}
data = {
"name": "fred",
"age": "32",
}
converted_data, errors = validate(data, schema)
assert not errors
assert converted_data == {'gender': 'female', 'age': 32, 'name': 'fred'}, converted_data
data = {
"name": "",
"age": "dsa32",
"extra": "extra",
}
converted_data, errors = validate(data, schema)
assert errors == {'age': [u'Please enter an integer value'], 'name': [u'Missing value']}, errors
assert converted_data == {'gender': 'female', 'age': 'dsa32', 'name': '', '__extras': {'extra': 'extra'}}
data = {"name": "fred",
"numbers": [{"number": "13221312"},
{"number": "432423432", "code": "+44"}]
}
schema = {
"name": [not_empty],
"numbers": {
"number": [convert_int],
"code": [not_empty],
"__extras": [ignore],
}
}
converted_data, errors = validate(data, schema)
print(errors)
assert errors == {'numbers': [{'code': [u'Missing value']}, {}]}
def test_simple():
schema = {
"name": [not_empty],
"age": [ignore_missing, convert_int],
"gender": [default("female")],
}
data = {
"name": "fred",
"age": "32",
}
converted_data, errors = validate(data, schema)
assert not errors
assert converted_data == {'gender': 'female', 'age': 32, 'name': 'fred'}, converted_data
data = {
"name": "",
"age": "dsa32",
"extra": "extra",
}
converted_data, errors = validate(data, schema)
assert errors == {'age': [u'Please enter an integer value'], 'name': [u'Missing value']}, errors
assert converted_data == {'gender': 'female', 'age': 'dsa32', 'name': '', '__extras': {'extra': 'extra'}}
data = {"name": "fred",
"numbers": [{"number": "13221312"},
{"number": "432423432", "code": "+44"}]
}
schema = {
"name": [not_empty],
"numbers": {
"number": [convert_int],
"code": [not_empty],
"__extras": [ignore],
}
}
converted_data, errors = validate(data, schema)
print(errors)
assert errors == {'numbers': [{'code': [u'Missing value']}, {}]}
def test_simple_converter_types():
schema = {
"name": [not_empty, unicode],
"age": [ignore_missing, int],
"gender": [default("female")],
}
data = {
"name": "fred",
"age": "32",
}
converted_data, errors = validate(data, schema)
assert not errors
assert converted_data == {'gender': 'female', 'age': 32, 'name': u'fred'}, converted_data
assert isinstance(converted_data["name"], unicode)
assert not isinstance(converted_data["gender"], unicode)
def test_simple_converter_types():
schema = {
"name": [not_empty, text_type],
"age": [ignore_missing, int],
"gender": [default("female")],
}
data = {
"name": "fred",
"age": "32",
}
converted_data, errors = validate(data, schema)
assert not errors
assert converted_data == {'gender': 'female', 'age': 32, 'name': u'fred'}, converted_data
assert isinstance(converted_data["name"], text_type)
assert isinstance(converted_data["gender"], str)
def test_simple_converter_types():
schema = {
"name": [not_empty, text_type],
"age": [ignore_missing, int],
"gender": [default("female")],
}
data = {"name": "fred", "age": "32"}
converted_data, errors = validate(data, schema)
assert not errors
assert converted_data == {
"gender": "female",
"age": 32,
"name": u"fred",
}, converted_data
assert isinstance(converted_data["name"], text_type)
assert isinstance(converted_data["gender"], str)
def test_already_has_a_value(self):
dict_ = {"key": "original"}
validators.default("default_value")("key", dict_, {}, {})
assert dict_ == {"key": "original"}
def test_value_is_false(self):
# False is a consciously set value, so should not be changed to the
# default
dict_ = {"key": False}
validators.default("default_value")("key", dict_, {}, {})
assert dict_ == {"key": False}
def test_value_is_empty_string(self):
dict_ = {"key": ""}
validators.default("default_value")("key", dict_, {}, {})
assert dict_ == {"key": "default_value"}
def test_value_is_none(self):
dict_ = {"key": None}
validators.default("default_value")("key", dict_, {}, {})
assert dict_ == {"key": "default_value"}
def test_key_doesnt_exist(self):
dict_ = {}
validators.default("default_value")("key", dict_, {}, {})
assert dict_ == {"key": "default_value"}
def _create_package_schema(cls):
""" Create common schema for dataset create and update. Used by user interfaces and harvesters.
"""
# Note: harvester schemas
schema = default_create_package_schema()
schema.pop('author')
for key in settings.KATA_FIELDS_REQUIRED:
schema[key] = [
not_empty, co.convert_to_extras_kata, unicode,
va.validate_general
]
for key in settings.KATA_FIELDS_RECOMMENDED:
schema[key] = [
ignore_missing, co.convert_to_extras_kata, unicode,
va.validate_general
]
schema['accept-terms'] = [va.usage_terms_accepted, ignore]
schema['__after'] = [
co.gen_translation_str_from_langtitle,
co.gen_translation_str_from_langnotes
]
schema['agent'] = {
'role': [
not_empty, va.check_agent_fields, va.validate_general, unicode,
co.flattened_to_extras
],
'name': [
ignore_empty, va.validate_general, unicode,
va.contains_alphanumeric, co.flattened_to_extras
],
'id': [
ignore_empty, va.validate_general, unicode,
co.flattened_to_extras
],
'organisation': [
ignore_empty, va.validate_general, unicode,
va.contains_alphanumeric, co.flattened_to_extras
],
'URL': [
ignore_empty, co.remove_trailing_spaces, url_validator,
va.validate_general, unicode, co.flattened_to_extras
],
'fundingid': [
ignore_empty, va.validate_general, unicode,
co.flattened_to_extras
]
}
schema['contact'] = {
'name': [
not_empty, va.validate_general, unicode,
va.contains_alphanumeric, co.flattened_to_extras
],
'email': [
not_empty, co.remove_trailing_spaces, unicode,
va.validate_email, co.flattened_to_extras
],
'URL': [
ignore_empty, co.remove_trailing_spaces, url_validator,
va.validate_general, unicode, co.flattened_to_extras
],
# phone number can be missing from the first users
'phone': [
ignore_missing, co.remove_trailing_spaces, unicode,
va.validate_phonenum, co.flattened_to_extras
]
}
schema['event'] = {
'type': [
ignore_missing, va.check_events, unicode,
co.flattened_to_extras, va.validate_general
],
'who': [
ignore_missing, unicode, co.flattened_to_extras,
va.validate_general, va.contains_alphanumeric
],
'when': [
ignore_missing, unicode, co.flattened_to_extras,
va.validate_kata_interval_date
],
'descr': [
ignore_missing, unicode, co.flattened_to_extras,
va.validate_general, va.contains_alphanumeric
]
}
schema['id'] = [not_empty, va.validate_package_id_format, unicode]
# Langtitle fields are used by the UI, to construct a 'title' field with translations in JSON format
# This is not necessarily needed for the API calls
schema['langtitle'] = {
'value': [
unicode, va.validate_title, va.validate_title_duplicates,
co.escape_quotes
],
'lang': [unicode, co.convert_languages]
}
# The title field contains all the title translations in JSON format.
# The converter gen_translation_str_from_langtitle
# needs to be called to construct the JSON string from the UI's langtitle fields.
schema['title'] = [va.not_empty_if_langtitle_empty]
# Description (notes) is a multilanguage field similar to title
schema['langnotes'] = {
'value': [unicode, va.validate_notes_duplicates, co.escape_quotes],
'lang': [unicode, co.convert_languages]
}
schema['notes'] = [ignore_empty]
schema['language'] = \
[ignore_missing, co.convert_languages, co.remove_disabled_languages, co.convert_to_extras_kata, unicode]
schema['license_id'] = [co.to_license_id, unicode]
schema['temporal_coverage_begin'] = \
[ignore_missing, va.validate_kata_date, co.convert_to_extras_kata, unicode]
schema['temporal_coverage_end'] = \
[ignore_missing, va.validate_kata_date, co.convert_to_extras_kata, unicode]
schema['pids'] = {
'provider': [ignore_missing, unicode, co.flattened_to_extras],
'id': [
not_empty, va.validate_general,
va.validate_primary_pid_uniqueness, unicode,
co.flattened_to_extras
],
'type': [
not_missing, co.remove_trailing_spaces, va.validate_pid_type,
unicode, co.flattened_to_extras
],
'relation': [
ignore_missing, co.remove_trailing_spaces, co.to_relation,
va.validate_pid_relation_type, unicode, co.flattened_to_extras
]
}
schema['tag_string'] = [
ignore_missing, not_empty, va.kata_tag_string_convert
]
# otherwise the tags would be validated with default tag validator during update
schema['tags'] = cls.tags_schema()
schema['xpaths'] = [ignore_missing, co.to_extras_json]
schema['version'] = [not_empty, unicode, va.validate_kata_date]
schema['availability'] = [
not_missing, va.validate_availability, co.convert_to_extras_kata
]
schema['langdis'] = [co.checkbox_to_boolean, co.convert_to_extras_kata]
schema['__extras'] = [
va.check_agent, va.check_contact, va.check_langtitle
]
schema['__junk'] = [va.check_junk]
schema['name'] = [
va.continue_if_missing, co.default_name_from_id, unicode,
package_name_validator, va.validate_general
]
schema['external_id'] = [
ignore_missing, co.remove_trailing_spaces, co.convert_external_id,
va.validate_external_id_uniqueness, unicode, va.validate_general,
co.convert_to_extras_kata
]
schema['access_application_download_URL'] = [
ignore_missing, co.remove_trailing_spaces,
va.validate_access_application_download_url, unicode,
va.validate_general, co.convert_to_extras_kata
]
schema['access_application_URL'] = [
ignore_missing, co.remove_trailing_spaces,
va.validate_access_application_url, unicode, va.validate_general,
co.convert_to_extras_kata
]
schema['access_request_URL'] = [
ignore_missing, co.remove_trailing_spaces,
va.check_access_request_url, url_validator, unicode,
va.validate_general, co.convert_to_extras_kata
]
schema['discipline'] = [
ignore_missing, va.validate_discipline, co.convert_to_extras_kata,
unicode
]
schema['geographic_coverage'] = [
ignore_missing, va.validate_spatial, co.convert_to_extras_kata,
unicode
]
schema['license_URL'] = [
va.continue_if_missing, va.validate_license_url,
co.populate_license_URL_if_license_id_not_resolved,
co.convert_to_extras_kata, unicode, va.validate_general
]
schema['owner_org'] = [va.kata_owner_org_validator, unicode]
schema['resources']['url'] = [
default(settings.DATASET_URL_UNKNOWN),
va.check_resource_url_for_direct_download_url, unicode,
va.validate_general
]
# Conversion (and validation) of direct_download_URL to resource['url'] is in utils.py:dataset_to_resource()
schema['resources']['algorithm'] = [
ignore_missing, unicode, va.validate_algorithm
]
schema['resources']['format'] = [
ignore_missing, unicode, va.validate_general
]
schema['resources']['hash'].append(va.validate_general)
schema['resources']['mimetype'].append(va.validate_mimetype)
return schema
def test_simple(self):
raise SkipTest()
schema = {"name": [not_empty], "age": [ignore_missing, convert_int], "gender": [default("female")]}
data = {"name": "fred", "age": "32"}
converted_data, errors = validate(data, schema)
assert not errors
assert converted_data == {"gender": "female", "age": 32, "name": "fred"}, converted_data
data = {"name": "", "age": "dsa32", "extra": "extra"}
converted_data, errors = validate(data, schema)
assert errors == {("age",): [u"Please enter an integer value"], ("name",): [u"Missing value"]}, errors
assert converted_data == {"gender": "female", "age": "dsa32", "name": "", "__extras": {"extra": "extra"}}
data = {"name": "fred", "numbers": [{"number": "13221312"}, {"number": "432423432", "code": "+44"}]}
schema = {"name": [not_empty], "numbers": {"number": [convert_int], "code": [not_empty], "__extras": [ignore]}}
converted_data, errors = validate(data, schema)
assert errors == {("numbers", 0, "code"): ["Missing value"]}
def test_simple_converter_types(self):
schema = {"name": [not_empty, unicode], "age": [ignore_missing, int], "gender": [default("female")]}
data = {"name": "fred", "age": "32"}
converted_data, errors = validate(data, schema)
assert not errors
assert converted_data == {"gender": "female", "age": 32, "name": u"fred"}, converted_data
assert isinstance(converted_data["name"], unicode)
assert not isinstance(converted_data["gender"], unicode)
def _create_package_schema(cls):
""" Create common schema for dataset create and update. Used by user interfaces and harvesters.
"""
# Note: harvester schemas
schema = default_create_package_schema()
schema.pop('author')
for key in settings.KATA_FIELDS_REQUIRED:
schema[key] = [not_empty, co.convert_to_extras_kata, unicode, va.validate_general]
for key in settings.KATA_FIELDS_RECOMMENDED:
schema[key] = [ignore_missing, co.convert_to_extras_kata, unicode, va.validate_general]
schema['accept-terms'] = [va.usage_terms_accepted, ignore]
schema['__after'] = [co.gen_translation_str_from_langtitle,
co.gen_translation_str_from_langnotes]
schema['agent'] = {'role': [not_empty, va.check_agent_fields, va.validate_general, unicode, co.flattened_to_extras],
'name': [ignore_empty, va.validate_general, unicode, va.contains_alphanumeric, co.flattened_to_extras],
'id': [ignore_empty, va.validate_general, unicode, co.flattened_to_extras],
'organisation': [ignore_empty, va.validate_general, unicode, va.contains_alphanumeric, co.flattened_to_extras],
'URL': [ignore_empty, co.remove_trailing_spaces, url_validator, va.validate_general, unicode, co.flattened_to_extras],
'fundingid': [ignore_empty, va.validate_general, unicode, co.flattened_to_extras]}
schema['contact'] = {'name': [not_empty, va.validate_general, unicode, va.contains_alphanumeric, co.flattened_to_extras],
'email': [not_empty, co.remove_trailing_spaces, unicode, va.validate_email, co.flattened_to_extras],
'URL': [ignore_empty, co.remove_trailing_spaces, url_validator, va.validate_general, unicode, co.flattened_to_extras],
# phone number can be missing from the first users
'phone': [ignore_missing, co.remove_trailing_spaces, unicode, va.validate_phonenum, co.flattened_to_extras]}
schema['event'] = {'type': [ignore_missing, va.check_events, unicode, co.flattened_to_extras, va.validate_general],
'who': [ignore_missing, unicode, co.flattened_to_extras, va.validate_general, va.contains_alphanumeric],
'when': [ignore_missing, unicode, co.flattened_to_extras, va.validate_kata_interval_date],
'descr': [ignore_missing, unicode, co.flattened_to_extras, va.validate_general, va.contains_alphanumeric]}
schema['id'] = [not_empty, va.validate_package_id_format, unicode]
# Langtitle fields are used by the UI, to construct a 'title' field with translations in JSON format
# This is not necessarily needed for the API calls
schema['langtitle'] = {'value': [unicode, va.validate_title, va.validate_title_duplicates, co.escape_quotes],
'lang': [unicode, co.convert_languages]}
# The title field contains all the title translations in JSON format.
# The converter gen_translation_str_from_langtitle
# needs to be called to construct the JSON string from the UI's langtitle fields.
schema['title'] = [va.not_empty_if_langtitle_empty]
# Description (notes) is a multilanguage field similar to title
schema['langnotes'] = {'value': [unicode, va.validate_notes_duplicates, co.escape_quotes],
'lang': [unicode, co.convert_languages]}
schema['notes'] = [ignore_empty]
schema['language'] = \
[ignore_missing, co.convert_languages, co.remove_disabled_languages, co.convert_to_extras_kata, unicode]
schema['license_id'] = [co.to_license_id, unicode]
schema['temporal_coverage_begin'] = \
[ignore_missing, va.validate_kata_date, co.convert_to_extras_kata, unicode]
schema['temporal_coverage_end'] = \
[ignore_missing, va.validate_kata_date, co.convert_to_extras_kata, unicode]
schema['pids'] = {'provider': [ignore_missing, unicode, co.flattened_to_extras],
'id': [not_empty, va.validate_general, va.validate_primary_pid_uniqueness,
unicode, co.flattened_to_extras],
'type': [not_missing, co.remove_trailing_spaces, va.validate_pid_type, unicode, co.flattened_to_extras],
'relation': [ignore_missing, co.remove_trailing_spaces, co.to_relation, va.validate_pid_relation_type,
unicode, co.flattened_to_extras]}
schema['tag_string'] = [ignore_missing, not_empty, va.kata_tag_string_convert]
# otherwise the tags would be validated with default tag validator during update
schema['tags'] = cls.tags_schema()
schema['xpaths'] = [ignore_missing, co.to_extras_json]
schema['version'] = [not_empty, unicode, va.validate_kata_date]
schema['availability'] = [not_missing, va.validate_availability, co.convert_to_extras_kata]
schema['langdis'] = [co.checkbox_to_boolean, co.convert_to_extras_kata]
schema['__extras'] = [va.check_agent, va.check_contact, va.check_langtitle]
schema['__junk'] = [va.check_junk]
schema['name'] = [va.continue_if_missing, co.default_name_from_id, unicode, package_name_validator,
va.validate_general]
schema['external_id'] = [ignore_missing, co.remove_trailing_spaces, co.convert_external_id, va.validate_external_id_uniqueness, unicode, va.validate_general,
co.convert_to_extras_kata]
schema['access_application_download_URL'] = [ignore_missing, co.remove_trailing_spaces, va.validate_access_application_download_url,
unicode, va.validate_general, co.convert_to_extras_kata]
schema['access_application_URL'] = [ignore_missing, co.remove_trailing_spaces, va.validate_access_application_url,
unicode, va.validate_general, co.convert_to_extras_kata]
schema['access_request_URL'] = [ignore_missing, co.remove_trailing_spaces, va.check_access_request_url, url_validator,
unicode, va.validate_general, co.convert_to_extras_kata]
schema['discipline'] = [ignore_missing, va.validate_discipline, co.convert_to_extras_kata, unicode]
schema['geographic_coverage'] = [ignore_missing, va.validate_spatial, co.convert_to_extras_kata, unicode]
schema['license_URL'] = [va.continue_if_missing, va.validate_license_url, co.populate_license_URL_if_license_id_not_resolved, co.convert_to_extras_kata, unicode,
va.validate_general]
schema['owner_org'] = [va.kata_owner_org_validator, unicode]
schema['resources']['url'] = [default(settings.DATASET_URL_UNKNOWN), va.check_resource_url_for_direct_download_url,
unicode, va.validate_general]
# Conversion (and validation) of direct_download_URL to resource['url'] is in utils.py:dataset_to_resource()
schema['resources']['algorithm'] = [ignore_missing, unicode, va.validate_algorithm]
schema['resources']['format'] = [ignore_missing, unicode, va.validate_general]
schema['resources']['hash'].append(va.validate_general)
schema['resources']['mimetype'].append(va.validate_mimetype)
return schema
def _create_package_schema(cls):
""" Create common schema for dataset create and update.
"""
# TODO: MIKKO: Use the general converter for lang_title and check that lang_title exists!
# Note: harvester schemas
schema = default_create_package_schema()
schema.pop('author')
for key in settings.KATA_FIELDS_REQUIRED:
schema[key] = [not_empty, co.convert_to_extras_kata, unicode, va.validate_general]
for key in settings.KATA_FIELDS_RECOMMENDED:
schema[key] = [ignore_missing, co.convert_to_extras_kata, unicode, va.validate_general]
schema['agent'] = {'role': [not_empty, va.check_agent_fields, va.validate_general, unicode, co.flattened_to_extras],
'name': [ignore_empty, va.validate_general, unicode, va.contains_alphanumeric, co.flattened_to_extras],
'id': [ignore_empty, va.validate_general, unicode, co.flattened_to_extras],
'organisation': [ignore_empty, va.validate_general, unicode, va.contains_alphanumeric, co.flattened_to_extras],
'URL': [ignore_empty, url_validator, va.validate_general, unicode, co.flattened_to_extras],
'fundingid': [ignore_empty, va.validate_general, unicode, co.flattened_to_extras]}
schema['contact'] = {'name': [not_empty, va.validate_general, unicode, va.contains_alphanumeric, co.flattened_to_extras],
'email': [not_empty, unicode, va.validate_email, co.flattened_to_extras],
'URL': [ignore_empty, url_validator, va.validate_general, unicode, co.flattened_to_extras],
# phone number can be missing from the first users
'phone': [ignore_missing, unicode, va.validate_phonenum, co.flattened_to_extras]}
# phone number can be missing from the first users
# schema['contact_phone'] = [ignore_missing, validate_phonenum, convert_to_extras_kata, unicode]
# schema['contact_URL'] = [ignore_missing, url_validator, convert_to_extras_kata, unicode, validate_general]
schema['event'] = {'type': [ignore_missing, va.check_events, unicode, co.flattened_to_extras, va.validate_general],
'who': [ignore_missing, unicode, co.flattened_to_extras, va.validate_general, va.contains_alphanumeric],
'when': [ignore_missing, unicode, co.flattened_to_extras, va.validate_kata_date],
'descr': [ignore_missing, unicode, co.flattened_to_extras, va.validate_general, va.contains_alphanumeric]}
schema['id'] = [default(u''), co.update_pid, unicode]
schema['langtitle'] = {'value': [not_missing, unicode, va.validate_title, va.validate_title_duplicates, co.ltitle_to_extras],
'lang': [not_missing, unicode, co.convert_languages]}
schema['language'] = \
[ignore_missing, co.convert_languages, co.remove_disabled_languages, co.convert_to_extras_kata, unicode]
schema['temporal_coverage_begin'] = \
[ignore_missing, va.validate_kata_date, co.convert_to_extras_kata, unicode]
schema['temporal_coverage_end'] = \
[ignore_missing, va.validate_kata_date, co.convert_to_extras_kata, unicode]
schema['pids'] = {'provider': [ignore_missing, unicode, co.flattened_to_extras],
'id': [not_empty, va.validate_general, unicode, co.flattened_to_extras],
'type': [not_missing, unicode, co.flattened_to_extras],
'primary': [ignore_missing, unicode, co.flattened_to_extras]}
schema['tag_string'] = [ignore_missing, not_empty, va.kata_tag_string_convert]
# otherwise the tags would be validated with default tag validator during update
schema['tags'] = cls.tags_schema()
schema['xpaths'] = [ignore_missing, co.to_extras_json]
# these two can be missing from the first Kata end users
# TODO: version date validation should be tighter, see metadata schema
schema['version'] = [not_empty, unicode, va.validate_kata_date]
schema['availability'] = [not_missing, co.convert_to_extras_kata]
schema['langdis'] = [co.checkbox_to_boolean, co.convert_to_extras_kata]
# TODO: MIKKO: __extras: check_langtitle needed? Its 'raise' seems to be unreachable
schema['__extras'] = [va.check_agent, va.check_langtitle, va.check_contact, va.check_pids]
schema['__junk'] = [va.check_junk]
schema['name'] = [ignore_missing, unicode, co.default_name_from_id, package_name_validator,
va.validate_general]
schema['access_application_download_URL'] = [ignore_missing, va.validate_access_application_download_url,
unicode, va.validate_general, co.convert_to_extras_kata]
schema['access_application_new_form'] = [co.checkbox_to_boolean, co.convert_to_extras_kata,
co.remove_access_application_new_form]
schema['access_application_URL'] = [ignore_missing, va.validate_access_application_url,
unicode, va.validate_general, co.convert_to_extras_kata]
schema['access_request_URL'] = [ignore_missing, va.check_access_request_url, url_validator,
unicode, va.validate_general, co.convert_to_extras_kata]
schema['through_provider_URL'] = [ignore_missing, va.check_through_provider_url, url_validator,
unicode, va.validate_general, co.convert_to_extras_kata]
schema['discipline'] = [ignore_missing, va.validate_discipline, co.convert_to_extras_kata, unicode]
schema['geographic_coverage'] = [ignore_missing, va.validate_spatial, co.convert_to_extras_kata, unicode]
schema['license_URL'] = [ignore_missing, co.convert_to_extras_kata, unicode, va.validate_general]
schema['owner_org'] = [ignore_missing, va.kata_owner_org_validator, unicode]
schema['resources']['url'] = [default(settings.DATASET_URL_UNKNOWN), va.check_direct_download_url,
unicode, va.validate_general]
# Conversion (and validation) of direct_download_URL to resource['url'] is in utils.py:dataset_to_resource()
schema['resources']['algorithm'] = [ignore_missing, unicode, va.validate_algorithm]
schema['resources']['hash'].append(va.validate_general)
schema['resources']['mimetype'].append(va.validate_mimetype)
return schema
def test_already_has_a_value(self):
dict_ = {'key': 'original'}
validators.default('default_value')('key', dict_, {}, {})
eq_(dict_, {'key': 'original'})
def test_value_is_false(self):
# False is a consciously set value, so should not be changed to the
# default
dict_ = {'key': False}
validators.default('default_value')('key', dict_, {}, {})
eq_(dict_, {'key': False})
def test_value_is_empty_string(self):
dict_ = {'key': ''}
validators.default('default_value')('key', dict_, {}, {})
eq_(dict_, {'key': 'default_value'})
def test_value_is_none(self):
dict_ = {'key': None}
validators.default('default_value')('key', dict_, {}, {})
eq_(dict_, {'key': 'default_value'})
def test_key_doesnt_exist(self):
dict_ = {}
validators.default('default_value')('key', dict_, {}, {})
eq_(dict_, {'key': 'default_value'})