def test_ignore_missing(self): schema = { "__junk": [ignore], "__extras": [ignore, default("weee")], "0": [default("default")], "1": [ignore_missing, default("default")], } converted_data, errors = validate_flattened(data, schema) assert not errors assert converted_data == {("0",): "0 value"}, converted_data
def test_ignore_missing(): schema = { "__junk": [ignore], "__extras": [ignore, default("weee")], "0": [default("default")], "1": [ignore_missing, default("default")], } converted_data, errors = validate_flattened(data, schema) assert not errors assert converted_data == {('0', ): '0 value'}, converted_data
def test_default(): schema = { "__junk": [ignore], "__extras": [ignore, default("weee")], "__before": [ignore], "__after": [ignore], "0": [default("default")], "1": [default("default")], } converted_data, errors = validate_flattened(data, schema) assert not errors assert converted_data == {('1',): 'default', ('0',): '0 value'}, converted_data
def test_ignore_missing(): log.debug('Starting test_ignore_missing()') schema = { "__junk": [ignore], "__extras": [ignore, default("weee")], "0": [default("default")], "1": [ignore_missing, default("default")], } converted_data, errors = validate_flattened(data, schema) log.debug('converted_data: {0}'.format(converted_data)) assert not errors assert converted_data == {('0', ): '0 value'}, converted_data
def test_ignore_missing(): log.debug('Starting test_ignore_missing()') schema = { "__junk": [ignore], "__extras": [ignore, default("weee")], "0": [default("default")], "1": [ignore_missing, default("default")], } converted_data, errors = validate_flattened(data, schema) log.debug('converted_data: {0}'.format(converted_data)) assert not errors assert converted_data == {('0',): '0 value'}, converted_data
def form_to_db_schema_options(self, package_type=None, options=None): schema = form_to_db_package_schema() for key in self.kata_fields_required: schema[key] = [not_empty, self.convert_to_extras_kata, unicode] for key in self.kata_fields_recommended: schema[key] = [ignore_missing, self.convert_to_extras_kata, unicode] schema['temporal_coverage_begin'].append(validate_lastmod) schema['temporal_coverage_end'].append(validate_lastmod) schema['language'] = [validate_language, self.convert_to_extras_kata, unicode] schema['phone'].append(validate_phonenum) schema['maintainer_email'].append(validate_email) schema['tag_string'].append(not_empty) schema.update({ 'version': [not_empty, unicode, validate_lastmod, check_last_and_update_pid], 'versionPID': [self.update_pid, unicode, self.pid_to_extras], 'author': {'value': [not_empty, unicode, org_auth_to_extras]}, 'organization': {'value': [not_empty, unicode, org_auth_to_extras]}, 'access': [not_missing, self.convert_to_extras_kata, validate_access], 'accessRights': [ignore_missing, self.convert_to_extras_kata, unicode], 'langdis': [default(False), unicode], '__extras': [check_author_org], 'projdis': [default(False), unicode, check_project], '__junk': [check_junk], 'name': [unicode, ignore_missing, self.update_name], 'accessRights': [check_accessrights, self.convert_to_extras_kata, unicode], 'accessrequestURL': [check_accessrequesturl, self.convert_to_extras_kata, unicode], 'project_name': [check_project_dis, unicode, self.convert_to_extras_kata], 'funder': [check_project_dis, unicode, self.convert_to_extras_kata], 'project_funding': [check_project_dis, unicode, self.convert_to_extras_kata], 'project_homepage': [check_project_dis, unicode, self.convert_to_extras_kata], 'resources': default_resource_schema(), 'discipline': [add_to_group], }) schema['title'] = {'value': [not_missing, ltitle_to_extras], 'lang': [not_missing]} schema['evtype'] = {'value': [ignore_missing, unicode, event_to_extras]} schema['evwho'] = {'value': [ignore_missing, unicode, event_to_extras]} schema['evwhen'] = {'value': [ignore_missing, unicode, event_to_extras]} schema['evdescr'] = {'value': [ignore_missing, unicode, event_to_extras]} schema['groups'] = { 'id': [ignore_missing, unicode], 'name': [ignore_missing, unicode], 'title': [ignore_missing, unicode], '__extras': [ignore], } return schema
def default_user_schema(schema): schema.update( { "name": [navl_validators.not_empty, core_validators.user_name_validator, unicode], "email": [navl_validators.default(u""), unicode], } ) return schema
def test_default(self): schema = { "__junk": [ignore], "__extras": [ignore, default("weee")], "__before": [ignore], "__after": [ignore], "0": [default("default")], "1": [default("default")], } converted_data, errors = validate_flattened(data, schema) assert not errors assert converted_data == { ('1', ): 'default', ('0', ): '0 value' }, converted_data
def db_to_form_schema_options(self, options = None): schema = db_to_form_package_schema() context = options['context'] for key in self.kata_field: schema[key] = [self.convert_from_extras_kata, ignore_missing, unicode] schema['versionPID'] = [pid_from_extras, ignore_missing, unicode] schema['author'] = [org_auth_from_extras, ignore_missing, unicode] schema['organization'] = [org_auth_from_extras, ignore_missing, unicode] schema['langdis'] = [default(False), unicode] schema['projdis'] = [default(False), unicode] schema['title'] = [ltitle_from_extras, ignore_missing] schema['evtype'] = [event_from_extras, ignore_missing, unicode] schema['evwho'] = [event_from_extras, ignore_missing, unicode] schema['evwhen'] = [event_from_extras, ignore_missing, unicode] schema['evdescr'] = [event_from_extras, ignore_missing, unicode] return schema
def default_user_schema(schema): schema.update({ 'name': [ navl_validators.not_empty, core_validators.user_name_validator, unicode ], 'email': [navl_validators.default(u''), unicode] }) return schema
def test_simple(): schema = { "name": [not_empty], "age": [ignore_missing, convert_int], "gender": [default("female")], } data = {"name": "fred", "age": "32"} converted_data, errors = validate(data, schema) assert not errors assert converted_data == { "gender": "female", "age": 32, "name": "fred", }, converted_data data = {"name": "", "age": "dsa32", "extra": "extra"} converted_data, errors = validate(data, schema) assert errors == { "age": [u"Please enter an integer value"], "name": [u"Missing value"], }, errors assert converted_data == { "gender": "female", "age": "dsa32", "name": "", "__extras": {"extra": "extra"}, } data = { "name": "fred", "numbers": [ {"number": "13221312"}, {"number": "432423432", "code": "+44"}, ], } schema = { "name": [not_empty], "numbers": { "number": [convert_int], "code": [not_empty], "__extras": [ignore], }, } converted_data, errors = validate(data, schema) assert errors == {"numbers": [{"code": [u"Missing value"]}]}
def default_user_schema(): # changes from core: # - username can be uppercase # - email is not required schema = core_schema.default_user_schema() schema['name'] = [navl_validators.not_empty, core_validators.user_name_validator, unicode] schema['email'] = [navl_validators.default(u''), unicode] return schema
def form_to_db_schema(self, package_type=None): schema = logic.schema.package_form_schema() schema.update({ 'name': [not_empty, unicode, ecportal_name_validator, package_name_validator], 'license_id': [ignore_missing, reduce_list, map_licenses, unicode], 'keyword_string': [ignore_missing, keyword_string_convert], 'alternative_title': [ignore_missing, unicode, convert_to_extras], 'description': [not_empty, unicode], 'status': [not_empty, convert_to_tags(STATUS_VOCAB_NAME)], 'identifier': [ignore_missing, unicode, convert_to_extras], 'interoperability_level': [ignore_missing, convert_to_tags(INTEROP_VOCAB_NAME)], 'type_of_dataset': [ignore_missing, convert_to_tags(DATASET_TYPE_VOCAB_NAME)], 'published_by': [not_empty, unicode, publisher_exists, convert_to_groups('name')], 'capacity': [ignore_missing, unicode, default(u'private'), convert_to_groups('capacity')], 'release_date': [ignore_missing, ecportal_date_to_db, convert_to_extras], 'modified_date': [ignore_missing, ecportal_date_to_db, convert_to_extras], 'accrual_periodicity': [ignore_missing, unicode, convert_to_extras], 'temporal_coverage_from': [ignore_missing, ecportal_date_to_db, convert_to_extras], 'temporal_coverage_to': [ignore_missing, ecportal_date_to_db, convert_to_extras], 'temporal_granularity': [ignore_missing, convert_to_tags(TEMPORAL_VOCAB_NAME)], 'geographical_coverage': [ignore_missing, convert_to_tags(GEO_VOCAB_NAME)], 'language': [ignore_missing, convert_to_tags(LANGUAGE_VOCAB_NAME)], 'metadata_language': [ignore_missing, member_of_vocab(LANGUAGE_VOCAB_NAME), convert_to_extras], 'version_description': [ignore_missing, unicode, convert_to_extras], 'rdf': [ignore_missing, unicode, update_rdf, convert_to_extras], 'contact_name': [ignore_missing, unicode, convert_to_extras], 'contact_email': [ignore_missing, requires_field('contact_name'), unicode, convert_to_extras], 'contact_address': [ignore_missing, requires_field('contact_name'), unicode, convert_to_extras], 'contact_telephone': [ignore_missing, requires_field('contact_name'), unicode, convert_to_extras], 'contact_webpage': [ignore_missing, requires_field('contact_name'), unicode, convert_to_extras], '__after': [duplicate_extras_key, rename('keywords', 'tags'), rename('description', 'notes')], }) schema['groups'].update({ 'capacity': [ignore_missing, unicode] }) schema['resources'].update({ 'type': [ignore_missing, unicode, convert_resource_type] }) return schema
def test_simple(): schema = { "name": [not_empty], "age": [ignore_missing, convert_int], "gender": [default("female")], } data = { "name": "fred", "age": "32", } converted_data, errors = validate(data, schema) assert not errors assert converted_data == {'gender': 'female', 'age': 32, 'name': 'fred'}, converted_data data = { "name": "", "age": "dsa32", "extra": "extra", } converted_data, errors = validate(data, schema) assert errors == {'age': [u'Please enter an integer value'], 'name': [u'Missing value']}, errors assert converted_data == {'gender': 'female', 'age': 'dsa32', 'name': '', '__extras': {'extra': 'extra'}} data = {"name": "fred", "numbers": [{"number": "13221312"}, {"number": "432423432", "code": "+44"}] } schema = { "name": [not_empty], "numbers": { "number": [convert_int], "code": [not_empty], "__extras": [ignore], } } converted_data, errors = validate(data, schema) print(errors) assert errors == {'numbers': [{'code': [u'Missing value']}, {}]}
def test_simple_converter_types(): schema = { "name": [not_empty, unicode], "age": [ignore_missing, int], "gender": [default("female")], } data = { "name": "fred", "age": "32", } converted_data, errors = validate(data, schema) assert not errors assert converted_data == {'gender': 'female', 'age': 32, 'name': u'fred'}, converted_data assert isinstance(converted_data["name"], unicode) assert not isinstance(converted_data["gender"], unicode)
def test_simple_converter_types(): schema = { "name": [not_empty, text_type], "age": [ignore_missing, int], "gender": [default("female")], } data = { "name": "fred", "age": "32", } converted_data, errors = validate(data, schema) assert not errors assert converted_data == {'gender': 'female', 'age': 32, 'name': u'fred'}, converted_data assert isinstance(converted_data["name"], text_type) assert isinstance(converted_data["gender"], str)
def test_simple_converter_types(): schema = { "name": [not_empty, text_type], "age": [ignore_missing, int], "gender": [default("female")], } data = {"name": "fred", "age": "32"} converted_data, errors = validate(data, schema) assert not errors assert converted_data == { "gender": "female", "age": 32, "name": u"fred", }, converted_data assert isinstance(converted_data["name"], text_type) assert isinstance(converted_data["gender"], str)
def test_already_has_a_value(self): dict_ = {"key": "original"} validators.default("default_value")("key", dict_, {}, {}) assert dict_ == {"key": "original"}
def test_value_is_false(self): # False is a consciously set value, so should not be changed to the # default dict_ = {"key": False} validators.default("default_value")("key", dict_, {}, {}) assert dict_ == {"key": False}
def test_value_is_empty_string(self): dict_ = {"key": ""} validators.default("default_value")("key", dict_, {}, {}) assert dict_ == {"key": "default_value"}
def test_value_is_none(self): dict_ = {"key": None} validators.default("default_value")("key", dict_, {}, {}) assert dict_ == {"key": "default_value"}
def test_key_doesnt_exist(self): dict_ = {} validators.default("default_value")("key", dict_, {}, {}) assert dict_ == {"key": "default_value"}
def _create_package_schema(cls): """ Create common schema for dataset create and update. Used by user interfaces and harvesters. """ # Note: harvester schemas schema = default_create_package_schema() schema.pop('author') for key in settings.KATA_FIELDS_REQUIRED: schema[key] = [ not_empty, co.convert_to_extras_kata, unicode, va.validate_general ] for key in settings.KATA_FIELDS_RECOMMENDED: schema[key] = [ ignore_missing, co.convert_to_extras_kata, unicode, va.validate_general ] schema['accept-terms'] = [va.usage_terms_accepted, ignore] schema['__after'] = [ co.gen_translation_str_from_langtitle, co.gen_translation_str_from_langnotes ] schema['agent'] = { 'role': [ not_empty, va.check_agent_fields, va.validate_general, unicode, co.flattened_to_extras ], 'name': [ ignore_empty, va.validate_general, unicode, va.contains_alphanumeric, co.flattened_to_extras ], 'id': [ ignore_empty, va.validate_general, unicode, co.flattened_to_extras ], 'organisation': [ ignore_empty, va.validate_general, unicode, va.contains_alphanumeric, co.flattened_to_extras ], 'URL': [ ignore_empty, co.remove_trailing_spaces, url_validator, va.validate_general, unicode, co.flattened_to_extras ], 'fundingid': [ ignore_empty, va.validate_general, unicode, co.flattened_to_extras ] } schema['contact'] = { 'name': [ not_empty, va.validate_general, unicode, va.contains_alphanumeric, co.flattened_to_extras ], 'email': [ not_empty, co.remove_trailing_spaces, unicode, va.validate_email, co.flattened_to_extras ], 'URL': [ ignore_empty, co.remove_trailing_spaces, url_validator, va.validate_general, unicode, co.flattened_to_extras ], # phone number can be missing from the first users 'phone': [ ignore_missing, co.remove_trailing_spaces, unicode, va.validate_phonenum, co.flattened_to_extras ] } schema['event'] = { 'type': [ ignore_missing, va.check_events, unicode, co.flattened_to_extras, va.validate_general ], 'who': [ ignore_missing, unicode, co.flattened_to_extras, va.validate_general, va.contains_alphanumeric ], 'when': [ ignore_missing, unicode, co.flattened_to_extras, va.validate_kata_interval_date ], 'descr': [ ignore_missing, unicode, co.flattened_to_extras, va.validate_general, va.contains_alphanumeric ] } schema['id'] = [not_empty, va.validate_package_id_format, unicode] # Langtitle fields are used by the UI, to construct a 'title' field with translations in JSON format # This is not necessarily needed for the API calls schema['langtitle'] = { 'value': [ unicode, va.validate_title, va.validate_title_duplicates, co.escape_quotes ], 'lang': [unicode, co.convert_languages] } # The title field contains all the title translations in JSON format. # The converter gen_translation_str_from_langtitle # needs to be called to construct the JSON string from the UI's langtitle fields. schema['title'] = [va.not_empty_if_langtitle_empty] # Description (notes) is a multilanguage field similar to title schema['langnotes'] = { 'value': [unicode, va.validate_notes_duplicates, co.escape_quotes], 'lang': [unicode, co.convert_languages] } schema['notes'] = [ignore_empty] schema['language'] = \ [ignore_missing, co.convert_languages, co.remove_disabled_languages, co.convert_to_extras_kata, unicode] schema['license_id'] = [co.to_license_id, unicode] schema['temporal_coverage_begin'] = \ [ignore_missing, va.validate_kata_date, co.convert_to_extras_kata, unicode] schema['temporal_coverage_end'] = \ [ignore_missing, va.validate_kata_date, co.convert_to_extras_kata, unicode] schema['pids'] = { 'provider': [ignore_missing, unicode, co.flattened_to_extras], 'id': [ not_empty, va.validate_general, va.validate_primary_pid_uniqueness, unicode, co.flattened_to_extras ], 'type': [ not_missing, co.remove_trailing_spaces, va.validate_pid_type, unicode, co.flattened_to_extras ], 'relation': [ ignore_missing, co.remove_trailing_spaces, co.to_relation, va.validate_pid_relation_type, unicode, co.flattened_to_extras ] } schema['tag_string'] = [ ignore_missing, not_empty, va.kata_tag_string_convert ] # otherwise the tags would be validated with default tag validator during update schema['tags'] = cls.tags_schema() schema['xpaths'] = [ignore_missing, co.to_extras_json] schema['version'] = [not_empty, unicode, va.validate_kata_date] schema['availability'] = [ not_missing, va.validate_availability, co.convert_to_extras_kata ] schema['langdis'] = [co.checkbox_to_boolean, co.convert_to_extras_kata] schema['__extras'] = [ va.check_agent, va.check_contact, va.check_langtitle ] schema['__junk'] = [va.check_junk] schema['name'] = [ va.continue_if_missing, co.default_name_from_id, unicode, package_name_validator, va.validate_general ] schema['external_id'] = [ ignore_missing, co.remove_trailing_spaces, co.convert_external_id, va.validate_external_id_uniqueness, unicode, va.validate_general, co.convert_to_extras_kata ] schema['access_application_download_URL'] = [ ignore_missing, co.remove_trailing_spaces, va.validate_access_application_download_url, unicode, va.validate_general, co.convert_to_extras_kata ] schema['access_application_URL'] = [ ignore_missing, co.remove_trailing_spaces, va.validate_access_application_url, unicode, va.validate_general, co.convert_to_extras_kata ] schema['access_request_URL'] = [ ignore_missing, co.remove_trailing_spaces, va.check_access_request_url, url_validator, unicode, va.validate_general, co.convert_to_extras_kata ] schema['discipline'] = [ ignore_missing, va.validate_discipline, co.convert_to_extras_kata, unicode ] schema['geographic_coverage'] = [ ignore_missing, va.validate_spatial, co.convert_to_extras_kata, unicode ] schema['license_URL'] = [ va.continue_if_missing, va.validate_license_url, co.populate_license_URL_if_license_id_not_resolved, co.convert_to_extras_kata, unicode, va.validate_general ] schema['owner_org'] = [va.kata_owner_org_validator, unicode] schema['resources']['url'] = [ default(settings.DATASET_URL_UNKNOWN), va.check_resource_url_for_direct_download_url, unicode, va.validate_general ] # Conversion (and validation) of direct_download_URL to resource['url'] is in utils.py:dataset_to_resource() schema['resources']['algorithm'] = [ ignore_missing, unicode, va.validate_algorithm ] schema['resources']['format'] = [ ignore_missing, unicode, va.validate_general ] schema['resources']['hash'].append(va.validate_general) schema['resources']['mimetype'].append(va.validate_mimetype) return schema
def test_simple(self): raise SkipTest() schema = {"name": [not_empty], "age": [ignore_missing, convert_int], "gender": [default("female")]} data = {"name": "fred", "age": "32"} converted_data, errors = validate(data, schema) assert not errors assert converted_data == {"gender": "female", "age": 32, "name": "fred"}, converted_data data = {"name": "", "age": "dsa32", "extra": "extra"} converted_data, errors = validate(data, schema) assert errors == {("age",): [u"Please enter an integer value"], ("name",): [u"Missing value"]}, errors assert converted_data == {"gender": "female", "age": "dsa32", "name": "", "__extras": {"extra": "extra"}} data = {"name": "fred", "numbers": [{"number": "13221312"}, {"number": "432423432", "code": "+44"}]} schema = {"name": [not_empty], "numbers": {"number": [convert_int], "code": [not_empty], "__extras": [ignore]}} converted_data, errors = validate(data, schema) assert errors == {("numbers", 0, "code"): ["Missing value"]}
def test_simple_converter_types(self): schema = {"name": [not_empty, unicode], "age": [ignore_missing, int], "gender": [default("female")]} data = {"name": "fred", "age": "32"} converted_data, errors = validate(data, schema) assert not errors assert converted_data == {"gender": "female", "age": 32, "name": u"fred"}, converted_data assert isinstance(converted_data["name"], unicode) assert not isinstance(converted_data["gender"], unicode)
def _create_package_schema(cls): """ Create common schema for dataset create and update. Used by user interfaces and harvesters. """ # Note: harvester schemas schema = default_create_package_schema() schema.pop('author') for key in settings.KATA_FIELDS_REQUIRED: schema[key] = [not_empty, co.convert_to_extras_kata, unicode, va.validate_general] for key in settings.KATA_FIELDS_RECOMMENDED: schema[key] = [ignore_missing, co.convert_to_extras_kata, unicode, va.validate_general] schema['accept-terms'] = [va.usage_terms_accepted, ignore] schema['__after'] = [co.gen_translation_str_from_langtitle, co.gen_translation_str_from_langnotes] schema['agent'] = {'role': [not_empty, va.check_agent_fields, va.validate_general, unicode, co.flattened_to_extras], 'name': [ignore_empty, va.validate_general, unicode, va.contains_alphanumeric, co.flattened_to_extras], 'id': [ignore_empty, va.validate_general, unicode, co.flattened_to_extras], 'organisation': [ignore_empty, va.validate_general, unicode, va.contains_alphanumeric, co.flattened_to_extras], 'URL': [ignore_empty, co.remove_trailing_spaces, url_validator, va.validate_general, unicode, co.flattened_to_extras], 'fundingid': [ignore_empty, va.validate_general, unicode, co.flattened_to_extras]} schema['contact'] = {'name': [not_empty, va.validate_general, unicode, va.contains_alphanumeric, co.flattened_to_extras], 'email': [not_empty, co.remove_trailing_spaces, unicode, va.validate_email, co.flattened_to_extras], 'URL': [ignore_empty, co.remove_trailing_spaces, url_validator, va.validate_general, unicode, co.flattened_to_extras], # phone number can be missing from the first users 'phone': [ignore_missing, co.remove_trailing_spaces, unicode, va.validate_phonenum, co.flattened_to_extras]} schema['event'] = {'type': [ignore_missing, va.check_events, unicode, co.flattened_to_extras, va.validate_general], 'who': [ignore_missing, unicode, co.flattened_to_extras, va.validate_general, va.contains_alphanumeric], 'when': [ignore_missing, unicode, co.flattened_to_extras, va.validate_kata_interval_date], 'descr': [ignore_missing, unicode, co.flattened_to_extras, va.validate_general, va.contains_alphanumeric]} schema['id'] = [not_empty, va.validate_package_id_format, unicode] # Langtitle fields are used by the UI, to construct a 'title' field with translations in JSON format # This is not necessarily needed for the API calls schema['langtitle'] = {'value': [unicode, va.validate_title, va.validate_title_duplicates, co.escape_quotes], 'lang': [unicode, co.convert_languages]} # The title field contains all the title translations in JSON format. # The converter gen_translation_str_from_langtitle # needs to be called to construct the JSON string from the UI's langtitle fields. schema['title'] = [va.not_empty_if_langtitle_empty] # Description (notes) is a multilanguage field similar to title schema['langnotes'] = {'value': [unicode, va.validate_notes_duplicates, co.escape_quotes], 'lang': [unicode, co.convert_languages]} schema['notes'] = [ignore_empty] schema['language'] = \ [ignore_missing, co.convert_languages, co.remove_disabled_languages, co.convert_to_extras_kata, unicode] schema['license_id'] = [co.to_license_id, unicode] schema['temporal_coverage_begin'] = \ [ignore_missing, va.validate_kata_date, co.convert_to_extras_kata, unicode] schema['temporal_coverage_end'] = \ [ignore_missing, va.validate_kata_date, co.convert_to_extras_kata, unicode] schema['pids'] = {'provider': [ignore_missing, unicode, co.flattened_to_extras], 'id': [not_empty, va.validate_general, va.validate_primary_pid_uniqueness, unicode, co.flattened_to_extras], 'type': [not_missing, co.remove_trailing_spaces, va.validate_pid_type, unicode, co.flattened_to_extras], 'relation': [ignore_missing, co.remove_trailing_spaces, co.to_relation, va.validate_pid_relation_type, unicode, co.flattened_to_extras]} schema['tag_string'] = [ignore_missing, not_empty, va.kata_tag_string_convert] # otherwise the tags would be validated with default tag validator during update schema['tags'] = cls.tags_schema() schema['xpaths'] = [ignore_missing, co.to_extras_json] schema['version'] = [not_empty, unicode, va.validate_kata_date] schema['availability'] = [not_missing, va.validate_availability, co.convert_to_extras_kata] schema['langdis'] = [co.checkbox_to_boolean, co.convert_to_extras_kata] schema['__extras'] = [va.check_agent, va.check_contact, va.check_langtitle] schema['__junk'] = [va.check_junk] schema['name'] = [va.continue_if_missing, co.default_name_from_id, unicode, package_name_validator, va.validate_general] schema['external_id'] = [ignore_missing, co.remove_trailing_spaces, co.convert_external_id, va.validate_external_id_uniqueness, unicode, va.validate_general, co.convert_to_extras_kata] schema['access_application_download_URL'] = [ignore_missing, co.remove_trailing_spaces, va.validate_access_application_download_url, unicode, va.validate_general, co.convert_to_extras_kata] schema['access_application_URL'] = [ignore_missing, co.remove_trailing_spaces, va.validate_access_application_url, unicode, va.validate_general, co.convert_to_extras_kata] schema['access_request_URL'] = [ignore_missing, co.remove_trailing_spaces, va.check_access_request_url, url_validator, unicode, va.validate_general, co.convert_to_extras_kata] schema['discipline'] = [ignore_missing, va.validate_discipline, co.convert_to_extras_kata, unicode] schema['geographic_coverage'] = [ignore_missing, va.validate_spatial, co.convert_to_extras_kata, unicode] schema['license_URL'] = [va.continue_if_missing, va.validate_license_url, co.populate_license_URL_if_license_id_not_resolved, co.convert_to_extras_kata, unicode, va.validate_general] schema['owner_org'] = [va.kata_owner_org_validator, unicode] schema['resources']['url'] = [default(settings.DATASET_URL_UNKNOWN), va.check_resource_url_for_direct_download_url, unicode, va.validate_general] # Conversion (and validation) of direct_download_URL to resource['url'] is in utils.py:dataset_to_resource() schema['resources']['algorithm'] = [ignore_missing, unicode, va.validate_algorithm] schema['resources']['format'] = [ignore_missing, unicode, va.validate_general] schema['resources']['hash'].append(va.validate_general) schema['resources']['mimetype'].append(va.validate_mimetype) return schema
def _create_package_schema(cls): """ Create common schema for dataset create and update. """ # TODO: MIKKO: Use the general converter for lang_title and check that lang_title exists! # Note: harvester schemas schema = default_create_package_schema() schema.pop('author') for key in settings.KATA_FIELDS_REQUIRED: schema[key] = [not_empty, co.convert_to_extras_kata, unicode, va.validate_general] for key in settings.KATA_FIELDS_RECOMMENDED: schema[key] = [ignore_missing, co.convert_to_extras_kata, unicode, va.validate_general] schema['agent'] = {'role': [not_empty, va.check_agent_fields, va.validate_general, unicode, co.flattened_to_extras], 'name': [ignore_empty, va.validate_general, unicode, va.contains_alphanumeric, co.flattened_to_extras], 'id': [ignore_empty, va.validate_general, unicode, co.flattened_to_extras], 'organisation': [ignore_empty, va.validate_general, unicode, va.contains_alphanumeric, co.flattened_to_extras], 'URL': [ignore_empty, url_validator, va.validate_general, unicode, co.flattened_to_extras], 'fundingid': [ignore_empty, va.validate_general, unicode, co.flattened_to_extras]} schema['contact'] = {'name': [not_empty, va.validate_general, unicode, va.contains_alphanumeric, co.flattened_to_extras], 'email': [not_empty, unicode, va.validate_email, co.flattened_to_extras], 'URL': [ignore_empty, url_validator, va.validate_general, unicode, co.flattened_to_extras], # phone number can be missing from the first users 'phone': [ignore_missing, unicode, va.validate_phonenum, co.flattened_to_extras]} # phone number can be missing from the first users # schema['contact_phone'] = [ignore_missing, validate_phonenum, convert_to_extras_kata, unicode] # schema['contact_URL'] = [ignore_missing, url_validator, convert_to_extras_kata, unicode, validate_general] schema['event'] = {'type': [ignore_missing, va.check_events, unicode, co.flattened_to_extras, va.validate_general], 'who': [ignore_missing, unicode, co.flattened_to_extras, va.validate_general, va.contains_alphanumeric], 'when': [ignore_missing, unicode, co.flattened_to_extras, va.validate_kata_date], 'descr': [ignore_missing, unicode, co.flattened_to_extras, va.validate_general, va.contains_alphanumeric]} schema['id'] = [default(u''), co.update_pid, unicode] schema['langtitle'] = {'value': [not_missing, unicode, va.validate_title, va.validate_title_duplicates, co.ltitle_to_extras], 'lang': [not_missing, unicode, co.convert_languages]} schema['language'] = \ [ignore_missing, co.convert_languages, co.remove_disabled_languages, co.convert_to_extras_kata, unicode] schema['temporal_coverage_begin'] = \ [ignore_missing, va.validate_kata_date, co.convert_to_extras_kata, unicode] schema['temporal_coverage_end'] = \ [ignore_missing, va.validate_kata_date, co.convert_to_extras_kata, unicode] schema['pids'] = {'provider': [ignore_missing, unicode, co.flattened_to_extras], 'id': [not_empty, va.validate_general, unicode, co.flattened_to_extras], 'type': [not_missing, unicode, co.flattened_to_extras], 'primary': [ignore_missing, unicode, co.flattened_to_extras]} schema['tag_string'] = [ignore_missing, not_empty, va.kata_tag_string_convert] # otherwise the tags would be validated with default tag validator during update schema['tags'] = cls.tags_schema() schema['xpaths'] = [ignore_missing, co.to_extras_json] # these two can be missing from the first Kata end users # TODO: version date validation should be tighter, see metadata schema schema['version'] = [not_empty, unicode, va.validate_kata_date] schema['availability'] = [not_missing, co.convert_to_extras_kata] schema['langdis'] = [co.checkbox_to_boolean, co.convert_to_extras_kata] # TODO: MIKKO: __extras: check_langtitle needed? Its 'raise' seems to be unreachable schema['__extras'] = [va.check_agent, va.check_langtitle, va.check_contact, va.check_pids] schema['__junk'] = [va.check_junk] schema['name'] = [ignore_missing, unicode, co.default_name_from_id, package_name_validator, va.validate_general] schema['access_application_download_URL'] = [ignore_missing, va.validate_access_application_download_url, unicode, va.validate_general, co.convert_to_extras_kata] schema['access_application_new_form'] = [co.checkbox_to_boolean, co.convert_to_extras_kata, co.remove_access_application_new_form] schema['access_application_URL'] = [ignore_missing, va.validate_access_application_url, unicode, va.validate_general, co.convert_to_extras_kata] schema['access_request_URL'] = [ignore_missing, va.check_access_request_url, url_validator, unicode, va.validate_general, co.convert_to_extras_kata] schema['through_provider_URL'] = [ignore_missing, va.check_through_provider_url, url_validator, unicode, va.validate_general, co.convert_to_extras_kata] schema['discipline'] = [ignore_missing, va.validate_discipline, co.convert_to_extras_kata, unicode] schema['geographic_coverage'] = [ignore_missing, va.validate_spatial, co.convert_to_extras_kata, unicode] schema['license_URL'] = [ignore_missing, co.convert_to_extras_kata, unicode, va.validate_general] schema['owner_org'] = [ignore_missing, va.kata_owner_org_validator, unicode] schema['resources']['url'] = [default(settings.DATASET_URL_UNKNOWN), va.check_direct_download_url, unicode, va.validate_general] # Conversion (and validation) of direct_download_URL to resource['url'] is in utils.py:dataset_to_resource() schema['resources']['algorithm'] = [ignore_missing, unicode, va.validate_algorithm] schema['resources']['hash'].append(va.validate_general) schema['resources']['mimetype'].append(va.validate_mimetype) return schema
def test_already_has_a_value(self): dict_ = {'key': 'original'} validators.default('default_value')('key', dict_, {}, {}) eq_(dict_, {'key': 'original'})
def test_value_is_false(self): # False is a consciously set value, so should not be changed to the # default dict_ = {'key': False} validators.default('default_value')('key', dict_, {}, {}) eq_(dict_, {'key': False})
def test_value_is_empty_string(self): dict_ = {'key': ''} validators.default('default_value')('key', dict_, {}, {}) eq_(dict_, {'key': 'default_value'})
def test_value_is_none(self): dict_ = {'key': None} validators.default('default_value')('key', dict_, {}, {}) eq_(dict_, {'key': 'default_value'})
def test_key_doesnt_exist(self): dict_ = {} validators.default('default_value')('key', dict_, {}, {}) eq_(dict_, {'key': 'default_value'})