def harvest_source_schema():
schema = {
'id': [ignore_missing, six.text_type, package_id_exists],
'type': [dataset_type_exists, six.text_type],
'url': [not_empty, six.text_type, harvest_source_url_validator],
'name': [not_empty, six.text_type, name_validator, package_name_validator],
'source_type': [not_empty, six.text_type, harvest_source_type_exists, convert_to_extras],
'title': [if_empty_same_as("name"), six.text_type],
'notes': [ignore_missing, six.text_type],
'owner_org': [owner_org_validator, six.text_type],
'private': [ignore_missing, boolean_validator],
'organization': [ignore_missing],
'frequency': [ignore_missing, six.text_type, harvest_source_frequency_exists, convert_to_extras],
'time': [ignore_missing, convert_to_extras],
'state': [ignore_missing],
'config': [ignore_missing, harvest_source_config_validator, convert_to_extras],
'extras': default_extras_schema(),
}
extras_schema = default_extras_schema()
extras_schema['__extras'] = [ignore]
schema['extras'] = extras_schema
if p.toolkit.check_ckan_version('2.2'):
from ckan.logic.validators import datasets_with_no_organization_cannot_be_private
schema['private'].append(datasets_with_no_organization_cannot_be_private)
return schema
def harvest_source_schema():
schema = {
'id': [ignore_missing, unicode, package_id_exists],
'type': [dataset_type_exists, unicode],
'url': [not_empty, unicode, harvest_source_url_validator],
'name': [not_empty, unicode, name_validator, package_name_validator],
'source_type':
[not_empty, unicode, harvest_source_type_exists, convert_to_extras],
'title': [if_empty_same_as("name"), unicode],
'notes': [ignore_missing, unicode],
'owner_org': [owner_org_validator, unicode],
'organization': [ignore_missing],
'frequency': [
ignore_missing, unicode, harvest_source_frequency_exists,
convert_to_extras
],
'state': [ignore_missing],
'config':
[ignore_missing, harvest_source_config_validator, convert_to_extras],
'extras':
default_extras_schema(),
}
extras_schema = default_extras_schema()
extras_schema['__extras'] = [ignore]
schema['extras'] = extras_schema
return schema
def default_package_schema():
schema = {
"id": [ignore_missing, unicode, package_id_exists],
"revision_id": [ignore],
"name": [not_empty, unicode, name_validator, package_name_validator],
"title": [if_empty_same_as("name"), unicode],
"author": [ignore_missing, unicode],
"author_email": [ignore_missing, unicode],
"maintainer": [ignore_missing, unicode],
"maintainer_email": [ignore_missing, unicode],
"license_id": [ignore_missing, unicode],
"notes": [ignore_missing, unicode],
"url": [ignore_missing, unicode], # , URL(add_http=False)],
"version": [ignore_missing, unicode],
"state": [ignore_not_admin, ignore_missing],
"__extras": [ignore],
"__junk": [empty],
"resources": default_resource_schema(),
"tags": default_tags_schema(),
"extras": default_extras_schema(),
"relationships_as_object": default_relationship_schema(),
"relationships_as_subject": default_relationship_schema(),
"groups": {"id": [ignore_missing, unicode], "name": [ignore_missing, unicode], "__extras": [ignore]},
}
return schema
def harvest_source_schema():
schema = {
"id": [ignore_missing, unicode, package_id_exists],
"type": [dataset_type_exists, unicode],
"url": [not_empty, unicode, harvest_source_url_validator],
"name": [not_empty, unicode, name_validator, package_name_validator],
"source_type": [not_empty, unicode, harvest_source_type_exists, convert_to_extras],
"title": [if_empty_same_as("name"), unicode],
"notes": [ignore_missing, unicode],
"owner_org": [owner_org_validator, unicode],
"private": [ignore_missing, boolean_validator],
"organization": [ignore_missing],
"frequency": [ignore_missing, unicode, harvest_source_frequency_exists, convert_to_extras],
"state": [ignore_missing],
"config": [ignore_missing, harvest_source_config_validator, convert_to_extras],
"extras": default_extras_schema(),
}
extras_schema = default_extras_schema()
extras_schema["__extras"] = [ignore]
schema["extras"] = extras_schema
if p.toolkit.check_ckan_version("2.2"):
from ckan.logic.validators import datasets_with_no_organization_cannot_be_private
schema["private"].append(datasets_with_no_organization_cannot_be_private)
return schema
def default_package_schema():
schema = {
'id': [ignore_missing, unicode, package_id_exists],
'revision_id': [ignore],
'name': [not_empty, unicode, name_validator, package_name_validator],
'title': [if_empty_same_as("name"), unicode],
'author': [ignore_missing, unicode],
'author_email': [ignore_missing, unicode],
'maintainer': [ignore_missing, unicode],
'maintainer_email': [ignore_missing, unicode],
'license_id': [ignore_missing, unicode],
'notes': [ignore_missing, unicode],
'url': [ignore_missing, unicode],#, URL(add_http=False)],
'version': [ignore_missing, unicode, package_version_validator],
'state': [ignore_not_package_admin, ignore_missing],
'type': [ignore_missing, unicode],
'owner_org': [owner_org_validator, unicode],
'__extras': [ignore],
'__junk': [empty],
'resources': default_resource_schema(),
'tags': default_tags_schema(),
'extras': default_extras_schema(),
'relationships_as_object': default_relationship_schema(),
'relationships_as_subject': default_relationship_schema(),
'groups': {
'id': [ignore_missing, unicode],
'name': [ignore_missing, unicode],
'title': [ignore_missing, unicode],
'__extras': [ignore],
}
}
return schema
def showcase_base_schema():
schema = {
'id': [empty],
'revision_id': [ignore],
'name': [not_empty, unicode, name_validator, package_name_validator],
'title': [if_empty_same_as("name"), unicode],
'author': [ignore_missing, unicode],
'author_email': [ignore_missing, unicode],
'notes': [ignore_missing, unicode],
'url': [ignore_missing, unicode],
'state': [ignore_not_package_admin, ignore_missing],
'type': [ignore_missing, unicode],
'log_message': [ignore_missing, unicode, no_http],
'__extras': [ignore],
'__junk': [empty],
'resources': default_resource_schema(),
'tags': default_tags_schema(),
'tag_string': [ignore_missing, tag_string_convert],
'extras': default_extras_schema(),
'save': [ignore],
'return_to': [ignore],
'image_url': [toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_extras')],
'original_related_item_id': [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_extras')]
}
return schema
def harvest_source_schema():
schema = {
'id': [ignore_missing, unicode, package_id_exists],
'type': [dataset_type_exists, unicode],
'url': [not_empty, unicode, harvest_source_url_validator],
'name': [not_empty, unicode, name_validator, package_name_validator],
'source_type': [not_empty, unicode, harvest_source_type_exists,
convert_to_extras],
'title': [if_empty_same_as("name"), unicode],
'notes': [ignore_missing, unicode],
'owner_org': [owner_org_validator, unicode],
'private': [ignore_missing, boolean_validator],
'organization': [ignore_missing],
'frequency': [ignore_missing, unicode, harvest_source_frequency_exists,
convert_to_extras],
'state': [ignore_missing],
'config': [ignore_missing, harvest_source_config_validator,
convert_to_extras],
'extras': default_extras_schema(),
}
extras_schema = default_extras_schema()
extras_schema['__extras'] = [ignore]
schema['extras'] = extras_schema
if p.toolkit.check_ckan_version('2.2'):
from ckan.logic.validators import \
datasets_with_no_organization_cannot_be_private
schema['private'].append(
datasets_with_no_organization_cannot_be_private)
return schema
def update_package_schema(self):
schema = super(EDC_DatasetForm, self).update_package_schema()
schema = self._modify_package_schema(schema)
schema.update({
'name': [not_empty, unicode, name_validator, package_name_validator, duplicate_pkg],
'title': [if_empty_same_as("name"), check_dashes, check_duplicates, unicode, duplicate_pkg]
})
return schema
def _schema_update(schema, purpose):
"""
:param schema: schema dict to update
:param purpose: 'create', 'update' or 'show'
"""
assert purpose in ('create', 'update', 'show')
# XXX: remove duplicate_extras_key validation
# to work around update failures on packages with
# multiple portal_release_date values mistakenly added
if '__before' in schema:
schema['__before'] = [v for v in schema['__before']
if v != duplicate_extras_key]
if purpose == 'create':
schema['id'] = [canada_validate_generate_uuid,
unicode, name_validator, package_id_doesnt_exist]
schema['name'] = [if_empty_same_as('id'), unicode, name_validator,
package_name_validator]
if purpose in ('create', 'update'):
schema['title'] = [not_empty, unicode]
schema['notes'] = [not_empty, unicode]
schema['owner_org'] = [not_empty, owner_org_validator_publisher, unicode]
schema['license_id'] = [not_empty, unicode]
else:
schema['author_email'] = [fixed_value(
schema_description.dataset_field_by_id['author_email'])]
schema['department_number'] = [get_department_number]
schema['license_title_fra'] = [get_license_field('title_fra')]
schema['license_url_fra'] = [get_license_field('url_fra')]
resources = schema['resources']
for name, lang, field in schema_description.dataset_field_iter():
if name in schema:
continue # don't modify other existing fields
v = _schema_field_validators(name, lang, field)
if v is not None:
schema[name] = v[0] if purpose != 'show' else v[1]
if field['type'] == 'choice' and purpose in ('create', 'update'):
schema[name].extend([
convert_pilot_uuid(field),
OneOf([c['key'] for c in field['choices']])])
for name, lang, field in schema_description.resource_field_iter():
if field['mandatory'] and purpose in ('create', 'update'):
resources[name] = [not_empty, unicode]
if field['type'] == 'choice' and purpose in ('create', 'update'):
resources[name].extend([
convert_pilot_uuid(field),
OneOf([c['key'] for c in field['choices']])])
def form_to_db_schema(self):
log.fatal("Enter form to db!!!")
schema = {
'title': [if_empty_same_as("name"), unicode],
'name': [not_empty, unicode, val.name_validator, val.package_name_validator],
'license_id': [not_empty, unicode],
'maintainer': [ignore_missing, unicode],
'schema_name': [ignore_missing, unicode, convert_to_extras],
'maintainer_link': [ignore_missing, unicode, convert_to_extras],
'schema_language': [ignore_missing, unicode,convert_to_extras],
'schema_characterset': [ignore_missing, unicode,convert_to_extras],
'date_released': [ignore_missing, unicode, convert_to_extras],
'begin_datetime': [ignore_missing, unicode, convert_to_extras],
'end_datetime': [ignore_missing, unicode, convert_to_extras],
'metadata_linkage': [ignore_missing, unicode, convert_to_extras],
'attribute_description': [ignore_missing, unicode, convert_to_extras],
'publisher': [ignore_missing, unicode, convert_to_extras],
'geographic_toponym': [ignore_missing, unicode, convert_to_extras],
'geographic_bbox': [ignore_missing, unicode, convert_to_extras],
'lineage_quality': [ignore_missing, unicode, convert_to_extras],
'en_title_and_desc': [ignore_missing, unicode, convert_to_extras],
'license_citation':[ignore_missing, unicode, convert_to_extras],
'metadata_identifier':[ignore_missing, unicode, convert_to_extras],
'metadata_modified':[ignore_missing, unicode, convert_to_extras],
'date_updated':[ignore_missing, unicode, convert_to_extras],
'publishing_date':[ignore_missing, unicode, convert_to_extras],
'publishing_state':[ignore_missing, unicode, convert_to_extras],
'url': [ignore_missing, unicode],
'resources': default_schema.default_resource_schema(),
'state': [val.ignore_not_admin, ignore_missing],
'log_message': [unicode, val.no_http],
'__extras': [ignore],
'revision_id': [ignore],
'update_frequency': [ignore_missing, unicode, convert_to_extras],
'categorization': [ignore_missing, unicode, self.add_to_extras],
'notes': [ignore_missing, unicode],
'tag_string': [ignore_missing, val.tag_string_convert],
'groups': {
'id': [ignore_missing, unicode],
'capacity': [ignore_missing, unicode],
'__extras': [ignore],
},
}
schema['resources'].update({
'language':[ignore_missing],
'characterset':[ignore_missing]
})
return schema
def update_package_schema(self):
schema = super(EDC_DatasetForm, self).update_package_schema()
schema = self._modify_package_schema(schema)
schema.update({
'name': [
not_empty, unicode, name_validator, package_name_validator,
duplicate_pkg
],
'title': [
if_empty_same_as("name"), check_dashes, check_duplicates,
unicode, duplicate_pkg
]
})
return schema
def default_create_package_schema():
schema = {
'__before': [duplicate_extras_key, ignore],
'id': [
empty_if_not_sysadmin, ignore_missing, unicode,
package_id_does_not_exist
],
'revision_id': [ignore],
'name': [not_empty, unicode, name_validator, package_name_validator],
'title': [if_empty_same_as("name"), unicode],
'author': [ignore_missing, unicode],
'author_email': [ignore_missing, unicode],
'maintainer': [ignore_missing, unicode],
'maintainer_email': [ignore_missing, unicode],
'license_id': [ignore_missing, unicode],
'notes': [ignore_missing, unicode],
'url': [ignore_missing, unicode], #, URL(add_http=False)],
'version': [ignore_missing, unicode, package_version_validator],
'state': [ignore_not_package_admin, ignore_missing],
'type': [ignore_missing, unicode],
'owner_org': [owner_org_validator, unicode],
'log_message': [ignore_missing, unicode, no_http],
'private': [
ignore_missing, boolean_validator,
datasets_with_no_organization_cannot_be_private
],
'__extras': [ignore],
'__junk': [empty],
'resources':
default_resource_schema(),
'tags':
default_tags_schema(),
'tag_string': [ignore_missing, tag_string_convert],
'extras':
default_extras_schema(),
'save': [ignore],
'return_to': [ignore],
'relationships_as_object':
default_relationship_schema(),
'relationships_as_subject':
default_relationship_schema(),
'groups': {
'id': [ignore_missing, unicode],
'name': [ignore_missing, unicode],
'title': [ignore_missing, unicode],
'__extras': [ignore],
}
}
return schema
def default_create_package_schema():
schema = {
'__before': [duplicate_extras_key, ignore],
'id': [empty_if_not_sysadmin, ignore_missing, unicode,
package_id_does_not_exist],
'revision_id': [ignore],
'name': [not_empty, unicode, name_validator, package_name_validator],
'title': [if_empty_same_as("name"), unicode],
'author': [ignore_missing, unicode],
'author_email': [ignore_missing, unicode],
'maintainer': [ignore_missing, unicode],
'maintainer_email': [ignore_missing, unicode],
'license_id': [ignore_missing, unicode],
'notes': [ignore_missing, unicode],
'url': [ignore_missing, unicode], # , URL(add_http=False)],
'version': [ignore_missing, unicode, package_version_validator],
'state': [ignore_not_package_admin, ignore_missing],
'type': [ignore_missing, unicode],
'owner_org': [owner_org_validator, unicode],
'log_message': [ignore_missing, unicode, no_http],
'private': [ignore_missing, boolean_validator,
datasets_with_no_organization_cannot_be_private],
'__extras': [ignore],
'__junk': [empty],
'resources': default_resource_schema(),
'tags': default_tags_schema(),
'tag_string': [ignore_missing, tag_string_convert],
'extras': default_extras_schema(),
'save': [ignore],
'return_to': [ignore],
'relationships_as_object': default_relationship_schema(),
'relationships_as_subject': default_relationship_schema(),
'groups': {
'id': [ignore_missing, unicode],
'name': [ignore_missing, unicode],
'title': [ignore_missing, unicode],
'__extras': [ignore],
}
}
return schema
def default_create_package_schema():
schema = {
"__before": [duplicate_extras_key, ignore],
"id": [empty_if_not_sysadmin, ignore_missing, unicode, package_id_does_not_exist],
"revision_id": [ignore],
"name": [not_empty, unicode, name_validator, package_name_validator],
"title": [if_empty_same_as("name"), unicode],
"author": [ignore_missing, unicode],
"author_email": [ignore_missing, unicode],
"maintainer": [ignore_missing, unicode],
"maintainer_email": [ignore_missing, unicode],
"license_id": [ignore_missing, unicode],
"notes": [ignore_missing, unicode],
"url": [ignore_missing, unicode], # , URL(add_http=False)],
"version": [ignore_missing, unicode, package_version_validator],
"state": [ignore_not_package_admin, ignore_missing],
"type": [ignore_missing, unicode],
"owner_org": [owner_org_validator, unicode],
"log_message": [ignore_missing, unicode, no_http],
"private": [ignore_missing, boolean_validator, datasets_with_no_organization_cannot_be_private],
"__extras": [ignore],
"__junk": [empty],
"resources": default_resource_schema(),
"tags": default_tags_schema(),
"tag_string": [ignore_missing, tag_string_convert],
"extras": default_extras_schema(),
"save": [ignore],
"return_to": [ignore],
"relationships_as_object": default_relationship_schema(),
"relationships_as_subject": default_relationship_schema(),
"groups": {
"id": [ignore_missing, unicode],
"name": [ignore_missing, unicode],
"title": [ignore_missing, unicode],
"__extras": [ignore],
},
}
return schema
def harvest_source_schema():
schema = {
'id': [ignore_missing, unicode, package_id_exists],
'type': [dataset_type_exists, unicode],
'url': [not_empty, unicode, harvest_source_url_validator],
'name': [not_empty, unicode, name_validator, package_name_validator],
'source_type': [not_empty, unicode, harvest_source_type_exists, convert_to_extras],
'title': [if_empty_same_as("name"), unicode],
'notes': [ignore_missing, unicode],
'owner_org': [owner_org_validator, unicode],
'organization': [ignore_missing],
'frequency': [ignore_missing, unicode, harvest_source_frequency_exists, convert_to_extras],
'state': [ignore_missing],
'config': [ignore_missing, harvest_source_config_validator, convert_to_extras],
'extras': default_extras_schema(),
}
extras_schema = default_extras_schema()
extras_schema['__extras'] = [ignore]
schema['extras'] = extras_schema
return schema
def showcase_base_schema():
schema = {
'id': [empty],
'revision_id': [ignore],
'name': [not_empty, unicode, name_validator, package_name_validator],
'title': [if_empty_same_as("name"), unicode],
'author': [ignore_missing, unicode],
'author_email': [ignore_missing, unicode],
'notes': [ignore_missing, unicode],
'url': [ignore_missing, unicode],
'state': [ignore_not_package_admin, ignore_missing],
'type': [ignore_missing, unicode],
'log_message': [ignore_missing, unicode, no_http],
'__extras': [ignore],
'__junk': [empty],
'resources': default_resource_schema(),
'tags': default_tags_schema(),
'tag_string': [ignore_missing, tag_string_convert],
'extras': default_extras_schema(),
'save': [ignore],
'return_to': [ignore],
'groups': [ignore_missing, convert_group_names_to_group_objects],
}
# Extras
schema['image_url'] = [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_extras')
]
schema['original_related_item_id'] = [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_extras')
]
schema['related_datasets'] = [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_extras')
]
schema['related_stories'] = [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_extras')
]
schema['story_date'] = [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_extras')
]
schema['story_type'] = [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_extras')
]
schema['methodology'] = [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_extras')
]
schema['spatial_text'] = [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_extras')
]
schema['author_title'] = [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_extras')
]
schema['author_workplace'] = [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_extras')
]
schema['author_profile'] = [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_extras')
]
# Extras (conditional)
if config.get('disqus.name'):
schema['allow_commenting'] = [
toolkit.get_validator('ignore_missing'),
toolkit.get_validator('boolean_validator'),
toolkit.get_converter('convert_to_extras')
]
return schema
def _schema_update(schema, purpose):
"""
:param schema: schema dict to update
:param purpose: 'create', 'update' or 'show'
"""
assert purpose in ('create', 'update', 'show')
# XXX: remove duplicate_extras_key validation
# to work around update failures on packages with
# multiple portal_release_date values mistakenly added
if '__before' in schema:
schema['__before'] = [
v for v in schema['__before'] if v != duplicate_extras_key
]
if purpose == 'create':
schema['id'] = [
protect_new_dataset_id, if_empty_generate_uuid, unicode,
name_validator, package_id_doesnt_exist
]
schema['name'] = [
if_empty_same_as('id'), unicode, name_validator,
package_name_validator
]
if purpose in ('create', 'update'):
schema['title'] = [not_empty, unicode]
schema['notes'] = [not_empty, unicode]
schema['owner_org'] = [
not_empty, owner_org_validator_publisher, unicode
]
schema['license_id'] = [not_empty, unicode]
else:
schema['author_email'] = [
fixed_value(schema_description.dataset_field_by_id['author_email'])
]
schema['department_number'] = [get_department_number]
schema['license_title_fra'] = [get_license_field('title_fra')]
schema['license_url_fra'] = [get_license_field('url_fra')]
resources = schema['resources']
for name, lang, field in schema_description.dataset_field_iter():
if name in schema:
continue # don't modify other existing fields
v = _schema_field_validators(name, lang, field)
if v is not None:
schema[name] = v[0] if purpose != 'show' else v[1]
if field['type'] == 'choice' and purpose in ('create', 'update'):
schema[name].extend([
convert_pilot_uuid(field),
OneOf([c['key'] for c in field['choices']])
])
for name, lang, field in schema_description.resource_field_iter():
if field['mandatory'] and purpose in ('create', 'update'):
resources[name] = [not_empty, unicode]
if field['type'] == 'choice' and purpose in ('create', 'update'):
resources[name].extend([
convert_pilot_uuid(field),
OneOf([c['key'] for c in field['choices']])
])
