def _trim_package(pkg):
"""
remove keys from pkg that we don't care about when comparing
or updating/creating packages. Also try to convert types and
create missing fields that will be present in package_show.
"""
# XXX full of custom hacks and deep knowledge of our schema :-(
if not pkg:
return
for k in ['extras', 'metadata_modified', 'metadata_created',
'revision_id', 'revision_timestamp', 'organization',
'version', 'tracking_summary',
'tags', # just because we don't use them
'num_tags', 'num_resources', 'maintainer',
'isopen', 'relationships_as_object', 'license_title',
'license_title_fra', 'license_url_fra', 'license_url',
'author',
'groups', # just because we don't use them
'relationships_as_subject', 'department_number',
# FIXME: remove these when we can:
'resource_type',
# new in 2.3:
'creator_user_id',
]:
if k in pkg:
del pkg[k]
for r in pkg['resources']:
for k in ['package_id', 'revision_id',
'revision_timestamp', 'cache_last_updated',
'webstore_last_updated', 'state', 'hash',
'description', 'tracking_summary', 'mimetype_inner',
'mimetype', 'cache_url', 'created', 'webstore_url',
'last_modified', 'position']:
if k in r:
del r[k]
for k in ['name', 'size']:
if k not in r:
r[k] = None
for k in ['private']:
pkg[k] = boolean_validator(unicode(pkg.get(k, '')), None)
if 'name' not in pkg:
pkg['name'] = pkg['id']
if 'type' not in pkg:
pkg['type'] = 'dataset'
if 'state' not in pkg:
pkg['state'] = 'active'
for k in ['url']:
if k not in pkg:
pkg[k] = ''
for name, lang, field in schema_description.dataset_field_iter():
if field['type'] == 'date':
try:
pkg[name] = str(isodate(pkg[name], None)) if pkg.get(name) else ''
except Invalid:
pass # not for us to fail validation
elif field['type'] == 'url':
if not pkg.get(name): # be consistent about what an empty url is
pkg[name] = ""
elif field['type'] == 'fixed' and name in pkg:
del pkg[name]
def _schema_update(schema, purpose):
"""
:param schema: schema dict to update
:param purpose: 'create', 'update' or 'show'
"""
assert purpose in ('create', 'update', 'show')
if purpose == 'create':
schema['id'] = [
ignore_missing, protect_new_dataset_id, unicode, name_validator,
package_id_doesnt_exist
]
schema['name'] = [
ignore_missing, unicode, name_validator, package_name_validator
]
if purpose in ('create', 'update'):
schema['title'] = [not_empty_allow_override, unicode]
schema['notes'] = [not_empty_allow_override, unicode]
schema['owner_org'] = [
not_empty, owner_org_validator_publisher, unicode
]
schema['license_id'] = [not_empty_allow_override, unicode]
else:
schema['author_email'] = [
fixed_value(schema_description.dataset_field_by_id['author_email'])
]
schema['license_id'] = [
fixed_value(schema_description.dataset_field_by_id['license_id'])
]
schema['department_number'] = [get_department_number]
schema['license_title_fra'] = [get_license_field('title_fra')]
schema['license_url_fra'] = [get_license_field('url_fra')]
resources = schema['resources']
for name, lang, field in schema_description.dataset_field_iter():
if name in schema:
continue # don't modify other existing fields
v = _schema_field_validators(name, lang, field)
if v is not None:
schema[name] = v[0] if purpose != 'show' else v[1]
if field['type'] == 'choice' and purpose in ('create', 'update'):
schema[name].extend([
convert_pilot_uuid(field),
OneOf([c['key'] for c in field['choices']])
])
for name, lang, field in schema_description.resource_field_iter():
if field['mandatory'] and purpose in ('create', 'update'):
resources[name] = [not_empty_allow_override, unicode]
if field['type'] == 'choice' and purpose in ('create', 'update'):
resources[name].extend([
convert_pilot_uuid(field),
OneOf([c['key'] for c in field['choices']])
])
if purpose in ('create', 'update'):
schema['validation_override'] = [ignore_missing]
def _schema_update(schema, purpose):
"""
:param schema: schema dict to update
:param purpose: 'create', 'update' or 'show'
"""
assert purpose in ('create', 'update', 'show')
# XXX: remove duplicate_extras_key validation
# to work around update failures on packages with
# multiple portal_release_date values mistakenly added
if '__before' in schema:
schema['__before'] = [v for v in schema['__before']
if v != duplicate_extras_key]
if purpose == 'create':
schema['id'] = [canada_validate_generate_uuid,
unicode, name_validator, package_id_doesnt_exist]
schema['name'] = [if_empty_same_as('id'), unicode, name_validator,
package_name_validator]
if purpose in ('create', 'update'):
schema['title'] = [not_empty, unicode]
schema['notes'] = [not_empty, unicode]
schema['owner_org'] = [not_empty, owner_org_validator_publisher, unicode]
schema['license_id'] = [not_empty, unicode]
else:
schema['author_email'] = [fixed_value(
schema_description.dataset_field_by_id['author_email'])]
schema['department_number'] = [get_department_number]
schema['license_title_fra'] = [get_license_field('title_fra')]
schema['license_url_fra'] = [get_license_field('url_fra')]
resources = schema['resources']
for name, lang, field in schema_description.dataset_field_iter():
if name in schema:
continue # don't modify other existing fields
v = _schema_field_validators(name, lang, field)
if v is not None:
schema[name] = v[0] if purpose != 'show' else v[1]
if field['type'] == 'choice' and purpose in ('create', 'update'):
schema[name].extend([
convert_pilot_uuid(field),
OneOf([c['key'] for c in field['choices']])])
for name, lang, field in schema_description.resource_field_iter():
if field['mandatory'] and purpose in ('create', 'update'):
resources[name] = [not_empty, unicode]
if field['type'] == 'choice' and purpose in ('create', 'update'):
resources[name].extend([
convert_pilot_uuid(field),
OneOf([c['key'] for c in field['choices']])])
def _schema_update(schema, purpose):
"""
:param schema: schema dict to update
:param purpose: 'create', 'update' or 'show'
"""
assert purpose in ('create', 'update', 'show')
if purpose == 'create':
schema['id'] = [ignore_missing, protect_new_dataset_id,
unicode, name_validator, package_id_doesnt_exist]
schema['name'] = [ignore_missing, unicode, name_validator,
package_name_validator]
if purpose in ('create', 'update'):
schema['title'] = [not_empty_allow_override, unicode]
schema['notes'] = [not_empty_allow_override, unicode]
schema['owner_org'] = [not_empty, owner_org_validator_publisher, unicode]
schema['license_id'] = [not_empty_allow_override, unicode]
else:
schema['author_email'] = [fixed_value(
schema_description.dataset_field_by_id['author_email'])]
schema['license_id'] = [fixed_value(
schema_description.dataset_field_by_id['license_id'])]
schema['department_number'] = [get_department_number]
schema['license_title_fra'] = [get_license_field('title_fra')]
schema['license_url_fra'] = [get_license_field('url_fra')]
resources = schema['resources']
for name, lang, field in schema_description.dataset_field_iter():
if name in schema:
continue # don't modify other existing fields
v = _schema_field_validators(name, lang, field)
if v is not None:
schema[name] = v[0] if purpose != 'show' else v[1]
if field['type'] == 'choice' and purpose in ('create', 'update'):
schema[name].extend([
convert_pilot_uuid(field),
OneOf([c['key'] for c in field['choices']])])
for name, lang, field in schema_description.resource_field_iter():
if field['mandatory'] and purpose in ('create', 'update'):
resources[name] = [not_empty_allow_override, unicode]
if field['type'] == 'choice' and purpose in ('create', 'update'):
resources[name].extend([
convert_pilot_uuid(field),
OneOf([c['key'] for c in field['choices']])])
if purpose in ('create', 'update'):
schema['validation_override'] = [ignore_missing]
def _trim_package(pkg):
"""
remove keys from pkg that we don't care about when comparing
or updating/creating packages. Also try to convert types and
create missing fields that will be present in package_show.
"""
# XXX full of custom hacks and deep knowledge of our schema :-(
if not pkg:
return
for k in ['extras', 'metadata_modified', 'metadata_created',
'revision_id', 'revision_timestamp', 'organization',
'version', 'tracking_summary',
'tags', # just because we don't use them
'num_tags', 'num_resources', 'maintainer',
'isopen', 'relationships_as_object', 'license_title',
'license_title_fra', 'license_url_fra', 'license_url',
'maintainer_email', 'author',
'groups', # just because we don't use them
'relationships_as_subject', 'department_number',
# FIXME: remove these when we can:
'resource_type',
]:
if k in pkg:
del pkg[k]
for r in pkg['resources']:
for k in ['resource_group_id', 'revision_id',
'revision_timestamp', 'cache_last_updated',
'webstore_last_updated', 'id', 'state', 'hash',
'description', 'tracking_summary', 'mimetype_inner',
'mimetype', 'cache_url', 'created', 'webstore_url',
'last_modified', 'position', ]:
if k in r:
del r[k]
for k in ['name', 'size']:
if k not in r:
r[k] = None
for k in ['ready_to_publish', 'private']:
pkg[k] = boolean_validator(unicode(pkg.get(k, '')), None)
if 'name' not in pkg:
pkg['name'] = pkg['id']
if 'type' not in pkg:
pkg['type'] = 'dataset'
if 'state' not in pkg:
pkg['state'] = 'active'
for k in ['url']:
if k not in pkg:
pkg[k] = ''
for name, lang, field in schema_description.dataset_field_iter():
if field['type'] == 'date':
try:
pkg[name] = str(isodate(pkg[name], None)) if pkg.get(name) else ''
except Invalid:
pass # not for us to fail validation
elif field['type'] == 'tag_vocabulary' and not isinstance(
pkg.get(name), list):
pkg[name] = convert_pilot_uuid_list(field)(pkg.get(name, []))
elif field['type'] == 'url':
if not pkg.get(name): # be consistent about what an empty url is
pkg[name] = ""
elif field['type'] == 'fixed' and name in pkg:
del pkg[name]
def _schema_update(schema, purpose):
"""
:param schema: schema dict to update
:param purpose: 'create', 'update' or 'show'
"""
assert purpose in ('create', 'update', 'show')
# XXX: remove duplicate_extras_key validation
# to work around update failures on packages with
# multiple portal_release_date values mistakenly added
if '__before' in schema:
schema['__before'] = [
v for v in schema['__before'] if v != duplicate_extras_key
]
if purpose == 'create':
schema['id'] = [
protect_new_dataset_id, if_empty_generate_uuid, unicode,
name_validator, package_id_doesnt_exist
]
schema['name'] = [
if_empty_same_as('id'), unicode, name_validator,
package_name_validator
]
if purpose in ('create', 'update'):
schema['title'] = [not_empty, unicode]
schema['notes'] = [not_empty, unicode]
schema['owner_org'] = [
not_empty, owner_org_validator_publisher, unicode
]
schema['license_id'] = [not_empty, unicode]
else:
schema['author_email'] = [
fixed_value(schema_description.dataset_field_by_id['author_email'])
]
schema['department_number'] = [get_department_number]
schema['license_title_fra'] = [get_license_field('title_fra')]
schema['license_url_fra'] = [get_license_field('url_fra')]
resources = schema['resources']
for name, lang, field in schema_description.dataset_field_iter():
if name in schema:
continue # don't modify other existing fields
v = _schema_field_validators(name, lang, field)
if v is not None:
schema[name] = v[0] if purpose != 'show' else v[1]
if field['type'] == 'choice' and purpose in ('create', 'update'):
schema[name].extend([
convert_pilot_uuid(field),
OneOf([c['key'] for c in field['choices']])
])
for name, lang, field in schema_description.resource_field_iter():
if field['mandatory'] and purpose in ('create', 'update'):
resources[name] = [not_empty, unicode]
if field['type'] == 'choice' and purpose in ('create', 'update'):
resources[name].extend([
convert_pilot_uuid(field),
OneOf([c['key'] for c in field['choices']])
])