def get_licenses():
licenses = [('', '')]
register = Package.get_license_register()
for l in register.values():
if l.status == 'active':
licenses.append((l.title, l.id))
return licenses
def get_licenses():
licenses = [('', '')]
register = Package.get_license_register()
for l in register.values():
if l.status == 'active':
licenses.append((l.title, l.id))
return licenses
def validate_config(self,config):
super(GuiaHarvesterPlugin, self).validate_config(config)
config_obj = json.loads(config)
if 'license_id' in config:
license_id = config_obj.get('license_id', None)
if license_id not in Package.get_license_register().keys():
raise ValueError('license_id not found')
return config
def validate_config(self, config):
super(GuiaHarvesterPlugin, self).validate_config(config)
config_obj = json.loads(config)
if 'license_id' in config:
license_id = config_obj.get('license_id', None)
if license_id not in Package.get_license_register().keys():
raise ValueError('license_id not found')
return config
def get_license(license_id):
return Package.get_license_register().get(license_id)
def get_license(license_id):
return Package.get_license_register().get(license_id)
def get_license(license_id):
'''Helper to return license based on id.
'''
return Package.get_license_register().get(license_id)
def build_metadata_dict(pkg_dict):
'''
Build/extract a basic dict of metadata that can then be passed to build_xml_dict.
:param pkg_dict: dict of package details
'''
metadata_dict = {}
# collect errors instead of throwing them immediately; some data may not be correctly handled
# by this base method but will be handled correctly by plugins that implement IDoi
errors = {}
# required fields first (identifier will be added later)
required = {
'creators': [],
'titles': [],
'publisher': None,
'publicationYear': None,
'resourceType': None
}
def _add_required(key, get_func):
try:
required[key] = get_func()
except Exception as e:
errors[key] = e
# CREATORS
_add_required('creators', lambda: [{
'full_name': pkg_dict.get('author')
}])
# TITLES
_add_required('titles', lambda: [{
'title': pkg_dict.get('title')
}])
# PUBLISHER
_add_required('publisher', lambda: toolkit.config.get('ckanext.doi.publisher'))
# PUBLICATION YEAR
_add_required('publicationYear', lambda: package_get_year(pkg_dict))
# TYPE
_add_required('resourceType', lambda: pkg_dict.get('type'))
# now the optional fields
optional = {
'subjects': [],
'contributors': [],
'dates': [],
'language': '',
'alternateIdentifiers': [],
'relatedIdentifiers': [],
'sizes': [],
'formats': [],
'version': '',
'rightsList': [],
'descriptions': [],
'geolocations': [],
'fundingReferences': []
}
# SUBJECTS
# use the tag list
try:
tags = pkg_dict.get('tag_string', '').split(',')
tags += [tag['name'] if isinstance(tag, dict) else tag for tag in pkg_dict.get('tags', [])]
optional['subjects'] = [{'subject': tag} for tag in sorted({t for t in tags if t != ''})]
except Exception as e:
errors['subjects'] = e
# CONTRIBUTORS
# use the author and maintainer; no splitting or parsing for either
# no try/except for this because it's just a simple .get() and if that doesn't work then we
# want to know
author = pkg_dict.get('author')
maintainer = pkg_dict.get('maintainer')
if author is not None:
optional['contributors'].append(
{
'contributor_type': 'Researcher',
'full_name': author
})
if maintainer is not None:
optional['contributors'].append({
'contributor_type': 'DataManager',
'full_name': maintainer
})
# DATES
# created, updated, and doi publish date
date_errors = {}
try:
optional['dates'].append({
'dateType': 'Created',
'date': date_or_none(pkg_dict.get('metadata_created'))
})
except Exception as e:
date_errors['created'] = e
try:
optional['dates'].append({
'dateType': 'Updated',
'date': date_or_none(pkg_dict.get('metadata_modified'))
})
except Exception as e:
date_errors['updated'] = e
if 'doi_date_published' in pkg_dict:
try:
optional['dates'].append({
'dateType': 'Issued',
'date': date_or_none(pkg_dict.get('doi_date_published'))
})
except Exception as e:
date_errors['doi_date_published'] = e
# LANGUAGE
# use language set in CKAN
try:
optional['language'] = ckan_lang()
except Exception as e:
errors['language'] = e
# ALTERNATE IDENTIFIERS
# add permalink back to this site
try:
permalink = f'{get_site_url()}/dataset/{pkg_dict["id"]}'
optional['alternateIdentifiers'] = [{
'alternateIdentifierType': 'URL',
'alternateIdentifier': permalink
}]
except Exception as e:
errors['alternateIdentifiers'] = e
# RELATED IDENTIFIERS
# nothing relevant in default schema
# SIZES
# sum up given sizes from resources in the package and convert from bytes to kilobytes
try:
resource_sizes = [r.get('size') or 0 for r in pkg_dict.get('resources', []) or []]
total_size = [f'{int(sum(resource_sizes) / 1024)} kb']
optional['sizes'] = total_size
except Exception as e:
errors['sizes'] = e
# FORMATS
# list unique formats from package resources
try:
formats = list(
set(filter(None, [r.get('format') for r in pkg_dict.get('resources', []) or []])))
optional['formats'] = formats
except Exception as e:
errors['formats'] = e
# VERSION
# doesn't matter if there's no version, it'll get filtered out later
optional['version'] = pkg_dict.get('version')
# RIGHTS
# use the package license and get details from CKAN's license register
license_id = pkg_dict.get('license_id', 'notspecified')
try:
if license_id != 'notspecified' and license_id is not None:
license_register = Package.get_license_register()
license = license_register.get(license_id)
if license is not None:
optional['rightsList'] = [
{
'url': license.url,
'identifier': license.id
}
]
except Exception as e:
errors['rightsList'] = e
# DESCRIPTIONS
# use package notes
optional['descriptions'] = [
{
'descriptionType': 'Other',
'description': pkg_dict.get('notes', '')
}
]
# GEOLOCATIONS
# nothing relevant in default schema
# FUNDING
# nothing relevant in default schema
metadata_dict.update(required)
metadata_dict.update(optional)
for plugin in PluginImplementations(IDoi):
# implementations should remove relevant errors from the errors dict if they successfully
# handle an item
metadata_dict, errors = plugin.build_metadata_dict(pkg_dict, metadata_dict, errors)
for k in required:
if metadata_dict.get(k) is None and errors.get(k) is None:
errors[k] = DOIMetadataException('Required field cannot be None')
required_errors = {k: e for k, e in errors.items() if k in required}
if len(required_errors) > 0:
error_msg = f'Could not extract metadata for the following required keys: ' \
f'{", ".join(required_errors)}'
log.exception(error_msg)
for k, e in required_errors.items():
log.exception(f'{k}: {e}')
raise DOIMetadataException(error_msg)
optional_errors = {k: e for k, e in errors.items() if k in optional}
if len(required_errors) > 0:
error_msg = f'Could not extract metadata for the following optional keys: ' \
f'{", ".join(optional_errors)}'
log.debug(error_msg)
for k, e in optional_errors.items():
log.debug(f'{k}: {e}')
return metadata_dict
def list_license(self):
return base.render(
'home/cmp/license-list.html',
extra_vars={'licenses': Package.get_license_register().items()})
def list_license(self):
return base.render('home/cmp/license-list.html', extra_vars = {'licenses': Package.get_license_register().items()})
