def get_dataset_revision_list(dataset_name):
'''List all revisions in metastore-lib for the given dataset.
'''
backend = get_metastore_backend()
return [rev.revision for rev in backend.revision_list(dataset_name)]
def before_view(self, pkg_dict):
try:
revision_ref = toolkit.request.view_args['revision_ref']
except (AttributeError, KeyError):
# TODO: How to correctly access to a request arg in CKAN?
return pkg_dict
if get_metastore_backend().is_valid_revision_id(revision_ref):
releases = action.dataset_release_list(
{"ignore_auth": True}, {'dataset': pkg_dict['name']})
revision = filter(lambda d: d['revision_ref'] == revision_ref,
releases)[0]
else:
revision = action.dataset_release_show({"ignore_auth": True}, {
'dataset': pkg_dict['name'],
'release': revision_ref
})
# current_release needs to be a release (eg, name and description).
# This assumes that there is a release for the given revision
toolkit.c.current_release = revision
# Hide package creation / update date if viewing a specific release
pkg_dict['metadata_created'] = None
pkg_dict['metadata_updated'] = None
return pkg_dict
def dataset_release_update(context, data_dict):
"""Update a release of the current dataset.
:param dataset: the id or name of the dataset
:type dataset: string
:param release: the id of the release
:type release: string
:param name: A short name for the release
:type name: string
:param description: A description for the release
:type description: string
:returns: the edited release
:rtype: dictionary
"""
release, name, dataset_name_or_id = toolkit.get_or_bust(
data_dict, ['release', 'name', 'dataset'])
toolkit.check_access('dataset_release_create', context, data_dict)
assert context.get('auth_user_obj') # Should be here after `check_access`
backend = get_metastore_backend()
author = create_author_from_context(context)
try:
release_info = backend.tag_update(
_get_dataset_name(dataset_name_or_id),
release,
new_name=name,
new_description=data_dict.get('description', None),
author=author)
except exc.NotFound:
raise toolkit.ObjectNotFound("Dataset release not found.")
log.info('Release "%s" with id %s modified successfully', name, release)
return tag_to_dict(release_info)
def after_update(self, context, pkg_dict):
"""Updates the datapackage.json using metastore-lib backend.
After updating the package it calls metastore-lib to update the
datapackage.json file in the GitHub repository.
"""
if pkg_dict['type'] == 'dataset':
# We need to get a complete dict to also update resources data.
# We need to save tracking_summary, required for templates.
pkg_dict = toolkit.get_action('package_show')(
{}, {
'id': pkg_dict['id'],
'include_tracking': True
})
datapackage = dataset_to_frictionless(pkg_dict)
backend = get_metastore_backend()
author = create_author_from_context(context)
pkg_info = backend.update(pkg_dict['name'],
datapackage,
author=author)
log.info(
'Package {} updated correctly. Revision {} created.'.format(
pkg_info.package_id, pkg_info.revision))
return pkg_dict
def get_dataset_current_revision(dataset_name):
'''Get the current revision in metastore-lib for the given dataset.
# TODO: This shouldn't be necessary. It is only used in tests.
'''
backend = get_metastore_backend()
return backend.fetch(dataset_name).revision
def dataset_release_show(context, data_dict):
"""Get a specific release by ID
:param dataset: the name of the dataset
:type dataset: string
:param release: the id of the release
:type release: string
:returns: The matched release
:rtype: dict
"""
dataset_name, release = toolkit.get_or_bust(data_dict,
['dataset', 'release'])
backend = get_metastore_backend()
with exception_mapper(exc.NotFound, toolkit.ObjectNotFound):
release_info = backend.tag_fetch(dataset_name, release)
return tag_to_dict(release_info)
def dataset_release_create(context, data_dict):
"""Create a new release from the current dataset's revision
Currently you must have editor level access on the dataset
to create a release.
:param dataset: the id or name of the dataset
:type dataset: string
:param name: A short name for the release
:type name: string
:param description: A description for the release
:type description: string
:returns: the newly created release
:rtype: dictionary
"""
model = context.get('model', core_model)
dataset_id_or_name, name = toolkit.get_or_bust(data_dict,
['dataset', 'name'])
dataset = model.Package.get(dataset_id_or_name)
if not dataset:
raise toolkit.ObjectNotFound('Dataset not found')
toolkit.check_access('dataset_release_create', context, data_dict)
assert context.get('auth_user_obj') # Should be here after `check_access`
# TODO: Names like 'Version 1.2' are not allowed as Github tags
backend = get_metastore_backend()
author = create_author_from_context(context)
current_revision = backend.fetch(dataset.name)
try:
release_info = backend.tag_create(dataset.name,
current_revision.revision,
name,
description=data_dict.get(
'description', None),
author=author)
except exc.Conflict as e:
# Name not unique
log.debug("Release already exists: %s", e)
raise toolkit.ValidationError(
'Release names must be unique per dataset')
log.info('Release "%s" created for package %s', name, dataset.id)
return tag_to_dict(release_info)
def dataset_release_delete(context, data_dict):
"""Delete a specific release of a dataset
:param dataset: name of the dataset
:type dataset: string
:param release: the id of the release
:type release: string
:returns: The matched release
:rtype: dict
"""
dataset_name, release = toolkit.get_or_bust(data_dict,
['dataset', 'release'])
backend = get_metastore_backend()
try:
backend.tag_delete(dataset_name, release)
except Exception:
raise toolkit.ObjectNotFound('Dataset release not found')
log.info('Release %s of dataset %s was deleted', release, dataset_name)
def dataset_release_list(context, data_dict):
"""List releases of a given dataset
:param dataset: the id or name of the dataset
:type dataset: string
:returns: list of matched releases
:rtype: list
"""
model = context.get('model', core_model)
dataset_id_or_name = toolkit.get_or_bust(data_dict, ['dataset'])
dataset = model.Package.get(dataset_id_or_name)
if not dataset:
raise toolkit.ObjectNotFound('Dataset not found')
backend = get_metastore_backend()
with exception_mapper(exc.NotFound, toolkit.ObjectNotFound):
release_list = backend.tag_list(dataset.name)
return [tag_to_dict(t) for t in release_list]
def after_create(self, context, pkg_dict):
"""Creates a datapackage.json using metastore-lib backend.
After creating the package, it calls metastore-lib to create a new
GitHub repository a store the package dict in a datapackage.json file.
"""
if pkg_dict['type'] == 'dataset':
datapackage = dataset_to_frictionless(pkg_dict)
backend = get_metastore_backend()
author = create_author_from_context(context)
pkg_info = backend.create(pkg_dict['name'],
datapackage,
author=author)
log.info(
'Package {} created correctly. Revision {} created.'.format(
pkg_info.package_id, pkg_info.revision))
return pkg_dict
def _get_package_in_revision(context, data_dict, revision_id):
"""Internal implementation of package_show_revision
"""
result = core_package_show(context, data_dict)
if revision_id:
backend = get_metastore_backend()
dataset_name = _get_dataset_name(data_dict.get('id'))
pkg_info = backend.fetch(dataset_name, revision_id)
dataset = frictionless_to_dataset(pkg_info.package)
result = update_ckan_dict(result, dataset)
for resource in result.get('resources', []):
resource['datastore_active'] = False
_fix_resource_data(resource, revision_id)
# Fetching the license_url, title from the license registry and validate
if 'license_id' in result and result['license_id']:
license_data = h.get_license(result['license_id'])
# Validate license has url and title both
result['license_url'] = license_data.url if license_data.url else ''
result['license_title'] = license_data.title if license_data.title \
else ''
return result
def dataset_purge(next_action, context, data_dict):
"""Purge a dataset.
.. warning:: Purging a dataset cannot be undone!
This wraps the core ``dataset_purge`` action with code that also removes
the datapackage from metastore.
:param id: the name or id of the dataset to be purged
:type id: string
"""
# We do not check permissions as we rely on core action to check them
next_action(context, data_dict)
assert 'package' in context
backend = get_metastore_backend()
try:
backend.delete(context['package'].name)
except exc.NotFound as e:
log.warning(
"Dataset deleted from DB but not found in metastore: %s; "
"Error: %s", context['package'].id, e)