def close_alert(self, dataset_name, alert_id):
'''Delete an alert'''
if not toolkit.request.method == 'POST':
raise toolkit.abort(400, 'Expected POST method')
user = toolkit.c.userobj
if not user:
raise toolkit.NotAuthorized(
'You need to be authenticated to delete an alert')
alert = DatasetAlert.get(alert_id)
if not alert.can_close(user):
raise toolkit.NotAuthorized(
'You are not allowed to close this alert')
alert.close(user, toolkit.request.POST.get('comment'))
DB.add(alert)
DB.commit()
alert.notify_response()
return self.json_response({
'id': alert.id,
'user_id': alert.user_id,
'dataset_id': alert.dataset_id,
'type': alert.type,
'comment': alert.comment,
'created': alert.created,
'closed': alert.closed,
'closed_by_id': alert.closed_by_id,
'close_comment': alert.close_comment,
})
def search_sql(self, data_dict):
'''
:param sql: the sql statement that will be
executed against the previously configured backend
(required, e.g. 'SELECT * FROM table_name')
:rtype: dictionary
:param fields: fields/columns and their extra metadata
:type fields: list of dictionaries
:param records: list of matching results
:type records: list of dictionaries
'''
try:
engine = self._get_engine()
connection = engine.connect()
except OperationalError as e:
log.error(e)
raise ValidationError({
'connection_parameters':
[_('Unable to connect to Database, please check!')]
})
sql = data_dict['sql']
try:
connection.execute(
u'SET LOCAL statement_timeout TO {0}'.format(_TIMEOUT))
results = connection.execute(sql)
return format_results(results)
except ProgrammingError as e:
if e.orig.pgcode == _PG_ERR_CODE['permission_denied']:
raise toolkit.NotAuthorized({
'permissions': ['Not authorized to read resource.']
})
def _remove_explain(msg):
return (msg.replace('EXPLAIN (VERBOSE, FORMAT JSON) ', '')
.replace('EXPLAIN ', ''))
raise ValidationError({
'query': [_remove_explain(str(e))],
'info': {
'statement': [_remove_explain(e.statement)],
'params': [e.params],
'orig': [_remove_explain(str(e.orig))]
}
})
except DBAPIError as e:
if e.orig.pgcode == _PG_ERR_CODE['query_canceled']:
raise ValidationError({
'query': ['Query took too long']
})
raise
finally:
connection.close()
def get_download_authz_token(context,
org_name,
package_name,
resource_id,
activity_id=None):
# type: (Dict[str, Any], str, str, str, str) -> str
"""Get an authorization token for getting the download URL from LFS
"""
authorize = toolkit.get_action('authz_authorize')
if not authorize:
raise RuntimeError(
"Cannot find authz_authorize; Is ckanext-authz-service installed?")
scope = helpers.resource_authz_scope(package_name,
org_name=org_name,
actions='read',
resource_id=resource_id,
activity_id=activity_id)
log.debug("Requesting authorization token for scope: %s", scope)
authz_result = authorize(context, {"scopes": [scope]})
if not authz_result or not authz_result.get('token', False):
raise RuntimeError("Failed to get authorization token for LFS server")
log.debug("Granted scopes: %s", authz_result['granted_scopes'])
if len(authz_result['granted_scopes']) == 0:
raise toolkit.NotAuthorized(
"You are not authorized to download this resource")
return ensure_text(authz_result['token'])
def search_sql(context, data_dict):
engine = _get_engine(data_dict)
context['connection'] = engine.connect()
timeout = context.get('query_timeout', _TIMEOUT)
_cache_types(context)
try:
context['connection'].execute(
u'SET LOCAL statement_timeout TO {0}'.format(timeout))
results = context['connection'].execute(
data_dict['sql'].replace('%', '%%')
)
return format_results(context, results, data_dict)
except ProgrammingError, e:
if e.orig.pgcode == _PG_ERR_CODE['permission_denied']:
raise toolkit.NotAuthorized({
'permissions': ['Not authorized to read resource.']
})
raise ValidationError({
'query': [str(e)],
'info': {
'statement': [e.statement],
'params': [e.params],
'orig': [str(e.orig)]
}
})
def _make_public(data_dict, context, type='package'):
try:
id_or_name = data_dict['id']
except KeyError:
raise toolkit.ValidationError({'id': 'missing id'})
dataset_dict = toolkit.get_action('package_show')(context, {'id': id_or_name})
# Check authorization
package_id = dataset_dict.get('package_id', dataset_dict.get('id', id_or_name))
if not authz.is_authorized(
'package_update', context,
{'id': package_id}).get('success', False):
raise toolkit.NotAuthorized({
'permissions': ['Not authorized to publish the dataset.']})
# Check state
if dataset_dict.get('publication_state', False):
return {'success': False, 'error': 'Dataset publication state is not empty'}
# set as public
dataset_dict['private'] = False
toolkit.get_action('package_update')(context=context, data_dict=dataset_dict)
log.info("success making public package {0}".format(package_id))
return {'success': True, 'error': None}
def alert(self, dataset_name):
'''
Put an alert aka. a signalement on a dataset.
'''
if not toolkit.request.method == 'POST':
raise toolkit.abort(400, 'Expected POST method')
user = toolkit.c.userobj
if not user:
raise toolkit.NotAuthorized('Alert creation requires an user')
dataset = Package.by_name(dataset_name)
alert_type = toolkit.request.POST['type']
comment = toolkit.request.POST['comment']
alert = DatasetAlert(dataset, user, alert_type, comment)
DB.add(alert)
DB.commit()
alert.notify_admins()
return self.json_response({
'id': alert.id,
'user_id': alert.user_id,
'dataset_id': alert.dataset_id,
'type': alert.type,
'comment': alert.comment,
'created': alert.created
})
def user_show(context, data_dict):
'''Forbid anonymous access to user info.
'''
if context.get('user') or context.get('ignore_auth'):
return logic.action.get.user_show(context, data_dict)
else:
raise toolkit.NotAuthorized(
'You must be logged in to perform this action.')
def search_sql(context, data_dict):
engine = _get_engine(data_dict)
context['connection'] = engine.connect()
timeout = context.get('query_timeout', _TIMEOUT)
_cache_types(context)
sql = data_dict['sql'].replace('%', '%%')
try:
context['connection'].execute(
u'SET LOCAL statement_timeout TO {0}'.format(timeout))
table_names = datastore_helpers.get_table_names_from_sql(context, sql)
log.debug('Tables involved in input SQL: {0!r}'.format(table_names))
system_tables = [t for t in table_names if t.startswith('pg_')]
if len(system_tables):
raise toolkit.NotAuthorized({
'permissions': ['Not authorized to access system tables']
})
results = context['connection'].execute(sql)
return format_results(context, results, data_dict)
except ProgrammingError, e:
if e.orig.pgcode == _PG_ERR_CODE['permission_denied']:
raise toolkit.NotAuthorized({
'permissions': ['Not authorized to read resource.']
})
def _remove_explain(msg):
return (msg.replace('EXPLAIN (FORMAT JSON) ', '')
.replace('EXPLAIN ', ''))
raise ValidationError({
'query': [_remove_explain(str(e))],
'info': {
'statement': [_remove_explain(e.statement)],
'params': [e.params],
'orig': [_remove_explain(str(e.orig))]
}
})
def user_show(context, data_dict):
'''Forbid anonymous access to user info.
API call works with POST request with authorization header and id
of desired user supplied.
'''
if context.get('user'):
return logic.action.get.user_show(context, data_dict)
else:
raise toolkit.NotAuthorized(
'You must be logged in to perform this action.')
def update_short_url(context, data_dict):
"""
Update a short url. Raises validation error if any
:param context:
:param data_dict:
:return:
"""
if toolkit.config.get('ckanext.url_shortner_key', None) != data_dict.get(
'token_key', '').strip():
raise toolkit.NotAuthorized("No token_key parameter provided")
md = UrlShorten.update_entry(**data_dict)
return md.as_dict()
def delete_short_url(context, data_dict):
"""
Delete an entry (soft delete) or raises validation error if any
:param context:
:param data_dict:
:return:
"""
if toolkit.config.get('ckanext.url_shortner_key', None) != data_dict.get(
'token_key', '').strip():
raise toolkit.NotAuthorized("No token_key parameter provided")
_ = UrlShorten.delete_entry(**data_dict)
return {u"msg": u"Entry {} deleted".format(data_dict.get(u'id', ''))}
def membership_accept(self, request_id):
'''Accept a membership request'''
if not toolkit.request.method == 'POST':
raise toolkit.abort(400, 'Expected POST method')
user = toolkit.c.userobj
if not user:
raise toolkit.NotAuthorized(
'Membership validation requires an user')
membership_request = MembershipRequest.get(request_id)
membership = membership_request.accept(user)
return self.json_response(membership)
def _toggle_featured(self, reuse_id, featured=None):
'''
Mark or unmark a reuse as featured
'''
user = toolkit.c.userobj
if not user or not user.sysadmin:
raise toolkit.NotAuthorized()
reuse = Related.get(reuse_id)
reuse.featured = featured if featured is not None else (
0 if reuse.featured else 1)
self.commit()
return reuse
return self.json_response(reuse)
def membership_refuse(self, request_id):
'''Refuse a membership request'''
if not toolkit.request.method == 'POST':
raise toolkit.abort(400, 'Expected POST method')
user = toolkit.c.userobj
if not user:
raise toolkit.NotAuthorized(
'Membership validation requires an user')
comment = toolkit.request.params.get('comment')
membership_request = MembershipRequest.get(request_id)
membership_request.refuse(user, comment)
return self.json_response({})
def _approve(data_dict, context, type='package'):
log.debug('_approve: Approving "{0}" ({1})'.format(data_dict['id'], data_dict.get('name', '')))
# a dataset id i s necessary
try:
id_or_name = data_dict['id']
except KeyError:
raise toolkit.ValidationError({'id': 'missing id'})
dataset_dict = toolkit.get_action('package_show')(context, {'id': id_or_name})
# DOI has to be already reserved (minted)
doi = dataset_dict.get('doi', '')
default_prefix = config.get('datacite_publication.doi_prefix', '10.xxxxx')
allowed_prefixes = config.get('datacite_publication.custom_prefix', '').split(' ') + [default_prefix]
log.debug('_approve: Doi "{0}" ({1})'.format(doi, ', '.join(allowed_prefixes)))
doi_prefix = doi.split('/')[0].strip()
if (not doi) or (len(doi) <= 0) or (doi_prefix not in allowed_prefixes):
raise toolkit.ValidationError(
{'doi': 'dataset has no valid minted DOI [' + ', '.join(allowed_prefixes) + ']/*'})
# Check authorization
package_id = dataset_dict.get('package_id', dataset_dict.get('id', id_or_name))
if not authz.is_authorized(
'package_update', context,
{'id': package_id}).get('success', False) or not helpers.datacite_publication_is_admin():
log.error('ERROR approving dataset, current user is not authorized: isAdmin = {0}'.format(
helpers.datacite_publication_is_admin()))
raise toolkit.NotAuthorized({
'permissions': ['Not authorized to approve the dataset (admin only).']})
# change publication state
dataset_dict['publication_state'] = 'approved'
dataset_dict['private'] = False
context['message'] = APPROVAL_MESSAGE + " for dataset {0}".format(package_id)
toolkit.get_action('package_update')(context=context, data_dict=dataset_dict)
# save activity
_add_activity(dataset_dict, APPROVAL_MESSAGE, context)
# notify owner and involved users
dataset_owner = dataset_dict.get('creator_user_id', '')
datacite_approved_mail(dataset_owner, dataset_dict, context)
return {'success': True, 'error': None}
def check_access(self, action_name, context, data_dict=None):
"""
Check whether the user has the privilege to perform the named action,
before calling the core check_access function.
"""
check_auth = not context.get('ignore_auth', False)
if check_auth:
check_context = context.copy()
check_context.setdefault('session', check_context['model'].Session) # because ckan.lib.helpers.check_access does not add the session to the context *smh*
check_data_dict = {
'user_id': context['user'],
'action': action_name,
}
privilege_check = action.user_privilege_check(check_context, check_data_dict)
if not privilege_check['success']:
raise tk.NotAuthorized(privilege_check['msg'])
return self.core_check_access(action_name, context, data_dict)
def _request_token(user_id):
if toolkit.c.user:
# Don't offer the reset form if already logged in
log.warning("User already logged in {}".format(toolkit.c.user))
raise toolkit.NotAuthorized('user already logged in, logout first')
context = {'user': toolkit.c.user}
data_dict = {'id': user_id}
user_obj = None
try:
user_dict = toolkit.get_action('user_show')(context, data_dict)
user_obj = context['user_obj']
except logic.NotFound:
# Try searching the user
del data_dict['id']
data_dict['q'] = user_id
if user_id and len(user_id) > 2:
user_list = toolkit.get_action('user_list')(context, data_dict)
if len(user_list) == 1:
# This is ugly, but we need the user object for the mailer,
# and user_list does not return them
del data_dict['q']
data_dict['id'] = user_list[0]['id']
user_dict = toolkit.get_action('user_show')(context, data_dict)
user_obj = context['user_obj']
elif len(user_list) > 1:
raise logic.NotFound('"%s" matched several users' % user_id)
else:
raise logic.NotFound('No such user: %s' % user_id)
else:
raise logic.NotFound('No such user: %s' % user_id)
if user_obj:
try:
passwordless_send_reset_link(user_obj)
except mailer.MailerException as e:
log.error('Could not send token link: %s' % str(e))
raise mailer.MailerException(
'could not send token link by mail: %s' % str(e))
return
def _reset(context, data_dict):
# No one is logged in already
if toolkit.c.user:
log.warning("User already logged in {}".format(toolkit.c.user))
raise toolkit.NotAuthorized('user already logged in, logout first')
# Check email is present
try:
email = data_dict['email']
email = email.lower()
except KeyError:
raise toolkit.ValidationError({'email': 'missing email'})
# Check email is valid
if not util.check_email(email):
raise toolkit.ValidationError({'email': 'invalid email'})
# control attempts (exception raised on fail)
_check_reset_attempts(email.encode())
# get existing user from email
user = util.get_user(email)
# log.debug('passwordless_request_reset: USER is = ' + str(user))
if not user:
# A user with this email address doesn't yet exist in CKAN,
# so create one.
user = _create_user(email)
log.debug('passwordless_request_reset: created user = '******'state') == 'deleted':
raise toolkit.ValidationError(
{'user': '******'.format(email)})
# token request
_request_token(user.get('id'))
else:
raise toolkit.ValidationError(
{'user': '******'})
log.debug("controller redirecting: user.login, email = " + str(email))
return "reset successful"
def membership_request(self, org_name):
'''Request membership for an organization'''
if not toolkit.request.method == 'POST':
raise toolkit.abort(400, 'Expected POST method')
user = toolkit.c.userobj
if not user:
raise toolkit.NotAuthorized('Membership request requires an user')
organization = Group.by_name(org_name)
comment = toolkit.request.params.get('comment')
membership_request = MembershipRequest(user, organization, comment)
DB.add(membership_request)
DB.commit()
membership_request.notify_admins()
return self.json_response({})
def get_upload_authz_token(self, dataset_id):
# type: (str) -> str
"""Get an authorization token to upload the file to LFS
"""
authorize = toolkit.get_action('authz_authorize')
if not authorize:
raise RuntimeError(
"Cannot find authz_authorize; Is ckanext-authz-service installed?"
)
context = {'ignore_auth': True, 'auth_user_obj': self._user}
scope = helpers.resource_authz_scope(dataset_id, actions='write')
authz_result = authorize(context, {"scopes": [scope]})
if not authz_result or not authz_result.get('token', False):
raise RuntimeError(
"Failed to get authorization token for LFS server")
if len(authz_result['granted_scopes']) == 0:
raise toolkit.NotAuthorized(
"You are not authorized to upload resources")
return authz_result['token']
def _finish_manually(data_dict, context, type='package'):
log.debug('_finish_manually: Finishing "{0}" ({1})'.format(data_dict['id'], data_dict.get('name', '')))
# a dataset id i s necessary
try:
id_or_name = data_dict['id']
except KeyError:
raise toolkit.ValidationError({'id': 'missing id'})
dataset_dict = toolkit.get_action('package_show')(context, {'id': id_or_name})
# DOI has to be already reserved (minted)
if not dataset_dict.get('doi', None):
raise toolkit.ValidationError({'doi': 'dataset has no valid minted DOI'})
# Check authorization
package_id = dataset_dict.get('package_id', dataset_dict.get('id', id_or_name))
if not authz.is_authorized(
'package_update', context,
{'id': package_id}).get('success', False) or not helpers.datacite_publication_is_admin():
log.error('ERROR finishing publication dataset, current user is not authorized: isAdmin = {0}'.format(
helpers.datacite_publication_is_admin()))
raise toolkit.NotAuthorized({
'permissions': ['Not authorized to finish manually the dataset publication (admin only).']})
# change publication state
dataset_dict['publication_state'] = 'published'
dataset_dict['private'] = False
context['message'] = FINISH_MESSAGE + " for dataset {0}".format(package_id)
toolkit.get_action('package_update')(context=context, data_dict=dataset_dict)
# save activity
_add_activity(dataset_dict, FINISH_MESSAGE, context)
# notify owner and involved users
dataset_owner = dataset_dict.get('creator_user_id', '')
datacite_finished_mail(dataset_owner, dataset_dict, context)
return {'success': True, 'error': None}
def package_get_microdata_collections(context, data_dict):
default_error = 'Unknown microdata error'
# Check access
toolkit.check_access('sysadmin', context)
api_key = config.get('ckanext.unhcr.microdata_api_key')
if not api_key:
raise toolkit.NotAuthorized('Microdata API Key is not set')
try:
# Get collections
headers = {'X-Api-Key': api_key}
url = 'https://microdata.unhcr.org/index.php/api/collections'
response = requests.get(url, headers=headers).json()
if response.get('status') != 'success':
raise RuntimeError(str(response.get('errors', default_error)))
collections = response['collections']
except requests.exceptions.HTTPError:
log.exception(exception)
raise RuntimeError('Microdata connection failed')
return collections
def _login(context, data_dict):
if toolkit.c.user:
# Don't offer the reset form if already logged in
log.warning("User already logged in")
raise toolkit.NotAuthorized('user already logged in, logout first')
# Check if parameters are present
try:
user_id = data_dict.get('id')
if not user_id:
email = data_dict['email'].lower()
# Check email is valid
if not util.check_email(email):
raise toolkit.ValidationError({'email': 'invalid email'})
# get the user id
user_id = util.get_user_id(email)
if not user_id:
raise toolkit.ValidationError({
'email':
'email does not correspond to a registered user'
})
except KeyError:
raise toolkit.ValidationError({'email': 'missing email'})
try:
orig_key = data_dict['key']
except KeyError:
raise toolkit.ValidationError({'key': 'missing token'})
if len(orig_key) <= 32 and not orig_key.startswith("b'"):
key = "b'{0}'".format(orig_key)
else:
key = orig_key
log.debug('login: {0} ({1}) => {2}'.format(user_id, orig_key, key))
# get whether to return context (UI) or just a message (API)
return_context = data_dict.get('return_context', False)
try:
data_dict = {'id': user_id}
user_dict = logic.get_action('user_show')(context, data_dict)
user_obj = context['user_obj']
email = user_dict.get('email', user_obj.email)
except logic.NotFound:
raise logic.NotFound('"%s" matched several users' % user_id)
except toolkit.NotAuthorized:
raise toolkit.NotAuthorized('Exception (Not Authorized) email = ' +
str(email) + 'id = ' + str(user_id))
if not user_obj or not mailer.verify_reset_link(user_obj, key):
raise toolkit.ValidationError({'key': 'token provided is not valid'})
flask.session['ckanext-passwordless-user'] = user_dict['name']
# remove token
mailer.create_reset_key(user_obj)
# log the user in programmatically
try:
_set_repoze_user_only(user_dict['name'])
except TypeError as e:
log.warning("Exception at login: {0}".format(e))
# delete attempts from Redis
log.debug("Redis: reset attempts for {0}".format(email))
redis_conn = connect_to_redis()
redis_conn.delete(email)
# make sure the master API key exists
apikey = util.renew_master_token(user_dict['name'])
# return message or context
if return_context:
return context
else:
user_obj = context.get('user_obj', None)
result_json = {
'user': {
'email': user_obj.email,
'id': user_obj.id,
'name': user_obj.name,
'apikey': apikey,
'fullname': user_obj.fullname
},
'message': "login success"
}
return result_json
def fork(self, dataset_name):
'''
Fork this package by duplicating it.
The new owner will be the user parameter.
The new package is created and the original will have a new related reference to the fork.
'''
if not toolkit.request.method == 'POST':
raise toolkit.abort(400, 'Expected POST method')
user = toolkit.c.userobj
if not user:
raise toolkit.NotAuthorized('Membership request requires an user')
dataset = Package.by_name(dataset_name)
name_width = min(len(dataset.name), 88)
forked_name = '{name}-fork-{hash}'.format(
name=dataset.name[:name_width],
hash=str(uuid1())[:6],
)
orgs = user.get_groups('organization')
resources = [{
'url': r.url,
'description': r.description,
'format': r.format,
'name': r.name,
'resource_type': r.resource_type,
'mimetype': r.mimetype,
} for r in dataset.resources]
groups = [{'id': g.id} for g in dataset.get_groups()]
tags = [{
'name': t.name,
'vocabulary_id': t.vocabulary_id
} for t in dataset.get_tags()]
extras = [{
'key': key,
'value': value
} for key, value in dataset.extras.items() if key != 'supplier_id']
forked = toolkit.get_action('package_create')(
data_dict={
'name': forked_name,
'title': dataset.title,
'maintainer': user.fullname,
'maintainer_email': user.email,
'license_id': dataset.license_id,
'notes': dataset.notes,
'url': dataset.url,
'version': dataset.version,
'type': dataset.type,
'owner_org': orgs[0].id if len(orgs) else None,
'resources': resources,
'groups': groups,
'tags': tags,
'extras': extras,
})
# PackageRole.add_user_to_role(user, model.Role.ADMIN, forked)
# Manually add the groups to bypass CKAN authorization
# TODO: Find a better way to handle open groups
for group in dataset.get_groups():
group.add_package_by_name(forked_name)
self.commit()
# Create the fork relationship
toolkit.get_action('package_relationship_create')(
data_dict={
'type': 'has_derivation',
'subject': dataset.id,
'object': forked['id'],
'comment': 'Fork',
})
return self.json_response(Package.by_name(forked_name))
def package_publish_microdata(context, data_dict):
default_error = 'Unknown microdata error'
# Get data
dataset_id = data_dict.get('id')
nation = data_dict.get('nation')
repoid = data_dict.get('repoid')
# Check access
toolkit.check_access('sysadmin', context)
api_key = config.get('ckanext.unhcr.microdata_api_key')
if not api_key:
raise toolkit.NotAuthorized('Microdata API Key is not set')
# Get dataset/survey
headers = {'X-Api-Key': api_key}
dataset = toolkit.get_action('package_show')(context, {'id': dataset_id})
survey = helpers.convert_dataset_to_microdata_survey(
dataset, nation, repoid)
idno = survey['study_desc']['title_statement']['idno']
try:
# Publish dataset
url = 'https://microdata.unhcr.org/index.php/api/datasets/create/survey/%s' % idno
response = requests.post(url, headers=headers, json=survey).json()
if response.get('status') != 'success':
raise RuntimeError(str(response.get('errors', default_error)))
template = 'https://microdata.unhcr.org/index.php/catalog/%s'
survey['url'] = template % response['dataset']['id']
survey['resources'] = []
survey['files'] = []
# Pubish resources/files
file_name_counter = {}
if dataset.get('resources', []):
url = 'https://microdata.unhcr.org/index.php/api/datasets/%s/%s'
for resource in dataset.get('resources', []):
# resource
resouce_url = url % (idno, 'resources')
md_resource = helpers.convert_resource_to_microdata_resource(
resource)
response = requests.post(resouce_url,
headers=headers,
json=md_resource).json()
if response.get('status') != 'success':
raise RuntimeError(
str(response.get('errors', default_error)))
survey['resources'].append(response['resource'])
# file
file_url = url % (idno, 'files')
file_name = resource['url'].split('/')[-1]
file_path = helpers.get_resource_file_path(resource)
file_mime = resource['mimetype']
if not file_name or not file_path:
continue
file_name_counter.setdefault(file_name, 0)
file_name_counter[file_name] += 1
if file_name_counter[file_name] > 1:
file_name = helpers.add_file_name_suffix(
file_name, file_name_counter[file_name] - 1)
with open(file_path, 'rb') as file_obj:
file = (file_name, file_obj, file_mime)
response = requests.post(file_url,
headers=headers,
files={
'file': file
}).json()
# TODO: update
# it's a hack to overcome incorrect Microdata responses
# usopported file types fail this way and we are skipping them
if not isinstance(response, dict):
continue
if response.get('status') != 'success':
raise RuntimeError(
str(response.get('errors', default_error)))
survey['files'].append(response)
except requests.exceptions.HTTPError:
log.exception(exception)
raise RuntimeError('Microdata connection failed')
return survey
def _publish(data_dict, context, type='package'):
try:
id_or_name = data_dict['id']
except KeyError:
raise toolkit.ValidationError({'id': 'missing id'})
dataset_dict = toolkit.get_action('package_show')(context, {'id': id_or_name})
# Check authorization
package_id = dataset_dict.get('package_id', dataset_dict.get('id', id_or_name))
if not authz.is_authorized(
'package_update', context,
{'id': package_id}).get('success', False):
raise toolkit.NotAuthorized({
'permissions': ['Not authorized to publish the dataset.']})
# get user
ckan_user = _get_username_from_context(context)
# check if dataset has a DOI already
existing_doi = dataset_dict.get('doi')
if existing_doi:
if helpers.datacite_publication_is_admin(ckan_user):
return _publish_custom_by_admin(dataset_dict, package_id, ckan_user, context, type)
else:
return {'success': False,
'error': 'Dataset has already a DOI. Registering of custom DOI is currently not allowed'}
# Check state
if dataset_dict.get('publication_state', False):
return {'success': False, 'error': 'Dataset publication state should be empty to request a new DOI'}
# mint doi mint_doi(self, ckan_id, ckan_user, prefix_id = None, suffix = None, entity='package')
minter_name = config.get('datacite_publication.minter', DEAFULT_MINTER)
package_name, class_name = minter_name.rsplit('.', 1)
module = importlib.import_module(package_name)
minter_class = getattr(module, class_name)
minter = minter_class()
prefix = config.get('datacite_publication.doi_prefix', '10.xxxxx')
doi, error = minter.mint(prefix, pkg=dataset_dict, user=ckan_user)
log.debug("minter got doi={0}, error={1}".format(doi, error))
if doi:
# update dataset
dataset_dict['doi'] = doi
dataset_dict['private'] = False
# TODO: check what is the proper state once workflow is complete
# dataset_dict['publication_state'] = 'reserved'
dataset_dict['publication_state'] = 'pub_pending'
context['message'] = REQUEST_MESSAGE + " for dataset {0}".format(package_id)
toolkit.get_action('package_update')(context=context, data_dict=dataset_dict)
# save activity
_add_activity(dataset_dict, REQUEST_MESSAGE, context)
# notify admin and user
datacite_publication_requested_mail(ckan_user, dataset_dict)
log.info("success minting DOI for package {0}, doi {1}".format(package_id, doi))
return {'success': True, 'error': None}
else:
log.error("error minting DOI for package {0}, error{1}".format(package_id, error))
return {'success': False, 'error': error}
return {'success': False, 'error': 'Internal error'}
def config(context, data_dict):
if toolkit.check_access(constants.CONFIG_PRIVILEGE, context, data_dict):
return {'success': True}
else:
raise toolkit.NotAuthorized()
def _publish_to_datacite(data_dict, context, type='package'):
log.debug(
'_publish_to_datacite: Publishing to datacite "{0}" ({1})'.format(data_dict['id'], data_dict.get('name', '')))
# a dataset id i s necessary
try:
id_or_name = data_dict['id']
except KeyError:
raise toolkit.ValidationError({'id': 'missing id'})
dataset_dict = toolkit.get_action('package_show')(context, {'id': id_or_name})
# state has to be approved
state = dataset_dict.get('publication_state', '')
if state != 'approved':
raise toolkit.ValidationError({'publication_state': 'dataset needs to be in state "approved" (by the admin)'})
# DOI has to be already reserved (minted)
doi = dataset_dict.get('doi', '')
default_prefix = config.get('datacite_publication.doi_prefix', '10.xxxxx')
allowed_prefixes = config.get('datacite_publication.custom_prefix', '').split(' ') + [default_prefix]
doi_prefix = doi.split('/')[0].strip()
if (not doi) or (len(doi) <= 0) or (doi_prefix not in allowed_prefixes):
raise toolkit.ValidationError(
{'doi': 'dataset has no valid minted DOI [' + ', '.join(allowed_prefixes) + ']/*'})
# Check authorization
package_id = dataset_dict.get('package_id', dataset_dict.get('id', id_or_name))
if not authz.is_authorized(
'package_update', context,
{'id': package_id}).get('success', False) or not helpers.datacite_publication_is_admin():
log.error(
'ERROR finishing publication dataset in datacite, current user is not authorized: isAdmin = {0}'.format(
helpers.datacite_publication_is_admin()))
raise toolkit.NotAuthorized({
'permissions': ['Not authorized to perform the dataset publication to datacite (admin only).']})
datacite_publisher = DatacitePublisher()
try:
doi, error = datacite_publisher.publish(doi, pkg=dataset_dict, context=context)
except Exception as e:
log.error("exception publishing package {0} to Datacite, error {1}".format(package_id, traceback.format_exc()))
return {'success': False, 'error': 'Exception when publishing to DataCite: {0}'.format(e)}
except:
log.error("error publishing package {0} to Datacite, error {1}".format(package_id, sys.exc_info()[0]))
return {'success': False, 'error': 'Unknown error when publishing to DataCite: {0}'.format(sys.exc_info()[0])}
if error:
log.error("error publishing package {0} to Datacite, error {1}".format(package_id, error))
return {'success': False, 'error': error}
# change publication state
dataset_dict['publication_state'] = 'published'
dataset_dict['private'] = False
context['message'] = FINISH_MESSAGE + " for dataset {0}".format(package_id)
toolkit.get_action('package_update')(context=context, data_dict=dataset_dict)
# save activity
_add_activity(dataset_dict, FINISH_MESSAGE, context)
# notify owner and involved users
dataset_owner = dataset_dict.get('creator_user_id', '')
datacite_finished_mail(dataset_owner, dataset_dict, context)
return {'success': True, 'error': None}
def _update_in_datacite(data_dict, context, type='package'):
log.debug(
'_update_in_datacite: Updating in datacite "{0}" ({1})'.format(data_dict['id'], data_dict.get('name', '')))
# a dataset id i s necessary
try:
id_or_name = data_dict['id']
except KeyError:
raise toolkit.ValidationError({'id': 'missing id'})
dataset_dict = toolkit.get_action('package_show')(context, {'id': id_or_name})
# state has to be approved
state = dataset_dict.get('publication_state', '')
if state != 'published':
raise toolkit.ValidationError({'publication_state': 'dataset needs to be in state "published" (in datacite)'})
# DOI has to be already present
doi = dataset_dict.get('doi', '')
default_prefix = config.get('datacite_publication.doi_prefix', '10.xxxxx')
allowed_prefixes = config.get('datacite_publication.custom_prefix', '').split(' ') + [default_prefix]
doi_prefix = doi.split('/')[0].strip()
if (not doi) or (len(doi) <= 0) or (doi_prefix not in allowed_prefixes):
raise toolkit.ValidationError(
{'doi': 'dataset has no valid minted DOI [' + ', '.join(allowed_prefixes) + ']/*'})
# Check authorization
package_id = dataset_dict.get('package_id', dataset_dict.get('id', id_or_name))
if not authz.is_authorized(
'package_update', context,
{'id': package_id}).get('success', False) or not helpers.datacite_publication_is_admin():
log.error(
'ERROR updating publication dataset in datacite, current user is not authorized: isAdmin = {0}'.format(
helpers.datacite_publication_is_admin()))
raise toolkit.NotAuthorized({
'permissions': ['Not authorized to perform the dataset update to datacite (admin only).']})
# Get the DOI minter
minter_name = config.get('datacite_publication.minter', DEAFULT_MINTER)
package_name, class_name = minter_name.rsplit('.', 1)
module = importlib.import_module(package_name)
minter_class = getattr(module, class_name)
minter = minter_class()
# check when possible if the association doi-ckan id is valid:
log.debug("CHECK DOI in minter")
is_doi_valid_op = getattr(minter, "is_doi_valid", None)
if callable(is_doi_valid_op):
valid_doi = minter.is_doi_valid(doi, package_id)
if not valid_doi:
return {'success': False, 'error': 'DOI and id do not match to the DOI realisation table in the DB'}
datacite_publisher = DatacitePublisher()
try:
doi, error = datacite_publisher.publish(doi, pkg=dataset_dict, context=context, update=True)
except Exception as e:
log.error("exception updating package {0} in Datacite, error {1}".format(package_id, traceback.format_exc()))
return {'success': False, 'error': 'Exception when updating in DataCite: {0}'.format(e)}
except:
log.error("error updating package {0} in Datacite, error {1}".format(package_id, sys.exc_info()[0]))
return {'success': False, 'error': 'Unknown error when updating in DataCite: {0}'.format(sys.exc_info()[0])}
if error:
log.error("error updating package {0} to Datacite, error {1}".format(package_id, error))
return {'success': False, 'error': error}
# save activity
_add_activity(dataset_dict, UPDATE_MESSAGE, context)
return {'success': True, 'error': None}
def _publish_resource(data_dict, context):
# validate the id and get the resource data
try:
id = data_dict['id']
except KeyError:
raise toolkit.ValidationError({'id': 'missing id'})
resource_dict = toolkit.get_action('resource_show')(context, {'id': id})
package_id = data_dict.get('package_id')
package_dict = toolkit.get_action('package_show')(context, {'id': package_id})
# Check authorization
ckan_user = _get_username_from_context(context)
if not helpers.datacite_publication_is_admin(ckan_user):
raise toolkit.NotAuthorized({
'permissions': ['Not authorized to publish the resource (admins only).']})
# check if it is an update
update = data_dict.get('update', False)
log.debug('_publish_resource: *UPDATE* = {0}'.format(update))
# state to publish has to be not yet published
state = resource_dict.get('publication_state', '')
if not update and state == 'published':
return {'success': False, 'error': 'Resource is already in state "published"'}
# state to update has to be not published
if update and state != 'published':
return {'success': False, 'error': 'Resource is should be in state "published" for updating'}
# DOI has to be already present and valid
custom_doi = resource_dict.get('doi', '')
if (not custom_doi) or (len(custom_doi) <= 0):
log.error('_publish_resource: resource has no minted DOI')
return {'success': False, 'error': 'Custom DOI not valid: empty'}
default_prefix = config.get('datacite_publication.doi_prefix', '')
allowed_prefixes = config.get('datacite_publication.custom_prefix', '').split(' ') + [default_prefix]
custom_prefix = custom_doi.split('/')[0].strip()
if custom_prefix not in allowed_prefixes:
log.error('_publish_resource: resource has no valid minted DOI [' + ', '.join(allowed_prefixes) + ']/*')
return {'success': False, 'error': 'Custom DOI not valid: prefix not allowed'}
custom_suffix = custom_doi.split('/')[1].strip()
if (not custom_suffix) or (len(custom_suffix) <= 0):
log.error('_publish_resource: resource has an empty DOI suffix')
return {'success': False, 'error': 'Custom DOI not valid: suffix empty'}
if update:
log.info("updating CUSTOM resource DOI by an Admin {0}/{1}, allowed: {2}".format(custom_prefix, custom_suffix,
allowed_prefixes))
else:
log.info("publishing CUSTOM resource DOI by an Admin {0}/{1}, allowed: {2}".format(custom_prefix, custom_suffix,
allowed_prefixes))
# Get the DOI minter
minter_name = config.get('datacite_publication.minter', DEAFULT_MINTER)
package_name, class_name = minter_name.rsplit('.', 1)
module = importlib.import_module(package_name)
minter_class = getattr(module, class_name)
minter = minter_class()
if update:
doi = custom_doi
# check when possible if the association doi-ckan id is valid:
log.debug("CHECK DOI in minter")
is_doi_valid_op = getattr(minter, "is_doi_valid", None)
if callable(is_doi_valid_op):
valid_doi = minter.is_doi_valid(doi, id, entity_type='resource')
if not valid_doi:
return {'success': False, 'error': 'DOI and id do not match to the DOI realisation table in the DB'}
else:
# Mint custom DOI
doi, error = minter.mint(custom_prefix, pkg=resource_dict, user=ckan_user,
suffix=custom_suffix, entity='resource')
log.debug("minter got doi={0}, error={1}".format(doi, error))
if error:
log.error("error minting DOI for resource {0}, error{1}".format(id, error))
return {'success': False, 'error': error.split('DETAIL')[0]}
log.info("success saving custom minted DOI for resource {0}, doi {1}".format(id, doi))
# Publish to DataCite
datacite_publisher = DatacitePublisher()
try:
doi, error = datacite_publisher.publish_resource(doi, resource=resource_dict, package=package_dict,
context=context, update=update)
except Exception as e:
log.error("exception publishing resource {0} to Datacite, error {1}".format(id, traceback.format_exc()))
return {'success': False, 'error': 'Exception when publishing to DataCite: {0}'.format(e)}
except:
log.error("error publishing resource {0} to Datacite, error {1}".format(id, sys.exc_info()[0]))
return {'success': False, 'error': 'Unknown error when publishing to DataCite: {0}'.format(sys.exc_info()[0])}
if error:
log.error("error publishing resource {0} to Datacite, error {1}".format(id, error))
return {'success': False, 'error': error}
if not update:
# update dataset publication state
resource_patch = {'id': id, 'publication_state': 'published'}
resource_patched = toolkit.get_action('resource_patch')(context, resource_patch)
return {'success': True, 'error': None}
