def resource_validation_delete(context, data_dict):
u'''
Remove the validation job result for a particular resource.
It also deletes the underlying Validation object.
:param resource_id: id of the resource to remove validation from
:type resource_id: string
:rtype: None
'''
t.check_access(u'resource_validation_delete', context, data_dict)
if not data_dict.get(u'resource_id'):
raise t.ValidationError({u'resource_id': u'Missing value'})
session = context['model'].Session
validation = ValidationStatusHelper().getValidationJob(session, data_dict['resource_id'])
if not validation:
raise t.ObjectNotFound(
'No validation report exists for this resource')
ValidationStatusHelper().deleteValidationJob(session, validation)
def create_datarequest(original_action, context, data_dict):
"""
Action to create a new data request. The function checks the access rights
of the user before creating the data request. If the user is not allowed
a NotAuthorized exception will be risen.
In addition, you should note that the parameters will be checked and an
exception (ValidationError) will be risen if some of these parameters are
not valid.
Data QLD modification
Will send email notification to users of assigned organisation with admin access
:param title: The title of the data request
:type title: string
:param description: A brief description for your data request
:type description: string
:param organiztion_id: The ID of the organization you want to asign the
data request (optional).
:type organization_id: string
:returns: A dict with the data request (id, user_id, title, description,
organization_id, open_time, accepted_dataset, close_time, closed,
followers)
:rtype: dict
"""
model = context['model']
session = context['session']
# Init the data base
db.init_db(model)
# Check access
tk.check_access(constants.CREATE_DATAREQUEST, context, data_dict)
# Validate data
validator.validate_datarequest(context, data_dict)
# Store the data
data_req = db.DataRequest()
_undictize_datarequest_basic(data_req, data_dict)
data_req.user_id = context['auth_user_obj'].id
data_req.open_time = datetime.datetime.utcnow()
session.add(data_req)
session.commit()
datarequest_dict = _dictize_datarequest(data_req)
if datarequest_dict['organization']:
# Data QLD modification
users = _get_admin_users_from_organisation(datarequest_dict)
users.discard(context['auth_user_obj'].id)
_send_mail(users, 'new_datarequest_organisation', datarequest_dict,
'Data Request Created Email')
return datarequest_dict
def resource_validation_show(context, data_dict):
u'''
Display the validation job result for a particular resource.
Returns a validation object, including the validation report or errors
and metadata about the validation like the timestamp and current status.
Validation status can be one of:
* `created`: The validation job is in the processing queue
* `running`: Validation is under way
* `error`: There was an error while performing the validation, eg the file
could not be downloaded or there was an error reading it
* `success`: Validation was performed, and no issues were found
* `failure`: Validation was performed, and there were issues found
:param resource_id: id of the resource to validate
:type resource_id: string
:rtype: dict
'''
t.check_access(u'resource_validation_show', context, data_dict)
if not data_dict.get(u'resource_id'):
raise t.ValidationError({u'resource_id': u'Missing value'})
session = context['model'].Session
validation = ValidationStatusHelper().getValidationJob(session, data_dict['resource_id'])
if not validation:
raise t.ObjectNotFound(
'No validation report exists for this resource')
return _validation_dictize(validation)
def open_datarequest(id):
data_dict = {'id': id}
context = _get_context()
# Basic initialization
c.datarequest = {}
try:
check_access(OPEN_DATAREQUEST, context, data_dict)
c.datarequest = get_action(SHOW_DATAREQUEST)(context, data_dict)
if c.datarequest.get('closed', False) is False:
return abort(403, _('This data request is already open'))
else:
data_dict = {}
data_dict['id'] = id
data_dict['organization_id'] = c.datarequest.get('organization_id')
get_action(OPEN_DATAREQUEST)(context, data_dict)
return redirect_to(url_for('datarequest.show', id=data_dict['id']))
except ValidationError as e:
log.warn(e)
errors_summary = _get_errors_summary(e.error_dict)
return abort(403, errors_summary)
except ObjectNotFound as e:
log.warn(e)
return abort(404, _('Data Request %s not found') % id)
except NotAuthorized as e:
log.warn(e)
return abort(
403, _('You are not authorized to open the Data Request %s' % id))
def organization_index(self):
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True,
'with_private': False}
data_dict = {'all_fields': True}
try:
check_access('site_read', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
# pass user info to context as needed to view private datasets of
# orgs correctly
if c.userobj:
context['user_id'] = c.userobj.id
context['user_is_admin'] = c.userobj.sysadmin
results = get_action('organization_list')(context, data_dict)
def org_key(org):
title = org['title'].split(' | ')[-1 if c.language == 'fr' else 0]
return normalize_strip_accents(title)
results.sort(key=org_key)
c.page = Page(
collection=results,
page=request.params.get('page', 1),
url=h.pager_url,
items_per_page=1000
)
return render('organization/index.html')
def delete_view(id):
if 'cancel' in tk.request.params:
tk.redirect_to('showcase_blueprint.edit' if tk.check_ckan_version(
min_version='2.9.0') else 'showcase_edit',
id=id)
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj
}
try:
tk.check_access('ckanext_showcase_delete', context, {'id': id})
except tk.NotAuthorized:
return tk.abort(401, _('Unauthorized to delete showcase'))
if tk.check_ckan_version(min_version='2.9.0'):
index_route = 'showcase_blueprint.index'
else:
index_route = 'showcase_index'
try:
if tk.request.method == 'POST':
tk.get_action('ckanext_showcase_delete')(context, {'id': id})
h.flash_notice(_('Showcase has been deleted.'))
return tk.redirect_to(index_route)
c.pkg_dict = tk.get_action('package_show')(context, {'id': id})
except tk.NotAuthorized:
tk.abort(401, _('Unauthorized to delete showcase'))
except tk.ObjectNotFound:
tk.abort(404, _('Showcase not found'))
return tk.render('showcase/confirm_delete.html',
extra_vars={'dataset_type': DATASET_TYPE_NAME})
def resource_validation_delete(context, data_dict):
u'''
Remove the validation job result for a particular resource.
It also deletes the underlying Validation object.
:param resource_id: id of the resource to remove validation from
:type resource_id: string
:rtype: None
'''
t.check_access(u'resource_validation_delete', context, data_dict)
if not data_dict.get(u'resource_id'):
raise t.ValidationError({u'resource_id': u'Missing value'})
Session = context['model'].Session
try:
validation = Session.query(Validation).filter(
Validation.resource_id == data_dict['resource_id']).one()
except NoResultFound:
validation = None
if not validation:
raise t.ObjectNotFound('No validation report exists for this resource')
Session.delete(validation)
Session.commit()
def organization_index(self):
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'for_view': True,
'with_private': False
}
data_dict = {'all_fields': True}
try:
check_access('site_read', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
# pass user info to context as needed to view private datasets of
# orgs correctly
if c.userobj:
context['user_id'] = c.userobj.id
context['user_is_admin'] = c.userobj.sysadmin
results = get_action('organization_list')(context, data_dict)
def org_key(org):
title = org['title'].split(' | ')[-1 if c.language == 'fr' else 0]
return normalize_strip_accents(title)
results.sort(key=org_key)
c.page = Page(collection=results,
page=request.params.get('page', 1),
url=h.pager_url,
items_per_page=1000)
return render('organization/index.html')
def check_user_access(permission, context=None):
data_dict = {
'permission': permission
}
check_access(
'has_user_permission_for_some_org',
context if context else get_context(),
data_dict
)
def check_user_org_access(org_id, permission='create_dataset'):
data_dict = {
'org_id': org_id,
'permission': permission
}
check_access(
'has_user_permission_for_org',
get_context(),
data_dict
)
def resource_validation_run(context, data_dict):
u'''
Start a validation job against a resource.
Returns the identifier for the job started.
Note that the resource format must be one of the supported ones,
currently CSV or Excel.
:param resource_id: id of the resource to validate
:type resource_id: string
:rtype: string
'''
t.check_access(u'resource_validation_run', context, data_dict)
resource_id = data_dict.get(u'resource_id')
if not resource_id:
raise t.ValidationError({u'resource_id': u'Missing value'})
resource = t.get_action(u'resource_show')(
{}, {u'id': resource_id})
# TODO: limit to sysadmins
async_job = data_dict.get(u'async', True)
# Ensure format is supported
if not resource.get(u'format', u'').lower() in settings.SUPPORTED_FORMATS:
raise t.ValidationError(
{u'format': u'Unsupported resource format.'
u'Must be one of {}'.format(
u','.join(settings.SUPPORTED_FORMATS))})
# Ensure there is a URL or file upload
if not resource.get(u'url') and not resource.get(u'url_type') == u'upload':
raise t.ValidationError(
{u'url': u'Resource must have a valid URL or an uploaded file'})
# Check if there was an existing validation for the resource
try:
session = context['model'].Session
ValidationStatusHelper().createValidationJob(session, resource_id)
except ValidationJobAlreadyEnqueued:
if async_job:
log.error("resource_validation_run: ValidationJobAlreadyEnqueued %s", data_dict['resource_id'])
return
if async_job:
package_id = resource['package_id']
enqueue_validation_job(package_id, resource_id)
else:
run_validation_job(resource)
def resource_create(context, data_dict):
'''Appends a new resource to a datasets list of resources.
This is duplicate of the CKAN core resource_create action, with just the
addition of a synchronous data validation step.
This is of course not ideal but it's the only way right now to hook
reliably into the creation process without overcomplicating things.
Hopefully future versions of CKAN will incorporate more flexible hook
points that will allow a better approach.
'''
model = context['model']
package_id = t.get_or_bust(data_dict, 'package_id')
if not data_dict.get('url'):
data_dict['url'] = ''
pkg_dict = t.get_action('package_show')(dict(context, return_type='dict'),
{
'id': package_id
})
t.check_access('resource_create', context, data_dict)
for plugin in plugins.PluginImplementations(plugins.IResourceController):
plugin.before_create(context, data_dict)
if 'resources' not in pkg_dict:
pkg_dict['resources'] = []
upload = uploader.get_resource_uploader(data_dict)
if 'mimetype' not in data_dict:
if hasattr(upload, 'mimetype'):
data_dict['mimetype'] = upload.mimetype
if 'size' not in data_dict:
if hasattr(upload, 'filesize'):
data_dict['size'] = upload.filesize
pkg_dict['resources'].append(data_dict)
try:
context['defer_commit'] = True
context['use_cache'] = False
t.get_action('package_update')(context, pkg_dict)
context.pop('defer_commit')
except t.ValidationError, e:
try:
raise t.ValidationError(e.error_dict['resources'][-1])
except (KeyError, IndexError):
raise t.ValidationError(e.error_dict)
def archive_download(id, resource_id, filename=None):
"""
Provides a direct download by either redirecting the user to the url
stored or downloading an uploaded file directly.
"""
context = {
'model': model,
'session': model.Session,
'user': c.user,
'auth_user_obj': c.userobj
}
try:
resource = toolkit.get_action('resource_show')(context, {
'id': resource_id
})
# Quick auth check to ensure you can access this resource
toolkit.check_access('package_show', context, {'id': id})
except (toolkit.ObjectNotFound, toolkit.NotAuthorized):
return toolkit.abort(404, _('Resource not found'))
# Archived files are only links not uploads
if resource.get('url_type') != 'upload':
# Return the key used for this resource in storage.
#
# Keys are in the form:
# <uploaderpath>/<upload_to>/<2 char from resource id >/<resource id>/<filename>
#
# e.g.:
# my_storage_path/archive/16/165900ba-3c60-43c5-9e9c-9f8acd0aa93f/data.csv
relative_archive_path = os.path.join(resource['id'][:2],
resource['id'])
# try to get a file name from the url
parsed_url = urlparse.urlparse(resource.get('url'))
try:
file_name = parsed_url.path.split('/')[-1] or 'resource'
file_name = file_name.strip() # trailing spaces cause problems
file_name = file_name.encode('ascii',
'ignore') # e.g. u'\xa3' signs
except Exception:
file_name = "resource"
try:
upload = uploader.get_uploader(
os.path.join('archive', relative_archive_path))
return upload.download(file_name)
except OSError:
# includes FileNotFoundError
return toolkit.abort(404, _('Resource data not found'))
return toolkit.abort(404, _('No download is available'))
def workflow_state_change(context, data_dict):
"""Update state of the dataset.
The only mandatory key in `data_dict` is `id`, which represents
updated package. Rest of items will be passed to the
`State.change` method.
:param id: id of updated package.
:type id: str
"""
id = tk.get_or_bust(data_dict, "id")
tk.check_access("workflow_state_change", context, data_dict)
pkg = tk.get_action("package_show")(context.copy(), {"id": id})
state = tk.h.workflow_get_state(pkg)
state = state.change(data_dict)
return state.save(context.copy())
def register(self, data=None, errors=None, error_summary=None):
'''GET to display a form for registering a new user.
or POST the form data to actually do the user registration.
The bulk of this code is pulled directly from
ckan/controlllers/user.py
'''
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'schema': schema.user_new_form_schema(),
'save': 'save' in request.params
}
try:
check_access('user_create', context)
except NotAuthorized:
abort(401, _('Unauthorized to create a user'))
if context['save'] and not data:
try:
return self._save_new(context)
except HTTPFound:
# redirected after successful user create
notify_ckan_user_create(
email=request.params.get('email', ''),
fullname=request.params.get('fullname', ''),
username=request.params.get('name', ''),
phoneno=request.params.get('phoneno', ''),
dept=request.params.get('department', ''))
notice_no_access()
raise
if c.user and not data and not is_sysadmin(c.user):
# #1799 Don't offer the registration form if already logged in
return render('user/logout_first.html')
data = data or {}
errors = errors or {}
error_summary = error_summary or {}
d = {'data': data, 'errors': errors, 'error_summary': error_summary}
c.is_sysadmin = is_sysadmin(c.user)
c.form = render('user/new_user_form.html', extra_vars=d)
return render('user/new.html')
def remove_showcase_admin():
'''
Remove a user from the Showcase Admin list.
'''
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author
}
try:
tk.check_access('sysadmin', context, {})
except tk.NotAuthorized:
return tk.abort(401, _('User not authorized to view page'))
form_data = tk.request.form if tk.check_ckan_version(
'2.9') else tk.request.params
if tk.check_ckan_version(min_version='2.9.0'):
admins_route = 'showcase_blueprint.admins'
else:
admins_route = 'showcase_admins'
if 'cancel' in form_data:
return tk.redirect_to(admins_route)
user_id = tk.request.params['user']
if tk.request.method == 'POST' and user_id:
user_id = tk.request.params['user']
try:
tk.get_action('ckanext_showcase_admin_remove')(data_dict={
'username': user_id
})
except tk.NotAuthorized:
return tk.abort(401, _('Unauthorized to perform that action'))
except tk.ObjectNotFound:
h.flash_error(_('The user is not a Showcase Admin'))
else:
h.flash_success(_('The user is no longer a Showcase Admin'))
return tk.redirect_to(h.url_for(admins_route))
c.user_dict = tk.get_action('user_show')(data_dict={'id': user_id})
c.user_id = user_id
return tk.render('admin/confirm_remove_showcase_admin.html')
def check_edit_view_auth(id):
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj,
'save': 'save' in tk.request.params,
'moderated': tk.config.get('moderated'),
'pending': True
}
try:
tk.check_access('ckanext_showcase_update', context)
except tk.NotAuthorized:
return tk.abort(
401,
_('User not authorized to edit {showcase_id}').format(
showcase_id=id))
def check_new_view_auth():
context = {
'model': model,
'session': model.Session,
'user': tk.c.user or tk.c.author,
'auth_user_obj': tk.c.userobj,
'save': 'save' in tk.request.params
}
# Check access here, then continue with PackageController.new()
# PackageController.new will also check access for package_create.
# This is okay for now, while only sysadmins can create Showcases, but
# may not work if we allow other users to create Showcases, who don't
# have access to create dataset package types. Same for edit below.
try:
tk.check_access('ckanext_showcase_create', context)
except tk.NotAuthorized:
return tk.abort(401, _('Unauthorized to create a package'))
def show_schema(dataset_id, resource_id):
data_dict = {'id': resource_id}
context = _get_context()
try:
check_access(RESOURCE_SHOW, context, data_dict)
resource = get_action(RESOURCE_SHOW)(context, data_dict)
schema_data = resource.get('schema')
c.schema_data = json.dumps(schema_data, indent=2, sort_keys=True)
return render('schema/show.html')
except ObjectNotFound as e:
log.warn(e)
return abort(404, _('Resource %s not found') % resource_id)
except NotAuthorized as e:
log.warn(e)
return abort(
403,
_('You are not authorized to view the Data Scheme for the resource %s'
% resource_id))
def register(self, data=None, errors=None, error_summary=None):
'''GET to display a form for registering a new user.
or POST the form data to actually do the user registration.
The bulk of this code is pulled directly from
ckan/controlllers/user.py
'''
context = {'model': model, 'session': model.Session,
'user': c.user or c.author,
'schema': schema.user_new_form_schema(),
'save': 'save' in request.params}
try:
check_access('user_create', context)
except NotAuthorized:
abort(401, _('Unauthorized to create a user'))
if context['save'] and not data:
try:
return self._save_new(context)
except HTTPFound:
# redirected after successful user create
notify_ckan_user_create(
email=request.params.get('email', ''),
fullname=request.params.get('fullname', ''),
username=request.params.get('name', ''),
phoneno=request.params.get('phoneno', ''),
dept=request.params.get('department', ''))
notice_no_access()
raise
if c.user and not data and not is_sysadmin(c.user):
# #1799 Don't offer the registration form if already logged in
return render('user/logout_first.html')
data = data or {}
errors = errors or {}
error_summary = error_summary or {}
d = {'data': data, 'errors': errors, 'error_summary': error_summary}
c.is_sysadmin = is_sysadmin(c.user)
c.form = render('user/new_user_form.html', extra_vars=d)
return render('user/new.html')
def manage_showcase_admins():
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author
}
try:
tk.check_access('sysadmin', context, {})
except tk.NotAuthorized:
return tk.abort(401, _('User not authorized to view page'))
form_data = tk.request.form if tk.check_ckan_version(
'2.9') else tk.request.params
if tk.check_ckan_version(min_version='2.9.0'):
admins_route = 'showcase_blueprint.admins'
else:
admins_route = 'showcase_admins'
# We're trying to add a user to the showcase admins list.
if tk.request.method == 'POST' and form_data['username']:
username = form_data['username']
try:
tk.get_action('ckanext_showcase_admin_add')(data_dict={
'username': username
})
except tk.NotAuthorized:
abort(401, _('Unauthorized to perform that action'))
except tk.ObjectNotFound:
h.flash_error(
_("User '{user_name}' not found.").format(user_name=username))
except tk.ValidationError as e:
h.flash_notice(e.error_summary)
else:
h.flash_success(_("The user is now a Showcase Admin"))
return tk.redirect_to(h.url_for(admins_route))
c.showcase_admins = tk.get_action('ckanext_showcase_admin_list')()
return tk.render('admin/manage_showcase_admins.html')
def ctp_list_types(context, data_dict):
with_lables = tk.asbool(data_dict.get("with_labels"))
tk.check_access("ctp_list_types", context, data_dict)
if _use_scheming():
types = _get_scheming_types()
else:
types = _get_native_types()
result = list(
set(types).union(_additional_types()).difference(_exclude_types()))
if with_lables:
labels = _labels_from_config()
result = sorted(
[{
"name": t,
"label": labels.get(t) or tk._(t)
} for t in result],
key=itemgetter("label"),
)
return result
def resource_validation_run_batch(context, data_dict):
u'''
Start asynchronous data validation on the site resources. If no
options are provided it will run validation on all resources of
the supported formats (`ckanext.validation.formats`). You can
specify particular datasets to run the validation on their
resources. You can also pass arbitrary search parameters to filter
the selected datasets.
Only sysadmins are allowed to run this action.
Examples::
curl -X POST http://localhost:5001/api/action/resource_validation_run_batch \
-d '{"dataset_ids": "ec9bfd88-f90a-45ca-b024-adc8854b49bd"}' \
-H Content-type:application/json \
-H Authorization:API_KEY
curl -X POST http://localhost:5001/api/action/resource_validation_run_batch \
-d '{"dataset_ids": ["passenger-data-2018", "passenger-data-2017]}}' \
-H Content-type:application/json \
-H Authorization:API_KEY
curl -X POST http://localhost:5001/api/action/resource_validation_run_batch \
-d '{"query": {"fq": "res_format:XLSX"}}' \
-H Content-type:application/json \
-H Authorization:API_KEY
:param dataset_ids: Run data validation on all resources for a
particular dataset or datasets. Not to be used with ``query``.
:type dataset_ids: string or list
:param query: Extra search parameters that will be used for getting
the datasets to run validation on. It must be a JSON object like
the one used by the `package_search` API call. Supported fields
are ``q``, ``fq`` and ``fq_list``. Check the documentation for
examples. Note that when using this you will have to specify
the resource formats to target your Not to be used with
``dataset_ids``.
:type query: dict
:rtype: string
'''
t.check_access(u'resource_validation_run_batch', context, data_dict)
page = 1
page_size = 100
count_resources = 0
dataset_ids = data_dict.get('dataset_ids')
if isinstance(dataset_ids, basestring):
try:
dataset_ids = json.loads(dataset_ids)
except ValueError as e:
dataset_ids = [dataset_ids]
search_params = data_dict.get('query')
if isinstance(search_params, basestring):
try:
search_params = json.loads(search_params)
except ValueError as e:
msg = 'Error parsing search parameters'.format(search_params)
return {'output': msg}
while True:
query = _search_datasets(page,
page_size=page_size,
dataset_ids=dataset_ids,
search_params=search_params)
if page == 1 and query['count'] == 0:
msg = 'No suitable datasets for validation'
return {'output': msg}
if query['results']:
for dataset in query['results']:
if not dataset.get('resources'):
continue
for resource in dataset['resources']:
if (not resource.get(u'format', u'').lower()
in settings.SUPPORTED_FORMATS):
continue
try:
t.get_action(u'resource_validation_run')(
{
u'ignore_auth': True
}, {
u'resource_id': resource['id'],
u'async': True
})
count_resources += 1
except t.ValidationError as e:
log.warning(
u'Could not run validation for resource {} ' +
u'from dataset {}: {}'.format(
resource['id'], dataset['name'], str(e)))
if len(query['results']) < page_size:
break
page += 1
else:
break
msg = 'Done. {} resources sent to the validation queue'.format(
count_resources)
log.info(msg)
return {'output': msg}
def resource_validation_run(context, data_dict):
u'''
Start a validation job against a resource.
Returns the identifier for the job started.
Note that the resource format must be one of the supported ones,
currently CSV or Excel.
:param resource_id: id of the resource to validate
:type resource_id: string
:rtype: string
'''
t.check_access(u'resource_validation_run', context, data_dict)
if not data_dict.get(u'resource_id'):
raise t.ValidationError({u'resource_id': u'Missing value'})
resource = t.get_action(u'resource_show')({}, {
u'id': data_dict[u'resource_id']
})
# TODO: limit to sysadmins
async_job = data_dict.get(u'async', True)
# Ensure format is supported
if not resource.get(u'format', u'').lower() in settings.SUPPORTED_FORMATS:
raise t.ValidationError({
u'format':
u'Unsupported resource format.' +
u'Must be one of {}'.format(u','.join(settings.SUPPORTED_FORMATS))
})
# Ensure there is a URL or file upload
if not resource.get(u'url') and not resource.get(u'url_type') == u'upload':
raise t.ValidationError(
{u'url': u'Resource must have a valid URL or an uploaded file'})
# Check if there was an existing validation for the resource
Session = context['model'].Session
try:
validation = Session.query(Validation).filter(
Validation.resource_id == data_dict['resource_id']).one()
except NoResultFound:
validation = None
if validation:
# Reset values
validation.finished = None
validation.report = None
validation.error = None
validation.created = datetime.datetime.utcnow()
validation.status = u'created'
else:
validation = Validation(resource_id=resource['id'])
Session.add(validation)
Session.commit()
if async_job:
enqueue_job(run_validation_job, [resource])
else:
run_validation_job(resource)
def auth_resource_validation_show(context, data_dict):
if t.check_access(u'resource_show', context,
{u'id': data_dict[u'resource_id']}):
return {u'success': True}
return {u'success': False}
def resource_update(context, data_dict):
'''Update a resource.
This is duplicate of the CKAN core resource_update action, with just the
addition of a synchronous data validation step.
This is of course not ideal but it's the only way right now to hook
reliably into the creation process without overcomplicating things.
Hopefully future versions of CKAN will incorporate more flexible hook
points that will allow a better approach.
'''
model = context['model']
id = t.get_or_bust(data_dict, "id")
if not data_dict.get('url'):
data_dict['url'] = ''
resource = model.Resource.get(id)
context["resource"] = resource
old_resource_format = resource.format
if not resource:
log.debug('Could not find resource %s', id)
raise t.ObjectNotFound(t._('Resource was not found.'))
t.check_access('resource_update', context, data_dict)
del context["resource"]
package_id = resource.package.id
pkg_dict = t.get_action('package_show')(dict(context, return_type='dict'),
{
'id': package_id
})
for n, p in enumerate(pkg_dict['resources']):
if p['id'] == id:
break
else:
log.error('Could not find resource %s after all', id)
raise t.ObjectNotFound(t._('Resource was not found.'))
# Persist the datastore_active extra if already present and not provided
if ('datastore_active' in resource.extras
and 'datastore_active' not in data_dict):
data_dict['datastore_active'] = resource.extras['datastore_active']
for plugin in plugins.PluginImplementations(plugins.IResourceController):
plugin.before_update(context, pkg_dict['resources'][n], data_dict)
upload = uploader.get_resource_uploader(data_dict)
if 'mimetype' not in data_dict:
if hasattr(upload, 'mimetype'):
data_dict['mimetype'] = upload.mimetype
if 'size' not in data_dict and 'url_type' in data_dict:
if hasattr(upload, 'filesize'):
data_dict['size'] = upload.filesize
pkg_dict['resources'][n] = data_dict
try:
context['defer_commit'] = True
context['use_cache'] = False
updated_pkg_dict = t.get_action('package_update')(context, pkg_dict)
context.pop('defer_commit')
except t.ValidationError, e:
try:
raise t.ValidationError(e.error_dict['resources'][-1])
except (KeyError, IndexError):
raise t.ValidationError(e.error_dict)
def update_pd_record(self, owner_org, resource_name, pk):
pk = [url_part_unescape(p) for p in pk.split(',')]
lc = LocalCKAN(username=c.user)
try:
chromo = h.recombinant_get_chromo(resource_name)
rcomb = lc.action.recombinant_show(
owner_org=owner_org,
dataset_type=chromo['dataset_type'])
[res] = [r for r in rcomb['resources'] if r['name'] == resource_name]
check_access(
'datastore_upsert',
{'user': c.user, 'auth_user_obj': c.userobj},
{'resource_id': res['id']})
except NotAuthorized:
abort(403, _('Unauthorized'))
choice_fields = {
f['datastore_id']: [
{'value': k, 'label': v} for (k, v) in f['choices']]
for f in h.recombinant_choice_fields(resource_name)}
pk_fields = aslist(chromo['datastore_primary_key'])
pk_filter = dict(zip(pk_fields, pk))
records = lc.action.datastore_search(
resource_id=res['id'],
filters=pk_filter)['records']
if len(records) == 0:
abort(404, _('Not found'))
if len(records) > 1:
abort(400, _('Multiple records found'))
record = records[0]
if request.method == 'POST':
post_data = parse_params(request.POST, ignore_keys=['save'] + pk_fields)
if 'cancel' in post_data:
return redirect(h.url_for(
controller='ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=resource_name,
owner_org=rcomb['owner_org'],
))
data = {}
for f in chromo['fields']:
f_id = f['datastore_id']
if not f.get('import_template_include', True):
continue
if f_id in pk_fields:
data[f_id] = record[f_id]
else:
val = post_data.get(f['datastore_id'], '')
if isinstance(val, list):
val = u','.join(val)
val = canonicalize(
val,
f['datastore_type'],
primary_key=False,
choice_field=f_id in choice_fields)
data[f['datastore_id']] = val
try:
lc.action.datastore_upsert(
resource_id=res['id'],
#method='update', FIXME not raising ValidationErrors
records=[data])
except ValidationError as ve:
err = {
k: [_(e) for e in v]
for (k, v) in ve.error_dict['records'][0].items()}
return render('recombinant/update_pd_record.html',
extra_vars={
'data': data,
'resource_name': resource_name,
'chromo_title': chromo['title'],
'choice_fields': choice_fields,
'pk_fields': pk_fields,
'owner_org': rcomb['owner_org'],
'errors': err,
})
h.flash_notice(_(u'Record %s Updated') % u','.join(pk) )
return redirect(h.url_for(
controller='ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=resource_name,
owner_org=rcomb['owner_org'],
))
data = {}
for f in chromo['fields']:
if not f.get('import_template_include', True):
continue
val = record[f['datastore_id']]
data[f['datastore_id']] = val
return render('recombinant/update_pd_record.html',
extra_vars={
'data': data,
'resource_name': resource_name,
'chromo_title': chromo['title'],
'choice_fields': choice_fields,
'pk_fields': pk_fields,
'owner_org': rcomb['owner_org'],
'errors': {},
})
def open_datarequest(context, data_dict):
"""
Action to open a data request. Access rights will be checked before
opening the data request. If the user is not allowed, a NotAuthorized
exception will be risen.
:param id: The ID of the data request to be closed
:type id: string
:returns: A dict with the data request (id, user_id, title, description,
organization_id, open_time, accepted_dataset, close_time, closed,
followers)
:rtype: dict
"""
model = context['model']
session = context['session']
datarequest_id = data_dict.get('id', '')
# Check id
if not datarequest_id:
raise tk.ValidationError(tk._('Data Request ID has not been included'))
# Init the data base
db.init_db(model)
# Check access
tk.check_access(constants.OPEN_DATAREQUEST, context, data_dict)
# Get the data request
result = db.DataRequest.get(id=datarequest_id)
if not result:
raise tk.ObjectNotFound(
tk._('Data Request %s not found in the data base') %
datarequest_id)
data_req = result[0]
data_req.closed = False
data_req.accepted_dataset_id = None
data_req.close_time = None
if tk.h.closing_circumstances_enabled:
data_req.close_circumstance = None
data_req.approx_publishing_date = None
session.add(data_req)
session.commit()
datarequest_dict = _dictize_datarequest(data_req)
# Mailing
users = [data_req.user_id]
# Creator email
_send_mail(users, 'open_datarequest_creator', datarequest_dict,
'Data Request Opened Creator Email')
if datarequest_dict['organization']:
users = _get_admin_users_from_organisation(datarequest_dict)
# Admins of organisation email
_send_mail(users, 'open_datarequest_organisation', datarequest_dict,
'Data Request Opened Admins Email')
return datarequest_dict
def close_datarequest(original_action, context, data_dict):
"""
Action to close a data request. Access rights will be checked before
closing the data request. If the user is not allowed, a NotAuthorized
exception will be risen.
Data QLD modification
Will send email notification to the data request creator
:param id: The ID of the data request to be closed
:type id: string
:param accepted_dataset_id: The ID of the dataset accepted as solution
for this data request
:type accepted_dataset_id: string
:returns: A dict with the data request (id, user_id, title, description,
organization_id, open_time, accepted_dataset, close_time, closed,
followers)
:rtype: dict
"""
model = context['model']
session = context['session']
datarequest_id = data_dict.get('id', '')
# Check id
if not datarequest_id:
raise tk.ValidationError(tk._('Data Request ID has not been included'))
# Init the data base
db.init_db(model)
# Check access
tk.check_access(constants.CLOSE_DATAREQUEST, context, data_dict)
# Get the data request
result = db.DataRequest.get(id=datarequest_id)
if not result:
raise tk.ObjectNotFound(
tk._('Data Request %s not found in the data base') %
datarequest_id)
# Validate data
validator.validate_datarequest_closing(context, data_dict)
data_req = result[0]
# Was the data request previously closed?
if data_req.closed:
raise tk.ValidationError([tk._('This Data Request is already closed')])
data_req.closed = True
data_req.accepted_dataset_id = data_dict.get('accepted_dataset_id') or None
data_req.close_time = datetime.datetime.utcnow()
_undictize_datarequest_closing_circumstances(data_req, data_dict)
session.add(data_req)
session.commit()
datarequest_dict = _dictize_datarequest(data_req)
# Mailing
users = [data_req.user_id]
_send_mail(users, 'close_datarequest_creator', datarequest_dict,
'Data Request Closed Send Email')
return datarequest_dict
def update_datarequest(original_action, context, data_dict):
"""
Action to update a data request. The function checks the access rights of
the user before updating the data request. If the user is not allowed
a NotAuthorized exception will be risen.
In addition, you should note that the parameters will be checked and an
exception (ValidationError) will be risen if some of these parameters are
invalid.
Data QLD modification
Will send email notification if organisation was changed to users of assigned organisation with admin access
:param id: The ID of the data request to be updated
:type id: string
:param title: The title of the data request
:type title: string
:param description: A brief description for your data request
:type description: string
:param organiztion_id: The ID of the organization you want to asign the
data request.
:type organization_id: string
:returns: A dict with the data request (id, user_id, title, description,
organization_id, open_time, accepted_dataset, close_time, closed,
followers)
:rtype: dict
"""
model = context['model']
session = context['session']
datarequest_id = data_dict.get('id', '')
if not datarequest_id:
raise tk.ValidationError(tk._('Data Request ID has not been included'))
# Init the data base
db.init_db(model)
# Check access
tk.check_access(constants.UPDATE_DATAREQUEST, context, data_dict)
# Get the initial data
result = db.DataRequest.get(id=datarequest_id)
if not result:
raise tk.ObjectNotFound(
tk._('Data Request %s not found in the data base') %
datarequest_id)
data_req = result[0]
# Avoid the validator to return an error when the user does not change the title
context['avoid_existing_title_check'] = data_req.title == data_dict[
'title']
# Validate data
validator.validate_datarequest(context, data_dict)
# Data QLD modification
organisation_updated = data_req.organization_id != data_dict[
'organization_id']
if organisation_updated:
unassigned_organisation_id = data_req.organization_id
# Set the data provided by the user in the data_red
_undictize_datarequest_basic(data_req, data_dict)
session.add(data_req)
session.commit()
datarequest_dict = _dictize_datarequest(data_req)
if datarequest_dict['organization'] and organisation_updated:
# Data QLD modification
# Email Admin users of the assigned organisation
users = _get_admin_users_from_organisation(datarequest_dict)
users.discard(context['auth_user_obj'].id)
_send_mail(users, 'new_datarequest_organisation', datarequest_dict,
'Data Request Assigned Email')
# Email Admin users of unassigned organisation
org_dict = {
'organization': _get_organization(unassigned_organisation_id)
}
users = _get_admin_users_from_organisation(org_dict)
users.discard(context['auth_user_obj'].id)
_send_mail(users, 'unassigned_datarequest_organisation',
datarequest_dict, 'Data Request Unassigned Email')
return datarequest_dict
def create_pd_record(self, owner_org, resource_name):
lc = LocalCKAN(username=c.user)
try:
chromo = h.recombinant_get_chromo(resource_name)
rcomb = lc.action.recombinant_show(
owner_org=owner_org, dataset_type=chromo['dataset_type'])
[res
] = [r for r in rcomb['resources'] if r['name'] == resource_name]
check_access('datastore_upsert', {
'user': c.user,
'auth_user_obj': c.userobj
}, {'resource_id': res['id']})
except NotAuthorized:
return abort(403, _('Unauthorized'))
choice_fields = {
f['datastore_id']: [{
'value': k,
'label': v
} for (k, v) in f['choices']]
for f in h.recombinant_choice_fields(resource_name)
}
pk_fields = aslist(chromo['datastore_primary_key'])
if request.method == 'POST':
post_data = parse_params(request.POST, ignore_keys=['save'])
if 'cancel' in post_data:
return redirect(
h.url_for(
controller=
'ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=resource_name,
owner_org=rcomb['owner_org'],
))
data, err = clean_check_type_errors(post_data, chromo['fields'],
pk_fields, choice_fields)
try:
lc.action.datastore_upsert(resource_id=res['id'],
method='insert',
records=[{
k: None if k in err else v
for (k, v) in data.items()
}],
dry_run=bool(err))
except ValidationError as ve:
if 'records' in ve.error_dict:
err = dict(
{
k: [_(e) for e in v]
for (k, v) in ve.error_dict['records'][0].items()
}, **err)
elif ve.error_dict.get('info', {}).get('pgcode',
'') == '23505':
err = dict(
{
k: [_("This record already exists")]
for k in pk_fields
}, **err)
if err:
return render('recombinant/create_pd_record.html',
extra_vars={
'data': data,
'resource_name': resource_name,
'chromo_title': chromo['title'],
'choice_fields': choice_fields,
'owner_org': rcomb['owner_org'],
'errors': err,
})
h.flash_notice(_(u'Record Created'))
return redirect(
h.url_for(
controller=
'ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=resource_name,
owner_org=rcomb['owner_org'],
))
return render('recombinant/create_pd_record.html',
extra_vars={
'data': {},
'resource_name': resource_name,
'chromo_title': chromo['title'],
'choice_fields': choice_fields,
'owner_org': rcomb['owner_org'],
'errors': {},
})
def update_pd_record(self, owner_org, resource_name, pk):
pk = [url_part_unescape(p) for p in pk.split(',')]
lc = LocalCKAN(username=c.user)
try:
chromo = h.recombinant_get_chromo(resource_name)
rcomb = lc.action.recombinant_show(
owner_org=owner_org, dataset_type=chromo['dataset_type'])
[res
] = [r for r in rcomb['resources'] if r['name'] == resource_name]
check_access('datastore_upsert', {
'user': c.user,
'auth_user_obj': c.userobj
}, {'resource_id': res['id']})
except NotAuthorized:
abort(403, _('Unauthorized'))
choice_fields = {
f['datastore_id']: [{
'value': k,
'label': v
} for (k, v) in f['choices']]
for f in h.recombinant_choice_fields(resource_name)
}
pk_fields = aslist(chromo['datastore_primary_key'])
pk_filter = dict(zip(pk_fields, pk))
records = lc.action.datastore_search(resource_id=res['id'],
filters=pk_filter)['records']
if len(records) == 0:
abort(404, _('Not found'))
if len(records) > 1:
abort(400, _('Multiple records found'))
record = records[0]
if request.method == 'POST':
post_data = parse_params(request.POST,
ignore_keys=['save'] + pk_fields)
if 'cancel' in post_data:
return redirect(
h.url_for(
controller=
'ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=resource_name,
owner_org=rcomb['owner_org'],
))
data, err = clean_check_type_errors(post_data, chromo['fields'],
pk_fields, choice_fields)
# can't change pk fields
for f_id in data:
if f_id in pk_fields:
data[f_id] = record[f_id]
try:
lc.action.datastore_upsert(
resource_id=res['id'],
#method='update', FIXME not raising ValidationErrors
records=[{
k: None if k in err else v
for (k, v) in data.items()
}],
dry_run=bool(err))
except ValidationError as ve:
err = dict(
{
k: [_(e) for e in v]
for (k, v) in ve.error_dict['records'][0].items()
}, **err)
if err:
return render('recombinant/update_pd_record.html',
extra_vars={
'data': data,
'resource_name': resource_name,
'chromo_title': chromo['title'],
'choice_fields': choice_fields,
'pk_fields': pk_fields,
'owner_org': rcomb['owner_org'],
'errors': err,
})
h.flash_notice(_(u'Record %s Updated') % u','.join(pk))
return redirect(
h.url_for(
controller=
'ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=resource_name,
owner_org=rcomb['owner_org'],
))
data = {}
for f in chromo['fields']:
if not f.get('import_template_include', True):
continue
val = record[f['datastore_id']]
data[f['datastore_id']] = val
return render('recombinant/update_pd_record.html',
extra_vars={
'data': data,
'resource_name': resource_name,
'chromo_title': chromo['title'],
'choice_fields': choice_fields,
'pk_fields': pk_fields,
'owner_org': rcomb['owner_org'],
'errors': {},
})
def ctp_list_types(context, data_dict=None):
tk.check_access("package_create", context, data_dict)
return {"success": True}
