def pages_delete(page, page_type='pages'):
if page.startswith('/'):
page = page[1:]
if 'cancel' in tk.request.params:
if ckan_29_or_higher:
return tk.redirect_to('pages.%s_edit' % page_type, page=page)
else:
tk.redirect_to('%s_edit' % page_type, page='/' + page)
try:
if tk.request.method == 'POST':
tk.get_action('ckanext_pages_delete')({}, {'page': page})
if ckan_29_or_higher:
endpoint = 'index' if page_type in (
'pages', 'page') else '%s_index' % page_type
return tk.redirect_to('pages.%s' % endpoint)
else:
tk.redirect_to('%s_index' % page_type)
else:
return tk.abort(404, _('Page Not Found'))
except tk.NotAuthorized:
return tk.abort(401, _('Unauthorized to delete page'))
except tk.ObjectNotFound:
return tk.abort(404, _('Group not found'))
return tk.render('ckanext_pages/confirm_delete.html', {'page': page})
def delete_view(id):
if 'cancel' in tk.request.params:
tk.redirect_to('showcase_blueprint.edit' if tk.check_ckan_version(
min_version='2.9.0') else 'showcase_edit',
id=id)
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj
}
try:
tk.check_access('ckanext_showcase_delete', context, {'id': id})
except tk.NotAuthorized:
return tk.abort(401, _('Unauthorized to delete showcase'))
if tk.check_ckan_version(min_version='2.9.0'):
index_route = 'showcase_blueprint.index'
else:
index_route = 'showcase_index'
try:
if tk.request.method == 'POST':
tk.get_action('ckanext_showcase_delete')(context, {'id': id})
h.flash_notice(_('Showcase has been deleted.'))
return tk.redirect_to(index_route)
c.pkg_dict = tk.get_action('package_show')(context, {'id': id})
except tk.NotAuthorized:
tk.abort(401, _('Unauthorized to delete showcase'))
except tk.ObjectNotFound:
tk.abort(404, _('Showcase not found'))
return tk.render('showcase/confirm_delete.html',
extra_vars={'dataset_type': DATASET_TYPE_NAME})
def organization_index(self):
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True,
'with_private': False}
data_dict = {'all_fields': True}
try:
check_access('site_read', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
# pass user info to context as needed to view private datasets of
# orgs correctly
if c.userobj:
context['user_id'] = c.userobj.id
context['user_is_admin'] = c.userobj.sysadmin
results = get_action('organization_list')(context, data_dict)
def org_key(org):
title = org['title'].split(' | ')[-1 if c.language == 'fr' else 0]
return normalize_strip_accents(title)
results.sort(key=org_key)
c.page = Page(
collection=results,
page=request.params.get('page', 1),
url=h.pager_url,
items_per_page=1000
)
return render('organization/index.html')
def datarequest_default_organisation():
"""Returns the default organisation for data request from the config file
Case insensitive.
:rtype: organisation
"""
default_organisation = config.get('ckan.datarequests.default_organisation')
try:
organisation = toolkit.get_action('organization_show')(
data_dict={
'id': default_organisation,
'include_datasets': False,
'include_dataset_count': False,
'include_extras': False,
'include_users': False,
'include_groups': False,
'include_tags': False,
'include_followers': False
})
except toolkit.ObjectNotFound:
toolkit.abort(
404,
toolkit.
_('Default Data Request Organisation not found. Please get the sysadmin to set one up'
))
return organisation
def open_datarequest(id):
data_dict = {'id': id}
context = _get_context()
# Basic initialization
c.datarequest = {}
try:
check_access(OPEN_DATAREQUEST, context, data_dict)
c.datarequest = get_action(SHOW_DATAREQUEST)(context, data_dict)
if c.datarequest.get('closed', False) is False:
return abort(403, _('This data request is already open'))
else:
data_dict = {}
data_dict['id'] = id
data_dict['organization_id'] = c.datarequest.get('organization_id')
get_action(OPEN_DATAREQUEST)(context, data_dict)
return redirect_to(url_for('datarequest.show', id=data_dict['id']))
except ValidationError as e:
log.warn(e)
errors_summary = _get_errors_summary(e.error_dict)
return abort(403, errors_summary)
except ObjectNotFound as e:
log.warn(e)
return abort(404, _('Data Request %s not found') % id)
except NotAuthorized as e:
log.warn(e)
return abort(
403, _('You are not authorized to open the Data Request %s' % id))
def organization_index(self):
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'for_view': True,
'with_private': False
}
data_dict = {'all_fields': True}
try:
check_access('site_read', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
# pass user info to context as needed to view private datasets of
# orgs correctly
if c.userobj:
context['user_id'] = c.userobj.id
context['user_is_admin'] = c.userobj.sysadmin
results = get_action('organization_list')(context, data_dict)
def org_key(org):
title = org['title'].split(' | ')[-1 if c.language == 'fr' else 0]
return normalize_strip_accents(title)
results.sort(key=org_key)
c.page = Page(collection=results,
page=request.params.get('page', 1),
url=h.pager_url,
items_per_page=1000)
return render('organization/index.html')
def read_view(id):
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'for_view': True,
'auth_user_obj': c.userobj
}
data_dict = {'id': id}
# check if showcase exists
try:
c.pkg_dict = tk.get_action('package_show')(context, data_dict)
except tk.ObjectNotFound:
return tk.abort(404, _('Showcase not found'))
except tk.NotAuthorized:
return tk.abort(401, _('Unauthorized to read showcase'))
# get showcase packages
c.showcase_pkgs = tk.get_action('ckanext_showcase_package_list')(
context, {
'showcase_id': c.pkg_dict['id']
})
package_type = DATASET_TYPE_NAME
return tk.render('showcase/read.html',
extra_vars={'dataset_type': package_type})
def _template_setup_group(id, group_type):
if not id:
return
context = {'for_view': True}
action = 'organization_show' if group_type == 'organization' else 'group_show'
try:
tk.c.group_dict = tk.get_action(action)(context, {'id': id})
except tk.ObjectNotFound:
tk.abort(404, _('{} not found'.format(group_type.title())))
except tk.NotAuthorized:
tk.abort(401, _('Unauthorized to read {} {}'.format(group_type, id)))
def upload():
if not tk.request.method == 'POST':
tk.abort(409, _('Only Posting is availiable'))
try:
url = tk.get_action('ckanext_showcase_upload')(None,
dict(tk.request.POST))
except tk.NotAuthorized:
tk.abort(401, _('Unauthorized to upload file %s') % id)
return json.dumps(url)
def _guess_package_type(self, expecting_name=False):
# this is a bit unortodox, but this method allows us to conveniently
# alter the search method without much code duplication.
if not is_sysadmin(c.user):
abort(401, _('Not authorized to see this page'))
# always set ready_to_publish to true for the publishing interface
request.GET['ready_to_publish'] = u'true'
# This MUST be None, otherwise the default filtering will apply and
# restrict to just dataset_type=dataset.
return None
def archive_download(id, resource_id, filename=None):
"""
Provides a direct download by either redirecting the user to the url
stored or downloading an uploaded file directly.
"""
context = {
'model': model,
'session': model.Session,
'user': c.user,
'auth_user_obj': c.userobj
}
try:
resource = toolkit.get_action('resource_show')(context, {
'id': resource_id
})
# Quick auth check to ensure you can access this resource
toolkit.check_access('package_show', context, {'id': id})
except (toolkit.ObjectNotFound, toolkit.NotAuthorized):
return toolkit.abort(404, _('Resource not found'))
# Archived files are only links not uploads
if resource.get('url_type') != 'upload':
# Return the key used for this resource in storage.
#
# Keys are in the form:
# <uploaderpath>/<upload_to>/<2 char from resource id >/<resource id>/<filename>
#
# e.g.:
# my_storage_path/archive/16/165900ba-3c60-43c5-9e9c-9f8acd0aa93f/data.csv
relative_archive_path = os.path.join(resource['id'][:2],
resource['id'])
# try to get a file name from the url
parsed_url = urlparse.urlparse(resource.get('url'))
try:
file_name = parsed_url.path.split('/')[-1] or 'resource'
file_name = file_name.strip() # trailing spaces cause problems
file_name = file_name.encode('ascii',
'ignore') # e.g. u'\xa3' signs
except Exception:
file_name = "resource"
try:
upload = uploader.get_uploader(
os.path.join('archive', relative_archive_path))
return upload.download(file_name)
except OSError:
# includes FileNotFoundError
return toolkit.abort(404, _('Resource data not found'))
return toolkit.abort(404, _('No download is available'))
def _guess_package_type(self, expecting_name=False):
# this is a bit unortodox, but this method allows us to conveniently
# alter the search method without much code duplication.
if not is_sysadmin(c.user):
abort(401, _('Not authorized to see this page'))
# always set ready_to_publish to true for the publishing interface
request.GET['ready_to_publish'] = u'true'
request.GET['imso_approval'] = u'true'
# This MUST be None, otherwise the default filtering will apply and
# restrict to just dataset_type=dataset.
return None
def reports(self, id=None):
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'auth_user_obj': c.userobj,
'for_view': True}
data_dict = {'id': id,
'user_obj': c.userobj,
'include_datasets': True,
'include_num_followers': True}
context['with_related'] = True
self._setup_template_variables(context, data_dict)
if c.is_myself:
return render('user/reports.html')
abort(403)
def reports(self, id=None):
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'auth_user_obj': c.userobj,
'for_view': True}
data_dict = {'id': id,
'user_obj': c.userobj,
'include_datasets': True,
'include_num_followers': True}
context['with_related'] = True
self._setup_template_variables(context, data_dict)
if c.is_myself:
return render('user/reports.html')
abort(403)
def type_redirect(self, resource_name):
orgs = h.organizations_available('read')
if not orgs:
abort(404, _('No organizations found'))
try:
chromo = get_chromo(resource_name)
except RecombinantException:
abort(404, _('Recombinant resource_name not found'))
# custom business logic
if is_sysadmin(c.user):
return redirect(h.url_for('recombinant_resource',
resource_name=resource_name, owner_org='tbs-sct'))
return redirect(h.url_for('recombinant_resource',
resource_name=resource_name, owner_org=orgs[0]['name']))
def register(self, data=None, errors=None, error_summary=None):
'''GET to display a form for registering a new user.
or POST the form data to actually do the user registration.
The bulk of this code is pulled directly from
ckan/controlllers/user.py
'''
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'schema': schema.user_new_form_schema(),
'save': 'save' in request.params
}
try:
check_access('user_create', context)
except NotAuthorized:
abort(401, _('Unauthorized to create a user'))
if context['save'] and not data:
try:
return self._save_new(context)
except HTTPFound:
# redirected after successful user create
notify_ckan_user_create(
email=request.params.get('email', ''),
fullname=request.params.get('fullname', ''),
username=request.params.get('name', ''),
phoneno=request.params.get('phoneno', ''),
dept=request.params.get('department', ''))
notice_no_access()
raise
if c.user and not data and not is_sysadmin(c.user):
# #1799 Don't offer the registration form if already logged in
return render('user/logout_first.html')
data = data or {}
errors = errors or {}
error_summary = error_summary or {}
d = {'data': data, 'errors': errors, 'error_summary': error_summary}
c.is_sysadmin = is_sysadmin(c.user)
c.form = render('user/new_user_form.html', extra_vars=d)
return render('user/new.html')
def pages_upload():
if not tk.request.method == 'POST':
tk.abort(409, _('Only Posting is availiable'))
if ckan_29_or_higher:
data_dict = logic.clean_dict(
dict_fns.unflatten(
logic.tuplize_dict(logic.parse_params(tk.request.files))))
else:
data_dict = tk.request.POST
try:
upload_info = tk.get_action('ckanext_pages_upload')(None, data_dict)
except tk.NotAuthorized:
return tk.abort(401, _('Unauthorized to upload file %s') % id)
if ckan_29_or_higher:
return upload_info
else:
return json.dumps(upload_info)
def remove_showcase_admin():
'''
Remove a user from the Showcase Admin list.
'''
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author
}
try:
tk.check_access('sysadmin', context, {})
except tk.NotAuthorized:
return tk.abort(401, _('User not authorized to view page'))
form_data = tk.request.form if tk.check_ckan_version(
'2.9') else tk.request.params
if tk.check_ckan_version(min_version='2.9.0'):
admins_route = 'showcase_blueprint.admins'
else:
admins_route = 'showcase_admins'
if 'cancel' in form_data:
return tk.redirect_to(admins_route)
user_id = tk.request.params['user']
if tk.request.method == 'POST' and user_id:
user_id = tk.request.params['user']
try:
tk.get_action('ckanext_showcase_admin_remove')(data_dict={
'username': user_id
})
except tk.NotAuthorized:
return tk.abort(401, _('Unauthorized to perform that action'))
except tk.ObjectNotFound:
h.flash_error(_('The user is not a Showcase Admin'))
else:
h.flash_success(_('The user is no longer a Showcase Admin'))
return tk.redirect_to(h.url_for(admins_route))
c.user_dict = tk.get_action('user_show')(data_dict={'id': user_id})
c.user_id = user_id
return tk.render('admin/confirm_remove_showcase_admin.html')
def validation(resource_id, id=None):
try:
validation = get_action(u'resource_validation_show')(
{
u'user': c.user
}, {
u'resource_id': resource_id
})
resource = get_action(u'resource_show')({
u'user': c.user
}, {
u'id': resource_id
})
package_id = resource[u'package_id']
if id and id != package_id:
raise ObjectNotFound("Resource {} not found in package {}".format(
resource_id, id))
dataset = get_action(u'package_show')(
{
u'user': c.user
}, {
u'id': id or resource[u'package_id']
})
# Needed for core resource templates
c.package = c.pkg_dict = dataset
c.resource = resource
return render(u'validation/validation_read.html',
extra_vars={
u'validation': validation,
u'resource': resource,
u'pkg_dict': dataset,
u'dataset': dataset,
})
except NotAuthorized:
return abort(403, _(u'Unauthorized to read this validation report'))
except ObjectNotFound:
return abort(404, _(u'No validation report exists for this resource'))
def register(self, data=None, errors=None, error_summary=None):
'''GET to display a form for registering a new user.
or POST the form data to actually do the user registration.
The bulk of this code is pulled directly from
ckan/controlllers/user.py
'''
context = {'model': model, 'session': model.Session,
'user': c.user or c.author,
'schema': schema.user_new_form_schema(),
'save': 'save' in request.params}
try:
check_access('user_create', context)
except NotAuthorized:
abort(401, _('Unauthorized to create a user'))
if context['save'] and not data:
try:
return self._save_new(context)
except HTTPFound:
# redirected after successful user create
notify_ckan_user_create(
email=request.params.get('email', ''),
fullname=request.params.get('fullname', ''),
username=request.params.get('name', ''),
phoneno=request.params.get('phoneno', ''),
dept=request.params.get('department', ''))
notice_no_access()
raise
if c.user and not data and not is_sysadmin(c.user):
# #1799 Don't offer the registration form if already logged in
return render('user/logout_first.html')
data = data or {}
errors = errors or {}
error_summary = error_summary or {}
d = {'data': data, 'errors': errors, 'error_summary': error_summary}
c.is_sysadmin = is_sysadmin(c.user)
c.form = render('user/new_user_form.html', extra_vars=d)
return render('user/new.html')
def show_schema(dataset_id, resource_id):
data_dict = {'id': resource_id}
context = _get_context()
try:
check_access(RESOURCE_SHOW, context, data_dict)
resource = get_action(RESOURCE_SHOW)(context, data_dict)
schema_data = resource.get('schema')
c.schema_data = json.dumps(schema_data, indent=2, sort_keys=True)
return render('schema/show.html')
except ObjectNotFound as e:
log.warn(e)
return abort(404, _('Resource %s not found') % resource_id)
except NotAuthorized as e:
log.warn(e)
return abort(
403,
_('You are not authorized to view the Data Scheme for the resource %s'
% resource_id))
def validation(self, resource_id):
try:
validation = get_action(u'resource_validation_show')(
{
u'user': c.user
}, {
u'resource_id': resource_id
})
resource = get_action(u'resource_show')({
u'user': c.user
}, {
u'id': resource_id
})
dataset = get_action(u'package_show')(
{
u'user': c.user
}, {
u'id': resource[u'package_id']
})
# Needed for core resource templates
c.package = c.pkg_dict = dataset
c.resource = resource
return render(u'validation/validation_read.html',
extra_vars={
u'validation': validation,
u'resource': resource,
u'pkg_dict': dataset,
})
except NotAuthorized:
abort(403, _(u'Unauthorized to read this validation report'))
except ObjectNotFound:
abort(404, _(u'No validation report exists for this resource'))
def check_edit_view_auth(id):
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj,
'save': 'save' in tk.request.params,
'moderated': tk.config.get('moderated'),
'pending': True
}
try:
tk.check_access('ckanext_showcase_update', context)
except tk.NotAuthorized:
return tk.abort(
401,
_('User not authorized to edit {showcase_id}').format(
showcase_id=id))
def check_new_view_auth():
context = {
'model': model,
'session': model.Session,
'user': tk.c.user or tk.c.author,
'auth_user_obj': tk.c.userobj,
'save': 'save' in tk.request.params
}
# Check access here, then continue with PackageController.new()
# PackageController.new will also check access for package_create.
# This is okay for now, while only sysadmins can create Showcases, but
# may not work if we allow other users to create Showcases, who don't
# have access to create dataset package types. Same for edit below.
try:
tk.check_access('ckanext_showcase_create', context)
except tk.NotAuthorized:
return tk.abort(401, _('Unauthorized to create a package'))
def manage_showcase_admins():
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author
}
try:
tk.check_access('sysadmin', context, {})
except tk.NotAuthorized:
return tk.abort(401, _('User not authorized to view page'))
form_data = tk.request.form if tk.check_ckan_version(
'2.9') else tk.request.params
if tk.check_ckan_version(min_version='2.9.0'):
admins_route = 'showcase_blueprint.admins'
else:
admins_route = 'showcase_admins'
# We're trying to add a user to the showcase admins list.
if tk.request.method == 'POST' and form_data['username']:
username = form_data['username']
try:
tk.get_action('ckanext_showcase_admin_add')(data_dict={
'username': username
})
except tk.NotAuthorized:
abort(401, _('Unauthorized to perform that action'))
except tk.ObjectNotFound:
h.flash_error(
_("User '{user_name}' not found.").format(user_name=username))
except tk.ValidationError as e:
h.flash_notice(e.error_summary)
else:
h.flash_success(_("The user is now a Showcase Admin"))
return tk.redirect_to(h.url_for(admins_route))
c.showcase_admins = tk.get_action('ckanext_showcase_admin_list')()
return tk.render('admin/manage_showcase_admins.html')
def export():
report_type = helpers.RequestHelper(request).get_first_query_param(
'report_type', '')
try:
report_permission = _get_report_type_permission(report_type)
reporting_helpers.check_user_access(report_permission)
error = _valid_report_type(report_type)
if error:
return error
if report_type == REPORT_TYPE_ENGAGEMENT:
return _export_engagement_report(report_type, report_permission)
elif report_type == REPORT_TYPE_ADMIN:
return _export_admin_report(report_type, report_permission)
except NotAuthorized as e: # Exception raised from check_user_access
log.warn(e)
return abort(
403,
_('You are not authorised to export the {0} report'.format(
report_type)))
def group_delete(id, group_type, page):
_template_setup_group(id, group_type)
if page.startswith('/'):
page = page[1:]
if 'cancel' in tk.request.params:
if ckan_29_or_higher:
return tk.redirect_to('pages.%s_edit' % group_type,
id=tk.c.group_dict['name'],
page=page)
else:
tk.redirect_to('%s_edit' % group_type,
id=tk.c.group_dict['name'],
page='/' + page)
try:
if tk.request.method == 'POST':
action = 'ckanext_org_pages_delete' if group_type == 'organization' else 'ckanext_group_pages_delete'
action = tk.get_action(action)
action({}, {'org_id': tk.c.group_dict['id'], 'page': page})
if ckan_29_or_higher:
endpoint = 'pages.{}_pages_index'.format(group_type)
return tk.redirect_to(endpoint, id=id)
else:
tk.redirect_to('{}_pages_index'.format(group_type), id=id)
else:
tk.abort(404, _('Page Not Found'))
except tk.NotAuthorized:
tk.abort(401, _('Unauthorized to delete page'))
except tk.ObjectNotFound:
tk.abort(404, _('{} not found'.format(group_type.title())))
context = {'for_view': True}
action = 'organization_show' if group_type == 'organization' else 'group_show'
group_dict = tk.get_action(action)(context, {'id': id})
return tk.render('ckanext_pages/confirm_delete.html', {
'page': page,
'group_type': group_type,
'group_dict': group_dict
})
def create_pd_record(self, owner_org, resource_name):
lc = LocalCKAN(username=c.user)
try:
chromo = h.recombinant_get_chromo(resource_name)
rcomb = lc.action.recombinant_show(
owner_org=owner_org, dataset_type=chromo['dataset_type'])
[res
] = [r for r in rcomb['resources'] if r['name'] == resource_name]
check_access('datastore_upsert', {
'user': c.user,
'auth_user_obj': c.userobj
}, {'resource_id': res['id']})
except NotAuthorized:
return abort(403, _('Unauthorized'))
choice_fields = {
f['datastore_id']: [{
'value': k,
'label': v
} for (k, v) in f['choices']]
for f in h.recombinant_choice_fields(resource_name)
}
pk_fields = aslist(chromo['datastore_primary_key'])
if request.method == 'POST':
post_data = parse_params(request.POST, ignore_keys=['save'])
if 'cancel' in post_data:
return redirect(
h.url_for(
controller=
'ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=resource_name,
owner_org=rcomb['owner_org'],
))
data, err = clean_check_type_errors(post_data, chromo['fields'],
pk_fields, choice_fields)
try:
lc.action.datastore_upsert(resource_id=res['id'],
method='insert',
records=[{
k: None if k in err else v
for (k, v) in data.items()
}],
dry_run=bool(err))
except ValidationError as ve:
if 'records' in ve.error_dict:
err = dict(
{
k: [_(e) for e in v]
for (k, v) in ve.error_dict['records'][0].items()
}, **err)
elif ve.error_dict.get('info', {}).get('pgcode',
'') == '23505':
err = dict(
{
k: [_("This record already exists")]
for k in pk_fields
}, **err)
if err:
return render('recombinant/create_pd_record.html',
extra_vars={
'data': data,
'resource_name': resource_name,
'chromo_title': chromo['title'],
'choice_fields': choice_fields,
'owner_org': rcomb['owner_org'],
'errors': err,
})
h.flash_notice(_(u'Record Created'))
return redirect(
h.url_for(
controller=
'ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=resource_name,
owner_org=rcomb['owner_org'],
))
return render('recombinant/create_pd_record.html',
extra_vars={
'data': {},
'resource_name': resource_name,
'chromo_title': chromo['title'],
'choice_fields': choice_fields,
'owner_org': rcomb['owner_org'],
'errors': {},
})
def update_pd_record(self, owner_org, resource_name, pk):
pk = [url_part_unescape(p) for p in pk.split(',')]
lc = LocalCKAN(username=c.user)
try:
chromo = h.recombinant_get_chromo(resource_name)
rcomb = lc.action.recombinant_show(
owner_org=owner_org,
dataset_type=chromo['dataset_type'])
[res] = [r for r in rcomb['resources'] if r['name'] == resource_name]
check_access(
'datastore_upsert',
{'user': c.user, 'auth_user_obj': c.userobj},
{'resource_id': res['id']})
except NotAuthorized:
abort(403, _('Unauthorized'))
choice_fields = {
f['datastore_id']: [
{'value': k, 'label': v} for (k, v) in f['choices']]
for f in h.recombinant_choice_fields(resource_name)}
pk_fields = aslist(chromo['datastore_primary_key'])
pk_filter = dict(zip(pk_fields, pk))
records = lc.action.datastore_search(
resource_id=res['id'],
filters=pk_filter)['records']
if len(records) == 0:
abort(404, _('Not found'))
if len(records) > 1:
abort(400, _('Multiple records found'))
record = records[0]
if request.method == 'POST':
post_data = parse_params(request.POST, ignore_keys=['save'] + pk_fields)
if 'cancel' in post_data:
return redirect(h.url_for(
controller='ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=resource_name,
owner_org=rcomb['owner_org'],
))
data = {}
for f in chromo['fields']:
f_id = f['datastore_id']
if not f.get('import_template_include', True):
continue
if f_id in pk_fields:
data[f_id] = record[f_id]
else:
val = post_data.get(f['datastore_id'], '')
if isinstance(val, list):
val = u','.join(val)
val = canonicalize(
val,
f['datastore_type'],
primary_key=False,
choice_field=f_id in choice_fields)
data[f['datastore_id']] = val
try:
lc.action.datastore_upsert(
resource_id=res['id'],
#method='update', FIXME not raising ValidationErrors
records=[data])
except ValidationError as ve:
err = {
k: [_(e) for e in v]
for (k, v) in ve.error_dict['records'][0].items()}
return render('recombinant/update_pd_record.html',
extra_vars={
'data': data,
'resource_name': resource_name,
'chromo_title': chromo['title'],
'choice_fields': choice_fields,
'pk_fields': pk_fields,
'owner_org': rcomb['owner_org'],
'errors': err,
})
h.flash_notice(_(u'Record %s Updated') % u','.join(pk) )
return redirect(h.url_for(
controller='ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=resource_name,
owner_org=rcomb['owner_org'],
))
data = {}
for f in chromo['fields']:
if not f.get('import_template_include', True):
continue
val = record[f['datastore_id']]
data[f['datastore_id']] = val
return render('recombinant/update_pd_record.html',
extra_vars={
'data': data,
'resource_name': resource_name,
'chromo_title': chromo['title'],
'choice_fields': choice_fields,
'pk_fields': pk_fields,
'owner_org': rcomb['owner_org'],
'errors': {},
})
def _valid_report_type(report_type):
if report_type not in REPORT_TYPES:
msg = _('Report type {0} not valid').format(report_type)
log.warn(msg)
return abort(404, msg)
def datasets(org_id, metric):
report_type = helpers.RequestHelper(request).get_first_query_param(
'report_type', '')
try:
report_permission = _get_report_type_permission(report_type)
reporting_helpers.check_user_org_access(org_id, report_permission)
error = _valid_report_type(report_type)
if error:
return error
get_validator('group_id_exists')(org_id,
reporting_helpers.get_context())
org = get_action('organization_show')({}, {'id': org_id})
data_dict = {
'org_id': org_id,
'org_title': org['title'],
'metric': metric,
'user_dict': helpers.get_user().as_dict(),
'report_type': report_type
}
if metric == 'no-reply':
start_date, end_date = reporting_helpers.get_report_date_range(
request)
utc_start_date, \
utc_end_date, \
utc_reply_expected_by_date = reporting_helpers.get_utc_dates(
start_date, end_date, COMMENT_NO_REPLY_MAX_DAYS)
data_dict.update({
'start_date':
start_date,
'end_date':
end_date,
'comment_no_reply_max_days':
COMMENT_NO_REPLY_MAX_DAYS,
'utc_start_date':
utc_start_date,
'utc_end_date':
utc_end_date,
'utc_reply_expected_by_date':
utc_reply_expected_by_date,
})
comments = get_action('dataset_comments_no_replies_after_x_days')(
{}, data_dict)
# Action `dataset_comments_no_replies_after_x_days` returns a
# collection of comments with no replies
# On this page we only need to display distinct datasets containing those comments
datasets = []
comment_ids = {}
for comment in comments:
if comment.package_name in comment_ids:
comment_ids[comment.package_name].append(
comment.comment_id)
else:
comment_ids[comment.package_name] = [comment.comment_id]
datasets.append(comment)
data_dict.update({
'datasets': datasets,
'total_comments': len(comments),
'comment_ids': comment_ids
})
elif metric == 'de-identified-datasets':
data_dict.update({
'return_count_only': False,
'permission': report_permission
})
datasets = get_action('de_identified_datasets')({}, data_dict)
data_dict.update({'datasets': datasets})
elif metric == 'overdue-datasets':
data_dict.update({
'return_count_only': False,
'permission': report_permission
})
datasets = get_action('overdue_datasets')({}, data_dict)
data_dict.update({'datasets': datasets})
elif metric == 'datasets_no_groups':
data_dict.update({
'return_count_only': False,
'permission': report_permission
})
datasets = get_action('datasets_no_groups')({}, data_dict)
data_dict.update({'datasets': datasets})
elif metric == 'datasets_no_tags':
data_dict.update({
'return_count_only': False,
'permission': report_permission
})
datasets = get_action('datasets_no_tags')({}, data_dict)
data_dict.update({'datasets': datasets})
return render('reporting/datasets.html', extra_vars=data_dict)
except Invalid as e: # Exception raised from get_validator('group_id_exists')
log.warn(e)
return abort(404, _('Organisation %s not found') % org_id)
except NotAuthorized as e: # Exception raised from check_user_access
log.warn(e)
return abort(
403,
_('You are not authorised to view the dataset {0} report for organisation {1}'
.format(report_type, org_id)))
def index():
request_helper = helpers.RequestHelper(request)
org_id = request_helper.get_first_query_param('organisation')
report_type = request_helper.get_first_query_param('report_type', '')
try:
report_permission = _get_report_type_permission(report_type)
reporting_helpers.check_user_access(report_permission)
extra_vars = {
'user_dict': helpers.get_user().as_dict(),
}
if report_type:
error = _valid_report_type(report_type)
if error:
return error
organisations = reporting_helpers.get_organisation_list(
report_permission)
extra_vars.update({
'organisations': organisations,
'report_type': report_type
})
if organisations and len(organisations) == 1:
org_id = organisations[0]['value']
if org_id:
org = get_action('organization_show')({}, {'id': org_id})
extra_vars.update({
'org_id': org_id,
'org_title': org['title'],
})
if report_type == REPORT_TYPE_ENGAGEMENT:
start_date, end_date = reporting_helpers.get_report_date_range(
request)
extra_vars.update({
'start_date':
start_date,
'end_date':
end_date,
'metrics':
reporting_helpers.gather_engagement_metrics(
org_id, start_date, end_date,
COMMENT_NO_REPLY_MAX_DAYS,
DATAREQUEST_OPEN_MAX_DAYS),
'datarequest_open_max_days':
DATAREQUEST_OPEN_MAX_DAYS,
'comment_no_reply_max_days':
COMMENT_NO_REPLY_MAX_DAYS
})
elif report_type == REPORT_TYPE_ADMIN:
extra_vars.update({
'metrics':
reporting_helpers.gather_admin_metrics(
org_id, report_permission)
})
return render('reporting/index.html', extra_vars=extra_vars)
except ObjectNotFound as e: # Exception raised from get_action('organization_show')
log.warn(e)
return abort(404, _('Organisation %s not found') % org_id)
except NotAuthorized as e: # Exception raised from check_user_access
log.warn(e)
organisation = request_helper.get_first_query_param('organisation')
if organisation:
msg = 'You are not authorised to view the {0} report for organisation {1}'.format(
report_type, organisation)
else:
msg = 'You are not authorised to view the {0} report'.format(
report_type)
return abort(403, _(msg))
def datarequests(org_id, metric):
"""Displays a list of data requests for the given organisation based on the desired metric"""
request_helper = helpers.RequestHelper(request)
report_type = request_helper.get_first_query_param('report_type', '')
try:
report_permission = _get_report_type_permission(report_type)
reporting_helpers.check_user_org_access(org_id, report_permission)
error = _valid_report_type(report_type)
if error:
return error
get_validator('group_id_exists')(org_id,
reporting_helpers.get_context())
start_date, end_date = reporting_helpers.get_report_date_range(request)
utc_start_date, \
utc_end_date, \
utc_reply_expected_by_date, \
utc_expected_closure_date = reporting_helpers.get_utc_dates(
start_date, end_date, COMMENT_NO_REPLY_MAX_DAYS, DATAREQUEST_OPEN_MAX_DAYS)
circumstance = request_helper.get_first_query_param('circumstance')
org = get_action('organization_show')({}, {'id': org_id})
data_dict = {
'org_id': org_id,
'org_title': org['title'],
'start_date': start_date,
'end_date': end_date,
'metric': metric,
'circumstance': circumstance,
'datarequest_open_max_days': DATAREQUEST_OPEN_MAX_DAYS,
'comment_no_reply_max_days': COMMENT_NO_REPLY_MAX_DAYS,
'user_dict': helpers.get_user().as_dict(),
'utc_start_date': utc_start_date,
'utc_end_date': utc_end_date,
'utc_reply_expected_by_date': utc_reply_expected_by_date,
'utc_expected_closure_date': utc_expected_closure_date,
'report_type': report_type
}
if metric == 'no-reply':
datarequests_comments = get_action(
'datarequests_no_replies_after_x_days')({}, data_dict)
# Action `datarequests_no_replies_after_x_days` returns a collection of comments with no replies
# On this page we only need to display distinct datarequests containing those comments
distinct_datarequests = []
comment_ids = {}
for datarequest in datarequests_comments:
if datarequest.datarequest_id in comment_ids:
comment_ids[datarequest.datarequest_id].append(
datarequest.comment_id)
else:
comment_ids[datarequest.datarequest_id] = [
datarequest.comment_id
]
distinct_datarequests.append(datarequest)
datarequests = distinct_datarequests
data_dict.update({
'total_comments': len(datarequests_comments),
'comment_ids': comment_ids
})
elif metric == 'no-comments':
datarequests = get_action(
'open_datarequests_no_comments_after_x_days')({}, data_dict)
elif metric == 'open-max-days':
datarequests = get_action('datarequests_open_after_x_days')(
{}, data_dict)
else:
closing_circumstances = [
x['circumstance']
for x in reporting_helpers.get_closing_circumstance_list()
]
if circumstance not in closing_circumstances:
raise Invalid(
_('Circumstance {0} is not valid'.format(circumstance)))
datarequests = get_action('datarequests_for_circumstance')(
{}, data_dict)
data_dict['datarequests'] = datarequests
return render('reporting/datarequests.html', extra_vars=data_dict)
except Invalid as e: # Exception raised from get_validator('group_id_exists')
log.warn(e)
return abort(404, _('Organisation %s not found') % org_id)
except NotAuthorized as e: # Exception raised from check_user_access
log.warn(e)
return abort(
403,
_('You are not authorised to view the datarequest {0} report for organisation {1}'
.format(report_type, org_id)))
def update_pd_record(self, owner_org, resource_name, pk):
pk = [url_part_unescape(p) for p in pk.split(',')]
lc = LocalCKAN(username=c.user)
try:
chromo = h.recombinant_get_chromo(resource_name)
rcomb = lc.action.recombinant_show(
owner_org=owner_org, dataset_type=chromo['dataset_type'])
[res
] = [r for r in rcomb['resources'] if r['name'] == resource_name]
check_access('datastore_upsert', {
'user': c.user,
'auth_user_obj': c.userobj
}, {'resource_id': res['id']})
except NotAuthorized:
abort(403, _('Unauthorized'))
choice_fields = {
f['datastore_id']: [{
'value': k,
'label': v
} for (k, v) in f['choices']]
for f in h.recombinant_choice_fields(resource_name)
}
pk_fields = aslist(chromo['datastore_primary_key'])
pk_filter = dict(zip(pk_fields, pk))
records = lc.action.datastore_search(resource_id=res['id'],
filters=pk_filter)['records']
if len(records) == 0:
abort(404, _('Not found'))
if len(records) > 1:
abort(400, _('Multiple records found'))
record = records[0]
if request.method == 'POST':
post_data = parse_params(request.POST,
ignore_keys=['save'] + pk_fields)
if 'cancel' in post_data:
return redirect(
h.url_for(
controller=
'ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=resource_name,
owner_org=rcomb['owner_org'],
))
data, err = clean_check_type_errors(post_data, chromo['fields'],
pk_fields, choice_fields)
# can't change pk fields
for f_id in data:
if f_id in pk_fields:
data[f_id] = record[f_id]
try:
lc.action.datastore_upsert(
resource_id=res['id'],
#method='update', FIXME not raising ValidationErrors
records=[{
k: None if k in err else v
for (k, v) in data.items()
}],
dry_run=bool(err))
except ValidationError as ve:
err = dict(
{
k: [_(e) for e in v]
for (k, v) in ve.error_dict['records'][0].items()
}, **err)
if err:
return render('recombinant/update_pd_record.html',
extra_vars={
'data': data,
'resource_name': resource_name,
'chromo_title': chromo['title'],
'choice_fields': choice_fields,
'pk_fields': pk_fields,
'owner_org': rcomb['owner_org'],
'errors': err,
})
h.flash_notice(_(u'Record %s Updated') % u','.join(pk))
return redirect(
h.url_for(
controller=
'ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=resource_name,
owner_org=rcomb['owner_org'],
))
data = {}
for f in chromo['fields']:
if not f.get('import_template_include', True):
continue
val = record[f['datastore_id']]
data[f['datastore_id']] = val
return render('recombinant/update_pd_record.html',
extra_vars={
'data': data,
'resource_name': resource_name,
'chromo_title': chromo['title'],
'choice_fields': choice_fields,
'pk_fields': pk_fields,
'owner_org': rcomb['owner_org'],
'errors': {},
})
def pages_edit(page=None,
data=None,
errors=None,
error_summary=None,
page_type='pages'):
page_dict = None
if page:
if page.startswith('/'):
page = page[1:]
page_dict = tk.get_action('ckanext_pages_show')(data_dict={
'org_id': None,
'page': page
})
if page_dict is None:
page_dict = {}
if tk.request.method == 'POST' and not data:
data = _parse_form_data(tk.request)
page_dict.update(data)
page_dict['org_id'] = None
page_dict['page'] = page
page_dict['page_type'] = 'page' if page_type == 'pages' else page_type
try:
tk.get_action('ckanext_pages_update')(data_dict=page_dict)
except tk.ValidationError as e:
errors = e.error_dict
error_summary = e.error_summary
tk.h.flash_error(error_summary)
return pages_edit(page,
data,
errors,
error_summary,
page_type=page_type)
if ckan_29_or_higher:
endpoint = 'show' if page_type in (
'pages', 'page') else '%s_show' % page_type
return tk.redirect_to('pages.%s' % endpoint,
page=page_dict['name'])
else:
endpoint = 'pages_show' if page_type == 'page' else '%s_show' % page_type
tk.redirect_to(endpoint, page='/' + page_dict['name'])
try:
tk.check_access('ckanext_pages_update',
{'user': tk.c.user or tk.c.author})
except tk.NotAuthorized:
return tk.abort(401, _('Unauthorized to create or edit a page'))
if not data:
data = page_dict
errors = errors or {}
error_summary = error_summary or {}
form_snippet = config.get('ckanext.pages.form',
'ckanext_pages/base_form.html')
vars = {
'data': data,
'errors': errors,
'error_summary': error_summary,
'page': page or '',
'form_snippet': form_snippet
}
return tk.render('ckanext_pages/%s_edit.html' % page_type, extra_vars=vars)