def open_datarequest(id):
data_dict = {'id': id}
context = _get_context()
# Basic initialization
c.datarequest = {}
try:
check_access(OPEN_DATAREQUEST, context, data_dict)
c.datarequest = get_action(SHOW_DATAREQUEST)(context, data_dict)
if c.datarequest.get('closed', False) is False:
return abort(403, _('This data request is already open'))
else:
data_dict = {}
data_dict['id'] = id
data_dict['organization_id'] = c.datarequest.get('organization_id')
get_action(OPEN_DATAREQUEST)(context, data_dict)
return redirect_to(url_for('datarequest.show', id=data_dict['id']))
except ValidationError as e:
log.warn(e)
errors_summary = _get_errors_summary(e.error_dict)
return abort(403, errors_summary)
except ObjectNotFound as e:
log.warn(e)
return abort(404, _('Data Request %s not found') % id)
except NotAuthorized as e:
log.warn(e)
return abort(
403, _('You are not authorized to open the Data Request %s' % id))
def run_validation(assume_yes, resource_ids, dataset_ids, search_params):
if resource_ids:
for resource_id in resource_ids:
resource = get_action('resource_show')({}, {'id': resource_id})
_run_validation_on_resource(resource['id'], resource['package_id'])
else:
query = _search_datasets()
if query['count'] == 0:
error('No suitable datasets, exiting...')
elif not assume_yes:
msg = ('\nYou are about to start validation for {0} datasets'
'.\n Do you want to continue?')
if not user_confirm(msg.format(query['count'])):
error('Command aborted by user')
result = get_action('resource_validation_run_batch')(
{
'ignore_auth': True
}, {
'dataset_ids': dataset_ids,
'query': search_params
})
print(result['output'])
def group_show(id, group_type, page=None):
if page and page.startswith('/'):
page = page[1:]
_template_setup_group(id, group_type)
context = {'for_view': True}
action = 'organization_show' if group_type == 'organization' else 'group_show'
group_dict = tk.get_action(action)(context, {'id': id})
if not page:
return group_list_pages(id, group_type, group_dict)
_page = tk.get_action('ckanext_pages_show')(data_dict={
'org_id': tk.c.group_dict['id'],
'page': page
})
if _page is None:
return group_list_pages(id, group_type, group_dict)
tk.c.page = _page
return tk.render('ckanext_pages/{}_page.html'.format(group_type), {
'group_type': group_type,
'group_dict': group_dict
})
def run_validation(self):
if self.options.resource_id:
for resource_id in self.options.resource_id:
resource = get_action('resource_show')({}, {'id': resource_id})
self._run_validation_on_resource(resource['id'],
resource['package_id'])
else:
query = _search_datasets()
if query['count'] == 0:
error('No suitable datasets, exiting...')
elif not self.options.assume_yes:
msg = ('\nYou are about to start validation for {0} datasets' +
'.\n Do you want to continue?')
confirm = query_yes_no(msg.format(query['count']))
if confirm == 'no':
error('Command aborted by user')
result = get_action('resource_validation_run_batch')(
{
'ignore_auth': True
}, {
'dataset_ids': self.options.dataset_id,
'query': self.options.search_params
})
print(result['output'])
def markdown_to_html():
''' Migrates the notes of all showcases from markdown to html.
When using CKEditor, notes on showcases are stored in html instead of
markdown, this command will migrate all nothes using CKAN's
render_markdown core helper.
'''
showcases = tk.get_action('ckanext_showcase_list')(data_dict={})
site_user = tk.get_action('get_site_user')({
'model': model,
'ignore_auth': True
}, {})
context = {
'model': model,
'session': model.Session,
'ignore_auth': True,
'user': site_user['name'],
}
for showcase in showcases:
tk.get_action('package_patch')(
context, {
'id': showcase['id'],
'notes': h.render_markdown(showcase['notes'])
})
log.info('All notes were migrated successfully.')
def pages_delete(page, page_type='pages'):
if page.startswith('/'):
page = page[1:]
if 'cancel' in tk.request.params:
if ckan_29_or_higher:
return tk.redirect_to('pages.%s_edit' % page_type, page=page)
else:
tk.redirect_to('%s_edit' % page_type, page='/' + page)
try:
if tk.request.method == 'POST':
tk.get_action('ckanext_pages_delete')({}, {'page': page})
if ckan_29_or_higher:
endpoint = 'index' if page_type in (
'pages', 'page') else '%s_index' % page_type
return tk.redirect_to('pages.%s' % endpoint)
else:
tk.redirect_to('%s_index' % page_type)
else:
return tk.abort(404, _('Page Not Found'))
except tk.NotAuthorized:
return tk.abort(401, _('Unauthorized to delete page'))
except tk.ObjectNotFound:
return tk.abort(404, _('Group not found'))
return tk.render('ckanext_pages/confirm_delete.html', {'page': page})
def read_view(id):
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'for_view': True,
'auth_user_obj': c.userobj
}
data_dict = {'id': id}
# check if showcase exists
try:
c.pkg_dict = tk.get_action('package_show')(context, data_dict)
except tk.ObjectNotFound:
return tk.abort(404, _('Showcase not found'))
except tk.NotAuthorized:
return tk.abort(401, _('Unauthorized to read showcase'))
# get showcase packages
c.showcase_pkgs = tk.get_action('ckanext_showcase_package_list')(
context, {
'showcase_id': c.pkg_dict['id']
})
package_type = DATASET_TYPE_NAME
return tk.render('showcase/read.html',
extra_vars={'dataset_type': package_type})
def delete_view(id):
if 'cancel' in tk.request.params:
tk.redirect_to('showcase_blueprint.edit' if tk.check_ckan_version(
min_version='2.9.0') else 'showcase_edit',
id=id)
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj
}
try:
tk.check_access('ckanext_showcase_delete', context, {'id': id})
except tk.NotAuthorized:
return tk.abort(401, _('Unauthorized to delete showcase'))
if tk.check_ckan_version(min_version='2.9.0'):
index_route = 'showcase_blueprint.index'
else:
index_route = 'showcase_index'
try:
if tk.request.method == 'POST':
tk.get_action('ckanext_showcase_delete')(context, {'id': id})
h.flash_notice(_('Showcase has been deleted.'))
return tk.redirect_to(index_route)
c.pkg_dict = tk.get_action('package_show')(context, {'id': id})
except tk.NotAuthorized:
tk.abort(401, _('Unauthorized to delete showcase'))
except tk.ObjectNotFound:
tk.abort(404, _('Showcase not found'))
return tk.render('showcase/confirm_delete.html',
extra_vars={'dataset_type': DATASET_TYPE_NAME})
def _push_file_to_logstash_folder(_file, _file_name, _dataset_id):
storage_path = config.get('ckan.storage_path')
try:
result = res = t.get_action('package_show')({
'ignore_auth':
True,
'user':
t.get_action('get_site_user')({
'ignore_auth': True
})['name']
}, {
'id': _dataset_id
})
package_name = result.get('name')
except ValueError as e:
log.debug(e)
# Create the directory if it didn't exist
extra_path = os.path.join(storage_path, 'storage', package_name)
full_path = os.path.join(extra_path, _file_name)
if not os.path.exists(extra_path):
try:
os.makedirs(extra_path)
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
f = open(_file, 'r')
w = open(full_path, 'w')
for lines in f:
w.write(lines)
w.close()
f.close()
log.debug('File saved to %s' % full_path)
def _get_site_user_api_key():
site_user_name = t.get_action('get_site_user')({'ignore_auth': True}, {})
site_user = t.get_action('get_site_user')({
'ignore_auth': True
}, {
'id': site_user_name
})
return site_user['apikey']
def _update_package(package_dict):
# Set some defaults
try:
print('Updating package {0}'.format(package_dict['id']))
get_action('package_patch')(context, package_dict)
return True
except Exception as ex:
print('Package exception: %s' % ex)
return False
def _update_resource(res_dict):
# Set some defaults
try:
print('Updating resource {0}'.format(res_dict['id']))
get_action('resource_patch')(context, res_dict)
return True
except Exception as ex:
print('Resource exception: %s' % ex)
return False
def test_harvest_source_create_twice_with_unique_url(self):
data_dict = SOURCE_DICT.copy()
factories.HarvestSourceObj(**data_dict)
site_user = get_action('get_site_user')({
'ignore_auth': True
}, {})['name']
data_dict['name'] = 'another-source'
data_dict['url'] = 'http://another-url'
get_action('harvest_source_create')({'user': site_user}, data_dict)
def test_harvest_source_create_twice_with_same_url(self):
data_dict = SOURCE_DICT.copy()
factories.HarvestSourceObj(**data_dict)
site_user = get_action('get_site_user')({
'ignore_auth': True
}, {})['name']
data_dict['name'] = 'another-source'
with pytest.raises(ValidationError):
get_action('harvest_source_create')({'user': site_user}, data_dict)
def test_harvest_source_create_twice_with_unique_url_and_config(self):
data_dict = SOURCE_DICT.copy()
factories.HarvestSourceObj(**data_dict)
site_user = get_action('get_site_user')({
'ignore_auth': True
}, {})['name']
data_dict['name'] = 'another-source'
data_dict['config'] = '{"something": "new"}'
get_action('harvest_source_create')({'user': site_user}, data_dict)
def _run_validation_on_resource(resource_id, dataset_id):
get_action(u'resource_validation_run')({
u'ignore_auth': True
}, {
u'resource_id': resource_id,
u'async': True
})
log.debug('Resource %s from dataset %s sent to the validation queue',
resource_id, dataset_id)
def _run_async_validation(resource_id):
try:
t.get_action(u'resource_validation_run')({
u'ignore_auth': True
}, {
u'resource_id': resource_id,
u'async': True
})
except t.ValidationError as e:
log.warning(u'Could not run validation for resource {}: {}'.format(
resource_id, str(e)))
def resource_create(context, data_dict):
'''Appends a new resource to a datasets list of resources.
This is duplicate of the CKAN core resource_create action, with just the
addition of a synchronous data validation step.
This is of course not ideal but it's the only way right now to hook
reliably into the creation process without overcomplicating things.
Hopefully future versions of CKAN will incorporate more flexible hook
points that will allow a better approach.
'''
model = context['model']
package_id = t.get_or_bust(data_dict, 'package_id')
if not data_dict.get('url'):
data_dict['url'] = ''
pkg_dict = t.get_action('package_show')(dict(context, return_type='dict'),
{
'id': package_id
})
t.check_access('resource_create', context, data_dict)
for plugin in plugins.PluginImplementations(plugins.IResourceController):
plugin.before_create(context, data_dict)
if 'resources' not in pkg_dict:
pkg_dict['resources'] = []
upload = uploader.get_resource_uploader(data_dict)
if 'mimetype' not in data_dict:
if hasattr(upload, 'mimetype'):
data_dict['mimetype'] = upload.mimetype
if 'size' not in data_dict:
if hasattr(upload, 'filesize'):
data_dict['size'] = upload.filesize
pkg_dict['resources'].append(data_dict)
try:
context['defer_commit'] = True
context['use_cache'] = False
t.get_action('package_update')(context, pkg_dict)
context.pop('defer_commit')
except t.ValidationError, e:
try:
raise t.ValidationError(e.error_dict['resources'][-1])
except (KeyError, IndexError):
raise t.ValidationError(e.error_dict)
def test_harvest_job_create_as_sysadmin(self):
source = factories.HarvestSource(**SOURCE_DICT.copy())
site_user = get_action('get_site_user')({
'ignore_auth': True
}, {})['name']
data_dict = {'source_id': source['id'], 'run': True}
job = get_action('harvest_job_create')({'user': site_user}, data_dict)
assert job['source_id'] == source['id']
assert job['status'] == 'Running'
assert job['gather_started'] is None
assert 'stats' in job.keys()
def _run_validation_on_resource(self, resource_id, dataset_id):
log = logging.getLogger(__name__)
get_action(u'resource_validation_run')({
u'ignore_auth': True
}, {
u'resource_id': resource_id,
u'async': True
})
log.debug('Resource %s from dataset %s sent to the validation queue',
resource_id, dataset_id)
def _run_validation_on_resource(self, resource_id, dataset_id):
log = logging.getLogger(__name__)
get_action(u'resource_validation_run')({
u'ignore_auth': True
}, {
u'resource_id': resource_id,
u'async': True
})
msg = ('Resource {} from dataset {} sent to ' + 'the validation queue')
log.debug(msg.format(resource_id, dataset_id))
def test_search_labels():
user = factories.User()
org = factories.Organization(user=user)
dataset = factories.Dataset(private=True,
state="draft",
owner_org=org["id"],
user=user)
context = {"user": user["name"]}
search_dict = {"include_drafts": True, "include_private": True}
query = search.query_for(model.Package)
result = query.run({"fq": "+state:*"}, permission_labels=["draft"])
assert result["count"] == 1
result = query.run({"fq": "+state:*"}, permission_labels=["review"])
assert result["count"] == 0
result = query.run({"fq": "+state:*"}, permission_labels=["published"])
assert result["count"] == 0
result = tk.get_action("package_search")(context, search_dict)
assert result["count"] == 1
helpers.call_action("workflow_state_change", id=dataset["id"], review=True)
search_dict = {"include_drafts": True, "include_private": True}
query = search.query_for(model.Package)
result = query.run({"fq": "+state:*"}, permission_labels=["draft"])
assert result["count"] == 0
result = query.run({"fq": "+state:*"}, permission_labels=["review"])
assert result["count"] == 1
result = query.run({"fq": "+state:*"}, permission_labels=["published"])
assert result["count"] == 0
result = tk.get_action("package_search")(context, search_dict)
assert result["count"] == 1
helpers.call_action("workflow_state_change",
id=dataset["id"],
publish=True)
search_dict = {"include_drafts": True, "include_private": True}
query = search.query_for(model.Package)
result = query.run({"fq": "+state:*"}, permission_labels=["draft"])
assert result["count"] == 0
result = query.run({"fq": "+state:*"}, permission_labels=["review"])
assert result["count"] == 0
result = query.run({"fq": "+state:*"}, permission_labels=["published"])
assert result["count"] == 1
result = tk.get_action("package_search")(context, search_dict)
assert result["count"] == 1
result = tk.get_action("package_search")(None, search_dict)
assert result["count"] == 0
def gather_admin_metrics(org_id, permission):
"""Collect admin statistics metrics for the provided organisation"""
data_dict = {
'org_id': org_id,
'return_count_only': True,
'permission': permission
}
return {
'de_identified_datasets': get_action('de_identified_datasets')({}, data_dict),
'overdue_datasets': get_action('overdue_datasets')({}, data_dict),
'datasets_no_groups': get_action('datasets_no_groups')({}, data_dict),
'datasets_no_tags': get_action('datasets_no_tags')({}, data_dict)
}
def post(self, id):
context = self._prepare(id)
utils.check_edit_view_auth(id)
data_dict = dataset.clean_dict(
dataset.dict_fns.unflatten(
dataset.tuplize_dict(dataset.parse_params(tk.request.form))))
data_dict.update(
dataset.clean_dict(
dataset.dict_fns.unflatten(
dataset.tuplize_dict(dataset.parse_params(
tk.request.files)))))
data_dict['id'] = id
try:
pkg = tk.get_action('ckanext_showcase_update')(context, data_dict)
except tk.ValidationError as e:
errors = e.error_dict
error_summary = e.error_summary
return self.get(id, data_dict, errors, error_summary)
tk.c.pkg_dict = pkg
# redirect to showcase details page
url = h.url_for('showcase_blueprint.read', id=pkg['name'])
return h.redirect_to(url)
def post(self):
data_dict = dataset.clean_dict(
dataset.dict_fns.unflatten(
dataset.tuplize_dict(dataset.parse_params(tk.request.form))))
data_dict.update(
dataset.clean_dict(
dataset.dict_fns.unflatten(
dataset.tuplize_dict(dataset.parse_params(
tk.request.files)))))
context = self._prepare()
data_dict['type'] = utils.DATASET_TYPE_NAME
context['message'] = data_dict.get('log_message', '')
try:
pkg_dict = tk.get_action('ckanext_showcase_create')(context,
data_dict)
except tk.ValidationError as e:
errors = e.error_dict
error_summary = e.error_summary
data_dict['state'] = 'none'
return self.get(data_dict, errors, error_summary)
# redirect to manage datasets
url = h.url_for('showcase_blueprint.manage_datasets',
id=pkg_dict['name'])
return h.redirect_to(url)
def organization_index(self):
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'for_view': True,
'with_private': False
}
data_dict = {'all_fields': True}
try:
check_access('site_read', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
# pass user info to context as needed to view private datasets of
# orgs correctly
if c.userobj:
context['user_id'] = c.userobj.id
context['user_is_admin'] = c.userobj.sysadmin
results = get_action('organization_list')(context, data_dict)
def org_key(org):
title = org['title'].split(' | ')[-1 if c.language == 'fr' else 0]
return normalize_strip_accents(title)
results.sort(key=org_key)
c.page = Page(collection=results,
page=request.params.get('page', 1),
url=h.pager_url,
items_per_page=1000)
return render('organization/index.html')
def test_harvest_abort_failed_jobs_with_include_and_exclude(self):
# prepare
data_dict = SOURCE_DICT.copy()
source = factories.HarvestSourceObj(**data_dict)
job = factories.HarvestJobObj(source=source)
# Simulate running job created 4 days ago
setattr(job, 'status', 'Running')
setattr(job, 'created',
datetime.datetime.utcnow() - datetime.timedelta(days=4))
model.Session.commit()
context = {
'model': model,
'session': model.Session,
'ignore_auth': True,
'user': ''
}
# include must prevaild over exclude
result = get_action('harvest_abort_failed_jobs')(context, {
'life_span': 3,
'exclude': source.id,
'include': source.id
})
job = harvest_model.HarvestJob.get(job.id)
assert job.status in ('Finished', 'Aborted')
assert job.source_id == source.id
assert 'Aborted jobs: 1' in result
def organization_index(self):
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True,
'with_private': False}
data_dict = {'all_fields': True}
try:
check_access('site_read', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
# pass user info to context as needed to view private datasets of
# orgs correctly
if c.userobj:
context['user_id'] = c.userobj.id
context['user_is_admin'] = c.userobj.sysadmin
results = get_action('organization_list')(context, data_dict)
def org_key(org):
title = org['title'].split(' | ')[-1 if c.language == 'fr' else 0]
return normalize_strip_accents(title)
results.sort(key=org_key)
c.page = Page(
collection=results,
page=request.params.get('page', 1),
url=h.pager_url,
items_per_page=1000
)
return render('organization/index.html')
def action(get_request_data_function, core_function, api_action, ver):
try:
function = get_action(api_action)
side_effect_free = getattr(function, 'side_effect_free', False)
request_data = get_request_data_function(
try_url_params=side_effect_free)
capture_api_actions = plugin.GoogleAnalyticsPlugin.capture_api_actions
# Only send api actions if it is in the capture_api_actions dictionary
if api_action in capture_api_actions and isinstance(
request_data, dict):
api_action_label = capture_api_actions.get(api_action)
parameter_value = request_data.get('id', '')
if parameter_value == '' and 'resource_id' in request_data:
parameter_value = request_data['resource_id']
if parameter_value == '' and 'q' in request_data:
parameter_value = request_data['q']
if parameter_value == '' and 'query' in request_data:
parameter_value = request_data['query']
if parameter_value == '' and 'sql' in request_data:
parameter_value = _alter_sql(request_data['sql'])
event_action = "{0} - {1}".format(
api_action,
request.environ['PATH_INFO'].replace('/api/3/', ''))
event_label = api_action_label.format(parameter_value)
_post_analytics(g.user, event_action, event_label, request_data)
except Exception as e:
log.debug(e)
pass
return core_function(api_action, ver=ver)
def _add_to_pkg_dict(self, context, pkg_dict):
'''
Add key/values to pkg_dict and return it.
'''
if pkg_dict['type'] != 'showcase':
return pkg_dict
if not context.get('reindexing'):
# Add a display url for the Showcase image to the pkg dict so template
# has access to it.
image_url = pkg_dict.get('image_url')
pkg_dict[u'image_display_url'] = image_url
if image_url and not image_url.startswith('http'):
pkg_dict[u'image_url'] = image_url
pkg_dict[u'image_display_url'] = \
h.url_for_static('uploads/{0}/{1}'
.format(DATASET_TYPE_NAME,
pkg_dict.get('image_url')),
qualified=True)
# Add dataset count
pkg_dict[u'num_datasets'] = len(
tk.get_action('ckanext_showcase_package_list')(
context, {'showcase_id': pkg_dict['id']}))
# Rendered notes
if showcase_helpers.get_wysiwyg_editor() == 'ckeditor':
pkg_dict[u'showcase_notes_formatted'] = pkg_dict['notes']
else:
pkg_dict[u'showcase_notes_formatted'] = \
h.render_markdown(pkg_dict['notes'])
return pkg_dict
def test_harvest_abort_failed_jobs_with_source_frequency(self):
# prepare
data_dict = SOURCE_DICT.copy()
source = factories.HarvestSourceObj(**data_dict)
job = factories.HarvestJobObj(source=source)
# Simulate running job created 4 days ago
setattr(job, 'status', 'Running')
setattr(job, 'created',
datetime.datetime.utcnow() - datetime.timedelta(days=4))
# set source update frequency to biweekly
# job will be aborted if it's runs more then 2 weeks
setattr(source, 'frequency', 'BIWEEKLY')
model.Session.commit()
context = {
'model': model,
'session': model.Session,
'ignore_auth': True,
'user': ''
}
result = get_action('harvest_abort_failed_jobs')(context, {
'life_span': 3
})
job = harvest_model.HarvestJob.get(job.id)
assert job.status == 'Running'
assert job.source_id == source.id
assert 'Aborted jobs: 0' in result
def logged_in(self):
# we need to set the language via a redirect
# Lang is not being retrieved properly by the Babel i18n lib in
# this redirect, so using this clunky workaround for now.
lang = session.pop('lang', None)
if lang is None:
came_from = request.params.get('came_from', '')
if came_from.startswith('/fr'):
lang = 'fr'
else:
lang = 'en'
session.save()
# we need to set the language explicitly here or the flash
# messages will not be translated.
i18n.set_lang(lang)
if c.user:
context = None
data_dict = {'id': c.user}
user_dict = get_action('user_show')(context, data_dict)
h.flash_success(
_('<strong>Note</strong><br>{0} is now logged in').format(
user_dict['display_name']
),
allow_html=True
)
notice_no_access()
return h.redirect_to(
controller='ckanext.canada.controller:CanadaController',
action='home',
locale=lang)
else:
h.flash_error(_('Login failed. Bad username or password.'))
return h.redirect_to(
controller='user',
action='login', locale=lang
)
def organization_autocomplete(self):
q = request.params.get('q', '')
limit = request.params.get('limit', 20)
organization_list = []
if q:
context = {'user': c.user, 'model': model}
data_dict = {'q': q, 'limit': limit}
organization_list = get_action(
'organization_autocomplete'
)(context, data_dict)
def _org_key(org):
return org['title'].split(' | ')[-1 if c.language == 'fr' else 0]
return [{
'id': o['id'],
'name': _org_key(o),
'title': _org_key(o)
} for o in organization_list]
