def _add_or_reply(self, dataset_name):
"""
Allows the user to add a comment to an existing dataset
"""
context = {'model': model, 'user': c.user}
# Auth check to make sure the user can see this package
ctx = context
ctx['id'] = dataset_name
check_access('package_show', ctx)
try:
c.pkg_dict = get_action('package_show')(context, {
'id': dataset_name
})
c.pkg = context['package']
except:
abort(403)
errors = {}
if request.method == 'POST':
data_dict = clean_dict(
unflatten(tuplize_dict(parse_params(request.POST))))
data_dict['parent_id'] = c.parent.id if c.parent else None
data_dict['url'] = '/dataset/%s' % c.pkg.name
success = False
try:
res = get_action('comment_create')(context, data_dict)
success = True
except ValidationError, ve:
errors = ve.error_dict
except Exception, e:
abort(403)
def delete(self, id):
"""Provide a delete action, but only for UKLP datasets"""
from ckan.lib.search import SearchIndexError
context = {
'model': model,
'session': model.Session,
'user': c.user,
}
pkg_dict = get_action('package_show')(context, {
'id': id
}) # has side-effect of populating context.get('package')
if request.params: # POST
if 'cancel' in request.params:
h.redirect_to(controller='package', action='read', id=id)
elif 'delete' in request.params:
try:
package_name = pkg_dict['name']
get_action('package_delete')(context, {'id': id})
h.flash_success(_('Successfully deleted package.'))
self._form_save_redirect(package_name, 'edit')
except NotAuthorized:
abort(401, _('Unauthorized to delete package %s') % id)
except ObjectNotFound, e:
abort(404, _('Package not found'))
except DataError:
abort(400, _(u'Integrity Error'))
except SearchIndexError, e:
abort(500,
_(u'Unable to update search index.') + repr(e.args))
def enqueue_document(user, filename, publisher):
"""
Uses the provided data to send a job to the priority celery queue so that
the spreadsheet is processed. We should create a job_started message header_row
to ensure that the user sees immediately that something is happening.
"""
from pylons import config
from ckan import model
from ckan.model.types import make_uuid
from ckan.lib.celery_app import celery
site_user = get_action('get_site_user')({
'model': model,
'ignore_auth': True,
'defer_commit': True
}, {})
task_id = make_uuid()
# Create the task for the queue
context = json.dumps({
'username':
user.name,
'site_url':
config.get('ckan.site_url_internally') or config.get('ckan.site_url'),
'apikey':
user.apikey,
'site_user_apikey':
site_user['apikey']
})
data = json.dumps({
'file': filename,
'publisher': publisher.name,
'publisher_id': publisher.id,
'jobid': task_id
})
celery.send_task("inventory.process",
args=[context, data],
task_id=task_id,
queue='priority')
# Create a task status.... and update it so that the user knows it has been started.
inventory_task_status = {
'entity_id': task_id,
'entity_type': u'inventory',
'task_type': u'inventory',
'key': u'celery_task_id',
'value': task_id,
'error': u'',
'state': 'Started',
'last_updated': datetime.datetime.now().isoformat()
}
inventory_task_context = {
'model': model,
'user': user.name,
'ignore_auth': True
}
res = get_action('task_status_update')(inventory_task_context,
inventory_task_status)
return res['id'], inventory_task_status['last_updated']
def add_field__openness(cls, pkg_dict):
'''Add the openness score (stars) to the search index'''
import ckan
from ckanext.dgu.plugins_toolkit import get_action
context = {'model': ckan.model, 'session': ckan.model.Session,
'ignore_auth': True}
data_dict = {'id': pkg_dict['id']}
try:
qa_openness = get_action('qa_package_openness_show')(context, data_dict)
except ObjectNotFound:
log.warning('No QA info for package %s', pkg_dict['name'])
return
except KeyError:
# occurs during tests or if you've not install ckanext-qa
log.warning('QA not installed - not indexing it.')
return
pkg_dict['openness_score'] = qa_openness.get('openness_score')
log.debug('Openness score %s: %s', pkg_dict['openness_score'], pkg_dict['name'])
try:
qa_broken = get_action('qa_package_broken_show')(context, data_dict)
except ObjectNotFound:
log.warning('No brokenness info for package %s', pkg_dict['name'])
return
if not hasattr(cls, 'broken_links_map'):
cls.broken_links_map = {
True: 'Broken',
'some': 'Partially broken',
False: 'OK',
None: 'TBC'
}
pkg_dict['broken_links'] = cls.broken_links_map[qa_broken.get('archival_is_broken')]
log.debug('Broken links %s: %s', pkg_dict['broken_links'], pkg_dict['name'])
def edit_item(self, id, data=None, errors=None, error_summary=None):
"""
Allows for the editing of a single item
"""
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'extras_as_string': True,
'save': 'save' in request.params}
if context['save'] and not data:
return self._save_edit(id, context)
try:
c.pkg_dict = get_action('package_show')(context, {'id': id})
context['for_edit'] = True
old_data = get_action('package_show')(context, {'id': id})
# old data is from the database and data is passed from the
# user if there is a validation error. Use users data if there.
data = data or old_data
except NotAuthorized:
abort(401, _('Unauthorized to read package %s') % '')
except ObjectNotFound:
abort(404, _('Dataset not found'))
c.pkg = context.get("package")
try:
check_access('package_update',context)
except NotAuthorized, e:
abort(401, _('User %r not authorized to edit %s') % (c.user, id))
def delete(self, id):
"""Provide a delete action, but only for UKLP datasets"""
from ckan.lib.search import SearchIndexError
context = {
'model': model,
'session': model.Session,
'user': c.user,
}
pkg_dict = get_action('package_show')(context, {'id':id}) # has side-effect of populating context.get('package')
if request.params: # POST
if 'cancel' in request.params:
h.redirect_to(controller='package', action='read', id=id)
elif 'delete' in request.params:
try:
package_name = pkg_dict['name']
get_action('package_delete')(context, {'id':id})
h.flash_success(_('Successfully deleted package.'))
self._form_save_redirect(package_name, 'edit')
except NotAuthorized:
abort(401, _('Unauthorized to delete package %s') % id)
except ObjectNotFound, e:
abort(404, _('Package not found'))
except DataError:
abort(400, _(u'Integrity Error'))
except SearchIndexError, e:
abort(500, _(u'Unable to update search index.') + repr(e.args))
def index(self):
c.q = request.params.get('q', '')
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'for_view': True
}
# This is the one difference from ckan core.
# No need for all vocab / translation stuff, so save the massive
# effort of dictizing every tag. This improves page load from
# 60s to 10s.
data_dict = {'all_fields': False}
if c.q:
try:
page = int(request.params.get('page', 1))
except ValueError:
abort(404, _('Not found'))
data_dict['q'] = c.q
data_dict['limit'] = LIMIT
data_dict['offset'] = (page - 1) * LIMIT
data_dict['return_objects'] = True
result_dict = get_action('tag_search')(context, data_dict)
def pager_url(q=None, page=None):
return h.url_for(
controller='ckanext.dgu.controllers.tag:TagController',
action='index',
q=request.params['q'],
page=page)
c.page = h.Page(
collection=result_dict['results'],
page=page,
item_count=result_dict['count'],
items_per_page=LIMIT,
url=pager_url,
)
c.page.items = [
tag_dict['name'] for tag_dict in result_dict['results']
]
else:
results = get_action('tag_list')(context, data_dict)
c.page = AlphaPageLarge(
collection=results,
page=request.params.get('page', 'A'),
alpha_attribute='name',
other_text=_('Other'),
controller_name='ckanext.dgu.controllers.tag:TagController',
)
return render('tag/index.html')
def index(self):
c.q = request.params.get("q", "")
context = {"model": model, "session": model.Session, "user": c.user or c.author, "for_view": True}
# This is the one difference from ckan core.
# No need for all vocab / translation stuff, so save the massive
# effort of dictizing every tag. This improves page load from
# 60s to 10s.
data_dict = {"all_fields": False}
if c.q:
try:
page = int(request.params.get("page", 1))
except ValueError:
abort(404, _("Not found"))
data_dict["q"] = c.q
data_dict["limit"] = LIMIT
data_dict["offset"] = (page - 1) * LIMIT
data_dict["return_objects"] = True
result_dict = get_action("tag_search")(context, data_dict)
def pager_url(q=None, page=None):
return h.url_for(
controller="ckanext.dgu.controllers.tag:TagController",
action="index",
q=request.params["q"],
page=page,
)
c.page = h.Page(
collection=result_dict["results"],
page=page,
item_count=result_dict["count"],
items_per_page=LIMIT,
url=pager_url,
)
c.page.items = [tag_dict["name"] for tag_dict in result_dict["results"]]
else:
results = get_action("tag_list")(context, data_dict)
c.page = AlphaPageLarge(
collection=results,
page=request.params.get("page", "A"),
alpha_attribute="name",
other_text=_("Other"),
controller_name="ckanext.dgu.controllers.tag:TagController",
)
return render("tag/index.html")
def enqueue_document(user, filename, publisher):
"""
Uses the provided data to send a job to the priority celery queue so that
the spreadsheet is processed. We should create a job_started message header_row
to ensure that the user sees immediately that something is happening.
"""
from pylons import config
from ckan import model
from ckan.model.types import make_uuid
from ckan.lib.celery_app import celery
site_user = get_action('get_site_user')(
{'model': model, 'ignore_auth': True, 'defer_commit': True}, {})
task_id = make_uuid()
# Create the task for the queue
context = json.dumps({
'username': user.name,
'site_url': config.get('ckan.site_url_internally') or config.get('ckan.site_url'),
'apikey': user.apikey,
'site_user_apikey': site_user['apikey']
})
data = json.dumps({
'file': filename,
'publisher': publisher.name,
'publisher_id': publisher.id,
'jobid': task_id
})
celery.send_task("inventory.process", args=[context, data], task_id=task_id, queue='priority')
# Create a task status.... and update it so that the user knows it has been started.
inventory_task_status = {
'entity_id': task_id,
'entity_type': u'inventory',
'task_type': u'inventory',
'key': u'celery_task_id',
'value': task_id,
'error': u'',
'state': 'Started',
'last_updated': datetime.datetime.now().isoformat()
}
inventory_task_context = {
'model': model,
'user': user.name,
'ignore_auth': True
}
res = get_action('task_status_update')(inventory_task_context, inventory_task_status)
return res['id'], inventory_task_status['last_updated']
def latest_datasets(self, published_only=True):
"""Designed for the dgu home page, shows lists the latest datasets
that got changed (exluding extra, group and tag changes) with lots
of details about each dataset.
"""
try:
limit = int(request.params.get("limit", default_limit))
except ValueError:
limit = default_limit
limit = min(100, limit) # max value
from ckan.lib.search import SearchError
fq = 'capacity:"public"'
if published_only:
fq = fq + " unpublished:false"
try:
# package search
context = {"model": model, "session": model.Session, "user": "******"}
data_dict = {
"q": "",
"fq": fq,
"facet": "false",
"start": 0,
"rows": limit,
"sort": "metadata_modified desc",
}
query = get_action("package_search")(context, data_dict)
except SearchError, se:
log.error("Search error: %s", se)
def setup(self):
"""
Creates a harvested UKLP dataset.
"""
_drop_sysadmin()
self.admin = _create_sysadmin()
CreateTestData.create_test_user()
self.tester = "tester"
CreateTestData.create_groups(_EXAMPLE_GROUPS, admin_user_name=self.tester, auth_profile="publisher")
CreateTestData.flag_for_deletion(group_names=[g["name"] for g in _EXAMPLE_GROUPS])
context = {
"model": ckan.model,
"session": ckan.model.Session,
"user": self.admin,
"api_version": 2,
"schema": ckan.logic.schema.default_package_schema(),
}
package_dict = _UKLP_DATASET.copy()
self.uklp_dataset = get_action("package_create_rest")(context, package_dict)
CreateTestData.flag_for_deletion(pkg_names=[self.uklp_dataset["name"]])
def home(self):
extra_vars = {}
# Get the dataset count using search
# (shouldn't cache, as it makes it more likely to be out of sync with
# the data page)
from ckan.lib.search import SearchError, SearchQueryError
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'for_view': True,
'auth_user_obj': c.userobj
}
data_dict = {
'q': '*:*',
'fq': '+dataset_type:dataset',
#'facet.field': facets.keys(),
'rows': 0,
}
query = get_action('package_search')(context, data_dict)
extra_vars['num_datasets'] = query['count']
extra_vars['themes'] = get_themes()
return render('data/home.html', extra_vars=extra_vars)
def index(self):
from ckan.lib.search import SearchError
try:
# package search
context = {'model': model, 'session': model.Session,
'user': c.user or c.author}
fq = 'capacity:"public"'
data_dict = {
'q':'',
'fq':fq,
'facet.field':g.facets,
'rows':0,
'start':0,
}
query = get_action('package_search')(context,data_dict)
c.package_count = query['count']
c.facets = query['facets']
c.search_facets = query['search_facets']
# group search
#data_dict = {'order_by': 'packages', 'all_fields': 1}
#c.groups = get_action('group_list')(context, data_dict)
except SearchError, se:
log.error('Search error: %s', se)
c.package_count = 0
c.groups = []
def _search(self,
theme='',
q='',
rows=5,
sort_string='last_major_modification desc'):
"""
Helper for retrieving just a handful of popular/recent datasets for the
current theme.
"""
raise NotImplementedError('Cannot view theme pages yet.')
from ckan.lib.search import SearchError
packages = []
try:
# package search
context = {
'model': model,
'session': model.Session,
'user': '******'
}
data_dict = {
'q': q,
'fq': 'theme-primary:"%s"' % theme,
'facet': 'true',
'rows': rows,
'start': 0,
'sort': sort_string,
}
query = get_action('package_search')(context, data_dict)
packages = query['results']
except SearchError, se:
log.error('Search error: %s', se)
def latest_datasets(self, published_only=True):
'''Designed for the dgu home page, shows lists the latest datasets
that got changed (exluding extra, group and tag changes) with lots
of details about each dataset.
'''
try:
limit = int(request.params.get('limit', default_limit))
except ValueError:
limit = default_limit
limit = min(100, limit) # max value
from ckan.lib.search import SearchError
fq = 'capacity:"public"'
if published_only:
fq = fq + ' unpublished:false'
try:
# package search
context = {'model': model, 'session': model.Session,
'user': '******'}
data_dict = {
'q':'',
'fq': fq,
'facet':'false',
'start':0,
'rows': limit,
'sort': 'last_major_modification desc'
}
query = get_action('package_search')(context,data_dict)
except SearchError, se:
log.error('Search error: %s', se)
count = 0
def commitments(self, id):
"""
Shows all of the commitments for the specified publisher
"""
from ckanext.dgu.model.commitment import Commitment
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'extras_as_string': True,
'save': 'save' in request.params
}
try:
c.publisher_dict = get_action('organization_show')(context, {
'id': id
})
except NotAuthorized:
abort(401, _('Unauthorized to read commitments'))
except ObjectNotFound:
abort(404, _('Publisher not found'))
c.publisher = context.get('group')
c.commitments = Commitment.get_for_publisher(c.publisher.name)\
.order_by('commitment.dataset_name').all()
return render('commitment/read.html')
def edit(self, id):
"""
The edit homepage to allow department admins to download and
upload their inventories
"""
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'for_view': True
}
try:
c.group_dict = get_action('organization_show')(context, {"id": id})
c.group = context['group']
except ObjectNotFound:
abort(404, 'Organisation not found')
except NotAuthorized:
abort(401, 'Unauthorized to read group %s' % id)
try:
context['group'] = c.group
check_access('organization_update', context)
except NotAuthorized, e:
abort(
401, 'User %r not authorized to view internal unpublished' %
(c.user))
def add_field__harvest_document(cls, pkg_dict):
'''Index a harvested dataset\'s XML content
(Given a low priority when searching)'''
if pkg_dict.get('UKLP', '') == 'True':
import ckan
from ckanext.dgu.plugins_toolkit import get_action
context = {
'model': ckan.model,
'session': ckan.model.Session,
'ignore_auth': True
}
data_dict = {'id': pkg_dict.get('harvest_object_id', '')}
try:
harvest_object = get_action('harvest_object_show')(context,
data_dict)
pkg_dict[
'extras_harvest_document_content'] = harvest_object.get(
'content', '')
except ObjectNotFound:
log.warning(
'Unable to find harvest object "%s" '
'referenced by dataset "%s"', data_dict['id'],
pkg_dict['id'])
def index(self):
from ckan.lib.search import SearchError
try:
# package search
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author
}
fq = 'capacity:"public"'
data_dict = {
'q': '',
'fq': fq,
'facet.field': g.facets,
'rows': 0,
'start': 0,
}
query = get_action('package_search')(context, data_dict)
c.package_count = query['count']
c.facets = query['facets']
c.search_facets = query['search_facets']
# group search
#data_dict = {'order_by': 'packages', 'all_fields': 1}
#c.groups = get_action('group_list')(context, data_dict)
except SearchError, se:
log.error('Search error: %s', se)
c.package_count = 0
c.groups = []
def download(self, id):
"""
Downloads all of the current datasets for a given publisher as a read-only
CSV file.
"""
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'for_view': True,
'group': id
}
try:
c.group_dict = get_action('organization_show')(context, {"id": id})
c.group = context['group']
except ObjectNotFound:
abort(404, 'Organization not found')
except NotAuthorized:
abort(401, 'Unauthorized to read Organization %s' % id)
try:
context['group'] = c.group
check_access('organization_update', context)
except NotAuthorized, e:
abort(401,
'User %r not authorized to download unpublished ' % (c.user))
def upload(self, id):
"""
Upload of an unpublished file, accepts a POST request with a file and
then renders the result of the import to the user.
"""
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'for_view': True
}
try:
c.group_dict = get_action('organization_show')(context, {"id": id})
c.group = context['group']
except ObjectNotFound:
abort(404, 'Organization not found')
except NotAuthorized:
abort(401, 'Unauthorized to read group %s' % id)
try:
context['group'] = c.group
check_access('organization_update', context)
except NotAuthorized, e:
abort(401, 'User %r not authorized to upload inventory' % (c.user))
def latest_datasets(self, published_only=True):
'''Designed for the dgu home page, shows lists the latest datasets
that got changed (exluding extra, group and tag changes) with lots
of details about each dataset.
'''
try:
limit = int(request.params.get('limit', default_limit))
except ValueError:
limit = default_limit
limit = min(100, limit) # max value
from ckan.lib.search import SearchError
fq = 'capacity:"public"'
if published_only:
fq = fq + ' unpublished:false'
try:
# package search
context = {'model': model, 'session': model.Session,
'user': '******'}
data_dict = {
'q':'',
'fq': fq,
'facet':'false',
'start':0,
'rows': limit,
'sort': 'metadata_modified desc'
}
query = get_action('package_search')(context,data_dict)
except SearchError, se:
log.error('Search error: %s', se)
def setup(self):
"""
Creates a harvested UKLP dataset.
"""
_drop_sysadmin()
self.admin = _create_sysadmin()
CreateTestData.create_test_user()
self.tester = 'tester'
CreateTestData.create_groups(_EXAMPLE_GROUPS, admin_user_name=self.tester, auth_profile='publisher')
CreateTestData.flag_for_deletion(group_names=[g['name'] for g in _EXAMPLE_GROUPS])
context = {
'model': ckan.model,
'session': ckan.model.Session,
'user': self.admin,
'api_version': 2,
'schema': ckan.logic.schema.default_package_schema(),
}
package_dict = _UKLP_DATASET.copy()
self.uklp_dataset = get_action('package_create_rest')(context, package_dict)
CreateTestData.flag_for_deletion(pkg_names=[self.uklp_dataset['name']])
def delete(self, id):
"""Provide a delete ('withdraw') action, but only for UKLP datasets"""
from ckan.lib.search import SearchIndexError
context = {
'model': model,
'session': model.Session,
'user': c.user,
}
try:
pkg_dict = get_action('package_show')(context, {
'id': id
}) # has side-effect of populating context.get('package')
except NotAuthorized:
abort(401, 'Not authorized to delete package')
except ObjectNotFound:
abort(404, 'Dataset not found')
if request.params: # POST
if 'cancel' in request.params:
h.redirect_to(controller='package', action='read', id=id)
elif 'delete' in request.params:
try:
package_name = pkg_dict['name']
get_action('package_delete')(context, {'id': id})
is_uklp = get_from_flat_dict(pkg_dict['extras'],
'UKLP') == 'True'
if is_uklp:
action = 'withdrawn'
resource_type = get_from_flat_dict(
pkg_dict['extras'], 'resource-type') + ' record'
else:
action = 'deleted'
resource_type = 'dataset'
h.flash_success('Successfully %s %s.' \
% (action, resource_type))
self._form_save_redirect(package_name, 'edit')
except NotAuthorized:
abort(401, _('Unauthorized to delete package %s') % id)
except ObjectNotFound, e:
abort(404, _('Package not found'))
except DataError:
abort(400, _(u'Integrity Error'))
except SearchIndexError, e:
abort(500,
_(u'Unable to update search index.') + repr(e.args))
def moderation(self):
context = {'model': model, 'user': c.user}
check_access('moderation_queue_show', context)
try:
res = get_action('moderation_queue_show')(context, {})
except Exception, e:
abort(403)
def flag(self, dataset_name, id):
context = {'model': model, 'user': c.user}
try:
c.pkg_dict = get_action('package_show')(context, {
'id': dataset_name
})
c.pkg = context['package']
except Exception, e:
abort(403)
def test_view_system_user(self):
# created on the API
user_dict = get_action('get_site_user')({'model': model, 'ignore_auth': True}, {})
user = user_dict['name']
c.is_an_official = False
assert_equal(str(dgu_linked_user(user)), 'System Process')
c.is_an_official = True
assert_equal(str(dgu_linked_user(user, maxlength=100)),
'<a href="/data/user/test.ckan.net">System Process (Site user)</a>')
def test_view_system_user(self):
# created on the API
user_dict = get_action('get_site_user')({'model': model, 'ignore_auth': True}, {})
user = user_dict['name']
c.is_an_official = False
assert_equal(str(dgu_linked_user(user)), 'System Process')
c.is_an_official = True
assert_equal(str(dgu_linked_user(user, maxlength=100)),
'<a href="/data/user/test.ckan.net">System Process (Site user)</a>')
def delete(self, id):
"""Provide a delete ('withdraw') action, but only for UKLP datasets"""
from ckan.lib.search import SearchIndexError
context = {
'model': model,
'session': model.Session,
'user': c.user,
}
try:
pkg_dict = get_action('package_show')(context, {'id':id}) # has side-effect of populating context.get('package')
except NotAuthorized:
abort(401, 'Not authorized to delete package')
except ObjectNotFound:
abort(404, 'Dataset not found')
if request.params: # POST
if 'cancel' in request.params:
h.redirect_to(controller='package', action='read', id=id)
elif 'delete' in request.params:
try:
package_name = pkg_dict['name']
get_action('package_delete')(context, {'id':id})
is_uklp = get_from_flat_dict(pkg_dict['extras'], 'UKLP') == 'True'
if is_uklp:
action = 'withdrawn'
resource_type = get_from_flat_dict(pkg_dict['extras'], 'resource-type') + ' record'
else:
action = 'deleted'
resource_type = 'dataset'
h.flash_success('Successfully %s %s.' \
% (action, resource_type))
self._form_save_redirect(package_name, 'edit')
except NotAuthorized:
abort(401, _('Unauthorized to delete package %s') % id)
except ObjectNotFound, e:
abort(404, _('Package not found'))
except DataError:
abort(400, _(u'Integrity Error'))
except SearchIndexError, e:
abort(500, _(u'Unable to update search index.') + repr(e.args))
def add_field__openness(cls, pkg_dict):
'''Add the openness score (stars) to the search index'''
import ckan
from ckanext.dgu.plugins_toolkit import get_action
context = {
'model': ckan.model,
'session': ckan.model.Session,
'ignore_auth': True
}
data_dict = {'id': pkg_dict['id']}
try:
qa_openness = get_action('qa_package_openness_show')(context,
data_dict)
except ObjectNotFound:
log.warning('No QA info for package %s', pkg_dict['name'])
return
except KeyError:
# occurs during tests or if you've not install ckanext-qa
log.warning('QA not installed - not indexing it.')
return
pkg_dict['openness_score'] = qa_openness.get('openness_score')
log.debug('Openness score %s: %s', pkg_dict['openness_score'],
pkg_dict['name'])
try:
qa_broken = get_action('qa_package_broken_show')(context,
data_dict)
except ObjectNotFound:
log.warning('No brokenness info for package %s', pkg_dict['name'])
return
if not hasattr(cls, 'broken_links_map'):
cls.broken_links_map = {
True: 'Broken',
'some': 'Partially broken',
False: 'OK',
None: 'TBC'
}
pkg_dict['broken_links'] = cls.broken_links_map[qa_broken.get(
'archival_is_broken')]
log.debug('Broken links %s: %s', pkg_dict['broken_links'],
pkg_dict['name'])
def all_packages(self):
ctx = {'model': model, 'session': model.Session}
package_list = get_action("package_list")(ctx, {})
def linkify_packages(pkgs):
x = 0
for pkg in pkgs:
yield '<a href="/dataset/{p}">{p}</a><br/>'.format(p=pkg)
c.datasets = linkify_packages(package_list)
return render("package/all_datasets.html")
def all_packages(self):
ctx = {'model': model, 'session': model.Session}
package_list = get_action("package_list")(ctx, {})
def linkify_packages(pkgs):
x = 0
for pkg in pkgs:
yield '<a href="/dataset/{p}">{p}</a><br/>'.format(p=pkg)
c.datasets = linkify_packages(package_list)
return render("package/all_datasets.html")
def test_view_system_user(self):
# created on the API
user_dict = get_action("get_site_user")({"model": model, "ignore_auth": True}, {})
user = user_dict["name"]
c.is_an_official = False
assert_equal(str(dgu_linked_user(user)), "System Process")
c.is_an_official = True
assert_equal(
str(dgu_linked_user(user, maxlength=100)),
'<a href="/data/user/test.ckan.net">System Process (Site user)</a>',
)
def dataset_count(self):
from ckan.lib.search import SearchError
try:
# package search
context = {"model": model, "session": model.Session, "user": "******"}
fq = 'capacity:"public" unpublished:false'
data_dict = {"q": "", "fq": fq, "facet": "false", "rows": 0, "start": 0}
query = get_action("package_search")(context, data_dict)
count = query["count"]
except SearchError, se:
log.error("Search error: %s", se)
count = 0
def reply(self, dataset_name, parent_id):
c.action = 'reply'
try:
data = {'id': parent_id}
c.parent_dict = get_action("comment_show")({
'model': model,
'user': c.user
}, data)
c.parent = data['comment']
except:
abort(404)
return self._add_or_reply(dataset_name)
def get_themes():
'''
Get the themes from ckanext-taxonomy
'''
context = {'model': model}
try:
log.debug('Refreshing home page themes')
terms = get_action('taxonomy_term_list')(
context, {'name': 'dgu-themes'})
except sqlalchemy.exc.OperationalError, e:
if 'no such table: taxonomy' in str(e):
model.Session.remove() # clear the erroring transaction
raise ImportError('ckanext-taxonomy tables not setup')
raise
def delete(self, id):
"""Provide a delete ('withdraw') action, but only for UKLP datasets"""
from ckan.lib.search import SearchIndexError
context = {"model": model, "session": model.Session, "user": c.user}
try:
pkg_dict = get_action("package_show")(
context, {"id": id}
) # has side-effect of populating context.get('package')
except NotAuthorized:
abort(401, "Not authorized to delete package")
if request.params: # POST
if "cancel" in request.params:
h.redirect_to(controller="package", action="read", id=id)
elif "delete" in request.params:
try:
package_name = pkg_dict["name"]
get_action("package_delete")(context, {"id": id})
is_uklp = get_from_flat_dict(pkg_dict["extras"], "UKLP") == "True"
if is_uklp:
action = "withdrawn"
resource_type = get_from_flat_dict(pkg_dict["extras"], "resource-type") + " record"
else:
action = "deleted"
resource_type = "dataset"
h.flash_success("Successfully %s %s." % (action, resource_type))
self._form_save_redirect(package_name, "edit")
except NotAuthorized:
abort(401, _("Unauthorized to delete package %s") % id)
except ObjectNotFound, e:
abort(404, _("Package not found"))
except DataError:
abort(400, _(u"Integrity Error"))
except SearchIndexError, e:
abort(500, _(u"Unable to update search index.") + repr(e.args))
def _save_edit(self, name_or_id, context):
try:
data_dict = clean_dict(
unflatten(tuplize_dict(parse_params(request.POST))))
context['message'] = data_dict.get('log_message', '')
data_dict['id'] = name_or_id
pkg = get_action('package_update')(context, data_dict)
c.pkg = context['package']
c.pkg_dict = pkg
h.redirect_to(controller='package', action='read', id=pkg['name'])
except NotAuthorized:
abort(401, 'Not authorized to save package')
except ObjectNotFound, e:
abort(404, _('Dataset not found'))
def test_view_system_user(self):
# created on the API
user_dict = get_action('get_site_user')({'model': model, 'ignore_auth': True}, {})
user = user_dict['name']
with regular_user():
assert_equal(str(dgu_linked_user(user)), 'System Process')
with publisher_user():
assert_equal(str(dgu_linked_user(user, maxlength=100)),
'<a href="/data/user/test.ckan.net">System Process (Site user)</a>')
with sysadmin_user():
assert_equal(str(dgu_linked_user(user, maxlength=100)),
'<a href="/data/user/test.ckan.net">System Process (Site user)</a>')
def get_themes():
'''
Get the themes from ckanext-taxonomy
'''
context = {'model': model}
try:
log.debug('Refreshing home page themes')
terms = get_action('taxonomy_term_list')(context, {
'name': 'dgu-themes'
})
except sqlalchemy.exc.OperationalError, e:
if 'no such table: taxonomy' in str(e):
model.Session.remove() # clear the erroring transaction
raise ImportError('ckanext-taxonomy tables not setup')
raise
def _save_edit(self, name_or_id, context):
try:
data_dict = clean_dict(unflatten(
tuplize_dict(parse_params(request.POST))))
context['message'] = data_dict.get('log_message', '')
data_dict['id'] = name_or_id
pkg = get_action('package_update')(context, data_dict)
c.pkg = context['package']
c.pkg_dict = pkg
h.redirect_to(controller='package', action='read', id=pkg['name'])
except NotAuthorized:
abort(401, 'Not authorized to save package')
except ObjectNotFound, e:
abort(404, _('Dataset not found'))
def __init__(self):
self.data = {}
self.topic_words = {} # topic:theme_name
self.topic_bigrams = {} # (topicword1, topicword2):theme_name
self.topic_trigrams = {} # (topicword1, topicword2, topicword3):theme_name
self.gemet = {} # gemet_keyword:theme_name
self.ons = {} # ons_keyword:theme_name
self.la_function = {} # LA functions extra
self.la_service = {} # LA services extra
self.odc = {} # OpenDataCommunities.org theme extra
context = {'model': model}
terms = get_action('taxonomy_term_list')(context, {'name': 'dgu-themes'})
for term in terms:
theme_dict = term['extras']
theme_dict['title'] = name = term['label']
theme_dict['description'] = term['description']
for key in ('topics', 'gemet', 'nscl', 'ons', 'la_function', 'la_service',
'odc'):
if key in theme_dict:
assert isinstance(theme_dict[key], list), (name, key)
for topic in theme_dict['topics']:
words = [normalize_token(word) for word in split_words(topic)]
if len(words) == 1:
self.topic_words[words[0]] = name
elif len(words) == 2:
self.topic_bigrams[tuple(words)] = name
elif len(words) == 3:
self.topic_trigrams[tuple(words)] = name
else:
assert 0, 'Too many words in topic: %s' % topic
for gemet_keyword in theme_dict.get('gemet', []):
self.gemet[normalize_keyword(gemet_keyword)] = name
for ons_keyword in theme_dict.get('nscl', []) + theme_dict.get('ons', []):
self.ons[tag_munge(ons_keyword)] = name
for function_id in theme_dict.get('la_functions', []):
self.la_function[function_id] = name
for service_id in theme_dict.get('la_service', []):
self.la_service[service_id] = name
for keyword in theme_dict.get('odc', []):
self.odc[keyword] = name
self.data[name] = theme_dict
self.topic_words_set = self.topic_words.viewkeys() # can do set-like operations on it
self.topic_bigrams_set = self.topic_bigrams.viewkeys()
self.topic_trigrams_set = self.topic_trigrams.viewkeys()
def upload_complete(self, id):
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True}
try:
c.group_dict = get_action('organization_show')(context, {"id": id})
c.group = context['group']
except ObjectNotFound:
abort(404, 'Group not found')
except NotAuthorized:
abort(401, 'Unauthorized to read group %s' % id)
try:
context['group'] = c.group
check_access('organization_update', context)
except NotAuthorized, e:
abort(401, 'User %r not authorized to upload unpublished' % (c.user))
def _get_package(self, id):
"""
Given an ID use the logic layer to fetch the Package and a
dict representation of it as well as adding formatted notes
and the publisher to the template context (c).
"""
import ckan.misc
import genshi
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'extras_as_string': True,
'for_view': True
}
try:
c.pkg_dict = get_action('package_show')(context, {'id': id})
c.pkg = context['package']
except ObjectNotFound:
abort(404, _('Dataset not found'))
except NotAuthorized:
abort(401, _('Unauthorized to read package %s') % id)
groups = c.pkg.get_groups('publisher')
if groups:
c.publisher = groups[0]
else:
log.warning("Package {0} is not a member of any group!".format(id))
# Try and render the notes as markdown for display on the page. Most
# unpublished items *won't* be markdown if they've come directly from the
# CSV - unless they've been edited.
try:
notes_formatted = ckan.misc.MarkdownFormat().to_html(c.pkg.notes)
c.pkg_notes_formatted = genshi.HTML(notes_formatted)
c.release_notes_formatted = None
notes = unpublished_release_notes(c.pkg_dict)
if notes and notes.strip():
c.release_notes_formatted = genshi.HTML(
ckan.misc.MarkdownFormat().to_html(notes))
except Exception:
c.pkg_notes_formatted = c.pkg.notes
def add_field__harvest_document(cls, pkg_dict):
"""Index a harvested dataset\'s XML content
(Given a low priority when searching)"""
if pkg_dict.get("UKLP", "") == "True":
import ckan
from ckanext.dgu.plugins_toolkit import get_action
context = {"model": ckan.model, "session": ckan.model.Session, "ignore_auth": True}
data_dict = {"id": pkg_dict.get("harvest_object_id", "")}
try:
harvest_object = get_action("harvest_object_show")(context, data_dict)
pkg_dict["extras_harvest_document_content"] = harvest_object.get("content", "")
except ObjectNotFound:
log.warning(
'Unable to find harvest object "%s" ' 'referenced by dataset "%s"', data_dict["id"], pkg_dict["id"]
)
def upload_status(self, id, upload_id):
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True}
try:
c.group_dict = get_action('group_show')(context, {"id": id})
c.group = context['group']
except ObjectNotFound:
self._redirect_if_previous_name(id)
abort(404, 'Group not found')
except NotAuthorized:
abort(401, 'Unauthorized to read group %s' % id)
try:
context['group'] = c.group
check_access('group_update', context)
except NotAuthorized, e:
abort(401, 'User %r not authorized to view internal inventory' % (c.user))
def upload_status(self, id, upload_id):
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True}
try:
c.group_dict = get_action('group_show')(context, {"id": id})
c.group = context['group']
except ObjectNotFound:
self._redirect_if_previous_name(id)
abort(404, 'Group not found')
except NotAuthorized:
abort(401, 'Unauthorized to read group %s' % id)
try:
context['group'] = c.group
check_access('group_update', context)
except NotAuthorized, e:
abort(401, 'User %r not authorized to view internal inventory' % (c.user))
def dataset_count(self):
from ckan.lib.search import SearchError
try:
# package search
context = {'model': model, 'session': model.Session,
'user': '******'}
fq = 'capacity:"public" unpublished:false'
data_dict = {
'q':'',
'fq':fq,
'facet':'false',
'rows':0,
'start':0,
}
query = get_action('package_search')(context,data_dict)
count = query['count']
except SearchError, se:
log.error('Search error: %s', se)
count = 0
def dataset_count(self):
from ckan.lib.search import SearchError
try:
# package search
context = {'model': model, 'session': model.Session,
'user': '******'}
fq = 'capacity:"public" unpublished:false'
data_dict = {
'q':'',
'fq':fq,
'facet':'false',
'rows':0,
'start':0,
}
query = get_action('package_search')(context,data_dict)
count = query['count']
except SearchError, se:
log.error('Search error: %s', se)
count = 0
def contact(self, name=None):
"""
This action allows users to create an issue by filling in a contact
form.
"""
import ckan.model as model
from ckanext.redmine.client import RedmineClient
print name
extra_vars = {"data": {}, "errors": {}}
client = RedmineClient(name)
c.categories = client.load_categories()
c.name = name
if request.method == 'POST':
data = clean_dict(
unflatten(tuplize_dict(parse_params(request.POST))))
context = {
'model': model,
'session': model.Session,
'user': c.user
}
# Create the issue with the data we were passed.
try:
newid = get_action('issue_create')(context, data)
if newid is None:
self._save_on_fail(data)
h.flash_success(
_('Thank you for contacting us'.format(newid)))
else:
h.flash_success(
_('Thank you for contacting us, please quote #{0} in future correspondence'
.format(newid)))
h.redirect_to(str(data.get('referer', '/')))
except ValidationError, e:
extra_vars["errors"] = e.error_dict
extra_vars["error_summary"] = e.error_summary
extra_vars["data"] = data
c.category_id = data.get('category', '')
def add_field__harvest_document(cls, pkg_dict):
'''Index a harvested dataset\'s XML content
(Given a low priority when searching)'''
if pkg_dict.get('UKLP', '') == 'True':
import ckan
from ckanext.dgu.plugins_toolkit import get_action
context = {'model': ckan.model,
'session': ckan.model.Session,
'ignore_auth': True}
data_dict = {'id': pkg_dict.get('harvest_object_id', '')}
try:
harvest_object = get_action('harvest_object_show')(context, data_dict)
pkg_dict['extras_harvest_document_content'] = harvest_object.get('content', '')
except ObjectNotFound:
log.warning('Unable to find harvest object "%s" '
'referenced by dataset "%s"',
data_dict['id'], pkg_dict['id'])
def __init__(self):
self.data = {}
self.topic_words = {} # topic:[theme_name]
self.topic_bigrams = {} # (topicword1, topicword2):[theme_name]
self.topic_trigrams = {} # (topicword1, topicword2, topicword3):[theme_name]
self.gemet = {} # gemet_keyword:theme_name
self.ons = {} # ons_keyword:theme_name
self.la_function = {} # LA functions extra
self.la_service = {} # LA services extra
self.odc = {} # OpenDataCommunities.org theme extra
context = {'model': model}
# Get the themes from ckanext-taxonomy
try:
terms = get_action('taxonomy_term_list')(context, {'name': 'dgu-themes'})
except sqlalchemy.exc.OperationalError, e:
if 'no such table: taxonomy' in str(e):
model.Session.remove() # clear the erroring transaction
raise ImportError('ckanext-taxonomy tables not setup')
raise
def commitments(self, id):
"""
Shows all of the commitments for the specified publisher
"""
from ckanext.dgu.model.commitment import Commitment
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'extras_as_string': True,
'save': 'save' in request.params}
try:
c.publisher_dict = get_action('organization_show')(context, {'id': id})
except NotAuthorized:
abort(401, _('Unauthorized to read commitments'))
except ObjectNotFound:
abort(404, _('Publisher not found'))
c.publisher = context.get('group')
c.commitments = Commitment.get_for_publisher(c.publisher.name).order_by('commitment.dataset_name').all()
return render('commitment/read.html')
def upload(self, id):
"""
Upload of an unpublished file, accepts a POST request with a file and
then renders the result of the import to the user.
"""
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True}
try:
c.group_dict = get_action('organization_show')(context, {"id": id})
c.group = context['group']
except ObjectNotFound:
abort(404, 'Organization not found')
except NotAuthorized:
abort(401, 'Unauthorized to read group %s' % id)
try:
context['group'] = c.group
check_access('organization_update', context)
except NotAuthorized, e:
abort(401, 'User %r not authorized to upload inventory' % (c.user))
def _get_package(self, id):
"""
Given an ID use the logic layer to fetch the Package and a
dict representation of it as well as adding formatted notes
and the publisher to the template context (c).
"""
import genshi
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'extras_as_string': True,
'for_view': True}
try:
c.pkg_dict = get_action('package_show')(context, {'id': id})
c.pkg = context['package']
except ObjectNotFound:
abort(404, _('Dataset not found'))
except NotAuthorized:
abort(401, _('Unauthorized to read package %s') % id)
groups = c.pkg.get_groups('organization')
if groups:
c.publisher = groups[0]
else:
log.warning("Package {0} is not a member of any group!".format(id))
# Try and render the notes as markdown for display on the page. Most
# unpublished items *won't* be markdown if they've come directly from the
# CSV - unless they've been edited.
try:
notes_formatted = render_markdown(c.pkg.notes)
c.pkg_notes_formatted = unicode(genshi.HTML(notes_formatted))
c.release_notes_formatted = None
notes = unpublished_release_notes(c.pkg_dict)
if notes and notes.strip():
c.release_notes_formatted = unicode(genshi.HTML(
render_markdown(notes)))
except Exception:
c.pkg_notes_formatted = c.pkg.notes
def edit(self, id):
"""
Allows editing of commitments for a specific publisher
"""
from ckanext.dgu.model.commitment import Commitment
from ckanext.dgu.lib import helpers as dgu_helpers
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'extras_as_string': True,
'save': 'save' in request.params}
if not dgu_helpers.is_sysadmin():
abort(401, "You are not allowed to edit the commitments for this publisher")
try:
c.publisher_dict = get_action('organization_show')(context, {'id': id})
except NotAuthorized:
abort(401, _('Unauthorized to read commitments'))
except ObjectNotFound:
abort(404, _('Publisher not found'))
c.publisher = context.get('group')
c.publishers = model.Session.query(model.Group).filter(model.Group.state=='active')\
.order_by(model.Group.title).all()
c.errors = {}
if request.method == "POST":
# need to flatten the request into some commitments, if there is an ID then
# we should update them, if there isn't then add them.
# TODO: Work out how to remove them. Perhaps get the current IDs and do a diff
# If successful redirect to read()
h.redirect_to(h.url_for(controller='ckanext.dgu.controllers.commitment:CommitmentController',
action='commitments', id=c.publisher.name))
c.commitments = Commitment.get_for_publisher(c.publisher.name).order_by('commitment.commitment_text')
return render('commitment/edit.html')
def download(self, id):
"""
Downloads all of the current datasets for a given publisher as a read-only
CSV file.
"""
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True,
'group': id}
try:
c.group_dict = get_action('organization_show')(context, {"id": id})
c.group = context['group']
except ObjectNotFound:
abort(404, 'Organization not found')
except NotAuthorized:
abort(401, 'Unauthorized to read Organization %s' % id)
try:
context['group'] = c.group
check_access('organization_update', context)
except NotAuthorized, e:
abort(401, 'User %r not authorized to download unpublished '% (c.user))
