def update_config(self, config):
u'''
Load config and set up the resource library,
public directory and template directory for the view
'''
# https://datatables.net/reference/option/lengthMenu
self.page_length_choices = toolkit.aslist(
config.get(u'ckan.datatables.page_length_choices',
DEFAULT_PAGE_LENGTH_CHOICES))
self.page_length_choices = [int(i) for i in self.page_length_choices]
self.top_pagination_controls = toolkit.asbool(
config.get(u'ckan.datatables.top_pagination_controls', False))
self.search_delay = toolkit.asint(
config.get(u'ckan.datatables.search_delay', DEFAULT_SEARCH_DELAY))
self.state_saving = toolkit.asbool(
config.get(u'ckan.datatables.state_saving', True))
# https://datatables.net/reference/option/stateDuration
self.state_duration = toolkit.asint(
config.get(u'ckan.datatables.state_duration',
DEFAULT_STATE_DURATION))
self.data_dictionary_labels = toolkit.asbool(
config.get(u'ckan.datatables.data_dictionary_labels', True))
toolkit.add_template_directory(config, u'templates')
toolkit.add_public_directory(config, u'public')
toolkit.add_resource(u'public', u'ckanext-datatablesview')
def parse_dataset(self, dataset_dict, dataset_ref):
dataset_dict['frequency'] = 'notPlanned'
dataset_dict['topic_category'] = 'location'
dataset_dict['lineage'] = '-'
dataset_dict['contact_name'] = 'Nial Mc Sorley'
dataset_dict[
'contact_email'] = '*****@*****.**'
dataset_dict['license_id'] = 'uk-ogl'
_remove_extra('contact_name', dataset_dict)
_remove_extra('contact_email', dataset_dict)
# Ping the ArcGIS server so the processing of the files
# starts
identifier = None
avoid = []
if toolkit.asbool(
config.get('ckanext.opendatani.harvest.ping_arcgis_urls')):
for extra in dataset_dict.get('extras', []):
if extra['key'] == 'identifier' and extra['value']:
identifier = extra['value']
if identifier:
query = toolkit.get_action('package_search')(
{}, {
'q': 'guid:"{0}"'.format(identifier)
})
if query['count']:
current_dataset = query['results'][0]
for current_resource in current_dataset.get(
'resources', []):
if ('requested' in current_resource and toolkit.asbool(
current_resource['requested'])):
avoid.append(current_resource['url'])
for resource in dataset_dict.get('resources', []):
if resource['format'] == 'OGC WMS':
resource['format'] = 'WMS'
if resource['format'] == 'ZIP':
resource['format'] = 'SHP'
resource['requested'] = False
file_formats = ('geojson', 'kml', 'zip', 'csv')
if resource['url'] in avoid:
resource['requested'] = True
elif resource['format'].lower() in file_formats:
try:
requests.head(resource['url'])
resource['requested'] = True
log.debug(
'Requested resource to start the processing: {0}'.
format(resource['url']))
except Exception, e:
log.debug('Error requesting resource: {0}\n{1}'.format(
resource['url'], e))
pass
def _can_edit(state: str, by_author: bool = False) -> bool:
if state == Comment.State.draft:
if by_author:
return tk.asbool(
tk.config.get(
const.CONFIG_DRAFT_EDITS_BY_AUTHOR,
const.DEFAULT_DRAFT_EDITS_BY_AUTHOR,
))
return tk.asbool(
tk.config.get(
const.CONFIG_DRAFT_EDITS,
const.DEFAULT_DRAFT_EDITS,
))
elif state == Comment.State.approved:
if by_author:
return tk.asbool(
tk.config.get(
const.CONFIG_APPROVED_EDITS_BY_AUTHOR,
const.DEFAULT_APPROVED_EDITS_BY_AUTHOR,
))
return tk.asbool(
tk.config.get(
const.CONFIG_APPROVED_EDITS,
const.DEFAULT_APPROVED_EDITS,
))
log.warning("Unexpected comment state: %s", state)
return False
def issues_enabled(dataset):
'''Returns whether issues are enabled for the given dataset (dict)'''
# config options allow you to only enable issues for particular datasets or
# organizations
datasets_with_issues_enabled = set(toolkit.aslist(
config.get('ckanext.issues.enabled_for_datasets')
))
organizations_with_issues_enabled = set(toolkit.aslist(
config.get('ckanext.issues.enabled_for_organizations')
))
if datasets_with_issues_enabled or organizations_with_issues_enabled:
if datasets_with_issues_enabled and \
dataset['name'] in datasets_with_issues_enabled:
return True
elif organizations_with_issues_enabled and \
(dataset.get('organization') or {}).get('name') in \
organizations_with_issues_enabled:
return True
return False
else:
extras = dataset.get('extras', [])
for extra in extras:
if extra.get('key') == 'issues_enabled':
return toolkit.asbool(extra.get('value'))
else:
return toolkit.asbool(
config.get('ckanext.issues.enabled_without_extra', True)
)
def update_config(self, config):
u'''
Set up the resource library, public directory and
template directory for the view
'''
# https://datatables.net/reference/option/lengthMenu
self.page_length_choices = toolkit.aslist(
config.get(u'ckan.datatables.page_length_choices',
DEFAULT_PAGE_LENGTH_CHOICES))
self.page_length_choices = [int(i) for i in self.page_length_choices]
self.state_saving = toolkit.asbool(
config.get(u'ckan.datatables.state_saving', True))
# https://datatables.net/reference/option/stateDuration
self.state_duration = toolkit.asint(
config.get(u'ckan.datatables.state_duration',
DEFAULT_STATE_DURATION))
self.data_dictionary_labels = toolkit.asbool(
config.get(u'ckan.datatables.data_dictionary_labels', True))
self.ellipsis_length = toolkit.asint(
config.get(u'ckan.datatables.ellipsis_length',
DEFAULT_ELLIPSIS_LENGTH))
self.date_format = config.get(u'ckan.datatables.date_format',
DEFAULT_DATE_FORMAT)
self.default_view = config.get(u'ckan.datatables.default_view',
'table')
toolkit.add_template_directory(config, u'templates')
toolkit.add_public_directory(config, u'public')
toolkit.add_resource(u'public', u'ckanext-datatablesview')
def issues_enabled(dataset):
'''Returns whether issues are enabled for the given dataset (dict)'''
# config options allow you to only enable issues for particular datasets or
# organizations
datasets_with_issues_enabled = set(
toolkit.aslist(config.get('ckanext.issues.enabled_for_datasets')))
organizations_with_issues_enabled = set(
toolkit.aslist(config.get('ckanext.issues.enabled_for_organizations')))
if datasets_with_issues_enabled or organizations_with_issues_enabled:
if datasets_with_issues_enabled and \
dataset['name'] in datasets_with_issues_enabled:
return True
elif organizations_with_issues_enabled and \
(dataset.get('organization') or {}).get('name') in \
organizations_with_issues_enabled:
return True
return False
else:
extras = dataset.get('extras', [])
for extra in extras:
if extra.get('key') == 'issues_enabled':
return toolkit.asbool(extra.get('value'))
else:
return toolkit.asbool(
config.get('ckanext.issues.enabled_without_extra', True))
def notify(self, entity, operation=None):
if isinstance(entity, model.Resource):
resource_dict = model.Resource.get(entity.id).as_dict()
if helpers.is_spatially_ingestible_resource(resource_dict):
d_type = model.domain_object.DomainObjectOperation
auto_ingest = toolkit.asbool(
config.get('ckan.spatialingestor.auto_ingest', 'False'))
is_spatial_parent = toolkit.asbool(
resource_dict.get('spatial_parent', 'False'))
if operation == d_type.deleted or entity.state == 'deleted':
helpers.log.error(">>>>>>> Registered Purge Trigger")
toolkit.get_action(
'spatialingestor_purge_resource_datastores')(
{}, resource_dict)
package_dict = model.Package.get(
resource_dict['package_id']).as_dict()
if package_dict['state'] != 'deleted':
helpers.log.error(
">>>>>>> Registered Orphan Delete Trigger")
toolkit.get_action(
'spatialingestor_delete_orphaned_resources')(
{}, package_dict)
elif (is_spatial_parent and
(operation == d_type.changed or not operation)) or (
operation == d_type.new and auto_ingest):
helpers.log.error(">>>>>>> Registered Ingest Trigger")
toolkit.get_action('spatialingestor_ingest_resource')(
{}, resource_dict)
def run(ctx, host, port, disable_reloader, threaded, extra_files, processes,
ssl_cert, ssl_key):
u"""Runs the Werkzeug development server"""
if tk.asbool(config.get("debug")):
warnings.filterwarnings("default", category=CkanDeprecationWarning)
# Reloading
use_reloader = not disable_reloader
config_extra_files = tk.aslist(
config.get(u"ckan.devserver.watch_patterns")
)
extra_files = list(extra_files) + [
config[u"__file__"]
] + config_extra_files
# Threads and processes
threaded = threaded or tk.asbool(config.get(u"ckan.devserver.threaded"))
processes = processes or tk.asint(
config.get(u"ckan.devserver.multiprocess", 1)
)
if threaded and processes > 1:
tk.error_shout(u"Cannot have a multithreaded and multi process server")
raise click.Abort()
# SSL
cert_file = ssl_cert or config.get('ckan.devserver.ssl_cert')
key_file = ssl_key or config.get('ckan.devserver.ssl_key')
if cert_file and key_file:
if cert_file == key_file == 'adhoc':
ssl_context = 'adhoc'
else:
ssl_context = (ssl_cert, ssl_key)
else:
ssl_context = None
host = host or config.get('ckan.devserver.host', DEFAULT_HOST)
port = port or config.get('ckan.devserver.port', DEFAULT_PORT)
try:
port = int(port)
except ValueError:
tk.error_shout(u"Server port must be an integer, not {}".format(port))
raise click.Abort()
log.info(u"Running CKAN on {scheme}://{host}:{port}".format(
scheme='https' if ssl_context else 'http', host=host, port=port))
run_simple(
host,
port,
ctx.obj.app,
use_reloader=use_reloader,
use_evalex=True,
threaded=threaded,
processes=processes,
extra_files=extra_files,
ssl_context=ssl_context,
)
def parse_dataset(self, dataset_dict, dataset_ref):
# TODO: if there is more than one source with different defaults,
# modify accordingly
dataset_dict['frequency'] = 'notPlanned'
dataset_dict['topic_category'] = 'location'
dataset_dict['lineage'] = '-'
dataset_dict['contact_name'] = 'OSNI Mapping Helpdesk'
dataset_dict['contact_email'] = '*****@*****.**'
dataset_dict['license_id'] = 'uk-ogl'
_remove_extra('contact_name', dataset_dict)
_remove_extra('contact_email', dataset_dict)
# Ping the ArcGIS server so the processing of the files
# starts
identifier = None
avoid = []
if toolkit.asbool(
config.get('ckanext.opendatani.harvest.ping_arcgis_urls')):
for extra in dataset_dict.get('extras', []):
if extra['key'] == 'identifier' and extra['value']:
identifier = extra['value']
if identifier:
query = toolkit.get_action('package_search')(
{}, {'q': 'guid:"{0}"'.format(identifier)})
if query['count']:
current_dataset = query['results'][0]
for current_resource in current_dataset.get('resources',
[]):
if ('requested' in current_resource and
toolkit.asbool(current_resource['requested'])):
avoid.append(current_resource['url'])
for resource in dataset_dict.get('resources', []):
if resource['format'] == 'OGC WMS':
resource['format'] = 'WMS'
resource['requested'] = False
file_formats = ('geojson', 'kml', 'zip', 'csv')
if resource['url'] in avoid:
resource['requested'] = True
elif resource['format'].lower() in file_formats:
try:
requests.head(resource['url'])
resource['requested'] = True
log.debug(
'Requested resource to start the processing: {0}'
.format(resource['url']))
except Exception, e:
log.debug(
'Error requesting resource: {0}\n{1}'
.format(resource['url'], e))
pass
def notify(context, issue, email_template):
notify_admin = toolkit.asbool(
config.get("ckanext.issues.notify_admin", False))
notify_owner = toolkit.asbool(
config.get("ckanext.issues.notify_owner", False))
if not notify_admin and not notify_owner:
return
user_obj = model.User.get(issue.user_id)
dataset = model.Package.get(issue.dataset_id)
extra_vars = {
'issue':
issue,
'title':
dataset.title,
'username':
user_obj.name,
'email':
user_obj.email,
'site_url':
h.url_for(controller='ckanext.issues.controller:IssueController',
action='show',
id=issue.id,
package_id=issue.dataset_id,
qualified=True)
}
if notify_owner:
contact_name = dataset.author or dataset.maintainer
contact_email = dataset.author_email or dataset.maintainer_email
email_msg = "TODO" #render(email_template,extra_vars=extra_vars,loader_class=NewTextTemplate)
send_email(contact_name, contact_email, email_msg)
if notify_admin:
# retrieve organization's admins (or the role specified on ckanext.issues.minimun_role_required) to notify
organization = action.get.organization_show(
context, data_dict={'id': dataset.owner_org})
minimun_role_required = config.get(
"ckanext.issues.minimun_role_required", "Anonymous")
for user in organization['users']:
if user['capacity'] == role_mapper[minimun_role_required] and user[
'activity_streams_email_notifications']:
admin_user = model.User.get(user['id'])
admin_name = admin_user.name
admin_email = admin_user.email
if admin_email != contact_email:
email_msg = "TODO" #render(email_template,extra_vars=extra_vars,loader_class=NewTextTemplate)
send_email(admin_name, admin_email, email_msg)
def build_pages_nav_main(*args):
about_menu = toolkit.asbool(config.get('ckanext.pages.about_menu', False))
group_menu = toolkit.asbool(config.get('ckanext.pages.group_menu', False))
org_menu = toolkit.asbool(
config.get('ckanext.pages.organization_menu', True))
new_args = []
for arg in args:
if arg[0] == 'home.about' and not about_menu:
continue
if arg[0] == 'organizations_index' and not org_menu:
continue
if arg[0] == 'group_index' and not group_menu:
continue
new_args.append(arg)
output = h.build_nav_main(*new_args)
# add forum link
forum_url = config.get('ckan.pages.forum.link', False)
if forum_url:
link = h.literal(u'<a href="/{}/{}">{}</a>'.format(
h.lang(), 'forum', "Forum"))
li = h.literal('<li>') + link + h.literal('</li>')
output = output + li
# do not display any private datasets in menu even for sysadmins
pages_list = toolkit.get_action('ckanext_pages_list')(None, {
'order': True,
'private': False
})
page_name = ''
if (toolkit.c.action in ('pages_show', 'blog_show')
and toolkit.c.controller
== 'ckanext.pages.controller:PagesController'):
page_name = toolkit.c.environ['routes.url'].current().split('/')[-1]
for page in pages_list:
type_ = 'blog' if page['page_type'] == 'blog' else 'pages'
name = urllib.quote(page['name'].encode('utf-8')).decode('utf-8')
title = cgi.escape(pages_page_title(h.lang(), page))
if h.lang():
link = h.literal(u'<a href="/{}/{}/{}">{}</a>'.format(
h.lang(), type_, name, title))
else:
link = h.literal(u'<a href="/{}/{}">{}</a>'.format(
type_, name, title))
if page['name'] == page_name:
li = h.literal('<li class="active">') + link + h.literal('</li>')
else:
li = h.literal('<li>') + link + h.literal('</li>')
output = output + li
return output
def update_config(self, config):
toolkit.add_template_directory(config, 'templates')
facets_inputs = config.get(self.ADDITIONAL_FACETS_CONFIG, '').split()
# get additional facets from the input
self.additional_facets = loader.get_additional_facets(facets_inputs[0])
self.display_facets_on_group_page = toolkit.asbool(config.get(self.DISPLAY_FACETS_ON_GROUPS_PAGE, False))
self.display_facets_on_org_page = toolkit.asbool(config.get(self.DISPLAY_FACETS_ON_ORG_PAGE, False))
self.clear_default_facets = toolkit.asbool(config.get(self.CLEAR_DEFAULT_FACETS, False))
def configure(self, config):
"""Load config settings for this extension from config file.
See IConfigurable.
"""
if "googleanalytics.id" not in config:
msg = "Missing googleanalytics.id in config"
raise GoogleAnalyticsException(msg)
self.googleanalytics_id = config["googleanalytics.id"]
self.googleanalytics_domain = config.get(
"googleanalytics.domain", "auto"
)
self.googleanalytics_fields = ast.literal_eval(
config.get("googleanalytics.fields", "{}")
)
googleanalytics_linked_domains = config.get(
"googleanalytics.linked_domains", ""
)
self.googleanalytics_linked_domains = [
x.strip() for x in googleanalytics_linked_domains.split(",") if x
]
if self.googleanalytics_linked_domains:
self.googleanalytics_fields["allowLinker"] = "true"
# If resource_prefix is not in config file then write the default value
# to the config dict, otherwise templates seem to get 'true' when they
# try to read resource_prefix from config.
if "googleanalytics_resource_prefix" not in config:
config[
"googleanalytics_resource_prefix"
] = DEFAULT_RESOURCE_URL_TAG
self.googleanalytics_resource_prefix = config[
"googleanalytics_resource_prefix"
]
self.show_downloads = tk.asbool(
config.get("googleanalytics.show_downloads", True)
)
self.track_events = tk.asbool(
config.get("googleanalytics.track_events", False)
)
self.enable_user_id = tk.asbool(
config.get("googleanalytics.enable_user_id", False)
)
p.toolkit.add_resource("../assets", "ckanext-googleanalytics")
# spawn a pool of 5 threads, and pass them queue instance
for i in range(5):
t = AnalyticsPostThread(self.analytics_queue)
t.setDaemon(True)
t.start()
def build_pages_nav_main(*args):
about_menu = toolkit.asbool(config.get('ckanext.pages.about_menu', True))
group_menu = toolkit.asbool(config.get('ckanext.pages.group_menu', True))
org_menu = toolkit.asbool(
config.get('ckanext.pages.organization_menu', True))
new_args = []
for arg in args:
if arg[0] == 'about' and not about_menu:
continue
if arg[0] == 'organizations_index' and not org_menu:
continue
if arg[0] == 'group_index' and not group_menu:
continue
new_args.append(arg)
output = h.build_nav_main(*new_args)
# do not display any private datasets in menu even for sysadmins
pages_list = toolkit.get_action('ckanext_pages_list')(None, {
'order': True,
'private': False
})
page_name = ''
if (toolkit.c.action in ('pages_show', 'blog_show')
and toolkit.c.controller
== 'ckanext.pages.controller:PagesController'):
page_name = toolkit.c.environ['routes.url'].current().split('/')[-1]
desired_lang_code = pylons.request.environ['CKAN_LANG']
acceptable_lang_codes = [
desired_lang_code,
desired_lang_code.split('_', 1)[0]
]
for page in pages_list:
if page.get('lang') and page.get('lang') not in acceptable_lang_codes:
continue
type_ = 'blog' if page['page_type'] == 'blog' else 'pages'
name = urllib.quote(page['name'].encode('utf-8')).decode('utf-8')
title = cgi.escape(page['title'])
link = h.literal(u'<a href="/{}/{}">{}</a>'.format(type_, name, title))
if page['name'] == page_name:
li = h.literal('<li class="active">') + link + h.literal('</li>')
else:
li = h.literal('<li>') + link + h.literal('</li>')
output = output + li
return output
def build_pages_nav_main(*args):
about_menu = toolkit.asbool(config.get('ckanext.pages.about_menu', True))
group_menu = toolkit.asbool(config.get('ckanext.pages.group_menu', True))
org_menu = toolkit.asbool(
config.get('ckanext.pages.organization_menu', True))
new_args = []
for arg in args:
if arg[0] == 'about' and not about_menu:
continue
if arg[0] == 'organizations_index' and not org_menu:
continue
if arg[0] == 'group_index' and not group_menu:
continue
new_args.append(arg)
output = h.build_nav_main(*new_args)
# do not display any private datasets in menu even for sysadmins
pages_list = toolkit.get_action('ckanext_pages_list')(None, {
'order': True,
'private': False
})
page_name = ''
try:
if (toolkit.c.action in ('pages_show', 'blog_show')
and toolkit.c.controller
== 'ckanext.pages.controller:PagesController'):
page_name = toolkit.c.environ['routes.url'].current().split(
'/')[-1]
except AttributeError:
# NOTE(e0ne): we don't have 'action' attribute in Flask context.
# We can igrore if it's Flask Bluprint-bases plugin
pass
for page in pages_list:
if page['page_type'] == 'blog':
link = h.literal('<a href="/blog/%s">%s</a>' %
(str(page['name']), str(page['title'])))
else:
link = h.literal('<a href="/pages/%s">%s</a>' %
(str(page['name']), str(page['title'])))
if page['name'] == page_name:
li = h.literal('<li class="active">') + link + h.literal('</li>')
else:
li = h.literal('<li>') + link + h.literal('</li>')
output = output + li
return output
def insert_or_update_pkg(self, pkg_dict, upload=None):
registry = ckanapi.LocalCKAN(username=self.username)
allow_duplicates = tk.asbool(
tk.config.get('ckanext.ddi.allow_duplicates', False)
)
override_datasets = tk.asbool(
tk.config.get('ckanext.ddi.override_datasets', False)
)
visibility = pkg_dict.pop('visibility', 'restricted')
try:
existing_pkg = registry.call_action('package_show', pkg_dict)
if not allow_duplicates and not override_datasets:
raise ContentDuplicateError(
'Dataset already exists and duplicates are not allowed.'
)
if override_datasets:
pkg_dict.pop('id', None)
pkg_dict.pop('name', None)
existing_pkg.update(pkg_dict)
pkg_dict = existing_pkg
registry.call_action('package_update', pkg_dict)
else:
raise ckanapi.NotFound()
except ckanapi.NotFound:
pkg_dict.pop('id', None)
pkg_dict['name'] = self._gen_new_name(pkg_dict['name'])
registry.call_action('package_create', pkg_dict)
if upload is not None:
try:
registry.call_action(
'resource_create',
{
'package_id': pkg_dict['name'],
'upload': upload,
'name': 'DDI XML',
'format': 'xml',
'url': '',
'type': 'attachment',
'file_type': 'other',
'visibility': visibility
}
)
except Exception as e:
raise UploadError(
'Could not upload file: %s' % str(e)
)
log.debug(pkg_dict['name'])
return pkg_dict['name']
def configure(self, config_):
'''Load config settings for this extension from config file.
See IConfigurable.
'''
if 'googleanalytics.id' not in config_:
msg = "Missing googleanalytics.id in config"
log.error(msg)
raise CscGAException(msg)
self.googleanalytics_id = config_['googleanalytics.id']
self.googleanalytics_domain = config_.get('googleanalytics.domain',
'auto')
self.googleanalytics_fields = ast.literal_eval(
config_.get('googleanalytics.fields', '{}'))
googleanalytics_linked_domains = config_.get(
'googleanalytics.linked_domains', '')
self.googleanalytics_linked_domains = [
x.strip() for x in googleanalytics_linked_domains.split(',') if x
]
if self.googleanalytics_linked_domains:
self.googleanalytics_fields['allowLinker'] = 'true'
# If resource_prefix is not in config file then write the default value
# to the config dict, otherwise templates seem to get 'true' when they
# try to read resource_prefix from config.
if 'googleanalytics_resource_prefix' not in config_:
config_['googleanalytics_resource_prefix'] = (
DEFAULT_RESOURCE_URL_TAG)
self.googleanalytics_resource_prefix = config_[
'googleanalytics_resource_prefix']
self.show_downloads = toolkit.asbool(
config_.get('googleanalytics.show_downloads', True))
self.track_events = toolkit.asbool(
config_.get('googleanalytics.track_events', False))
self.ga_send_stats = toolkit.asbool(
config_.get('ckanext.csc_ga.send_stats', True))
log.debug('ga_send_stats=%s', self.ga_send_stats)
toolkit.add_resource('fanstatic', 'csc_ga')
# spawn a pool of 5 threads, and pass them queue instance
for i in range(5):
t = AnalyticsPostThread(self.analytics_queue)
t.setDaemon(True)
t.start()
def build_pages_nav_main(*args):
about_menu = toolkit.asbool(config.get('ckanext.pages.about_menu', True))
group_menu = toolkit.asbool(config.get('ckanext.pages.group_menu', True))
org_menu = toolkit.asbool(
config.get('ckanext.pages.organization_menu', True))
# Different CKAN versions use different route names - gotta catch em all!
about_menu_routes = ['about', 'home.about']
group_menu_routes = ['group_index', 'home.group_index']
org_menu_routes = ['organizations_index', 'home.organizations_index']
new_args = []
for arg in args:
if arg[0] in about_menu_routes and not about_menu:
continue
if arg[0] in org_menu_routes and not org_menu:
continue
if arg[0] in group_menu_routes and not group_menu:
continue
new_args.append(arg)
output = h.build_nav_main(*new_args)
# do not display any private datasets in menu even for sysadmins
pages_list = toolkit.get_action('ckanext_pages_list')(None, {
'order': True,
'private': False
})
page_name = ''
if (hasattr(toolkit.c, 'action')
and toolkit.c.action in ('pages_show', 'blog_show')
and toolkit.c.controller
== 'ckanext.pages.controller:PagesController'):
page_name = toolkit.c.environ['routes.url'].current().split('/')[-1]
for page in pages_list:
type_ = 'blog' if page['page_type'] == 'blog' else 'pages'
name = urllib.quote(page['name'].encode('utf-8')).decode('utf-8')
title = cgi.escape(page['title'])
link = h.literal(u'<a href="/{}/{}">{}</a>'.format(type_, name, title))
if page['name'] == page_name:
li = h.literal('<li class="active">') + link + h.literal('</li>')
else:
li = h.literal('<li>') + link + h.literal('</li>')
output = output + li
return output
def notify(context,issue,email_template):
notify_admin = toolkit.asbool(config.get("ckanext.issues.notify_admin", False))
notify_owner = toolkit.asbool(config.get("ckanext.issues.notify_owner", False))
if not notify_admin and not notify_owner:
return
user_obj = model.User.get(issue.user_id)
dataset = model.Package.get(issue.dataset_id)
extra_vars = {
'issue': issue,
'title': dataset.title,
'username': user_obj.name,
'email': user_obj.email,
'site_url': h.url_for(
controller='ckanext.issues.controller:IssueController',
action='show',
id=issue.id,
package_id=issue.dataset_id,
qualified=True
)
}
if notify_owner:
contact_name = dataset.author or dataset.maintainer
contact_email = dataset.author_email or dataset.maintainer_email
email_msg = render(email_template,extra_vars=extra_vars)
send_email(contact_name,contact_email,email_msg)
if notify_admin:
# retrieve organization's admins (or the role specified on ckanext.issues.minimun_role_required) to notify
organization = action.get.organization_show(context,data_dict={'id':dataset.owner_org})
minimun_role_required = config.get("ckanext.issues.minimun_role_required", "Anonymous")
for user in organization['users']:
if user['capacity'] == role_mapper[minimun_role_required] and user['activity_streams_email_notifications']:
admin_user = model.User.get(user['id'])
admin_name = admin_user.name
admin_email = admin_user.email
if admin_email != contact_email:
email_msg = render(email_template,extra_vars=extra_vars)
send_email(admin_name,admin_email,email_msg)
def resource_create(context, data_dict):
if not tk.asbool(config.get('ckan.cloud_storage_enable')) or data_dict.get('url'):
return origin.resource_create(context, data_dict)
model = context['model']
user = context['user']
package_id = _get_or_bust(data_dict, 'package_id')
data_dict.pop('package_id')
pkg_dict = _get_action('package_show')(context, {'id': package_id})
_check_access('resource_create', context, data_dict)
if not 'resources' in pkg_dict:
pkg_dict['resources'] = []
upload = uploader.S3Upload(data_dict)
pkg_dict['resources'].append(data_dict)
try:
context['defer_commit'] = True
context['use_cache'] = False
_get_action('package_update')(context, pkg_dict)
context.pop('defer_commit')
except ValidationError, e:
errors = e.error_dict['resources'][-1]
raise ValidationError(errors)
def update_config(self, config):
self.organization_pages = toolkit.asbool(config.get('ckanext.pages.organization', False))
self.group_pages = toolkit.asbool(config.get('ckanext.pages.group', False))
toolkit.add_template_directory(config, 'theme/templates_main')
if self.group_pages:
toolkit.add_template_directory(config, 'theme/templates_group')
if self.organization_pages:
toolkit.add_template_directory(config, 'theme/templates_organization')
toolkit.add_resource('fanstatic', 'pages')
toolkit.add_public_directory(config, 'public')
toolkit.add_resource('theme/public', 'ckanext-pages')
toolkit.add_resource('theme/resources', 'pages-theme')
toolkit.add_public_directory(config, 'theme/public')
def fetch_stage(self, harvest_object):
api_url = config.get('ckanext.glasgow.metadata_api', '').rstrip('/')
# NB: this end point does not seem to support the $skip parameter
api_endpoint = api_url + '/Metadata/Organisation/{0}/Dataset/{1}/File'
try:
content = json.loads(harvest_object.content)
org = content['OrganisationId']
dataset = content['Id']
verify_ssl = toolkit.asbool(
config.get('ckanext.glasgow.verify_ssl_certs', True)
)
request = requests.get(api_endpoint.format(org, dataset),
verify=verify_ssl)
if request.status_code == 404:
result = False
log.debug('No files for dataset {0}'.format(dataset))
else:
result = _fetch_from_ec(request)
except requests.exceptions.RequestException, e:
self._save_object_error(
'Error fetching file metadata for package {0}: {1}'.format(
harvest_object.guid, str(e)),
harvest_object, 'Fetch')
return False
def cadasta_api_action(context, data_dict):
# we actually always want to call check access
# development option that should be removed later
if toolkit.asbool(config.get('ckanext.cadasta.enforce_permissions',
True)):
toolkit.check_access(action, context, data_dict)
string_arguments = [a[1] for a in
string.Formatter().parse(cadasta_endpoint.url)
if a[1]]
cadasta_dict = {}
for k, v in data_dict.items():
cadasta_dict[k] = cadasta_endpoint.convert_argument(k, v)
error_dict = {}
endpoint = cadasta_endpoint.url
for arg in string_arguments:
if arg not in data_dict.keys() or not data_dict.get(arg):
error_dict[arg] = ['Missing value']
else:
arg_value = ''
if cadasta_endpoint.keep_param_key is True:
arg_value = cadasta_dict.get(arg, '')
else:
arg_value = cadasta_dict.pop(arg, '')
arg_value = re.sub('[^\-\.0-9a-zA-Z]+', '', str(arg_value))
endpoint_arg = ''.join(['{', arg, '}'])
endpoint = endpoint.replace(endpoint_arg, arg_value)
if error_dict:
raise toolkit.ValidationError(error_dict)
return cadasta_api_func(endpoint, cadasta_dict,
cadasta_endpoint.upload_fields)
def create_orgs(organization_id, site_user):
api_url = config.get('ckanext.glasgow.metadata_api', '').rstrip('/')
api_endpoint = '{}/Metadata/Organisation/{}'.format(api_url, organization_id)
verify_ssl = toolkit.asbool(config.get('ckanext.glasgow.verify_ssl_certs', True))
request = requests.get(api_endpoint, verify=verify_ssl)
try:
result = _fetch_from_ec(request)
org = result['MetadataResultSet'][0]
except (KeyError, IndexError):
print 'failed to fetch org {} from EC. Response {}'.format(organization_id, str(result))
return
context = {
'model': model,
'session': model.Session,
'user': site_user,
'local_action': True,
}
org_name = get_org_name(org, 'Title')
data_dict = {
'id': org['Id'],
'title': org['Title'],
'name': org_name,
}
try:
toolkit.get_action('organization_create')(context, data_dict)
context.pop('local_action', None)
return toolkit.get_action('organization_show')(context, {id: 'organization_id'})
except toolkit.ValidationError:
print 'failed to create org {}'.format(organization_id)
def cadasta_api_action(context, data_dict):
# we actually always want to call check access
# development option that should be removed later
if toolkit.asbool(config.get('ckanext.cadasta.enforce_permissions',
True)):
toolkit.check_access(action, context, data_dict)
string_arguments = [a[1] for a in
string.Formatter().parse(cadasta_endpoint.url)
if a[1]]
cadasta_dict = {}
for k, v in data_dict.items():
cadasta_dict[k] = cadasta_endpoint.convert_argument(k, v)
error_dict = {}
for arg in string_arguments:
if arg not in data_dict.keys():
error_dict[arg] = ['Missing value']
cadasta_dict.pop(arg, None)
if error_dict:
raise toolkit.ValidationError(error_dict)
endpoint = cadasta_endpoint.url.format(**data_dict)
return cadasta_api_func(endpoint, cadasta_dict,
cadasta_endpoint.upload_fields)
def comment_create(context, data_dict):
tk.check_access("comments_comment_create", context, data_dict)
thread_data = {
"subject_id": data_dict["subject_id"],
"subject_type": data_dict["subject_type"],
}
try:
thread_dict = tk.get_action("comments_thread_show")(
context.copy(), thread_data
)
except tk.ObjectNotFound:
if not data_dict["create_thread"]:
raise
thread_dict = tk.get_action("comments_thread_create")(
context.copy(), thread_data
)
author_id = data_dict.get("author_id")
can_set_author_id = (
context.get("ignore_auth") or context["auth_user_obj"].sysadmin
)
if not author_id or not can_set_author_id:
author_id = context["user"]
reply_to_id = data_dict.get("reply_to_id")
if reply_to_id:
parent = tk.get_action("comments_comment_show")(
context.copy(), {"id": reply_to_id}
)
if parent["thread_id"] != thread_dict["id"]:
raise tk.ValidationError(
{"reply_to_id": ["Coment is owned by different thread"]}
)
comment = Comment(
thread_id=thread_dict["id"],
content=data_dict["content"],
author_type=data_dict["author_type"],
author_id=author_id,
reply_to_id=reply_to_id,
)
author = comment.get_author()
if author is None:
raise tk.ObjectNotFound("Cannot find author for comment")
# make sure we are not messing up with name_or_id
comment.author_id = author.id
if not tk.asbool(
tk.config.get(
const.CONFIG_REQUIRE_APPROVAL, const.DEFAULT_REQUIRE_APPROVAL
)
):
comment.approve()
context['session'].add(comment)
context['session'].commit()
comment_dict = get_dictizer(type(comment))(comment, context)
return comment_dict
def render_content(content):
allow_html = toolkit.asbool(config.get('ckanext.pages.allow_html', False))
try:
return h.render_markdown(content, allow_html=allow_html)
except TypeError:
# allow_html is only available in CKAN >= 2.3
return h.render_markdown(content)
def __init__(self, **kwargs):
for k, v in kwargs.items():
setattr(self, k, v)
# Auto-set some values based on configuration
from pylons import config
if toolkit.asbool(config.get('ckan.comments.moderation', 'true')):
self.approval_status = COMMENT_PENDING
else:
# If user wants first comment moderated and the user who wrote this hasn't
# got another comment, put it into moderation, otherwise approve
if toolkit.asbool(config.get('ckan.comments.moderation.first_only', 'true')) and \
Comment.count_for_user(self.user, COMMENT_APPROVED) == 0:
self.approval_status = COMMENT_PENDING
else:
self.approval_status = COMMENT_APPROVED
def before_search(self, search_params):
deployment_mode = toolkit.asbool(
config.get('ckan.ab_scheming.deployment', False))
if deployment_mode:
return search_params
user_member_of_orgs = [
org['id'] for org in h.organizations_available('read')
]
if (c.group and c.group.id in user_member_of_orgs):
# added for more control on result datasets
# based on our requirement
search_params.update({
'include_private': True,
'include_drafts': True
})
if c.process_state:
# for the filters in user dashboard and organization.
if search_params.get("q"):
search_params['q'] += ' %s: "%s"' % ('process_state',
c.process_state)
else:
search_params['q'] = '%s: "%s"' % ('process_state',
c.process_state)
if search_params.get('fq'):
if re.search(r'dataset_type', search_params['fq']):
# this is for general search
search_params.update({
'include_private': True,
'include_drafts': True
})
return search_params
def _get_filepath_for_resource(res):
"""Returns a filepath for a resource that will be indexed"""
res_id = res['id']
res_url = res['url']
if res["url_type"] == "upload":
uploader = get_resource_uploader(res)
# TODO temporary workaround for ckanext-cloudstorage support
if p.plugin_loaded('cloudstorage'):
url = uploader.get_url_from_filename(res_id, res_url)
filepath = _download_remote_file(res_id, url)
return filepath
path = uploader.get_path(res_id)
if not os.path.exists(path):
log.warn('Resource "{res_id}" refers to unexisting path "{path}"')
return
return path
if not tk.asbool(tk.config.get("ckanext.resource_indexer.allow_remote")):
return
filepath = _download_remote_file(res_id, res_url)
return filepath
def harvest_source_index_clear(context, data_dict):
'''
Clears all datasets, jobs and objects related to a harvest source, but
keeps the source itself. This is useful to clean history of long running
harvest sources to start again fresh.
:param id: the id of the harvest source to clear
:type id: string
'''
check_access('harvest_source_clear', context, data_dict)
harvest_source_id = data_dict.get('id')
source = HarvestSource.get(harvest_source_id)
if not source:
log.error('Harvest source %s does not exist', harvest_source_id)
raise NotFound('Harvest source %s does not exist' % harvest_source_id)
harvest_source_id = source.id
conn = make_connection()
query = ''' +%s:"%s" +site_id:"%s" ''' % (
'harvest_source_id', harvest_source_id, config.get('ckan.site_id'))
solr_commit = toolkit.asbool(config.get('ckan.search.solr_commit', 'true'))
if toolkit.check_ckan_version(max_version='2.5.99'):
# conn is solrpy
try:
conn.delete_query(query)
if solr_commit:
conn.commit()
except Exception, e:
log.exception(e)
raise SearchIndexError(e)
finally:
def __init__(self, **kwargs):
for k, v in kwargs.items():
setattr(self, k, v)
# Auto-set some values based on configuration
from pylons import config
if toolkit.asbool(config.get('ckan.comments.moderation', 'true')):
self.approval_status = COMMENT_PENDING
else:
# If user wants first comment moderated and the user who wrote this hasn't
# got another comment, put it into moderation, otherwise approve
if toolkit.asbool(config.get('ckan.comments.moderation.first_only', 'true')) and \
Comment.count_for_user(self.user, COMMENT_APPROVED) == 0:
self.approval_status = COMMENT_PENDING
else:
self.approval_status = COMMENT_APPROVED
def harvest_source_index_clear(context, data_dict):
'''
Clears all datasets, jobs and objects related to a harvest source, but
keeps the source itself. This is useful to clean history of long running
harvest sources to start again fresh.
:param id: the id of the harvest source to clear
:type id: string
'''
check_access('harvest_source_clear', context, data_dict)
harvest_source_id = data_dict.get('id')
source = HarvestSource.get(harvest_source_id)
if not source:
log.error('Harvest source %s does not exist', harvest_source_id)
raise NotFound('Harvest source %s does not exist' % harvest_source_id)
harvest_source_id = source.id
conn = make_connection()
query = ''' +%s:"%s" +site_id:"%s" ''' % (
'harvest_source_id', harvest_source_id, config.get('ckan.site_id'))
solr_commit = toolkit.asbool(config.get('ckan.search.solr_commit', 'true'))
if toolkit.check_ckan_version(max_version='2.5.99'):
# conn is solrpy
try:
conn.delete_query(query)
if solr_commit:
conn.commit()
except Exception, e:
log.exception(e)
raise SearchIndexError(e)
finally:
def add_defaults_to_options(options, defaults):
'''Returns the options, using option values passed in and falling back
to the default values for that report.
When a option needs a boolean, an option passed in as 0 or 1 are
converted to True/False, which suits when the options passed in are URL
parameters.
'''
defaulted_options = copy.deepcopy(defaults)
for key in defaulted_options:
if key not in options:
if defaulted_options[key] is True:
# Checkboxes don't submit a value when False, so cannot
# default to True. i.e. to get a True value, you always
# need be explicit in the params.
defaulted_options[key] = False
continue
value = options[key]
if isinstance(defaulted_options[key], bool):
try:
defaulted_options[key] = asbool(value)
except ValueError:
pass # leave it as default
else:
defaulted_options[key] = value
for key in set(options) - set(defaulted_options):
defaulted_options[key] = options[key]
return defaulted_options
def update_config(self, config):
self.organization_pages = toolkit.asbool(config.get('ckanext.pages.organization', False))
self.group_pages = toolkit.asbool(config.get('ckanext.pages.group', False))
toolkit.add_template_directory(config, 'theme/templates_main')
if self.group_pages:
toolkit.add_template_directory(config, 'theme/templates_group')
if self.organization_pages:
toolkit.add_template_directory(config, 'theme/templates_organization')
toolkit.add_resource('fanstatic', 'pages')
toolkit.add_public_directory(config, 'public')
toolkit.add_resource('theme/public', 'ckanext-pages')
toolkit.add_resource('theme/resources', 'pages-theme')
toolkit.add_public_directory(config, 'theme/public')
def issue_deleted_from_dataset(data_dict):
review_system = toolkit.asbool(
config.get("ckanext.issues.review_system", False))
log.debug("review_system issue_deleted: %s %s", data_dict, review_system)
if review_system:
issue_count = toolkit.get_action('issue_count')(
data_dict={
'dataset_id': data_dict['dataset_id'],
'status': issuemodel.ISSUE_STATUS.open
})
if issue_count == 0:
try:
logic.get_action('package_patch')(data_dict={
'id': data_dict['dataset_id'],
'private': False
})
except logic.NotAuthorized:
abort(401, _('Not authorized to modify the dataset'))
h.flash_success(_('The dataset has now been made public.'))
log.debug("Dataset %s made public", data_dict['dataset_id'])
def ingest_resource(context, resource_dict):
if toolkit.asbool(resource_dict.get('spatial_parent', 'False')):
try:
task = toolkit.get_action('task_status_show')(
{
'ignore_auth': True
}, {
'entity_id': resource_dict['id'],
'task_type': 'spatial_ingest',
'key': 'spatialingestor'
})
if task.get('state') in ['pending']:
# There already is a pending Spatialingestor submission,
# skip this one ...
log.debug(
'Skipping Spatial Ingestor submission for resource {0}'.
format(resource_dict['id']))
return
except toolkit.ObjectNotFound:
pass
try:
log.debug('Submitting resource {0} to Spatial Ingestor'.format(
resource_dict['id']))
toolkit.get_action('spatialingestor_job_submit')(
context, {
'resource_id': resource_dict['id'],
'job_type': 'spatial_ingest'
})
except toolkit.ValidationError, e:
log.error(e)
def update(context, data_dict):
'''Update a PowerView.
By default, only sysadmins can update PowerViews. But the setting
`ckanext.powerview.allow_user_create` allows any logged in user to
update powerviews they own.
'''
if not toolkit.asbool(config.get('ckanext.powerview.allow_user_create',
False)):
return {'success': False}
else:
user = context.get('user')
user_obj = model.User.get(user)
powerview = PowerView.get(id=data_dict['id'])
# Check resources
for res_id in data_dict.get('resources', []):
try:
toolkit.check_access('resource_show', context,
data_dict={'id': res_id})
except NotAuthorized:
return {
'success': False,
'msg': _('User {0} not authorized to read resource {1}'
.format(user, res_id))
}
if powerview and user and powerview.created_by == user_obj.id:
return {'success': True}
return {'success': False}
def check_access_user(context, data_dict):
if c.user:
return {'success': True}
else:
allow_rating = toolkit.asbool(
config.get('rating.enabled_for_unauthenticated_users', True))
return {'success': allow_rating}
def create(context, data_dict):
'''Create a PowerView.
By default, only sysadmins can create a PowerView. But the setting
`ckanext.powerview.allow_user_create` allows any logged in user to
create powerviews.
'''
allow_user_create = toolkit.asbool(
config.get('ckanext.powerview.allow_user_create', False))
if not allow_user_create:
return {'success': False}
user = context.get('user')
# Check resources
for res_id in data_dict.get('resources', []):
try:
toolkit.check_access('resource_show', context,
data_dict={'id': res_id})
except NotAuthorized:
return {
'success': False,
'msg': _('User {0} not authorized to read resource {1}'
.format(user, res_id))
}
return {'success': True}
def download(id, resource_id, filename=None):
"""Download resource blueprint
This calls all registered download handlers in order, until
a response is returned to the user
"""
context = get_context()
resource = None
try:
resource = toolkit.get_action('resource_show')(context, {
'id': resource_id
})
if id != resource['package_id']:
return toolkit.abort(
404, toolkit._('Resource not found belonging to package'))
package = toolkit.get_action('package_show')(context, {'id': id})
except toolkit.ObjectNotFound:
return toolkit.abort(404, toolkit._('Resource not found'))
except toolkit.NotAuthorized:
return toolkit.abort(
401, toolkit._('Not authorized to read resource {0}'.format(id)))
activity_id = request.args.get('activity_id')
inline = toolkit.asbool(request.args.get('preview'))
if activity_id and toolkit.check_ckan_version(min_version='2.9'):
try:
activity = toolkit.get_action(u'activity_show')(
context, {
u'id': activity_id,
u'include_data': True
})
activity_dataset = activity['data']['package']
assert activity_dataset['id'] == id
activity_resources = activity_dataset['resources']
for r in activity_resources:
if r['id'] == resource_id:
resource = r
package = activity_dataset
break
except toolkit.NotFound:
toolkit.abort(404, toolkit._(u'Activity not found'))
try:
resource = call_pre_download_handlers(resource,
package,
activity_id=activity_id)
return call_download_handlers(resource,
package,
filename,
inline,
activity_id=activity_id)
except toolkit.ObjectNotFound:
return toolkit.abort(404, toolkit._('Resource not found'))
except toolkit.NotAuthorized:
return toolkit.abort(
401,
toolkit._(
'Not authorized to read resource {0}'.format(resource_id)))
def run(ctx, host, port, disable_reloader, threaded, extra_files, processes):
u"""Runs the Werkzeug development server"""
use_reloader = not disable_reloader
threaded = threaded or tk.asbool(config.get(u"ckan.devserver.threaded"))
processes = processes or tk.asint(
config.get(u"ckan.devserver.multiprocess", 1)
)
if threaded and processes > 1:
tk.error_shout(u"Cannot have a multithreaded and multi process server")
raise click.Abort()
log.info(u"Running server {0} on port {1}".format(host, port))
config_extra_files = tk.aslist(
config.get(u"ckan.devserver.watch_patterns")
)
extra_files = list(extra_files) + [
config[u"__file__"]
] + config_extra_files
run_simple(
host,
port,
ctx.obj.app,
use_reloader=use_reloader,
use_evalex=True,
threaded=threaded,
processes=processes,
extra_files=extra_files,
)
def render_content(content):
allow_html = toolkit.asbool(config.get('ckanext.pages.allow_html', False))
try:
return h.render_markdown(content, allow_html=allow_html)
except TypeError:
# allow_html is only available in CKAN >= 2.3
return h.render_markdown(content)
def comment_update(context, data_dict):
model = context['model']
logic.check_access("comment_update", context, data_dict)
cid = logic.get_or_bust(data_dict, 'id')
comment = comment_model.Comment.get(cid)
if not comment:
abort(404)
# Validate that we have the required fields.
if not all([data_dict.get('comment')]):
raise logic.ValidationError("Comment text is required")
# Cleanup the comment
cleaned_comment = util.clean_input(data_dict.get('comment'))
# Run profanity check
if toolkit.asbool(config.get('ckan.comments.check_for_profanity', False)) \
and (helpers.profanity_check(cleaned_comment) or helpers.profanity_check(data_dict.get('subject', ''))):
raise logic.ValidationError("Comment blocked due to profanity.")
comment.subject = data_dict.get('subject')
comment.comment = cleaned_comment
comment.modified_date = datetime.datetime.utcnow()
comment.flagged = data_dict.get('flagged')
model.Session.add(comment)
model.Session.commit()
return comment.as_dict()
def create_zip(ckan_ini_filepath, package_id, queue='bulk'):
load_config(ckan_ini_filepath)
register_translator()
log = create_zip.get_logger()
context = {'model': model, 'ignore_auth': True, 'session': model.Session}
pkg = get_action('package_show')(context, {'id': package_id})
extras = dict([(d['key'], d['value'],) for d in pkg['extras']])
unpublished = t.asbool(extras.get('unpublished', False))
if unpublished:
log.info("Skipping unpublished dataset: %s", pkg['name'])
return
directory = config.get('ckanext.packagezip.destination_dir')
if not os.path.exists(directory):
log.info('Creating packagezip directory: %s' % directory)
os.mkdir(directory)
filename = "{0}.zip".format(pkg['name'])
filepath = os.path.join(directory, filename)
with zipfile.ZipFile(filepath, 'w', zipfile.ZIP_DEFLATED, allowZip64=True) as zipf:
try:
datapackage = get_action('datapackage_show')(context, {'id': package_id})
except KeyError, e:
log.error('Cannot find action - check this plugin is enabled: %s',
e)
raise
any_have_data = False
for res in datapackage['resources']:
if res['has_data']:
any_have_data = True
if res['cache_filepath'] and os.path.exists(res['cache_filepath']):
zipf.write(res['cache_filepath'], res['path'])
res['included_in_zip'] = True
else:
res['included_in_zip'] = False
env = Environment(loader=PackageLoader('ckanext.packagezip', 'templates'))
env.filters['datetimeformat'] = datetimeformat
template = env.get_template('index.html')
zipf.writestr('index.html',
template.render(datapackage=datapackage,
date=datetime.datetime.now()).encode('utf8'))
# Strip out unnecessary data from datapackage
for res in datapackage['resources']:
del res['has_data']
if 'cache_filepath' in res:
del res['cache_filepath']
if 'reason' in res:
del res['reason']
if 'detected_format' in res:
del res['detected_format']
zipf.writestr('datapackage.json', json.dumps(datapackage, indent=4))
def counter_on_off(context, data_dict=None):
# Get the value of the ckan.open_alberta.counter_on
# setting from the CKAN config file as a string, or False if the setting
# isn't in the config file.
counter_on = config.get('ckan.open_alberta.counter_on', False)
# Convert the value from a string to a boolean.
counter_on = toolkit.asbool(counter_on)
return {"counter_on": counter_on}
def package_create(context, data_dict):
deployment_mode = toolkit.asbool(config.get('ckan.ab_scheming.deployment', False))
# need to change data_dict if import from ckanapi
if deployment_mode:
data_dict = change_pkg_dict_for_import_deployment(data_dict, 'create')
if data_dict['type'] in ['publications', 'opendata']:
context['defer_commit'] = True
return create.package_create(context, data_dict)
def issues_enabled(dataset):
datasets_with_issues_enabled = set(toolkit.aslist(
config.get('ckanext.issues.enabled_for_datasets')
))
# if the config option 'ckanext.issues.enabled_for_dataset' is
# set with a list of datasets, only enabled for the listed datasets
if datasets_with_issues_enabled:
if dataset['name'] in datasets_with_issues_enabled:
return True
else:
extras = dataset.get('extras')
if extras is not None:
for extra in extras:
if extra.get('key') == 'issues_enabled':
return toolkit.asbool(extra.get('value'))
else:
return toolkit.asbool(config.get('ckanext.issues.enabled_per_dataset_default', True))
def _create_user_on_login():
'''Return whether to automatically create users on login defaults to
true'''
try:
return toolkit.asbool(
pylons.config['ckanext.oauth2waad.create_users_on_login']
)
except KeyError:
return True
def command(self):
self._load_config()
import ckan.model as model
model.Session.remove()
model.Session.configure(bind=model.meta.engine)
model.repo.new_revision()
self.log.info("Database access initialised")
root_url = config.get('ckan.site_url', 'http://data.gov.uk')
self.log.debug("Root url is " + root_url)
if toolkit.asbool(config.get('debug')):
root_url = "http://data.gov.uk"
self.log.debug("Overriding root_url in DEBUG ")
data = self._make_request()
for d in data:
# Eventually we might handle other types.
if d['type'] != 'App':
continue
related_url = urljoin(root_url, d['path'])
package = model.Package.get(d['ckan_id'])
if not package:
continue
found = False
current_related = model.Related.get_for_dataset(package)
for current in current_related:
if current.related.url == related_url:
self.log.info("Skipping existing related")
found = True
if not found:
self.log.info("Adding related item [%s] to dataset [%s]" % (d['title'], package.name))
related = model.Related()
related.type = 'App'
related.title = d['title']
related.description = ""
related.url = related_url
thumb = d.get('thumbnail', '').replace("://", "/")
if thumb:
thumb_url = urljoin(root_url, "/sites/default/files/styles/medium/")
thumb_url = urljoin(thumb_url, thumb)
else:
thumb_url = ''
related.image_url = thumb_url
model.Session.add(related)
model.Session.commit()
related_item = model.RelatedDataset(dataset_id=package.id, related_id=related.id)
model.Session.add(related_item)
model.Session.commit()
def _setup_webassets(app):
app.use_x_sendfile = toolkit.asbool(
config.get('ckan.webassets.use_x_sendfile')
)
webassets_folder = get_webassets_path()
@app.route('/webassets/<path:path>')
def webassets(path):
return send_from_directory(webassets_folder, path)
def update_config(self, config):
if self.instance:
# reloading plugins, probably in WebTest
_SchemingMixin._helpers_loaded = False
self._store_instance(self)
self._add_template_directory(config)
self._is_fallback = asbool(config.get(self.FALLBACK_OPTION, False))
self._schema_urls = config.get(self.SCHEMA_OPTION, "").split()
self._schemas = _load_schemas(self._schema_urls, self.SCHEMA_TYPE_FIELD)
def resources(self, id=None):
try:
c.include_sub_publishers = t.asbool(t.request.params.get('include_sub_publishers') or False)
except ValueError:
abort(400, 'include_sub_publishers parameter value must be boolean')
c.publisher_name = id or 'department-for-culture-media-and-sport'
c.query = reports.organisation_resources
c.data = c.query(organisation_name=c.publisher_name,
include_sub_organisations=c.include_sub_publishers)
return t.render('reports/resources.html')
def geo_resource_form(context, data):
"""
Returns the form for the OFI Manager create/edit of OFI resource.
"""
if toolkit.asbool(ofi_config.get('convert_to_single_res', False)):
file_formats = toolkit.get_action(u'file_formats')({}, {})
data.update(file_formats=file_formats)
return toolkit.render('ofi/snippets/geo_resource_form.html',
extra_vars=data)
def configure(self, config):
'''Store the YouCKAN configuration'''
if not toolkit.check_ckan_version('2.1'):
log.warn('This extension has only been tested on CKAN 2.1!')
self.use_auth = toolkit.asbool(config.get('youckan.use_auth', False))
if self.use_auth:
self.logout_url = config['youckan.logout_url']
model_setup()
def _setup_ofi(id, context=None, pkg_dict=None, open_modal=True):
# Setup for OFI
if not context:
context = {}
try:
if not pkg_dict:
pkg_dict = get_action('package_show')(context, {'id': id, 'include_tracking':True})
except NotFound:
abort(404, _('Dataset not found'))
except NotAuthorized:
abort(401, _('Unauthorized to read dataset %s') % id)
ofi_data = {}
if 'type' in pkg_dict and pkg_dict[u'type'] == 'Geographic':
ofi_config = edc_h.get_ofi_config()
# dev_secure_call is for working around missing siteminer session, for vagrant use only
secure_call = toolkit.asbool(ofi_config.get('dev_secure_call', True))
ofi_resources = []
for resource in pkg_dict[u'resources']:
if 'ofi' in resource and resource[u'ofi']:
ofi_resources.append(resource)
if len(ofi_resources) > 0:
projections = get_action(u'crs_types')(context, {})
ofi_data.update({
u'object_name': pkg_dict.get(u'object_name', ""),
u'ofi_resources': ofi_resources,
u'ofi_exists': True,
u'secure': secure_call,
u'projections': projections,
u'open_modal': open_modal
})
elif pkg_dict.get('object_name', False):
obj_data = {
u'object_name': pkg_dict[u'object_name'],
u'secure': secure_call,
}
ofi_data.update(get_action('check_object_name')(context, obj_data))
ofi_data.update(open_modal=open_modal)
else:
ofi_data.update({
u'object_name': '',
u'secure': False,
u'open_modal': False
})
return ofi_data
def command(self):
#if not correct args, show help
if not self.args or self.args[0] in ['--help', '-h', 'help']:
print self.usage
return
cmd = self.args[0]
self._load_config()
#update command
if cmd == 'update':
url_id = config.get('ckan.piwik.url').split('/piwik.php')
piwik_url = url_id[0]
use_https = toolkit.asbool(config.get('ckan.piwik.https', False))
if use_https:
piwik_url = 'https:' + piwik_url
else:
piwik_url = 'http:' + piwik_url
recent_date = date.today() - timedelta(days=int(config.get('ckan.piwik.recent_days', '14')))
piwik_date_opts = {'total': '2011-01-01,today',
'recent': recent_date.strftime('%Y-%m-%d') + ',today'}
#setup piwik api GET request params
piwik_params = {
'idSite': config.get('ckan.piwik.site_id'),
'token_auth': config.get('ckan.piwik.auth_token'),
'module': 'API',
'method': 'Actions.getPageUrl',
'format': 'json',
'period': 'range',
'pageUrl': None,
'date': None
}
if len(self.args) > 1:
#update a single package
self.update_package_stats(self.args[1],
piwik_params,
piwik_date_opts,
piwik_url)
else:
#get all the packages
pkgs_list = toolkit.get_action('package_list')(context=None, data_dict={})
for pkg in pkgs_list:
self.update_package_stats(pkg,
piwik_params,
piwik_date_opts,
piwik_url)
else:
print 'command: {cmd} not recognised'.format(cmd=cmd)
def build_pages_nav_main(*args):
about_menu = toolkit.asbool(config.get('ckanext.pages.about_menu', True))
group_menu = toolkit.asbool(config.get('ckanext.pages.group_menu', True))
org_menu = toolkit.asbool(config.get('ckanext.pages.organization_menu', True))
new_args = []
for arg in args:
if arg[0] == 'about' and not about_menu:
continue
if arg[0] == 'organizations_index' and not org_menu:
continue
if arg[0] == 'group_index' and not group_menu:
continue
new_args.append(arg)
output = h.build_nav_main(*new_args)
# do not display any private datasets in menu even for sysadmins
pages_list = toolkit.get_action('ckanext_pages_list')(None, {'order': True, 'private': False})
page_name = ''
if (toolkit.c.action in ('pages_show', 'blog_show')
and toolkit.c.controller == 'ckanext.pages.controller:PagesController'):
page_name = toolkit.c.environ['routes.url'].current().split('/')[-1]
for page in pages_list:
type_ = 'blog' if page['page_type'] == 'blog' else 'pages'
name = urllib.quote(page['name'].encode('utf-8')).decode('utf-8')
title = cgi.escape(page['title'])
link = h.literal(u'<a href="/{}/{}">{}</a>'.format(type_, name, title))
if page['name'] == page_name:
li = h.literal('<li class="active">') + link + h.literal('</li>')
else:
li = h.literal('<li>') + link + h.literal('</li>')
output = output + li
return output
def related_pkgs(cls):
# Parameter
extractNum = int(config.get('ckan.data_recommended.extract_num', '5'))
byTag = asbool(config.get('ckan.data_recommended.by_tag', 'true'))
byTitle = asbool(config.get('ckan.data_recommended.by_title', 'true'))
renderMod = 'front_end'
# fetch pkg info
pkg_name = request.environ['PATH_INFO'].split('/')[-1]
pkg_title = toolkit.get_action('package_show')({}, {'id':pkg_name})['title']
pkg_tags = [pkg_tag['name'] for pkg_tag in toolkit.get_action('package_show')({}, {'id':pkg_name})['tags']]
# related_tag_titles
related_tag_titles = set()
if byTag:
related_tag_titles.update(set(pkg_tags))
if byTitle:
related_tag_titles.update(
set(
jieba.analyse.extract_tags(pkg_title, topK=extractNum)
)
)
related_pkgs = {}
if renderMod == 'back_end':
related_pkgs['mod'] = renderMod
related_pkgs['results'] = dict()
for related_tag_title in related_tag_titles:
related_pkg_results = toolkit.get_action('package_search')({}, {'q': related_tag_title, 'rows': 3})['results']
related_pkgs['results'][related_tag_title] = related_pkg_results
return related_pkgs
elif renderMod == 'front_end':
related_pkgs['results'] = related_tag_titles
related_pkgs['mod'] = renderMod
return related_pkgs
def after_create(self, context, pkg_dict):
dataset_type = context['package'].type if 'package' in context else pkg_dict['type']
if dataset_type == 'dataset':
log.info('after_create: %s', pkg_dict['name'])
odm_dataset_helper.session['last_dataset'] = pkg_dict
odm_dataset_helper.session.save()
# Create default Issue
review_system = toolkit.asbool(config.get("ckanext.issues.review_system", False))
if review_system:
if pkg_dict['type'] == 'dataset':
odm_dataset_helper.create_default_issue_dataset(pkg_dict)
