def update_config(self, config):
tk.add_template_directory(config, '../templates')
tk.add_public_directory(config, '../public')
tk.add_resource('../fanstatic', 'showcase')
if tk.check_ckan_version(min_version='2.4', max_version='2.9.0'):
tk.add_ckan_admin_tab(config, 'showcase_admins',
'Showcase Config')
elif tk.check_ckan_version(min_version='2.9.0'):
tk.add_ckan_admin_tab(config, 'showcase_blueprint.admins',
'Showcase Config')
if tk.check_ckan_version(min_version='2.9.0'):
mappings = config.get('ckan.legacy_route_mappings', {})
if isinstance(mappings, string_types):
mappings = json.loads(mappings)
bp_routes = [
'index', 'new', 'delete',
'read', 'edit', 'manage_datasets',
'dataset_showcase_list', 'admins', 'admin_remove'
]
mappings.update({
'showcase_' + route: 'showcase_blueprint.' + route
for route in bp_routes
})
# https://github.com/ckan/ckan/pull/4521
config['ckan.legacy_route_mappings'] = json.dumps(mappings)
def delete_view(id):
if 'cancel' in tk.request.params:
tk.redirect_to('showcase_blueprint.edit' if tk.check_ckan_version(
min_version='2.9.0') else 'showcase_edit',
id=id)
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj
}
try:
tk.check_access('ckanext_showcase_delete', context, {'id': id})
except tk.NotAuthorized:
return tk.abort(401, _('Unauthorized to delete showcase'))
if tk.check_ckan_version(min_version='2.9.0'):
index_route = 'showcase_blueprint.index'
else:
index_route = 'showcase_index'
try:
if tk.request.method == 'POST':
tk.get_action('ckanext_showcase_delete')(context, {'id': id})
h.flash_notice(_('Showcase has been deleted.'))
return tk.redirect_to(index_route)
c.pkg_dict = tk.get_action('package_show')(context, {'id': id})
except tk.NotAuthorized:
tk.abort(401, _('Unauthorized to delete showcase'))
except tk.ObjectNotFound:
tk.abort(404, _('Showcase not found'))
return tk.render('showcase/confirm_delete.html',
extra_vars={'dataset_type': DATASET_TYPE_NAME})
def test_choice_field_only_accepts_given_choices(self):
lc = LocalCKAN()
try:
lc.action.package_create(
type="test-schema", name="fred_choices1", category="rocker"
)
except ValidationError as e:
if check_ckan_version("2.9"):
expected = "Value must be one of {}".format(
[
u"bactrian",
u"hybrid",
u"f2hybrid",
u"snowwhite",
u"black",
]
)
else:
expected = (
"Value must be one of: bactrian; hybrid; f2hybrid; "
"snowwhite; black (not 'rocker')"
)
assert e.error_dict["category"] == [expected]
else:
raise AssertionError("ValidationError not raised")
def setup_template_variables(self, context, data_dict):
import ckanext.resourceproxy.plugin as proxy
same_domain = on_same_domain(data_dict)
if not data_dict['resource'].get('format'):
data_dict['resource']['format'] = \
self._guess_format_from_extension(data_dict['resource']['url'])
if self.proxy_enabled and not same_domain:
proxy_url = proxy.get_proxified_resource_url(data_dict)
proxy_service_url = get_proxified_service_url(data_dict)
else:
proxy_url = data_dict['resource']['url']
proxy_service_url = data_dict['resource']['url']
gapi_key = toolkit.config.get('ckanext.geoview.gapi_key')
if not toolkit.check_ckan_version(min_version='2.3'):
toolkit.c.resource['proxy_url'] = proxy_url
toolkit.c.resource['proxy_service_url'] = proxy_service_url
toolkit.c.resource['gapi_key'] = gapi_key
return {
'resource_view_json':
'resource_view' in data_dict
and json.dumps(data_dict['resource_view']),
'proxy_service_url':
proxy_service_url,
'proxy_url':
proxy_url,
'gapi_key':
gapi_key,
'basemapsConfig':
self.basemapsConfig
}
def _post(app, url, data, resource_id='', upload=None):
args = []
env = _get_extra_env_as_sysadmin()
# from the form
data['id'] = resource_id
data['save'] = ''
if check_ckan_version('2.9'):
if upload:
for entry in upload:
data[entry[0]] = (io.BytesIO(entry[2]), entry[1])
kwargs = {
'url': url,
'data': data,
'extra_environ': env
}
else:
args.append(url)
kwargs = {
'params': data,
'extra_environ': env,
'upload_files': upload
}
return app.post(*args, **kwargs)
def setup_template_variables(self, context, data_dict):
import ckanext.resourceproxy.plugin as proxy
same_domain = on_same_domain(data_dict)
if not data_dict['resource'].get('format'):
data_dict['resource']['format'] = \
self._guess_format_from_extension(data_dict['resource']['url'])
if self.proxy_enabled and not same_domain:
proxy_url = proxy.get_proxified_resource_url(data_dict)
proxy_service_url = get_proxified_service_url(data_dict)
else:
proxy_url = data_dict['resource']['url']
proxy_service_url = data_dict['resource']['url']
gapi_key = toolkit.config.get('ckanext.geoview.gapi_key')
if not toolkit.check_ckan_version(min_version='2.3'):
toolkit.c.resource['proxy_url'] = proxy_url
toolkit.c.resource['proxy_service_url'] = proxy_service_url
toolkit.c.resource['gapi_key'] = gapi_key
return {'resource_view_json': 'resource_view' in data_dict and json.dumps(data_dict['resource_view']),
'proxy_service_url': proxy_service_url,
'proxy_url': proxy_url,
'gapi_key': gapi_key,
'basemapsConfig' : self.basemapsConfig}
def _search_url(params, name):
if tk.check_ckan_version(min_version='2.9.0'):
manage_route = 'showcase_blueprint.manage_datasets'
else:
manage_route = 'showcase_manage_datasets'
url = h.url_for(manage_route, id=name)
return url_with_params(url, params)
def user_confirm(msg):
if check_ckan_version(min_version='2.9'):
import click
return click.confirm(msg)
else:
from ckan.lib.cli import query_yes_no
return query_yes_no(msg) == 'yes'
def populate_revision(resource):
if 'revision_timestamp' in resource \
or toolkit.check_ckan_version(min_version='2.9'):
return
current_revision = latest_revision(resource['id'])
if current_revision is not None:
resource['revision_timestamp'] = current_revision.revision_timestamp
class GeoViewBase(p.SingletonPlugin):
'''This base class is for view extensions. '''
if toolkit.check_ckan_version(min_version='2.3'):
p.implements(p.IResourceView, inherit=True)
else:
p.implements(p.IResourcePreview, inherit=True)
p.implements(p.IConfigurer, inherit=True)
p.implements(p.IConfigurable, inherit=True)
proxy_enabled = False
same_domain = False
def configure(self, config):
basemapConfigFile = toolkit.config.get('ckanext.geoview.basemaps',
None)
self.basemapsConfig = basemapConfigFile and load_basemaps(
basemapConfigFile)
def update_config(self, config):
toolkit.add_public_directory(config, 'public')
toolkit.add_template_directory(config, 'templates')
toolkit.add_resource('public', 'ckanext-geoview')
self.proxy_enabled = 'resource_proxy' in toolkit.config.get(
'ckan.plugins', '')
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key,
value=value,
replace=replace,
controller='dataset' if
tk.check_ckan_version('2.9') else 'package',
action='search')
def remove_showcase_admin():
'''
Remove a user from the Showcase Admin list.
'''
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author
}
try:
tk.check_access('sysadmin', context, {})
except tk.NotAuthorized:
return tk.abort(401, _('User not authorized to view page'))
form_data = tk.request.form if tk.check_ckan_version(
'2.9') else tk.request.params
if tk.check_ckan_version(min_version='2.9.0'):
admins_route = 'showcase_blueprint.admins'
else:
admins_route = 'showcase_admins'
if 'cancel' in form_data:
return tk.redirect_to(admins_route)
user_id = tk.request.params['user']
if tk.request.method == 'POST' and user_id:
user_id = tk.request.params['user']
try:
tk.get_action('ckanext_showcase_admin_remove')(data_dict={
'username': user_id
})
except tk.NotAuthorized:
return tk.abort(401, _('Unauthorized to perform that action'))
except tk.ObjectNotFound:
h.flash_error(_('The user is not a Showcase Admin'))
else:
h.flash_success(_('The user is no longer a Showcase Admin'))
return tk.redirect_to(h.url_for(admins_route))
c.user_dict = tk.get_action('user_show')(data_dict={'id': user_id})
c.user_id = user_id
return tk.render('admin/confirm_remove_showcase_admin.html')
def configure(self, config):
self.search_backend = config.get('ckanext.spatial.search_backend',
'postgis')
if self.search_backend != 'postgis' and not tk.check_ckan_version(
'2.0.1'):
msg = 'The Solr backends for the spatial search require CKAN 2.0.1 or higher. ' + \
'Please upgrade CKAN or select the \'postgis\' backend.'
raise tk.CkanVersionException(msg)
def test_group_form_includes_custom_field(self):
if not ckantoolkit.check_ckan_version(min_version='2.8.0'):
raise SkipTest
app = self._get_test_app()
env, response = _get_group_new_page_as_sysadmin(app, type='theme')
form = response.forms[1]
assert_true('status' in form.fields)
def test_group_form_includes_custom_field(self):
if not ckantoolkit.check_ckan_version(min_version='2.7.0'):
raise SkipTest
app = self._get_test_app()
env, response = _get_group_new_page_as_sysadmin(app)
form = response.forms[1] # FIXME: add an id to this form
assert_true('bookface' in form.fields)
def enqueue_validation_job(package_id, resource_id):
enqueue_args = {
'fn': run_validation_job,
'title': "run_validation_job: package_id: {} resource: {}".format(package_id, resource_id),
'kwargs': {'resource': resource_id},
}
if t.check_ckan_version('2.8'):
ttl = 24 * 60 * 60 # 24 hour ttl.
rq_kwargs = {
'ttl': ttl
}
if t.check_ckan_version('2.9'):
rq_kwargs['failure_ttl'] = ttl
enqueue_args['rq_kwargs'] = rq_kwargs
# Optional variable, if not set, default queue is used
queue = t.config.get('ckanext.validation.queue', None)
if queue:
enqueue_args['queue'] = queue
t.enqueue_job(**enqueue_args)
def _get_resource_new_page_as_sysadmin(app, id):
user = Sysadmin()
env = {"REMOTE_USER": user["name"].encode("ascii")}
if ckantoolkit.check_ckan_version(min_version="2.9"):
url = '/dataset/{}/resource/new'.format(id)
else:
url = '/dataset/new_resource/{}'.format(id)
response = app.get(url, extra_environ=env)
return env, response
def manage_showcase_admins():
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author
}
try:
tk.check_access('sysadmin', context, {})
except tk.NotAuthorized:
return tk.abort(401, _('User not authorized to view page'))
form_data = tk.request.form if tk.check_ckan_version(
'2.9') else tk.request.params
if tk.check_ckan_version(min_version='2.9.0'):
admins_route = 'showcase_blueprint.admins'
else:
admins_route = 'showcase_admins'
# We're trying to add a user to the showcase admins list.
if tk.request.method == 'POST' and form_data['username']:
username = form_data['username']
try:
tk.get_action('ckanext_showcase_admin_add')(data_dict={
'username': username
})
except tk.NotAuthorized:
abort(401, _('Unauthorized to perform that action'))
except tk.ObjectNotFound:
h.flash_error(
_("User '{user_name}' not found.").format(user_name=username))
except tk.ValidationError as e:
h.flash_notice(e.error_summary)
else:
h.flash_success(_("The user is now a Showcase Admin"))
return tk.redirect_to(h.url_for(admins_route))
c.showcase_admins = tk.get_action('ckanext_showcase_admin_list')()
return tk.render('admin/manage_showcase_admins.html')
def test_org_form_includes_custom_field(self):
if not ckantoolkit.check_ckan_version(min_version='2.8.0'):
raise SkipTest
app = self._get_test_app()
env, response = _get_organization_new_page_as_sysadmin(
app, type='publisher')
form = response.forms[1]
assert_true('address' in form.fields)
class GoogleAnalyticsPlugin(p.SingletonPlugin):
p.implements(p.IConfigurable, inherit=True)
if check_ckan_version('2.8'):
p.implements(p.IBlueprint)
# workaround for https://github.com/ckan/ckan/issues/6678
import ckan.views.api as core_api
else:
p.implements(p.IRoutes, inherit=True)
analytics_queue = Queue.Queue()
capture_api_actions = {}
google_analytics_id = None
def configure(self, config):
'''Load config settings for this extension from config file.
See IConfigurable.
'''
# Load capture_api_actions from JSON file
here = path.abspath(path.dirname(__file__))
with open(path.join(here, 'capture_api_actions.json')) as json_file:
GoogleAnalyticsPlugin.capture_api_actions = json.load(json_file)
# Get google_analytics_id from config file
GoogleAnalyticsPlugin.google_analytics_id = config.get('ckan.data_qld_googleanalytics.id')
# spawn a pool of 5 threads, and pass them queue instance
for i in range(5):
t = AnalyticsPostThread(self.analytics_queue)
t.setDaemon(True)
t.start()
# IRoutes
def before_map(self, map):
'''Add new routes that this extension's controllers handle.
'''
from routes.mapper import SubMapper
# /api ver 3 or none
with SubMapper(map, controller='ckanext.data_qld.google_analytics.controller:GoogleAnalyticsApiController',
path_prefix='/api{ver:/3|}', ver='/3') as m:
m.connect('/action/{api_action}', action='action', conditions={'method': ['GET', 'POST']})
return map
# IBlueprint
def get_blueprint(self):
import blueprints
return [blueprints.blueprint]
class TestCustomGroupFormNew(object):
@pytest.mark.skipif(
not ckantoolkit.check_ckan_version(min_version="2.8.0"),
reason="Unspecified")
def test_group_form_includes_custom_field(self, app):
env, response = _get_group_new_page_as_sysadmin(app, type="theme")
form = BeautifulSoup(response.body).select("form")[1]
assert form.select("input[name=status]")
def test_group_form_slug_uses_custom_type(self, app):
env, response = _get_group_new_page_as_sysadmin(app, type="theme")
assert "/theme/" in response.body
def test_resource_form_includes_custom_fields(self, app, sysadmin_env):
dataset = Dataset(type="test-schema", name="resource-includes-custom")
if ckantoolkit.check_ckan_version(min_version="2.9"):
url = '/dataset/{}/resource/new'.format(dataset["id"])
else:
url = '/dataset/new_resource/{}'.format(dataset["id"])
response = app.get(
url,
extra_environ=sysadmin_env,
)
form = BeautifulSoup(response.body).select_one("#resource-edit")
assert form.select("input[name=camels_in_photo]")
class TestCustomOrgFormNew(object):
@pytest.mark.skipif(
not ckantoolkit.check_ckan_version(min_version="2.8.0"),
reason="Unspecified")
def test_org_form_includes_custom_field(self, app):
env, response = _get_organization_new_page_as_sysadmin(
app, type="publisher")
form = BeautifulSoup(response.body).select("form")[1]
assert form.select("input[name=address]")
def test_org_form_slug_uses_custom_type(self, app):
env, response = _get_organization_new_page_as_sysadmin(
app, type="publisher")
assert "/publisher/" in response.body
def get_validation_badge(resource, in_listing=False):
if in_listing and not asbool(
config.get('ckanext.validation.show_badges_in_listings', True)):
return ''
if not resource.get('validation_status'):
return ''
statuses = {
'success': _('success'),
'failure': _('failure'),
'invalid': _('invalid'),
'error': _('error'),
'unknown': _('unknown'),
}
if resource['validation_status'] in ['success', 'failure', 'error']:
status = resource['validation_status']
if status == 'failure':
status = 'invalid'
else:
status = 'unknown'
if check_ckan_version(min_version='2.9.0'):
action = 'validation.read'
else:
action = 'validation_read'
validation_url = url_for(
action,
id=resource['package_id'],
resource_id=resource['id'])
return u'''
<a href="{validation_url}" class="validation-badge" title="{title}">
<span class="prefix">{prefix}</span><span class="status {status}">{status_title}</span>
</a>'''.format(
validation_url=validation_url,
prefix=_('data'),
status=status,
status_title=statuses[status],
title=resource.get('validation_timestamp', ''))
class TestGroupFormNew(object):
@pytest.mark.skipif(
not ckantoolkit.check_ckan_version(min_version="2.7.0"),
reason="Unspecified")
def test_group_form_includes_custom_field(self, app):
env, response = _get_group_new_page_as_sysadmin(app)
# FIXME: add an id to this form
form = BeautifulSoup(response.body).select("form")[1]
assert form.select("input[name=bookface]")
def test_group_form_slug_says_group(self, app):
"""The default prefix shouldn't be /packages?id="""
env, response = _get_group_new_page_as_sysadmin(app)
# Commenting until ckan/ckan#4208 is fixed
# assert_true('packages?id=' not in response.body)
assert "/group/" in response.body
def test_widget_loaded(app):
dataset = factories.Dataset()
for i in range(1, 11):
resource = factories.Resource(
name='Resource {}'.format(i),
url='https://example.com',
package_id=dataset['id'],
)
if check_ckan_version(min_version='2.9.0'):
url = url_for('dataset.read', id=dataset['id'])
else:
url = url_for(controller='package', action='read', id=dataset['id'])
res = app.get(url)
assert 'resources-list-filter-container' in res.get_data(as_text=True)
assert 'script type="text/template" id="resources-list-template"' in res.get_data(as_text=True)
def can_preview(self, data_dict):
format_lower = data_dict['resource']['format'].lower()
correct_format = format_lower in self.WMTS
can_preview_from_domain = (self.proxy_enabled or
data_dict['resource'].get('on_same_domain'))
quality = 2
if toolkit.check_ckan_version('2.1'):
if correct_format:
if can_preview_from_domain:
return {'can_preview': True, 'quality': quality}
else:
return {'can_preview': False,
'fixable': 'Enable resource_proxy',
'quality': quality}
else:
return {'can_preview': False, 'quality': quality}
return correct_format and can_preview_from_domain
def can_preview(self, data_dict):
format_lower = data_dict['resource']['format'].lower()
correct_format = format_lower in self.WMTS
can_preview_from_domain = (self.proxy_enabled or
data_dict['resource'].get('on_same_domain'))
quality = 2
if toolkit.check_ckan_version('2.1'):
if correct_format:
if can_preview_from_domain:
return {'can_preview': True, 'quality': quality}
else:
return {
'can_preview': False,
'fixable': 'Enable resource_proxy',
'quality': quality
}
else:
return {'can_preview': False, 'quality': quality}
return correct_format and can_preview_from_domain
def check_geoalchemy_requirement():
'''Checks if a suitable geoalchemy version installed
Checks if geoalchemy2 is present when using CKAN >= 2.3, and raises
an ImportError otherwise so users can upgrade manually.
'''
msg = ('This version of ckanext-spatial requires {0}. ' +
'Please install it by running `pip install {0}`.\n' +
'For more details see the "Troubleshooting" section of the ' +
'install documentation')
if tk.check_ckan_version(min_version='2.3'):
try:
import geoalchemy2
except ImportError:
raise ImportError(msg.format('geoalchemy2'))
else:
try:
import geoalchemy
except ImportError:
raise ImportError(msg.format('geoalchemy'))
import os
import cgi
import logging
import datetime
import mimetypes
import boto3
import botocore
import ckantoolkit as toolkit
import ckan.model as model
import ckan.lib.munge as munge
if toolkit.check_ckan_version(min_version='2.7.0'):
from werkzeug.datastructures import FileStorage as FlaskFileStorage
ALLOWED_UPLOAD_TYPES = (cgi.FieldStorage, FlaskFileStorage)
else:
ALLOWED_UPLOAD_TYPES = (cgi.FieldStorage)
config = toolkit.config
log = logging.getLogger(__name__)
_storage_path = None
_max_resource_size = None
_max_image_size = None
def _get_underlying_file(wrapper):
if isinstance(wrapper, FlaskFileStorage):
return wrapper.stream
return wrapper.file
import os
import cgi
import logging
import datetime
import mimetypes
import boto3
import botocore
import ckantoolkit as toolkit
import ckan.model as model
import ckan.lib.munge as munge
if toolkit.check_ckan_version(min_version='2.7.0'):
from werkzeug.datastructures import FileStorage as FlaskFileStorage
ALLOWED_UPLOAD_TYPES = (cgi.FieldStorage, FlaskFileStorage)
else:
ALLOWED_UPLOAD_TYPES = (cgi.FieldStorage)
config = toolkit.config
log = logging.getLogger(__name__)
_storage_path = None
_max_resource_size = None
_max_image_size = None
def _get_underlying_file(wrapper):
if isinstance(wrapper, FlaskFileStorage):
return wrapper.stream
import os
import re
import mimetypes
from logging import getLogger
import six
import ckantoolkit as tk
from ckan import plugins as p
from ckan.lib.helpers import json
if tk.check_ckan_version(min_version="2.9.0"):
from ckanext.spatial.plugin.flask_plugin import (SpatialQueryMixin,
HarvestMetadataApiMixin)
else:
from ckanext.spatial.plugin.pylons_plugin import (SpatialQueryMixin,
HarvestMetadataApiMixin)
config = tk.config
def check_geoalchemy_requirement():
'''Checks if a suitable geoalchemy version installed
Checks if geoalchemy2 is present when using CKAN >= 2.3, and raises
an ImportError otherwise so users can upgrade manually.
'''
msg = ('This version of ckanext-spatial requires {0}. ' +
'Please install it by running `pip install {0}`.\n' +
def proxy_service_url(req, url):
parts = urlsplit(url)
if not parts.scheme or not parts.netloc:
base.abort(409, detail="Invalid URL.")
try:
method = req.environ["REQUEST_METHOD"]
params = parse_qs(parts.query)
if not p.toolkit.asbool(
base.config.get("ckanext.geoview.forward_ogc_request_params",
"False")):
# remove query parameters that may conflict with OGC protocols
for key in dict(params):
if key.lower() in OGC_EXCLUDED_PARAMS:
del params[key]
parts = parts._replace(query=urlencode(params))
parts = parts._replace(fragment="") # remove potential fragment
url = parts.geturl()
if method == "POST":
length = int(req.environ["CONTENT_LENGTH"])
headers = {"Content-Type": req.environ["CONTENT_TYPE"]}
body = req.body
r = requests.post(url, data=body, headers=headers, stream=True)
else:
r = requests.get(url, params=req.query_string, stream=True)
# log.info('Request: {req}'.format(req=r.request.url))
# log.info('Request Headers: {h}'.format(h=r.request.headers))
cl = r.headers.get("content-length")
if cl and int(cl) > MAX_FILE_SIZE:
base.abort(
409,
("""Content is too large to be proxied. Allowed
file size: {allowed}, Content-Length: {actual}. Url: """ +
url).format(allowed=MAX_FILE_SIZE, actual=cl),
)
if toolkit.check_ckan_version("2.9"):
from flask import make_response
response = make_response()
else:
response = base.response
response.content_type = r.headers["content-type"]
response.charset = r.encoding
length = 0
for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
if toolkit.check_ckan_version("2.9"):
response.data += chunk
else:
response.body_file.write(chunk)
length += len(chunk)
if length >= MAX_FILE_SIZE:
base.abort(
409,
("""Content is too large to be proxied. Allowed
file size: {allowed}, Content-Length: {actual}. Url: """ +
url).format(allowed=MAX_FILE_SIZE, actual=length),
)
except requests.exceptions.HTTPError as error:
details = "Could not proxy resource. Server responded with %s %s" % (
error.response.status_code,
error.response.reason,
)
base.abort(409, detail=details)
except requests.exceptions.ConnectionError as error:
details = ("""Could not proxy resource because a
connection error occurred. %s""" % error)
base.abort(502, detail=details)
except requests.exceptions.Timeout as error:
details = "Could not proxy resource because the connection timed out."
base.abort(504, detail=details)
return response
