def update_config(self, config): tk.add_template_directory(config, '../templates') tk.add_public_directory(config, '../public') tk.add_resource('../fanstatic', 'showcase') if tk.check_ckan_version(min_version='2.4', max_version='2.9.0'): tk.add_ckan_admin_tab(config, 'showcase_admins', 'Showcase Config') elif tk.check_ckan_version(min_version='2.9.0'): tk.add_ckan_admin_tab(config, 'showcase_blueprint.admins', 'Showcase Config') if tk.check_ckan_version(min_version='2.9.0'): mappings = config.get('ckan.legacy_route_mappings', {}) if isinstance(mappings, string_types): mappings = json.loads(mappings) bp_routes = [ 'index', 'new', 'delete', 'read', 'edit', 'manage_datasets', 'dataset_showcase_list', 'admins', 'admin_remove' ] mappings.update({ 'showcase_' + route: 'showcase_blueprint.' + route for route in bp_routes }) # https://github.com/ckan/ckan/pull/4521 config['ckan.legacy_route_mappings'] = json.dumps(mappings)
def delete_view(id): if 'cancel' in tk.request.params: tk.redirect_to('showcase_blueprint.edit' if tk.check_ckan_version( min_version='2.9.0') else 'showcase_edit', id=id) context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj } try: tk.check_access('ckanext_showcase_delete', context, {'id': id}) except tk.NotAuthorized: return tk.abort(401, _('Unauthorized to delete showcase')) if tk.check_ckan_version(min_version='2.9.0'): index_route = 'showcase_blueprint.index' else: index_route = 'showcase_index' try: if tk.request.method == 'POST': tk.get_action('ckanext_showcase_delete')(context, {'id': id}) h.flash_notice(_('Showcase has been deleted.')) return tk.redirect_to(index_route) c.pkg_dict = tk.get_action('package_show')(context, {'id': id}) except tk.NotAuthorized: tk.abort(401, _('Unauthorized to delete showcase')) except tk.ObjectNotFound: tk.abort(404, _('Showcase not found')) return tk.render('showcase/confirm_delete.html', extra_vars={'dataset_type': DATASET_TYPE_NAME})
def test_choice_field_only_accepts_given_choices(self): lc = LocalCKAN() try: lc.action.package_create( type="test-schema", name="fred_choices1", category="rocker" ) except ValidationError as e: if check_ckan_version("2.9"): expected = "Value must be one of {}".format( [ u"bactrian", u"hybrid", u"f2hybrid", u"snowwhite", u"black", ] ) else: expected = ( "Value must be one of: bactrian; hybrid; f2hybrid; " "snowwhite; black (not 'rocker')" ) assert e.error_dict["category"] == [expected] else: raise AssertionError("ValidationError not raised")
def setup_template_variables(self, context, data_dict): import ckanext.resourceproxy.plugin as proxy same_domain = on_same_domain(data_dict) if not data_dict['resource'].get('format'): data_dict['resource']['format'] = \ self._guess_format_from_extension(data_dict['resource']['url']) if self.proxy_enabled and not same_domain: proxy_url = proxy.get_proxified_resource_url(data_dict) proxy_service_url = get_proxified_service_url(data_dict) else: proxy_url = data_dict['resource']['url'] proxy_service_url = data_dict['resource']['url'] gapi_key = toolkit.config.get('ckanext.geoview.gapi_key') if not toolkit.check_ckan_version(min_version='2.3'): toolkit.c.resource['proxy_url'] = proxy_url toolkit.c.resource['proxy_service_url'] = proxy_service_url toolkit.c.resource['gapi_key'] = gapi_key return { 'resource_view_json': 'resource_view' in data_dict and json.dumps(data_dict['resource_view']), 'proxy_service_url': proxy_service_url, 'proxy_url': proxy_url, 'gapi_key': gapi_key, 'basemapsConfig': self.basemapsConfig }
def _post(app, url, data, resource_id='', upload=None): args = [] env = _get_extra_env_as_sysadmin() # from the form data['id'] = resource_id data['save'] = '' if check_ckan_version('2.9'): if upload: for entry in upload: data[entry[0]] = (io.BytesIO(entry[2]), entry[1]) kwargs = { 'url': url, 'data': data, 'extra_environ': env } else: args.append(url) kwargs = { 'params': data, 'extra_environ': env, 'upload_files': upload } return app.post(*args, **kwargs)
def setup_template_variables(self, context, data_dict): import ckanext.resourceproxy.plugin as proxy same_domain = on_same_domain(data_dict) if not data_dict['resource'].get('format'): data_dict['resource']['format'] = \ self._guess_format_from_extension(data_dict['resource']['url']) if self.proxy_enabled and not same_domain: proxy_url = proxy.get_proxified_resource_url(data_dict) proxy_service_url = get_proxified_service_url(data_dict) else: proxy_url = data_dict['resource']['url'] proxy_service_url = data_dict['resource']['url'] gapi_key = toolkit.config.get('ckanext.geoview.gapi_key') if not toolkit.check_ckan_version(min_version='2.3'): toolkit.c.resource['proxy_url'] = proxy_url toolkit.c.resource['proxy_service_url'] = proxy_service_url toolkit.c.resource['gapi_key'] = gapi_key return {'resource_view_json': 'resource_view' in data_dict and json.dumps(data_dict['resource_view']), 'proxy_service_url': proxy_service_url, 'proxy_url': proxy_url, 'gapi_key': gapi_key, 'basemapsConfig' : self.basemapsConfig}
def _search_url(params, name): if tk.check_ckan_version(min_version='2.9.0'): manage_route = 'showcase_blueprint.manage_datasets' else: manage_route = 'showcase_manage_datasets' url = h.url_for(manage_route, id=name) return url_with_params(url, params)
def user_confirm(msg): if check_ckan_version(min_version='2.9'): import click return click.confirm(msg) else: from ckan.lib.cli import query_yes_no return query_yes_no(msg) == 'yes'
def populate_revision(resource): if 'revision_timestamp' in resource \ or toolkit.check_ckan_version(min_version='2.9'): return current_revision = latest_revision(resource['id']) if current_revision is not None: resource['revision_timestamp'] = current_revision.revision_timestamp
class GeoViewBase(p.SingletonPlugin): '''This base class is for view extensions. ''' if toolkit.check_ckan_version(min_version='2.3'): p.implements(p.IResourceView, inherit=True) else: p.implements(p.IResourcePreview, inherit=True) p.implements(p.IConfigurer, inherit=True) p.implements(p.IConfigurable, inherit=True) proxy_enabled = False same_domain = False def configure(self, config): basemapConfigFile = toolkit.config.get('ckanext.geoview.basemaps', None) self.basemapsConfig = basemapConfigFile and load_basemaps( basemapConfigFile) def update_config(self, config): toolkit.add_public_directory(config, 'public') toolkit.add_template_directory(config, 'templates') toolkit.add_resource('public', 'ckanext-geoview') self.proxy_enabled = 'resource_proxy' in toolkit.config.get( 'ckan.plugins', '')
def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='dataset' if tk.check_ckan_version('2.9') else 'package', action='search')
def remove_showcase_admin(): ''' Remove a user from the Showcase Admin list. ''' context = { 'model': model, 'session': model.Session, 'user': c.user or c.author } try: tk.check_access('sysadmin', context, {}) except tk.NotAuthorized: return tk.abort(401, _('User not authorized to view page')) form_data = tk.request.form if tk.check_ckan_version( '2.9') else tk.request.params if tk.check_ckan_version(min_version='2.9.0'): admins_route = 'showcase_blueprint.admins' else: admins_route = 'showcase_admins' if 'cancel' in form_data: return tk.redirect_to(admins_route) user_id = tk.request.params['user'] if tk.request.method == 'POST' and user_id: user_id = tk.request.params['user'] try: tk.get_action('ckanext_showcase_admin_remove')(data_dict={ 'username': user_id }) except tk.NotAuthorized: return tk.abort(401, _('Unauthorized to perform that action')) except tk.ObjectNotFound: h.flash_error(_('The user is not a Showcase Admin')) else: h.flash_success(_('The user is no longer a Showcase Admin')) return tk.redirect_to(h.url_for(admins_route)) c.user_dict = tk.get_action('user_show')(data_dict={'id': user_id}) c.user_id = user_id return tk.render('admin/confirm_remove_showcase_admin.html')
def configure(self, config): self.search_backend = config.get('ckanext.spatial.search_backend', 'postgis') if self.search_backend != 'postgis' and not tk.check_ckan_version( '2.0.1'): msg = 'The Solr backends for the spatial search require CKAN 2.0.1 or higher. ' + \ 'Please upgrade CKAN or select the \'postgis\' backend.' raise tk.CkanVersionException(msg)
def test_group_form_includes_custom_field(self): if not ckantoolkit.check_ckan_version(min_version='2.8.0'): raise SkipTest app = self._get_test_app() env, response = _get_group_new_page_as_sysadmin(app, type='theme') form = response.forms[1] assert_true('status' in form.fields)
def test_group_form_includes_custom_field(self): if not ckantoolkit.check_ckan_version(min_version='2.7.0'): raise SkipTest app = self._get_test_app() env, response = _get_group_new_page_as_sysadmin(app) form = response.forms[1] # FIXME: add an id to this form assert_true('bookface' in form.fields)
def enqueue_validation_job(package_id, resource_id): enqueue_args = { 'fn': run_validation_job, 'title': "run_validation_job: package_id: {} resource: {}".format(package_id, resource_id), 'kwargs': {'resource': resource_id}, } if t.check_ckan_version('2.8'): ttl = 24 * 60 * 60 # 24 hour ttl. rq_kwargs = { 'ttl': ttl } if t.check_ckan_version('2.9'): rq_kwargs['failure_ttl'] = ttl enqueue_args['rq_kwargs'] = rq_kwargs # Optional variable, if not set, default queue is used queue = t.config.get('ckanext.validation.queue', None) if queue: enqueue_args['queue'] = queue t.enqueue_job(**enqueue_args)
def _get_resource_new_page_as_sysadmin(app, id): user = Sysadmin() env = {"REMOTE_USER": user["name"].encode("ascii")} if ckantoolkit.check_ckan_version(min_version="2.9"): url = '/dataset/{}/resource/new'.format(id) else: url = '/dataset/new_resource/{}'.format(id) response = app.get(url, extra_environ=env) return env, response
def manage_showcase_admins(): context = { 'model': model, 'session': model.Session, 'user': c.user or c.author } try: tk.check_access('sysadmin', context, {}) except tk.NotAuthorized: return tk.abort(401, _('User not authorized to view page')) form_data = tk.request.form if tk.check_ckan_version( '2.9') else tk.request.params if tk.check_ckan_version(min_version='2.9.0'): admins_route = 'showcase_blueprint.admins' else: admins_route = 'showcase_admins' # We're trying to add a user to the showcase admins list. if tk.request.method == 'POST' and form_data['username']: username = form_data['username'] try: tk.get_action('ckanext_showcase_admin_add')(data_dict={ 'username': username }) except tk.NotAuthorized: abort(401, _('Unauthorized to perform that action')) except tk.ObjectNotFound: h.flash_error( _("User '{user_name}' not found.").format(user_name=username)) except tk.ValidationError as e: h.flash_notice(e.error_summary) else: h.flash_success(_("The user is now a Showcase Admin")) return tk.redirect_to(h.url_for(admins_route)) c.showcase_admins = tk.get_action('ckanext_showcase_admin_list')() return tk.render('admin/manage_showcase_admins.html')
def test_org_form_includes_custom_field(self): if not ckantoolkit.check_ckan_version(min_version='2.8.0'): raise SkipTest app = self._get_test_app() env, response = _get_organization_new_page_as_sysadmin( app, type='publisher') form = response.forms[1] assert_true('address' in form.fields)
class GoogleAnalyticsPlugin(p.SingletonPlugin): p.implements(p.IConfigurable, inherit=True) if check_ckan_version('2.8'): p.implements(p.IBlueprint) # workaround for https://github.com/ckan/ckan/issues/6678 import ckan.views.api as core_api else: p.implements(p.IRoutes, inherit=True) analytics_queue = Queue.Queue() capture_api_actions = {} google_analytics_id = None def configure(self, config): '''Load config settings for this extension from config file. See IConfigurable. ''' # Load capture_api_actions from JSON file here = path.abspath(path.dirname(__file__)) with open(path.join(here, 'capture_api_actions.json')) as json_file: GoogleAnalyticsPlugin.capture_api_actions = json.load(json_file) # Get google_analytics_id from config file GoogleAnalyticsPlugin.google_analytics_id = config.get('ckan.data_qld_googleanalytics.id') # spawn a pool of 5 threads, and pass them queue instance for i in range(5): t = AnalyticsPostThread(self.analytics_queue) t.setDaemon(True) t.start() # IRoutes def before_map(self, map): '''Add new routes that this extension's controllers handle. ''' from routes.mapper import SubMapper # /api ver 3 or none with SubMapper(map, controller='ckanext.data_qld.google_analytics.controller:GoogleAnalyticsApiController', path_prefix='/api{ver:/3|}', ver='/3') as m: m.connect('/action/{api_action}', action='action', conditions={'method': ['GET', 'POST']}) return map # IBlueprint def get_blueprint(self): import blueprints return [blueprints.blueprint]
class TestCustomGroupFormNew(object): @pytest.mark.skipif( not ckantoolkit.check_ckan_version(min_version="2.8.0"), reason="Unspecified") def test_group_form_includes_custom_field(self, app): env, response = _get_group_new_page_as_sysadmin(app, type="theme") form = BeautifulSoup(response.body).select("form")[1] assert form.select("input[name=status]") def test_group_form_slug_uses_custom_type(self, app): env, response = _get_group_new_page_as_sysadmin(app, type="theme") assert "/theme/" in response.body
def test_resource_form_includes_custom_fields(self, app, sysadmin_env): dataset = Dataset(type="test-schema", name="resource-includes-custom") if ckantoolkit.check_ckan_version(min_version="2.9"): url = '/dataset/{}/resource/new'.format(dataset["id"]) else: url = '/dataset/new_resource/{}'.format(dataset["id"]) response = app.get( url, extra_environ=sysadmin_env, ) form = BeautifulSoup(response.body).select_one("#resource-edit") assert form.select("input[name=camels_in_photo]")
class TestCustomOrgFormNew(object): @pytest.mark.skipif( not ckantoolkit.check_ckan_version(min_version="2.8.0"), reason="Unspecified") def test_org_form_includes_custom_field(self, app): env, response = _get_organization_new_page_as_sysadmin( app, type="publisher") form = BeautifulSoup(response.body).select("form")[1] assert form.select("input[name=address]") def test_org_form_slug_uses_custom_type(self, app): env, response = _get_organization_new_page_as_sysadmin( app, type="publisher") assert "/publisher/" in response.body
def get_validation_badge(resource, in_listing=False): if in_listing and not asbool( config.get('ckanext.validation.show_badges_in_listings', True)): return '' if not resource.get('validation_status'): return '' statuses = { 'success': _('success'), 'failure': _('failure'), 'invalid': _('invalid'), 'error': _('error'), 'unknown': _('unknown'), } if resource['validation_status'] in ['success', 'failure', 'error']: status = resource['validation_status'] if status == 'failure': status = 'invalid' else: status = 'unknown' if check_ckan_version(min_version='2.9.0'): action = 'validation.read' else: action = 'validation_read' validation_url = url_for( action, id=resource['package_id'], resource_id=resource['id']) return u''' <a href="{validation_url}" class="validation-badge" title="{title}"> <span class="prefix">{prefix}</span><span class="status {status}">{status_title}</span> </a>'''.format( validation_url=validation_url, prefix=_('data'), status=status, status_title=statuses[status], title=resource.get('validation_timestamp', ''))
class TestGroupFormNew(object): @pytest.mark.skipif( not ckantoolkit.check_ckan_version(min_version="2.7.0"), reason="Unspecified") def test_group_form_includes_custom_field(self, app): env, response = _get_group_new_page_as_sysadmin(app) # FIXME: add an id to this form form = BeautifulSoup(response.body).select("form")[1] assert form.select("input[name=bookface]") def test_group_form_slug_says_group(self, app): """The default prefix shouldn't be /packages?id=""" env, response = _get_group_new_page_as_sysadmin(app) # Commenting until ckan/ckan#4208 is fixed # assert_true('packages?id=' not in response.body) assert "/group/" in response.body
def test_widget_loaded(app): dataset = factories.Dataset() for i in range(1, 11): resource = factories.Resource( name='Resource {}'.format(i), url='https://example.com', package_id=dataset['id'], ) if check_ckan_version(min_version='2.9.0'): url = url_for('dataset.read', id=dataset['id']) else: url = url_for(controller='package', action='read', id=dataset['id']) res = app.get(url) assert 'resources-list-filter-container' in res.get_data(as_text=True) assert 'script type="text/template" id="resources-list-template"' in res.get_data(as_text=True)
def can_preview(self, data_dict): format_lower = data_dict['resource']['format'].lower() correct_format = format_lower in self.WMTS can_preview_from_domain = (self.proxy_enabled or data_dict['resource'].get('on_same_domain')) quality = 2 if toolkit.check_ckan_version('2.1'): if correct_format: if can_preview_from_domain: return {'can_preview': True, 'quality': quality} else: return {'can_preview': False, 'fixable': 'Enable resource_proxy', 'quality': quality} else: return {'can_preview': False, 'quality': quality} return correct_format and can_preview_from_domain
def can_preview(self, data_dict): format_lower = data_dict['resource']['format'].lower() correct_format = format_lower in self.WMTS can_preview_from_domain = (self.proxy_enabled or data_dict['resource'].get('on_same_domain')) quality = 2 if toolkit.check_ckan_version('2.1'): if correct_format: if can_preview_from_domain: return {'can_preview': True, 'quality': quality} else: return { 'can_preview': False, 'fixable': 'Enable resource_proxy', 'quality': quality } else: return {'can_preview': False, 'quality': quality} return correct_format and can_preview_from_domain
def check_geoalchemy_requirement(): '''Checks if a suitable geoalchemy version installed Checks if geoalchemy2 is present when using CKAN >= 2.3, and raises an ImportError otherwise so users can upgrade manually. ''' msg = ('This version of ckanext-spatial requires {0}. ' + 'Please install it by running `pip install {0}`.\n' + 'For more details see the "Troubleshooting" section of the ' + 'install documentation') if tk.check_ckan_version(min_version='2.3'): try: import geoalchemy2 except ImportError: raise ImportError(msg.format('geoalchemy2')) else: try: import geoalchemy except ImportError: raise ImportError(msg.format('geoalchemy'))
import os import cgi import logging import datetime import mimetypes import boto3 import botocore import ckantoolkit as toolkit import ckan.model as model import ckan.lib.munge as munge if toolkit.check_ckan_version(min_version='2.7.0'): from werkzeug.datastructures import FileStorage as FlaskFileStorage ALLOWED_UPLOAD_TYPES = (cgi.FieldStorage, FlaskFileStorage) else: ALLOWED_UPLOAD_TYPES = (cgi.FieldStorage) config = toolkit.config log = logging.getLogger(__name__) _storage_path = None _max_resource_size = None _max_image_size = None def _get_underlying_file(wrapper): if isinstance(wrapper, FlaskFileStorage): return wrapper.stream return wrapper.file
import os import cgi import logging import datetime import mimetypes import boto3 import botocore import ckantoolkit as toolkit import ckan.model as model import ckan.lib.munge as munge if toolkit.check_ckan_version(min_version='2.7.0'): from werkzeug.datastructures import FileStorage as FlaskFileStorage ALLOWED_UPLOAD_TYPES = (cgi.FieldStorage, FlaskFileStorage) else: ALLOWED_UPLOAD_TYPES = (cgi.FieldStorage) config = toolkit.config log = logging.getLogger(__name__) _storage_path = None _max_resource_size = None _max_image_size = None def _get_underlying_file(wrapper): if isinstance(wrapper, FlaskFileStorage): return wrapper.stream
import os import re import mimetypes from logging import getLogger import six import ckantoolkit as tk from ckan import plugins as p from ckan.lib.helpers import json if tk.check_ckan_version(min_version="2.9.0"): from ckanext.spatial.plugin.flask_plugin import (SpatialQueryMixin, HarvestMetadataApiMixin) else: from ckanext.spatial.plugin.pylons_plugin import (SpatialQueryMixin, HarvestMetadataApiMixin) config = tk.config def check_geoalchemy_requirement(): '''Checks if a suitable geoalchemy version installed Checks if geoalchemy2 is present when using CKAN >= 2.3, and raises an ImportError otherwise so users can upgrade manually. ''' msg = ('This version of ckanext-spatial requires {0}. ' + 'Please install it by running `pip install {0}`.\n' +
def proxy_service_url(req, url): parts = urlsplit(url) if not parts.scheme or not parts.netloc: base.abort(409, detail="Invalid URL.") try: method = req.environ["REQUEST_METHOD"] params = parse_qs(parts.query) if not p.toolkit.asbool( base.config.get("ckanext.geoview.forward_ogc_request_params", "False")): # remove query parameters that may conflict with OGC protocols for key in dict(params): if key.lower() in OGC_EXCLUDED_PARAMS: del params[key] parts = parts._replace(query=urlencode(params)) parts = parts._replace(fragment="") # remove potential fragment url = parts.geturl() if method == "POST": length = int(req.environ["CONTENT_LENGTH"]) headers = {"Content-Type": req.environ["CONTENT_TYPE"]} body = req.body r = requests.post(url, data=body, headers=headers, stream=True) else: r = requests.get(url, params=req.query_string, stream=True) # log.info('Request: {req}'.format(req=r.request.url)) # log.info('Request Headers: {h}'.format(h=r.request.headers)) cl = r.headers.get("content-length") if cl and int(cl) > MAX_FILE_SIZE: base.abort( 409, ("""Content is too large to be proxied. Allowed file size: {allowed}, Content-Length: {actual}. Url: """ + url).format(allowed=MAX_FILE_SIZE, actual=cl), ) if toolkit.check_ckan_version("2.9"): from flask import make_response response = make_response() else: response = base.response response.content_type = r.headers["content-type"] response.charset = r.encoding length = 0 for chunk in r.iter_content(chunk_size=CHUNK_SIZE): if toolkit.check_ckan_version("2.9"): response.data += chunk else: response.body_file.write(chunk) length += len(chunk) if length >= MAX_FILE_SIZE: base.abort( 409, ("""Content is too large to be proxied. Allowed file size: {allowed}, Content-Length: {actual}. Url: """ + url).format(allowed=MAX_FILE_SIZE, actual=length), ) except requests.exceptions.HTTPError as error: details = "Could not proxy resource. Server responded with %s %s" % ( error.response.status_code, error.response.reason, ) base.abort(409, detail=details) except requests.exceptions.ConnectionError as error: details = ("""Could not proxy resource because a connection error occurred. %s""" % error) base.abort(502, detail=details) except requests.exceptions.Timeout as error: details = "Could not proxy resource because the connection timed out." base.abort(504, detail=details) return response