def read_dataset_page(_id, _format): if not _format: _format = check_access_header() if not _format: if toolkit.check_ckan_version(max_version='2.8.99'): return read_endpoint(_id) else: return read_endpoint(_get_package_type(_id), _id) _profiles = toolkit.request.params.get('profiles') if _profiles: _profiles = _profiles.split(',') try: response = toolkit.get_action('dcat_dataset_show')( {}, { 'id': _id, 'format': _format, 'profiles': _profiles }) except toolkit.ObjectNotFound: toolkit.abort(404) except (toolkit.ValidationError, RDFProfileException) as e: toolkit.abort(409, str(e)) if toolkit.check_ckan_version(max_version='2.8.99'): toolkit.response.headers.update( {'Content-type': CONTENT_TYPES[_format]}) else: from flask import make_response response = make_response(response) response.headers['Content-type'] = CONTENT_TYPES[_format] return response
def update_config(self, config): tk.add_template_directory(config, '../templates') tk.add_public_directory(config, '../public') tk.add_resource('../fanstatic', 'showcase') if tk.check_ckan_version(min_version='2.4', max_version='2.9.0'): tk.add_ckan_admin_tab(config, 'showcase_admins', 'Showcase Config') elif tk.check_ckan_version(min_version='2.9.0'): tk.add_ckan_admin_tab(config, 'showcase_blueprint.admins', 'Showcase Config') if tk.check_ckan_version(min_version='2.9.0'): mappings = config.get('ckan.legacy_route_mappings', {}) if isinstance(mappings, string_types): mappings = json.loads(mappings) bp_routes = [ 'index', 'new', 'delete', 'read', 'edit', 'manage_datasets', 'dataset_showcase_list', 'admins', 'admin_remove' ] mappings.update({ 'showcase_' + route: 'showcase_blueprint.' + route for route in bp_routes }) # https://github.com/ckan/ckan/pull/4521 config['ckan.legacy_route_mappings'] = json.dumps(mappings)
def test_spatial_extra_bad_geojson(self, app): user = factories.User() env = {"REMOTE_USER": user["name"].encode("ascii")} dataset = factories.Dataset(user=user) if tk.check_ckan_version(min_version="2.9"): offset = url_for("dataset.edit", id=dataset["id"]) else: offset = url_for(controller="package", action="edit", id=dataset["id"]) res = app.get(offset, extra_environ=env) if tk.check_ckan_version(min_version="2.9"): data = { "name": dataset['name'], "extras__0__key": u"spatial", "extras__0__value": u'{"Type":"Bad_GeoJSON","a":2}' } res = app.post(offset, environ_overrides=env, data=data) else: form = res.forms[1] form['extras__0__key'] = u'spatial' form['extras__0__value'] = u'{"Type":"Bad_GeoJSON","a":2}' res = helpers.webtest_submit(form, extra_environ=env, name='save') assert "Error" in res, res assert "Spatial" in res assert "Error creating geometry" in res
def update_config(self, ckan_config): toolkit.add_template_directory(ckan_config, '../templates') toolkit.add_public_directory(ckan_config, '../assets') toolkit.add_resource('../assets', 'opengov_custom_css_resource') if toolkit.check_ckan_version(min_version='2.4', max_version='2.9'): toolkit.add_ckan_admin_tab(ckan_config, 'custom_css', 'Custom CSS') elif toolkit.check_ckan_version(min_version='2.9'): toolkit.add_ckan_admin_tab(ckan_config, 'custom-css.custom_css', 'Custom CSS')
def update_config(self, ckan_config): toolkit.add_template_directory(ckan_config, '../templates') if toolkit.check_ckan_version(min_version='2.4', max_version='2.9'): toolkit.add_ckan_admin_tab(ckan_config, 'custom_home_page', 'Home Page Layout') elif toolkit.check_ckan_version(min_version='2.9'): toolkit.add_ckan_admin_tab(ckan_config, 'custom-homepage.custom_home_page', 'Home Page Layout')
def update_config(self, ckan_config): toolkit.add_template_directory(ckan_config, '../templates') if toolkit.check_ckan_version(min_version='2.4', max_version='2.9'): toolkit.add_ckan_admin_tab(ckan_config, 'custom_header', 'Custom Header') elif toolkit.check_ckan_version(min_version='2.9'): toolkit.add_ckan_admin_tab(ckan_config, 'custom-header.custom_header', 'Custom Header')
def test_core_ckan_envvar_values_in_config(self): if not toolkit.check_ckan_version('2.4.0'): raise SkipTest('CKAN version 2.4 or above needed') core_ckan_env_var_list = [ ('CKAN_SQLALCHEMY_URL', 'postgresql://mynewsqlurl/'), ('CKAN_DATASTORE_WRITE_URL', 'http://mynewdbwriteurl/'), ('CKAN_DATASTORE_READ_URL', 'http://mynewdbreadurl/'), ('CKAN_SITE_ID', 'my-site'), ('CKAN_DB', 'postgresql://mydeprectatesqlurl/'), ('CKAN_SMTP_SERVER', 'mail.example.com'), ('CKAN_SMTP_STARTTLS', 'True'), ('CKAN_SMTP_USER', 'my_user'), ('CKAN_SMTP_PASSWORD', 'password'), ('CKAN_SMTP_MAIL_FROM', '*****@*****.**') ] self._setup_env_vars(core_ckan_env_var_list) assert_equal(config['sqlalchemy.url'], 'postgresql://mynewsqlurl/') assert_equal(config['ckan.datastore.write_url'], 'http://mynewdbwriteurl/') assert_equal(config['ckan.datastore.read_url'], 'http://mynewdbreadurl/') assert_equal(config['ckan.site_id'], 'my-site') assert_equal(config['smtp.server'], 'mail.example.com') assert_equal(config['smtp.starttls'], 'True') assert_equal(config['smtp.user'], 'my_user') assert_equal(config['smtp.password'], 'password') assert_equal(config['smtp.mail_from'], '*****@*****.**') self._teardown_env_vars(core_ckan_env_var_list)
class StatisticsPlugin(plugins.SingletonPlugin, DefaultTranslation): plugins.implements(plugins.IConfigurer) plugins.implements(plugins.IRoutes, inherit=True) if toolkit.check_ckan_version(min_version='2.5.0'): plugins.implements(plugins.ITranslation, inherit=True) plugins.implements(plugins.IActions) # IConfigurer def update_config(self, config_): toolkit.add_template_directory(config_, 'templates') toolkit.add_resource('fanstatic', 'statistics') # IRoutes def before_map(self, map): map.connect( '/statistics', controller='ckanext.statistics.controller:StatisticsController', action='statistics_read') map.connect( '/statistics/internal', controller='ckanext.statistics.controller:StatisticsController', action='reports_read') return map # IActions def get_actions(self): return {'get_all_public_datasets': get_all_public_datasets}
def harvest_source_index_clear(context, data_dict): ''' Clears all datasets, jobs and objects related to a harvest source, but keeps the source itself. This is useful to clean history of long running harvest sources to start again fresh. :param id: the id of the harvest source to clear :type id: string ''' check_access('harvest_source_clear', context, data_dict) harvest_source_id = data_dict.get('id') source = HarvestSource.get(harvest_source_id) if not source: log.error('Harvest source %s does not exist', harvest_source_id) raise NotFound('Harvest source %s does not exist' % harvest_source_id) harvest_source_id = source.id conn = make_connection() query = ''' +%s:"%s" +site_id:"%s" ''' % ( 'harvest_source_id', harvest_source_id, config.get('ckan.site_id')) solr_commit = toolkit.asbool(config.get('ckan.search.solr_commit', 'true')) if toolkit.check_ckan_version(max_version='2.5.99'): # conn is solrpy try: conn.delete_query(query) if solr_commit: conn.commit() except Exception, e: log.exception(e) raise SearchIndexError(e) finally:
def update_config(self, config): tk.add_template_directory(config, 'templates') tk.add_public_directory(config, 'public') # If ckan is more than 2.3, use the 2.4+ toolkit method if not tk.check_ckan_version(max_version='2.3'): tk.add_ckan_admin_tab(config, 'ckanext_showcase_admins', 'Showcase Config')
def test_distribution_format_format_normalized(self): g = Graph() dataset1 = URIRef("http://example.org/datasets/1") g.add((dataset1, RDF.type, DCAT.Dataset)) distribution1_1 = URIRef("http://example.org/datasets/1/ds/1") g.add((distribution1_1, RDF.type, DCAT.Distribution)) g.add((distribution1_1, DCAT.mediaType, Literal('text/csv'))) g.add((distribution1_1, DCT['format'], Literal('Comma Separated Values'))) g.add((dataset1, DCAT.distribution, distribution1_1)) p = RDFParser(profiles=['euro_dcat_ap']) p.g = g datasets = [d for d in p.datasets()] resource = datasets[0]['resources'][0] if toolkit.check_ckan_version(min_version='2.3'): assert resource['format'] == u'CSV' assert resource['mimetype'] == u'text/csv' else: assert resource['format'] == u'Comma Separated Values'
def update_config(self, config): tk.add_template_directory(config, 'templates') tk.add_public_directory(config, 'public') tk.add_resource('fanstatic', 'showcase') if tk.check_ckan_version(min_version='2.4'): tk.add_ckan_admin_tab(config, 'ckanext_showcase_admins', 'Showcase Config')
def update_config(self, config): tk.add_template_directory(config, 'templates') tk.add_public_directory(config, 'public') tk.add_resource('fanstatic', 'project') # If ckan is more than 2.3, use the 2.4+ toolkit method if not tk.check_ckan_version(max_version='2.3'): tk.add_ckan_admin_tab(config, 'ckanext_project_admins', 'project Config')
def test_distribution_format_format_normalized(self): g = Graph() dataset1 = URIRef("http://example.org/datasets/1") g.add((dataset1, RDF.type, DCAT.Dataset)) distribution1_1 = URIRef("http://example.org/datasets/1/ds/1") g.add((distribution1_1, RDF.type, DCAT.Distribution)) g.add((distribution1_1, DCAT.mediaType, Literal('text/csv'))) g.add((distribution1_1, DCT['format'], Literal('Comma Separated Values'))) g.add((dataset1, DCAT.distribution, distribution1_1)) p = RDFParser(profiles=['euro_dcat_ap']) p.g = g datasets = [d for d in p.datasets()] resource = datasets[0]['resources'][0] if toolkit.check_ckan_version(min_version='2.3'): eq_(resource['format'], u'CSV') eq_(resource['mimetype'], u'text/csv') else: eq_(resource['format'], u'Comma Separated Values')
def test_user_show_no_additional_filters(self): ''' Perform package_search with no additional filters should not include showcases. ''' if not toolkit.check_ckan_version(min_version='2.4'): raise SkipTest( 'Filtering out showcases requires CKAN 2.4+ (ckan/ckan/issues/2380)' ) user = factories.User() factories.Dataset(user=user) factories.Dataset(user=user) factories.Dataset(user=user, type='showcase') factories.Dataset(user=user, type='custom') search_results = helpers.call_action('user_show', context={}, include_datasets=True, id=user['name'])['datasets'] types = [result['type'] for result in search_results] nosetools.assert_equal(len(search_results), 3) nosetools.assert_true('showcase' not in types) nosetools.assert_true('custom' in types)
def test_unicode(self, app): user = factories.Sysadmin() env = {'REMOTE_USER': user['name'].encode('ascii')} page = 'test_html_page_div' page = '/' + page if not ckan_29_or_higher else page response = app.post( url=toolkit.url_for('pages_edit', page=page), params={ 'title': u'Tïtlé'.encode('utf-8'), 'name': 'page_unicode', 'content': u'Çöñtéñt'.encode('utf-8'), 'order': 1, 'private': False, }, extra_environ=env, ) if not ckan_29_or_higher: response = response.follow(extra_environ=env) if toolkit.check_ckan_version(min_version='2.9.0'): assert u'<p>Çöñtéñt</p>' in response.get_data(as_text=True) assert u'<title>Tïtlé - CKAN</title>' in response.get_data(as_text=True) assert u'<a href="/pages/page_unicode">Tïtlé</a>' in response.get_data(as_text=True) assert u'<h1 class="page-heading">Tïtlé</h1>' in response.get_data(as_text=True) else: assert u'<title>Tïtlé - CKAN</title>' in response.unicode_body assert u'<a href="/pages/page_unicode">Tïtlé</a>' in response.unicode_body assert u'<h1 class="page-heading">Tïtlé</h1>' in response.unicode_body try: assert u'<p>Çöñtéñt</p>' in response.unicode_body except AssertionError: assert u'<p>Çöñtéñt</p>' in response.unicode_body
def read_catalog_page(_format): if not _format: _format = check_access_header() if not _format: return index_endpoint() _profiles = toolkit.request.params.get('profiles') if _profiles: _profiles = _profiles.split(',') data_dict = { 'page': toolkit.request.params.get('page'), 'modified_since': toolkit.request.params.get('modified_since'), 'q': toolkit.request.params.get('q'), 'fq': toolkit.request.params.get('fq'), 'format': _format, 'profiles': _profiles, } try: response = toolkit.get_action('dcat_catalog_show')({}, data_dict) except (toolkit.ValidationError, RDFProfileException) as e: toolkit.abort(409, str(e)) if toolkit.check_ckan_version(max_version='2.8.99'): toolkit.response.headers.update( {'Content-type': CONTENT_TYPES[_format]}) else: from flask import make_response response = make_response(response) response.headers['Content-type'] = CONTENT_TYPES[_format] return response
def read_dataset(self, _id, _format=None): if not _format: _format = check_access_header() if not _format: if toolkit.check_ckan_version(max_version='2.8.99'): return read_endpoint(_id) else: return read_endpoint(_get_package_type(_id), _id) _profiles = toolkit.request.params.get('profiles') if _profiles: _profiles = _profiles.split(',') toolkit.response.headers.update( {'Content-type': CONTENT_TYPES[_format]}) try: result = toolkit.get_action('dcat_dataset_show')({}, {'id': _id, 'format': _format, 'profiles': _profiles}) except toolkit.ObjectNotFound: toolkit.abort(404) except (toolkit.ValidationError, RDFProfileException) as e: toolkit.abort(409, str(e)) return result
def populate_revision(resource): if 'revision_timestamp' in resource \ or toolkit.check_ckan_version(min_version='2.9'): return current_revision = latest_revision(resource['id']) if current_revision is not None: resource['revision_timestamp'] = current_revision.revision_timestamp
def download(id, resource_id, filename=None): """Download resource blueprint This calls all registered download handlers in order, until a response is returned to the user """ context = get_context() resource = None try: resource = toolkit.get_action('resource_show')(context, { 'id': resource_id }) if id != resource['package_id']: return toolkit.abort( 404, toolkit._('Resource not found belonging to package')) package = toolkit.get_action('package_show')(context, {'id': id}) except toolkit.ObjectNotFound: return toolkit.abort(404, toolkit._('Resource not found')) except toolkit.NotAuthorized: return toolkit.abort( 401, toolkit._('Not authorized to read resource {0}'.format(id))) activity_id = request.args.get('activity_id') inline = toolkit.asbool(request.args.get('preview')) if activity_id and toolkit.check_ckan_version(min_version='2.9'): try: activity = toolkit.get_action(u'activity_show')( context, { u'id': activity_id, u'include_data': True }) activity_dataset = activity['data']['package'] assert activity_dataset['id'] == id activity_resources = activity_dataset['resources'] for r in activity_resources: if r['id'] == resource_id: resource = r package = activity_dataset break except toolkit.NotFound: toolkit.abort(404, toolkit._(u'Activity not found')) try: resource = call_pre_download_handlers(resource, package, activity_id=activity_id) return call_download_handlers(resource, package, filename, inline, activity_id=activity_id) except toolkit.ObjectNotFound: return toolkit.abort(404, toolkit._('Resource not found')) except toolkit.NotAuthorized: return toolkit.abort( 401, toolkit._( 'Not authorized to read resource {0}'.format(resource_id)))
def test_distribution_format_format_normalized(self): g = Graph() dataset1 = URIRef("http://example.org/datasets/1") g.add((dataset1, RDF.type, DCAT.Dataset)) distribution1_1 = URIRef("http://example.org/datasets/1/ds/1") g.add((distribution1_1, RDF.type, DCAT.Distribution)) g.add((distribution1_1, DCAT.mediaType, Literal("text/csv"))) g.add((distribution1_1, DCT["format"], Literal("Comma Separated Values"))) g.add((dataset1, DCAT.distribution, distribution1_1)) p = RDFParser(profiles=["euro_dcat_ap"]) p.g = g datasets = [d for d in p.datasets()] resource = datasets[0]["resources"][0] if toolkit.check_ckan_version(min_version="2.3"): eq_(resource["format"], u"CSV") eq_(resource["mimetype"], u"text/csv") else: eq_(resource["format"], u"Comma Separated Values")
def test_dataset_showcase_page_add_to_showcase_dropdown_submit(self, app): """ Submitting 'Add to showcase' form with selected showcase value creates a sc/pkg association. """ sysadmin = factories.Sysadmin() dataset = factories.Dataset(name="my-dataset") showcase_one = factories.Dataset(name="my-first-showcase", type="showcase") factories.Dataset(name="my-second-showcase", type="showcase") factories.Dataset(name="my-third-showcase", type="showcase") assert model.Session.query(ShowcasePackageAssociation).count() == 0 if tk.check_ckan_version("2.9"): pytest.skip("submit_and_follow not supported") env = {"REMOTE_USER": sysadmin["name"].encode("ascii")} response = app.get( url=url_for("showcase_dataset_showcase_list", id=dataset["id"]), extra_environ=env, ) form = response.forms["showcase-add"] form["showcase_added"] = showcase_one["id"] showcase_add_response = helpers.submit_and_follow(app, form, env) # returns to the correct page assert (showcase_add_response.request.path == "/dataset/showcases/my-dataset") # an association is created assert model.Session.query(ShowcasePackageAssociation).count() == 1
def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='dataset' if tk.check_ckan_version('2.9') else 'package', action='search')
def test_showcase_new_redirects_to_manage_datasets(self, app): """Creating a new showcase redirects to the manage datasets form.""" if tk.check_ckan_version("2.9"): pytest.skip("submit_and_follow not supported") sysadmin = factories.Sysadmin() # need a dataset for the 'bulk_action.showcase_add' button to show factories.Dataset() env = {"REMOTE_USER": sysadmin["name"].encode("ascii")} response = app.get( url=url_for("showcase_new"), extra_environ=env, ) # create showcase form = response.forms["dataset-edit"] form["name"] = u"my-showcase" create_response = helpers.submit_and_follow(app, form, env, "save") # Unique to manage_datasets page assert "bulk_action.showcase_add" in create_response # Requested page is the manage_datasets url. assert (url_for("showcase_manage_datasets", id="my-showcase") == create_response.request.path)
def read_dataset(self, _id, _format=None): if not _format: _format = check_access_header() if not _format: if toolkit.check_ckan_version(max_version='2.8.99'): return read_endpoint(_id) else: return read_endpoint(_get_package_type(_id), _id) _profiles = toolkit.request.params.get('profiles') if _profiles: _profiles = _profiles.split(',') toolkit.response.headers.update( {'Content-type': CONTENT_TYPES[_format]}) try: result = toolkit.get_action('dcat_dataset_show')({}, {'id': _id, 'format': _format, 'profiles': _profiles}) except toolkit.ObjectNotFound: toolkit.abort(404) return result
def update_config(self, config_): toolkit.add_template_directory(config_, 'templates') toolkit.add_resource('theme/resources', 'deadoralive') if toolkit.check_ckan_version(max_version='2.2.999'): # Add CKAN version 2.2 support templates. toolkit.add_template_directory(config_, '2.2_templates')
def test_geojson_view_is_rendered(app): view_default_title = GeoJSONView().info()["title"] dataset = factories.Dataset() for format in GeoJSONView.GeoJSON: resource = factories.Resource(name='My Resource', format=format, package_id=dataset['id']) if toolkit.check_ckan_version("2.9"): url = toolkit.url_for( "{}_resource.read".format(dataset["type"]), id=dataset["name"], resource_id=resource["id"], ) else: url = toolkit.url_for( controller="package", action="resource_read", id=resource["package_id"], resource_id=resource["id"], ) res = app.get(url) assert 'class="resource-view"' in res.body assert 'data-title="{}"'.format(view_default_title) in res.body assert 'id="view-' in res.body
def relative_url_for(**kwargs): '''Return the existing URL but amended for the given url_for-style parameters. Much easier than calling h.add_url_param etc. For switching to URLs inside the current controller action only. ''' # Restrict the request params to the same action & controller, to avoid # being an open redirect. disallowed_params = set( ('controller', 'action', 'anchor', 'host', 'protocol', 'qualified')) user_specified_params = [(k, v) for k, v in list(tk.request.params.items()) if k not in disallowed_params] if tk.check_ckan_version(min_version="2.9.0"): from flask import request args = dict( list(request.args.items()) + user_specified_params + list(kwargs.items())) # remove blanks for k, v in list(args.items()): if not v: del args[k] return tk.url_for(request.url_rule.rule, **args) else: args = dict( list(tk.request.environ['pylons.routes_dict'].items()) + user_specified_params + list(kwargs.items())) # remove blanks for k, v in list(args.items()): if not v: del args[k] return tk.url_for(**args)
def test_spatial_extra_base(self, app): user = factories.User() env = {"REMOTE_USER": user["name"].encode("ascii")} dataset = factories.Dataset(user=user) if tk.check_ckan_version(min_version="2.9"): offset = url_for("dataset.edit", id=dataset["id"]) else: offset = url_for(controller="package", action="edit", id=dataset["id"]) res = app.get(offset, extra_environ=env) if tk.check_ckan_version(min_version="2.9"): data = { "name": dataset['name'], "extras__0__key": u"spatial", "extras__0__value": self.geojson_examples["point"] } res = app.post(offset, environ_overrides=env, data=data) else: form = res.forms[1] form['extras__0__key'] = u'spatial' form['extras__0__value'] = self.geojson_examples['point'] res = helpers.submit_and_follow(app, form, env, 'save') assert "Error" not in res, res package_extent = ( Session.query(PackageExtent) .filter(PackageExtent.package_id == dataset["id"]) .first() ) geojson = json.loads(self.geojson_examples["point"]) assert package_extent.package_id == dataset["id"] from sqlalchemy import func assert ( Session.query(func.ST_X(package_extent.the_geom)).first()[0] == geojson["coordinates"][0] ) assert ( Session.query(func.ST_Y(package_extent.the_geom)).first()[0] == geojson["coordinates"][1] ) assert package_extent.the_geom.srid == self.db_srid
def remove_showcase_admin(): ''' Remove a user from the Showcase Admin list. ''' context = { 'model': model, 'session': model.Session, 'user': tk.g.user or tk.g.author } try: tk.check_access('sysadmin', context, {}) except tk.NotAuthorized: return tk.abort(401, _('User not authorized to view page')) form_data = tk.request.form if tk.check_ckan_version( '2.9') else tk.request.params if tk.check_ckan_version(min_version='2.9.0'): admins_route = 'showcase_blueprint.admins' else: admins_route = 'showcase_admins' if 'cancel' in form_data: return tk.redirect_to(admins_route) user_id = tk.request.params['user'] if tk.request.method == 'POST' and user_id: user_id = tk.request.params['user'] try: tk.get_action('ckanext_showcase_admin_remove')({}, { 'username': user_id }) except tk.NotAuthorized: return tk.abort(401, _('Unauthorized to perform that action')) except tk.ObjectNotFound: h.flash_error(_('The user is not a Showcase Admin')) else: h.flash_success(_('The user is no longer a Showcase Admin')) return tk.redirect_to(h.url_for(admins_route)) tk.g.user_dict = tk.get_action('user_show')({}, {'id': user_id}) tk.g.user_id = user_id return tk.render('admin/confirm_remove_showcase_admin.html')
def test_spatial_search_widget(self, app): if tk.check_ckan_version(min_version="2.9"): offset = url_for("dataset.search") else: offset = url_for(controller="package", action="search") res = app.get(offset) assert 'data-module="spatial-query"' in res assert "spatial_query.js" in res
def auth_allow_anonymous_access(auth_function): ''' Local version of the auth_allow_anonymous_access decorator that only calls the actual toolkit decorator if the CKAN version supports it ''' if pt.check_ckan_version(min_version='2.2'): auth_function = pt.auth_allow_anonymous_access(auth_function) return auth_function
def test_check_ckan_version(version, bound, value, expected, monkeypatch): # test name numbers refer to: # * number of numbers in the ckan version # * number of numbers in the checked version # * the index of the number being tested in the checked version monkeypatch.setattr(tk.ckan, u"__version__", version) kwargs = {bound + u"_version": value} assert tk.check_ckan_version(**kwargs) is expected
def test_dataset_count_no_datasets(self): ''' Dataset and showcase count is 0 when no datasets, and no showcases. ''' if not tk.check_ckan_version(min_version='2.5'): raise SkipTest('get_site_statistics without user broken in CKAN 2.4') stats = showcase_helpers.get_site_statistics() nosetools.assert_equal(stats['dataset_count'], 0) nosetools.assert_equal(stats['showcase_count'], 0)
def test_dataset_count_no_datasets(self): ''' Dataset and experience count is 0 when no datasets, and no experiences. ''' if not tk.check_ckan_version(min_version='2.5'): raise SkipTest('get_site_statistics without user broken in CKAN 2.4') stats = experience_helpers.get_site_statistics() nosetools.assert_equal(stats['dataset_count'], 0) nosetools.assert_equal(stats['experience_count'], 0)
class ESDStandardsPlugin(plugins.SingletonPlugin): plugins.implements(plugins.IConfigurer) plugins.implements(plugins.IConfigurable) plugins.implements(plugins.IActions) plugins.implements(plugins.IAuthFunctions) plugins.implements(plugins.ITemplateHelpers) if toolkit.check_ckan_version(min_version='2.3'): plugins.implements(plugins.IValidators) # IConfigurer def update_config(self, config_): toolkit.add_template_directory(config_, 'templates') toolkit.add_public_directory(config_, 'public') toolkit.add_resource('fanstatic', 'esd') # IConfigurable def configure(self, config): model_setup() # IActions def get_actions(self): return { 'esd_function_show': esd_function_show, 'esd_service_show': esd_service_show, 'esd_function_autocomplete': esd_function_autocomplete, 'esd_service_autocomplete': esd_service_autocomplete, } # IAuthFunctions def get_auth_functions(self): return { 'esd_function_show': esd_auth, 'esd_service_show': esd_auth, 'esd_function_autocomplete': esd_auth, 'esd_service_autocomplete': esd_auth, } # ITemplateHelpers def get_helpers(self): return { 'get_esd_function_titles': get_esd_function_titles, 'get_esd_service_titles': get_esd_service_titles, } # IValidators (CKAN >= 2.3) def get_validators(self): return { 'esd_function_validator': esd_function_validator, 'esd_service_validator': esd_service_validator, }
def configure(self, config): '''Store the YouCKAN configuration''' if not toolkit.check_ckan_version('2.1'): log.warn('This extension has only been tested on CKAN 2.1!') self.use_auth = toolkit.asbool(config.get('youckan.use_auth', False)) if self.use_auth: self.logout_url = config['youckan.logout_url'] model_setup()
def create_qa_update_task(resource, queue): if check_ckan_version(max_version='2.2.99'): package = resource.resource_group.package else: package = resource.package compat_enqueue('qa.update', update, queue, kwargs={'resource_id': resource.id}) log.debug('QA of resource put into celery queue %s: %s/%s url=%r', queue, package.name, resource.id, resource.url)
def test_dataset_count_no_datasets_some_showcases(self): ''' Dataset and showcase count is 0 when no datasets, but some showcases. ''' if not tk.check_ckan_version(min_version='2.5'): raise SkipTest('get_site_statistics without user broken in CKAN 2.4') for i in xrange(0, 10): factories.Dataset(type='showcase') stats = showcase_helpers.get_site_statistics() nosetools.assert_equal(stats['dataset_count'], 0) nosetools.assert_equal(stats['showcase_count'], 10)
def get_auth_functions(self): auth_functions = {'package_show': auth.package_show, 'package_update': auth.package_update, # 'resource_show': auth.resource_show, constants.PACKAGE_ACQUIRED: auth.package_acquired, constants.ACQUISITIONS_LIST: auth.acquisitions_list} # resource_show is not required in CKAN 2.3 because it delegates to # package_show if not tk.check_ckan_version(min_version='2.3'): auth_functions['resource_show'] = auth.resource_show return auth_functions
def create(cls, resource_id): c = cls() c.resource_id = resource_id # Find the package_id for the resource. q = model.Session.query(model.Package.id) if toolkit.check_ckan_version(max_version="2.2.99"): q = q.join(model.ResourceGroup) q = q.join(model.Resource).filter_by(id=c.resource_id) result = q.first() if not result or not result[0]: raise Exception("Missing dataset") c.package_id = result[0] return c
def _get_assigned_user(assignee_id, session): context = {'session': session, 'model': cmodel} data_dict = {'id': assignee_id} # we only need the basic properties of the user, not its datasets etc if toolkit.check_ckan_version(min_version='2.3'): # these are the defaults, but just in case... data_dict.update({'include_datasets': False, 'include_num_followers': False}) else: context = {'return_minimal': True} try: return toolkit.get_action('user_show')(context, data_dict) except toolkit.ObjectNotFound: return None except toolkit.NotAuthorized: return None
def test_ckan_admin_has_showcase_config_tab(self): ''' ckan-admin index page has a showcase config tab. ''' if not tk.check_ckan_version(min_version='2.4'): raise SkipTest('Showcase config tab only available for CKAN 2.4+') app = self._get_test_app() sysadmin = factories.Sysadmin() env = {'REMOTE_USER': sysadmin['name'].encode('ascii')} response = app.get( url=url_for(controller='admin', action='index'), extra_environ=env ) # response contains link to dataset's showcase list nosetools.assert_true('/ckan-admin/showcase_admins' in response)
def test_core_ckan_envvar_values_in_config_take_precedence(self): '''Core CKAN env var transformations take precedence over this extension''' if not toolkit.check_ckan_version('2.4.0'): raise SkipTest('CKAN version 2.4 or above needed') combined_list = [ ('CKAN___SQLALCHEMY__URL', 'postgresql://thisextensionformat/'), ('CKAN_SQLALCHEMY_URL', 'postgresql://coreckanformat/'), ] self._setup_env_vars(combined_list) assert_equal(config['sqlalchemy.url'], 'postgresql://coreckanformat/') self._teardown_env_vars(combined_list)
def test_rendering_with_html_allowed(self): env = {'REMOTE_USER': self.user['name'].encode('ascii')} response = self.app.post( url=toolkit.url_for('pages_edit', page='/test_html_page'), params={ 'title': 'Allowed', 'name': 'page_html_allowed', 'content': '<a href="/test">Test Link</a>', }, extra_environ=env, ) response = response.follow(extra_environ=env) assert_in('<h1 class="page-heading">Allowed</h1>', response.body) if toolkit.check_ckan_version(min_version='2.3'): assert_in('<a href="/test">Test Link</a>', response.body) else: assert_in('Test Link', response.body)
def update(ckan_ini_filepath, resource_id): """ Given a resource, calculates an openness score. data - details of the resource that is to be scored is JSON dict with keys: 'package', 'position', 'id', 'format', 'url', 'is_open' Returns a JSON dict with keys: 'openness_score': score (int) 'openness_score_reason': the reason for the score (string) """ log = update.get_logger() load_config(ckan_ini_filepath) register_translator() from ckan import model try: resource = model.Resource.get(resource_id) if not resource: raise QAError('Resource ID not found: %s' % resource_id) qa_result = resource_score(resource, log) log.info('Openness scoring: \n%r\n%r\n%r\n\n', qa_result, resource, resource.url) save_qa_result(resource.id, qa_result, log) log.info('CKAN updated with openness score') if toolkit.check_ckan_version(max_version='2.2.99'): package = resource.resource_group.package else: package = resource.package if package: # Refresh the index for this dataset, so that it contains the latest # qa info _update_search_index(package.id, log) else: log.warning('Resource not connected to a package. Res: %r', resource) return json.dumps(qa_result) #, cls=DateTimeJsonEncoder) except Exception, e: log.error('Exception occurred during QA update: %s: %s', e.__class__.__name__, unicode(e)) raise
def update_resource_(resource_id): from ckan import model resource = model.Resource.get(resource_id) if not resource: raise QAError('Resource ID not found: %s' % resource_id) qa_result = resource_score(resource) log.info('Openness scoring: \n%r\n%r\n%r\n\n', qa_result, resource, resource.url) save_qa_result(resource, qa_result) log.info('CKAN updated with openness score') if toolkit.check_ckan_version(max_version='2.2.99'): package = resource.resource_group.package else: package = resource.package if package: # Refresh the index for this dataset, so that it contains the latest # qa info _update_search_index(package.id) else: log.warning('Resource not connected to a package. Res: %r', resource) return json.dumps(qa_result)
def test_user_show_no_additional_filters(self): ''' Perform package_search with no additional filters should not include showcases. ''' if not toolkit.check_ckan_version(min_version='2.4'): raise SkipTest('Filtering out showcases requires CKAN 2.4+ (ckan/ckan/issues/2380)') user = factories.User() factories.Dataset(user=user) factories.Dataset(user=user) factories.Dataset(user=user, type='showcase') factories.Dataset(user=user, type='custom') search_results = helpers.call_action('user_show', context={}, include_datasets=True, id=user['name'])['datasets'] types = [result['type'] for result in search_results] nosetools.assert_equal(len(search_results), 3) nosetools.assert_true('showcase' not in types) nosetools.assert_true('custom' in types)
def test_min_322_lt(self): tk.ckan.__version__ = '1.5.1' assert_equal(tk.check_ckan_version(min_version='1.5'), True)
""" CKAN Issue Extension """ from logging import getLogger log = getLogger(__name__) import ckan.plugins as p from ckan.plugins import implements, toolkit # Imports are done in methods to speed up paster. # Please don't move back up to here. if toolkit.check_ckan_version(min_version='2.5'): from ckan.lib.plugins import DefaultTranslation class IssuesPluginBase(p.SingletonPlugin, DefaultTranslation): p.implements(p.ITranslation, inherit=True) else: class IssuesPluginBase(p.SingletonPlugin): pass class IssuesPlugin(IssuesPluginBase): """ CKAN Issues Extension """ implements(p.IConfigurer, inherit=True) implements(p.ITemplateHelpers, inherit=True) implements(p.IRoutes, inherit=True) implements(p.IActions) implements(p.IAuthFunctions)
def update_config(self, config): tk.add_template_directory(config, 'templates') tk.add_public_directory(config, 'public') if tk.check_ckan_version(min_version='2.4'): tk.add_ckan_admin_tab(config, 'ckanext_showcase_admins', 'Showcase Config')
def __init__(self): self.action = 'harvest_source_patch' if toolkit.check_ckan_version(max_version='2.2.99'): # harvest_source_patch only came in with ckan 2.3 raise SkipTest()
def harvest_source_clear(context,data_dict): ''' Clears all datasets, jobs and objects related to a harvest source, but keeps the source itself. This is useful to clean history of long running harvest sources to start again fresh. :param id: the id of the harvest source to clear :type id: string ''' check_access('harvest_source_clear',context,data_dict) harvest_source_id = data_dict.get('id',None) source = HarvestSource.get(harvest_source_id) if not source: log.error('Harvest source %s does not exist', harvest_source_id) raise NotFound('Harvest source %s does not exist' % harvest_source_id) harvest_source_id = source.id # Clear all datasets from this source from the index harvest_source_index_clear(context, data_dict) model = context['model'] sql = "select id from related where id in (select related_id from related_dataset where dataset_id in (select package_id from harvest_object where harvest_source_id = '{harvest_source_id}'));".format(harvest_source_id=harvest_source_id) result = model.Session.execute(sql) ids = [] for row in result: ids.append(row[0]) related_ids = "('" + "','".join(ids) + "')" sql = '''begin; update package set state = 'to_delete' where id in (select package_id from harvest_object where harvest_source_id = '{harvest_source_id}');'''.format( harvest_source_id=harvest_source_id) # CKAN-2.3 or above: delete resource views, resource revisions & resources if check_ckan_version(min_version='2.3'): sql += ''' delete from resource_view where resource_id in (select id from resource where package_id in (select id from package where state = 'to_delete' )); delete from resource_revision where package_id in (select id from package where state = 'to_delete' ); delete from resource where package_id in (select id from package where state = 'to_delete' ); ''' # Backwards-compatibility: support ResourceGroup (pre-CKAN-2.3) else: sql += ''' delete from resource_revision where resource_group_id in (select id from resource_group where package_id in (select id from package where state = 'to_delete')); delete from resource where resource_group_id in (select id from resource_group where package_id in (select id from package where state = 'to_delete')); delete from resource_group_revision where package_id in (select id from package where state = 'to_delete'); delete from resource_group where package_id in (select id from package where state = 'to_delete'); ''' sql += ''' delete from harvest_object_error where harvest_object_id in (select id from harvest_object where harvest_source_id = '{harvest_source_id}'); delete from harvest_object_extra where harvest_object_id in (select id from harvest_object where harvest_source_id = '{harvest_source_id}'); delete from harvest_object where harvest_source_id = '{harvest_source_id}'; delete from harvest_gather_error where harvest_job_id in (select id from harvest_job where source_id = '{harvest_source_id}'); delete from harvest_job where source_id = '{harvest_source_id}'; delete from package_role where package_id in (select id from package where state = 'to_delete' ); delete from user_object_role where id not in (select user_object_role_id from package_role) and context = 'Package'; delete from package_tag_revision where package_id in (select id from package where state = 'to_delete'); delete from member_revision where table_id in (select id from package where state = 'to_delete'); delete from package_extra_revision where package_id in (select id from package where state = 'to_delete'); delete from package_revision where id in (select id from package where state = 'to_delete'); delete from package_tag where package_id in (select id from package where state = 'to_delete'); delete from package_extra where package_id in (select id from package where state = 'to_delete'); delete from member where table_id in (select id from package where state = 'to_delete'); delete from related_dataset where dataset_id in (select id from package where state = 'to_delete'); delete from related where id in {related_ids}; delete from package where id in (select id from package where state = 'to_delete'); commit; '''.format( harvest_source_id=harvest_source_id, related_ids=related_ids) model.Session.execute(sql) # Refresh the index for this source to update the status object get_action('harvest_source_reindex')(context, {'id': harvest_source_id}) return {'id': harvest_source_id}
import routes from ckan.common import _ from ckan.lib import i18n from ckan.plugins import toolkit import ckan.lib.helpers as ckan_helpers from sniff_format import sniff_file_format import lib from ckanext.archiver.model import Archival, Status import logging log = logging.getLogger(__name__) if toolkit.check_ckan_version(max_version='2.6.99'): from ckan.lib import celery_app @celery_app.celery.task(name="qa.update_package") def update_package_celery(*args, **kwargs): update_package(*args, **kwargs) @celery_app.celery.task(name="qa.update") def update_celery(*args, **kwargs): update(*args, **kwargs) class QAError(Exception): pass
def test_min_233_gt(self): tk.ckan.__version__ = '2.2' assert_equal(tk.check_ckan_version(min_version='2.2.1'), False)
def test_min_322_gt(self): tk.ckan.__version__ = '1.5.1' assert_equal(tk.check_ckan_version(min_version='1.6'), False)
def _distribution_format(self, distribution, normalize_ckan_format=True): ''' Returns the Internet Media Type and format label for a distribution Given a reference (URIRef or BNode) to a dcat:Distribution, it will try to extract the media type (previously knowm as MIME type), eg `text/csv`, and the format label, eg `CSV` Values for the media type will be checked in the following order: 1. literal value of dcat:mediaType 2. literal value of dct:format if it contains a '/' character 3. value of dct:format if it is an instance of dct:IMT, eg: <dct:format> <dct:IMT rdf:value="text/html" rdfs:label="HTML"/> </dct:format> Values for the label will be checked in the following order: 1. literal value of dct:format if it not contains a '/' character 2. label of dct:format if it is an instance of dct:IMT (see above) If `normalize_ckan_format` is True and using CKAN>=2.3, the label will be tried to match against the standard list of formats that is included with CKAN core (https://github.com/ckan/ckan/blob/master/ckan/config/resource_formats.json) This allows for instance to populate the CKAN resource format field with a format that view plugins, etc will understand (`csv`, `xml`, etc.) Return a tuple with the media type and the label, both set to None if they couldn't be found. ''' imt = None label = None imt = self._object_value(distribution, DCAT.mediaType) _format = self._object(distribution, DCT['format']) if isinstance(_format, Literal): if not imt and '/' in _format: imt = unicode(_format) else: label = unicode(_format) elif isinstance(_format, (BNode, URIRef)): if self._object(_format, RDF.type) == DCT.IMT: if not imt: imt = unicode(self.g.value(_format, default=None)) label = unicode(self.g.label(_format, default=None)) if ((imt or label) and normalize_ckan_format and toolkit.check_ckan_version(min_version='2.3')): import ckan.config from ckan.lib import helpers format_registry = helpers.resource_formats() if imt in format_registry: label = format_registry[imt][1] elif label in format_registry: label = format_registry[label][1] return imt, label