def read_dataset_page(_id, _format):
if not _format:
_format = check_access_header()
if not _format:
if toolkit.check_ckan_version(max_version='2.8.99'):
return read_endpoint(_id)
else:
return read_endpoint(_get_package_type(_id), _id)
_profiles = toolkit.request.params.get('profiles')
if _profiles:
_profiles = _profiles.split(',')
try:
response = toolkit.get_action('dcat_dataset_show')(
{}, {
'id': _id,
'format': _format,
'profiles': _profiles
})
except toolkit.ObjectNotFound:
toolkit.abort(404)
except (toolkit.ValidationError, RDFProfileException) as e:
toolkit.abort(409, str(e))
if toolkit.check_ckan_version(max_version='2.8.99'):
toolkit.response.headers.update(
{'Content-type': CONTENT_TYPES[_format]})
else:
from flask import make_response
response = make_response(response)
response.headers['Content-type'] = CONTENT_TYPES[_format]
return response
def update_config(self, config):
tk.add_template_directory(config, '../templates')
tk.add_public_directory(config, '../public')
tk.add_resource('../fanstatic', 'showcase')
if tk.check_ckan_version(min_version='2.4', max_version='2.9.0'):
tk.add_ckan_admin_tab(config, 'showcase_admins', 'Showcase Config')
elif tk.check_ckan_version(min_version='2.9.0'):
tk.add_ckan_admin_tab(config, 'showcase_blueprint.admins',
'Showcase Config')
if tk.check_ckan_version(min_version='2.9.0'):
mappings = config.get('ckan.legacy_route_mappings', {})
if isinstance(mappings, string_types):
mappings = json.loads(mappings)
bp_routes = [
'index', 'new', 'delete', 'read', 'edit', 'manage_datasets',
'dataset_showcase_list', 'admins', 'admin_remove'
]
mappings.update({
'showcase_' + route: 'showcase_blueprint.' + route
for route in bp_routes
})
# https://github.com/ckan/ckan/pull/4521
config['ckan.legacy_route_mappings'] = json.dumps(mappings)
def test_spatial_extra_bad_geojson(self, app):
user = factories.User()
env = {"REMOTE_USER": user["name"].encode("ascii")}
dataset = factories.Dataset(user=user)
if tk.check_ckan_version(min_version="2.9"):
offset = url_for("dataset.edit", id=dataset["id"])
else:
offset = url_for(controller="package", action="edit", id=dataset["id"])
res = app.get(offset, extra_environ=env)
if tk.check_ckan_version(min_version="2.9"):
data = {
"name": dataset['name'],
"extras__0__key": u"spatial",
"extras__0__value": u'{"Type":"Bad_GeoJSON","a":2}'
}
res = app.post(offset, environ_overrides=env, data=data)
else:
form = res.forms[1]
form['extras__0__key'] = u'spatial'
form['extras__0__value'] = u'{"Type":"Bad_GeoJSON","a":2}'
res = helpers.webtest_submit(form, extra_environ=env, name='save')
assert "Error" in res, res
assert "Spatial" in res
assert "Error creating geometry" in res
def update_config(self, ckan_config):
toolkit.add_template_directory(ckan_config, '../templates')
toolkit.add_public_directory(ckan_config, '../assets')
toolkit.add_resource('../assets', 'opengov_custom_css_resource')
if toolkit.check_ckan_version(min_version='2.4', max_version='2.9'):
toolkit.add_ckan_admin_tab(ckan_config, 'custom_css', 'Custom CSS')
elif toolkit.check_ckan_version(min_version='2.9'):
toolkit.add_ckan_admin_tab(ckan_config, 'custom-css.custom_css', 'Custom CSS')
def update_config(self, ckan_config):
toolkit.add_template_directory(ckan_config, '../templates')
if toolkit.check_ckan_version(min_version='2.4', max_version='2.9'):
toolkit.add_ckan_admin_tab(ckan_config, 'custom_home_page',
'Home Page Layout')
elif toolkit.check_ckan_version(min_version='2.9'):
toolkit.add_ckan_admin_tab(ckan_config,
'custom-homepage.custom_home_page',
'Home Page Layout')
def update_config(self, ckan_config):
toolkit.add_template_directory(ckan_config, '../templates')
if toolkit.check_ckan_version(min_version='2.4', max_version='2.9'):
toolkit.add_ckan_admin_tab(ckan_config, 'custom_header',
'Custom Header')
elif toolkit.check_ckan_version(min_version='2.9'):
toolkit.add_ckan_admin_tab(ckan_config,
'custom-header.custom_header',
'Custom Header')
def test_core_ckan_envvar_values_in_config(self):
if not toolkit.check_ckan_version('2.4.0'):
raise SkipTest('CKAN version 2.4 or above needed')
core_ckan_env_var_list = [
('CKAN_SQLALCHEMY_URL', 'postgresql://mynewsqlurl/'),
('CKAN_DATASTORE_WRITE_URL', 'http://mynewdbwriteurl/'),
('CKAN_DATASTORE_READ_URL', 'http://mynewdbreadurl/'),
('CKAN_SITE_ID', 'my-site'),
('CKAN_DB', 'postgresql://mydeprectatesqlurl/'),
('CKAN_SMTP_SERVER', 'mail.example.com'),
('CKAN_SMTP_STARTTLS', 'True'),
('CKAN_SMTP_USER', 'my_user'),
('CKAN_SMTP_PASSWORD', 'password'),
('CKAN_SMTP_MAIL_FROM', '*****@*****.**')
]
self._setup_env_vars(core_ckan_env_var_list)
assert_equal(config['sqlalchemy.url'], 'postgresql://mynewsqlurl/')
assert_equal(config['ckan.datastore.write_url'],
'http://mynewdbwriteurl/')
assert_equal(config['ckan.datastore.read_url'],
'http://mynewdbreadurl/')
assert_equal(config['ckan.site_id'], 'my-site')
assert_equal(config['smtp.server'], 'mail.example.com')
assert_equal(config['smtp.starttls'], 'True')
assert_equal(config['smtp.user'], 'my_user')
assert_equal(config['smtp.password'], 'password')
assert_equal(config['smtp.mail_from'], '*****@*****.**')
self._teardown_env_vars(core_ckan_env_var_list)
class StatisticsPlugin(plugins.SingletonPlugin, DefaultTranslation):
plugins.implements(plugins.IConfigurer)
plugins.implements(plugins.IRoutes, inherit=True)
if toolkit.check_ckan_version(min_version='2.5.0'):
plugins.implements(plugins.ITranslation, inherit=True)
plugins.implements(plugins.IActions)
# IConfigurer
def update_config(self, config_):
toolkit.add_template_directory(config_, 'templates')
toolkit.add_resource('fanstatic', 'statistics')
# IRoutes
def before_map(self, map):
map.connect(
'/statistics',
controller='ckanext.statistics.controller:StatisticsController',
action='statistics_read')
map.connect(
'/statistics/internal',
controller='ckanext.statistics.controller:StatisticsController',
action='reports_read')
return map
# IActions
def get_actions(self):
return {'get_all_public_datasets': get_all_public_datasets}
def harvest_source_index_clear(context, data_dict):
'''
Clears all datasets, jobs and objects related to a harvest source, but
keeps the source itself. This is useful to clean history of long running
harvest sources to start again fresh.
:param id: the id of the harvest source to clear
:type id: string
'''
check_access('harvest_source_clear', context, data_dict)
harvest_source_id = data_dict.get('id')
source = HarvestSource.get(harvest_source_id)
if not source:
log.error('Harvest source %s does not exist', harvest_source_id)
raise NotFound('Harvest source %s does not exist' % harvest_source_id)
harvest_source_id = source.id
conn = make_connection()
query = ''' +%s:"%s" +site_id:"%s" ''' % (
'harvest_source_id', harvest_source_id, config.get('ckan.site_id'))
solr_commit = toolkit.asbool(config.get('ckan.search.solr_commit', 'true'))
if toolkit.check_ckan_version(max_version='2.5.99'):
# conn is solrpy
try:
conn.delete_query(query)
if solr_commit:
conn.commit()
except Exception, e:
log.exception(e)
raise SearchIndexError(e)
finally:
def update_config(self, config):
tk.add_template_directory(config, 'templates')
tk.add_public_directory(config, 'public')
# If ckan is more than 2.3, use the 2.4+ toolkit method
if not tk.check_ckan_version(max_version='2.3'):
tk.add_ckan_admin_tab(config, 'ckanext_showcase_admins',
'Showcase Config')
def test_distribution_format_format_normalized(self):
g = Graph()
dataset1 = URIRef("http://example.org/datasets/1")
g.add((dataset1, RDF.type, DCAT.Dataset))
distribution1_1 = URIRef("http://example.org/datasets/1/ds/1")
g.add((distribution1_1, RDF.type, DCAT.Distribution))
g.add((distribution1_1, DCAT.mediaType, Literal('text/csv')))
g.add((distribution1_1, DCT['format'],
Literal('Comma Separated Values')))
g.add((dataset1, DCAT.distribution, distribution1_1))
p = RDFParser(profiles=['euro_dcat_ap'])
p.g = g
datasets = [d for d in p.datasets()]
resource = datasets[0]['resources'][0]
if toolkit.check_ckan_version(min_version='2.3'):
assert resource['format'] == u'CSV'
assert resource['mimetype'] == u'text/csv'
else:
assert resource['format'] == u'Comma Separated Values'
def update_config(self, config):
tk.add_template_directory(config, 'templates')
tk.add_public_directory(config, 'public')
tk.add_resource('fanstatic', 'showcase')
if tk.check_ckan_version(min_version='2.4'):
tk.add_ckan_admin_tab(config, 'ckanext_showcase_admins',
'Showcase Config')
def update_config(self, config):
tk.add_template_directory(config, 'templates')
tk.add_public_directory(config, 'public')
tk.add_resource('fanstatic', 'project')
# If ckan is more than 2.3, use the 2.4+ toolkit method
if not tk.check_ckan_version(max_version='2.3'):
tk.add_ckan_admin_tab(config, 'ckanext_project_admins', 'project Config')
def test_distribution_format_format_normalized(self):
g = Graph()
dataset1 = URIRef("http://example.org/datasets/1")
g.add((dataset1, RDF.type, DCAT.Dataset))
distribution1_1 = URIRef("http://example.org/datasets/1/ds/1")
g.add((distribution1_1, RDF.type, DCAT.Distribution))
g.add((distribution1_1, DCAT.mediaType, Literal('text/csv')))
g.add((distribution1_1, DCT['format'], Literal('Comma Separated Values')))
g.add((dataset1, DCAT.distribution, distribution1_1))
p = RDFParser(profiles=['euro_dcat_ap'])
p.g = g
datasets = [d for d in p.datasets()]
resource = datasets[0]['resources'][0]
if toolkit.check_ckan_version(min_version='2.3'):
eq_(resource['format'], u'CSV')
eq_(resource['mimetype'], u'text/csv')
else:
eq_(resource['format'], u'Comma Separated Values')
def test_user_show_no_additional_filters(self):
'''
Perform package_search with no additional filters should not include
showcases.
'''
if not toolkit.check_ckan_version(min_version='2.4'):
raise SkipTest(
'Filtering out showcases requires CKAN 2.4+ (ckan/ckan/issues/2380)'
)
user = factories.User()
factories.Dataset(user=user)
factories.Dataset(user=user)
factories.Dataset(user=user, type='showcase')
factories.Dataset(user=user, type='custom')
search_results = helpers.call_action('user_show',
context={},
include_datasets=True,
id=user['name'])['datasets']
types = [result['type'] for result in search_results]
nosetools.assert_equal(len(search_results), 3)
nosetools.assert_true('showcase' not in types)
nosetools.assert_true('custom' in types)
def test_unicode(self, app):
user = factories.Sysadmin()
env = {'REMOTE_USER': user['name'].encode('ascii')}
page = 'test_html_page_div'
page = '/' + page if not ckan_29_or_higher else page
response = app.post(
url=toolkit.url_for('pages_edit', page=page),
params={
'title': u'Tïtlé'.encode('utf-8'),
'name': 'page_unicode',
'content': u'Çöñtéñt'.encode('utf-8'),
'order': 1,
'private': False,
},
extra_environ=env,
)
if not ckan_29_or_higher:
response = response.follow(extra_environ=env)
if toolkit.check_ckan_version(min_version='2.9.0'):
assert u'<p>Çöñtéñt</p>' in response.get_data(as_text=True)
assert u'<title>Tïtlé - CKAN</title>' in response.get_data(as_text=True)
assert u'<a href="/pages/page_unicode">Tïtlé</a>' in response.get_data(as_text=True)
assert u'<h1 class="page-heading">Tïtlé</h1>' in response.get_data(as_text=True)
else:
assert u'<title>Tïtlé - CKAN</title>' in response.unicode_body
assert u'<a href="/pages/page_unicode">Tïtlé</a>' in response.unicode_body
assert u'<h1 class="page-heading">Tïtlé</h1>' in response.unicode_body
try:
assert u'<p>Çöñtéñt</p>' in response.unicode_body
except AssertionError:
assert u'<p>Çöñtéñt</p>' in response.unicode_body
def read_catalog_page(_format):
if not _format:
_format = check_access_header()
if not _format:
return index_endpoint()
_profiles = toolkit.request.params.get('profiles')
if _profiles:
_profiles = _profiles.split(',')
data_dict = {
'page': toolkit.request.params.get('page'),
'modified_since': toolkit.request.params.get('modified_since'),
'q': toolkit.request.params.get('q'),
'fq': toolkit.request.params.get('fq'),
'format': _format,
'profiles': _profiles,
}
try:
response = toolkit.get_action('dcat_catalog_show')({}, data_dict)
except (toolkit.ValidationError, RDFProfileException) as e:
toolkit.abort(409, str(e))
if toolkit.check_ckan_version(max_version='2.8.99'):
toolkit.response.headers.update(
{'Content-type': CONTENT_TYPES[_format]})
else:
from flask import make_response
response = make_response(response)
response.headers['Content-type'] = CONTENT_TYPES[_format]
return response
def read_dataset(self, _id, _format=None):
if not _format:
_format = check_access_header()
if not _format:
if toolkit.check_ckan_version(max_version='2.8.99'):
return read_endpoint(_id)
else:
return read_endpoint(_get_package_type(_id), _id)
_profiles = toolkit.request.params.get('profiles')
if _profiles:
_profiles = _profiles.split(',')
toolkit.response.headers.update(
{'Content-type': CONTENT_TYPES[_format]})
try:
result = toolkit.get_action('dcat_dataset_show')({}, {'id': _id,
'format': _format, 'profiles': _profiles})
except toolkit.ObjectNotFound:
toolkit.abort(404)
except (toolkit.ValidationError, RDFProfileException) as e:
toolkit.abort(409, str(e))
return result
def populate_revision(resource):
if 'revision_timestamp' in resource \
or toolkit.check_ckan_version(min_version='2.9'):
return
current_revision = latest_revision(resource['id'])
if current_revision is not None:
resource['revision_timestamp'] = current_revision.revision_timestamp
def download(id, resource_id, filename=None):
"""Download resource blueprint
This calls all registered download handlers in order, until
a response is returned to the user
"""
context = get_context()
resource = None
try:
resource = toolkit.get_action('resource_show')(context, {
'id': resource_id
})
if id != resource['package_id']:
return toolkit.abort(
404, toolkit._('Resource not found belonging to package'))
package = toolkit.get_action('package_show')(context, {'id': id})
except toolkit.ObjectNotFound:
return toolkit.abort(404, toolkit._('Resource not found'))
except toolkit.NotAuthorized:
return toolkit.abort(
401, toolkit._('Not authorized to read resource {0}'.format(id)))
activity_id = request.args.get('activity_id')
inline = toolkit.asbool(request.args.get('preview'))
if activity_id and toolkit.check_ckan_version(min_version='2.9'):
try:
activity = toolkit.get_action(u'activity_show')(
context, {
u'id': activity_id,
u'include_data': True
})
activity_dataset = activity['data']['package']
assert activity_dataset['id'] == id
activity_resources = activity_dataset['resources']
for r in activity_resources:
if r['id'] == resource_id:
resource = r
package = activity_dataset
break
except toolkit.NotFound:
toolkit.abort(404, toolkit._(u'Activity not found'))
try:
resource = call_pre_download_handlers(resource,
package,
activity_id=activity_id)
return call_download_handlers(resource,
package,
filename,
inline,
activity_id=activity_id)
except toolkit.ObjectNotFound:
return toolkit.abort(404, toolkit._('Resource not found'))
except toolkit.NotAuthorized:
return toolkit.abort(
401,
toolkit._(
'Not authorized to read resource {0}'.format(resource_id)))
def test_distribution_format_format_normalized(self):
g = Graph()
dataset1 = URIRef("http://example.org/datasets/1")
g.add((dataset1, RDF.type, DCAT.Dataset))
distribution1_1 = URIRef("http://example.org/datasets/1/ds/1")
g.add((distribution1_1, RDF.type, DCAT.Distribution))
g.add((distribution1_1, DCAT.mediaType, Literal("text/csv")))
g.add((distribution1_1, DCT["format"], Literal("Comma Separated Values")))
g.add((dataset1, DCAT.distribution, distribution1_1))
p = RDFParser(profiles=["euro_dcat_ap"])
p.g = g
datasets = [d for d in p.datasets()]
resource = datasets[0]["resources"][0]
if toolkit.check_ckan_version(min_version="2.3"):
eq_(resource["format"], u"CSV")
eq_(resource["mimetype"], u"text/csv")
else:
eq_(resource["format"], u"Comma Separated Values")
def test_dataset_showcase_page_add_to_showcase_dropdown_submit(self, app):
"""
Submitting 'Add to showcase' form with selected showcase value creates
a sc/pkg association.
"""
sysadmin = factories.Sysadmin()
dataset = factories.Dataset(name="my-dataset")
showcase_one = factories.Dataset(name="my-first-showcase",
type="showcase")
factories.Dataset(name="my-second-showcase", type="showcase")
factories.Dataset(name="my-third-showcase", type="showcase")
assert model.Session.query(ShowcasePackageAssociation).count() == 0
if tk.check_ckan_version("2.9"):
pytest.skip("submit_and_follow not supported")
env = {"REMOTE_USER": sysadmin["name"].encode("ascii")}
response = app.get(
url=url_for("showcase_dataset_showcase_list", id=dataset["id"]),
extra_environ=env,
)
form = response.forms["showcase-add"]
form["showcase_added"] = showcase_one["id"]
showcase_add_response = helpers.submit_and_follow(app, form, env)
# returns to the correct page
assert (showcase_add_response.request.path ==
"/dataset/showcases/my-dataset")
# an association is created
assert model.Session.query(ShowcasePackageAssociation).count() == 1
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key,
value=value,
replace=replace,
controller='dataset' if
tk.check_ckan_version('2.9') else 'package',
action='search')
def harvest_source_index_clear(context, data_dict):
'''
Clears all datasets, jobs and objects related to a harvest source, but
keeps the source itself. This is useful to clean history of long running
harvest sources to start again fresh.
:param id: the id of the harvest source to clear
:type id: string
'''
check_access('harvest_source_clear', context, data_dict)
harvest_source_id = data_dict.get('id')
source = HarvestSource.get(harvest_source_id)
if not source:
log.error('Harvest source %s does not exist', harvest_source_id)
raise NotFound('Harvest source %s does not exist' % harvest_source_id)
harvest_source_id = source.id
conn = make_connection()
query = ''' +%s:"%s" +site_id:"%s" ''' % (
'harvest_source_id', harvest_source_id, config.get('ckan.site_id'))
solr_commit = toolkit.asbool(config.get('ckan.search.solr_commit', 'true'))
if toolkit.check_ckan_version(max_version='2.5.99'):
# conn is solrpy
try:
conn.delete_query(query)
if solr_commit:
conn.commit()
except Exception, e:
log.exception(e)
raise SearchIndexError(e)
finally:
def test_showcase_new_redirects_to_manage_datasets(self, app):
"""Creating a new showcase redirects to the manage datasets form."""
if tk.check_ckan_version("2.9"):
pytest.skip("submit_and_follow not supported")
sysadmin = factories.Sysadmin()
# need a dataset for the 'bulk_action.showcase_add' button to show
factories.Dataset()
env = {"REMOTE_USER": sysadmin["name"].encode("ascii")}
response = app.get(
url=url_for("showcase_new"),
extra_environ=env,
)
# create showcase
form = response.forms["dataset-edit"]
form["name"] = u"my-showcase"
create_response = helpers.submit_and_follow(app, form, env, "save")
# Unique to manage_datasets page
assert "bulk_action.showcase_add" in create_response
# Requested page is the manage_datasets url.
assert (url_for("showcase_manage_datasets",
id="my-showcase") == create_response.request.path)
def read_dataset(self, _id, _format=None):
if not _format:
_format = check_access_header()
if not _format:
if toolkit.check_ckan_version(max_version='2.8.99'):
return read_endpoint(_id)
else:
return read_endpoint(_get_package_type(_id), _id)
_profiles = toolkit.request.params.get('profiles')
if _profiles:
_profiles = _profiles.split(',')
toolkit.response.headers.update(
{'Content-type': CONTENT_TYPES[_format]})
try:
result = toolkit.get_action('dcat_dataset_show')({}, {'id': _id,
'format': _format, 'profiles': _profiles})
except toolkit.ObjectNotFound:
toolkit.abort(404)
return result
def update_config(self, config_):
toolkit.add_template_directory(config_, 'templates')
toolkit.add_resource('theme/resources', 'deadoralive')
if toolkit.check_ckan_version(max_version='2.2.999'):
# Add CKAN version 2.2 support templates.
toolkit.add_template_directory(config_, '2.2_templates')
def test_geojson_view_is_rendered(app):
view_default_title = GeoJSONView().info()["title"]
dataset = factories.Dataset()
for format in GeoJSONView.GeoJSON:
resource = factories.Resource(name='My Resource',
format=format,
package_id=dataset['id'])
if toolkit.check_ckan_version("2.9"):
url = toolkit.url_for(
"{}_resource.read".format(dataset["type"]),
id=dataset["name"],
resource_id=resource["id"],
)
else:
url = toolkit.url_for(
controller="package",
action="resource_read",
id=resource["package_id"],
resource_id=resource["id"],
)
res = app.get(url)
assert 'class="resource-view"' in res.body
assert 'data-title="{}"'.format(view_default_title) in res.body
assert 'id="view-' in res.body
def relative_url_for(**kwargs):
'''Return the existing URL but amended for the given url_for-style
parameters. Much easier than calling h.add_url_param etc. For switching to
URLs inside the current controller action only.
'''
# Restrict the request params to the same action & controller, to avoid
# being an open redirect.
disallowed_params = set(
('controller', 'action', 'anchor', 'host', 'protocol', 'qualified'))
user_specified_params = [(k, v) for k, v in list(tk.request.params.items())
if k not in disallowed_params]
if tk.check_ckan_version(min_version="2.9.0"):
from flask import request
args = dict(
list(request.args.items()) + user_specified_params +
list(kwargs.items()))
# remove blanks
for k, v in list(args.items()):
if not v:
del args[k]
return tk.url_for(request.url_rule.rule, **args)
else:
args = dict(
list(tk.request.environ['pylons.routes_dict'].items()) +
user_specified_params + list(kwargs.items()))
# remove blanks
for k, v in list(args.items()):
if not v:
del args[k]
return tk.url_for(**args)
def update_config(self, config_):
toolkit.add_template_directory(config_, 'templates')
toolkit.add_resource('theme/resources', 'deadoralive')
if toolkit.check_ckan_version(max_version='2.2.999'):
# Add CKAN version 2.2 support templates.
toolkit.add_template_directory(config_, '2.2_templates')
def test_spatial_extra_base(self, app):
user = factories.User()
env = {"REMOTE_USER": user["name"].encode("ascii")}
dataset = factories.Dataset(user=user)
if tk.check_ckan_version(min_version="2.9"):
offset = url_for("dataset.edit", id=dataset["id"])
else:
offset = url_for(controller="package", action="edit", id=dataset["id"])
res = app.get(offset, extra_environ=env)
if tk.check_ckan_version(min_version="2.9"):
data = {
"name": dataset['name'],
"extras__0__key": u"spatial",
"extras__0__value": self.geojson_examples["point"]
}
res = app.post(offset, environ_overrides=env, data=data)
else:
form = res.forms[1]
form['extras__0__key'] = u'spatial'
form['extras__0__value'] = self.geojson_examples['point']
res = helpers.submit_and_follow(app, form, env, 'save')
assert "Error" not in res, res
package_extent = (
Session.query(PackageExtent)
.filter(PackageExtent.package_id == dataset["id"])
.first()
)
geojson = json.loads(self.geojson_examples["point"])
assert package_extent.package_id == dataset["id"]
from sqlalchemy import func
assert (
Session.query(func.ST_X(package_extent.the_geom)).first()[0]
== geojson["coordinates"][0]
)
assert (
Session.query(func.ST_Y(package_extent.the_geom)).first()[0]
== geojson["coordinates"][1]
)
assert package_extent.the_geom.srid == self.db_srid
def remove_showcase_admin():
'''
Remove a user from the Showcase Admin list.
'''
context = {
'model': model,
'session': model.Session,
'user': tk.g.user or tk.g.author
}
try:
tk.check_access('sysadmin', context, {})
except tk.NotAuthorized:
return tk.abort(401, _('User not authorized to view page'))
form_data = tk.request.form if tk.check_ckan_version(
'2.9') else tk.request.params
if tk.check_ckan_version(min_version='2.9.0'):
admins_route = 'showcase_blueprint.admins'
else:
admins_route = 'showcase_admins'
if 'cancel' in form_data:
return tk.redirect_to(admins_route)
user_id = tk.request.params['user']
if tk.request.method == 'POST' and user_id:
user_id = tk.request.params['user']
try:
tk.get_action('ckanext_showcase_admin_remove')({}, {
'username': user_id
})
except tk.NotAuthorized:
return tk.abort(401, _('Unauthorized to perform that action'))
except tk.ObjectNotFound:
h.flash_error(_('The user is not a Showcase Admin'))
else:
h.flash_success(_('The user is no longer a Showcase Admin'))
return tk.redirect_to(h.url_for(admins_route))
tk.g.user_dict = tk.get_action('user_show')({}, {'id': user_id})
tk.g.user_id = user_id
return tk.render('admin/confirm_remove_showcase_admin.html')
def test_spatial_search_widget(self, app):
if tk.check_ckan_version(min_version="2.9"):
offset = url_for("dataset.search")
else:
offset = url_for(controller="package", action="search")
res = app.get(offset)
assert 'data-module="spatial-query"' in res
assert "spatial_query.js" in res
def auth_allow_anonymous_access(auth_function):
'''
Local version of the auth_allow_anonymous_access decorator that only
calls the actual toolkit decorator if the CKAN version supports it
'''
if pt.check_ckan_version(min_version='2.2'):
auth_function = pt.auth_allow_anonymous_access(auth_function)
return auth_function
def test_check_ckan_version(version, bound, value, expected, monkeypatch):
# test name numbers refer to:
# * number of numbers in the ckan version
# * number of numbers in the checked version
# * the index of the number being tested in the checked version
monkeypatch.setattr(tk.ckan, u"__version__", version)
kwargs = {bound + u"_version": value}
assert tk.check_ckan_version(**kwargs) is expected
def test_dataset_count_no_datasets(self):
'''
Dataset and showcase count is 0 when no datasets, and no showcases.
'''
if not tk.check_ckan_version(min_version='2.5'):
raise SkipTest('get_site_statistics without user broken in CKAN 2.4')
stats = showcase_helpers.get_site_statistics()
nosetools.assert_equal(stats['dataset_count'], 0)
nosetools.assert_equal(stats['showcase_count'], 0)
def auth_allow_anonymous_access(auth_function):
'''
Local version of the auth_allow_anonymous_access decorator that only
calls the actual toolkit decorator if the CKAN version supports it
'''
if pt.check_ckan_version(min_version='2.2'):
auth_function = pt.auth_allow_anonymous_access(auth_function)
return auth_function
def test_dataset_count_no_datasets(self):
'''
Dataset and experience count is 0 when no datasets, and no experiences.
'''
if not tk.check_ckan_version(min_version='2.5'):
raise SkipTest('get_site_statistics without user broken in CKAN 2.4')
stats = experience_helpers.get_site_statistics()
nosetools.assert_equal(stats['dataset_count'], 0)
nosetools.assert_equal(stats['experience_count'], 0)
class ESDStandardsPlugin(plugins.SingletonPlugin):
plugins.implements(plugins.IConfigurer)
plugins.implements(plugins.IConfigurable)
plugins.implements(plugins.IActions)
plugins.implements(plugins.IAuthFunctions)
plugins.implements(plugins.ITemplateHelpers)
if toolkit.check_ckan_version(min_version='2.3'):
plugins.implements(plugins.IValidators)
# IConfigurer
def update_config(self, config_):
toolkit.add_template_directory(config_, 'templates')
toolkit.add_public_directory(config_, 'public')
toolkit.add_resource('fanstatic', 'esd')
# IConfigurable
def configure(self, config):
model_setup()
# IActions
def get_actions(self):
return {
'esd_function_show': esd_function_show,
'esd_service_show': esd_service_show,
'esd_function_autocomplete': esd_function_autocomplete,
'esd_service_autocomplete': esd_service_autocomplete,
}
# IAuthFunctions
def get_auth_functions(self):
return {
'esd_function_show': esd_auth,
'esd_service_show': esd_auth,
'esd_function_autocomplete': esd_auth,
'esd_service_autocomplete': esd_auth,
}
# ITemplateHelpers
def get_helpers(self):
return {
'get_esd_function_titles': get_esd_function_titles,
'get_esd_service_titles': get_esd_service_titles,
}
# IValidators (CKAN >= 2.3)
def get_validators(self):
return {
'esd_function_validator': esd_function_validator,
'esd_service_validator': esd_service_validator,
}
def configure(self, config):
'''Store the YouCKAN configuration'''
if not toolkit.check_ckan_version('2.1'):
log.warn('This extension has only been tested on CKAN 2.1!')
self.use_auth = toolkit.asbool(config.get('youckan.use_auth', False))
if self.use_auth:
self.logout_url = config['youckan.logout_url']
model_setup()
def create_qa_update_task(resource, queue):
if check_ckan_version(max_version='2.2.99'):
package = resource.resource_group.package
else:
package = resource.package
compat_enqueue('qa.update', update, queue, kwargs={'resource_id': resource.id})
log.debug('QA of resource put into celery queue %s: %s/%s url=%r',
queue, package.name, resource.id, resource.url)
def test_dataset_count_no_datasets_some_showcases(self):
'''
Dataset and showcase count is 0 when no datasets, but some showcases.
'''
if not tk.check_ckan_version(min_version='2.5'):
raise SkipTest('get_site_statistics without user broken in CKAN 2.4')
for i in xrange(0, 10):
factories.Dataset(type='showcase')
stats = showcase_helpers.get_site_statistics()
nosetools.assert_equal(stats['dataset_count'], 0)
nosetools.assert_equal(stats['showcase_count'], 10)
def get_auth_functions(self):
auth_functions = {'package_show': auth.package_show,
'package_update': auth.package_update,
# 'resource_show': auth.resource_show,
constants.PACKAGE_ACQUIRED: auth.package_acquired,
constants.ACQUISITIONS_LIST: auth.acquisitions_list}
# resource_show is not required in CKAN 2.3 because it delegates to
# package_show
if not tk.check_ckan_version(min_version='2.3'):
auth_functions['resource_show'] = auth.resource_show
return auth_functions
def create(cls, resource_id):
c = cls()
c.resource_id = resource_id
# Find the package_id for the resource.
q = model.Session.query(model.Package.id)
if toolkit.check_ckan_version(max_version="2.2.99"):
q = q.join(model.ResourceGroup)
q = q.join(model.Resource).filter_by(id=c.resource_id)
result = q.first()
if not result or not result[0]:
raise Exception("Missing dataset")
c.package_id = result[0]
return c
def _get_assigned_user(assignee_id, session):
context = {'session': session, 'model': cmodel}
data_dict = {'id': assignee_id}
# we only need the basic properties of the user, not its datasets etc
if toolkit.check_ckan_version(min_version='2.3'):
# these are the defaults, but just in case...
data_dict.update({'include_datasets': False,
'include_num_followers': False})
else:
context = {'return_minimal': True}
try:
return toolkit.get_action('user_show')(context, data_dict)
except toolkit.ObjectNotFound:
return None
except toolkit.NotAuthorized:
return None
def test_ckan_admin_has_showcase_config_tab(self):
'''
ckan-admin index page has a showcase config tab.
'''
if not tk.check_ckan_version(min_version='2.4'):
raise SkipTest('Showcase config tab only available for CKAN 2.4+')
app = self._get_test_app()
sysadmin = factories.Sysadmin()
env = {'REMOTE_USER': sysadmin['name'].encode('ascii')}
response = app.get(
url=url_for(controller='admin', action='index'),
extra_environ=env
)
# response contains link to dataset's showcase list
nosetools.assert_true('/ckan-admin/showcase_admins' in response)
def test_core_ckan_envvar_values_in_config_take_precedence(self):
'''Core CKAN env var transformations take precedence over this
extension'''
if not toolkit.check_ckan_version('2.4.0'):
raise SkipTest('CKAN version 2.4 or above needed')
combined_list = [
('CKAN___SQLALCHEMY__URL', 'postgresql://thisextensionformat/'),
('CKAN_SQLALCHEMY_URL', 'postgresql://coreckanformat/'),
]
self._setup_env_vars(combined_list)
assert_equal(config['sqlalchemy.url'], 'postgresql://coreckanformat/')
self._teardown_env_vars(combined_list)
def test_rendering_with_html_allowed(self):
env = {'REMOTE_USER': self.user['name'].encode('ascii')}
response = self.app.post(
url=toolkit.url_for('pages_edit', page='/test_html_page'),
params={
'title': 'Allowed',
'name': 'page_html_allowed',
'content': '<a href="/test">Test Link</a>',
},
extra_environ=env,
)
response = response.follow(extra_environ=env)
assert_in('<h1 class="page-heading">Allowed</h1>', response.body)
if toolkit.check_ckan_version(min_version='2.3'):
assert_in('<a href="/test">Test Link</a>', response.body)
else:
assert_in('Test Link', response.body)
def update(ckan_ini_filepath, resource_id):
"""
Given a resource, calculates an openness score.
data - details of the resource that is to be scored
is JSON dict with keys: 'package', 'position', 'id', 'format', 'url', 'is_open'
Returns a JSON dict with keys:
'openness_score': score (int)
'openness_score_reason': the reason for the score (string)
"""
log = update.get_logger()
load_config(ckan_ini_filepath)
register_translator()
from ckan import model
try:
resource = model.Resource.get(resource_id)
if not resource:
raise QAError('Resource ID not found: %s' % resource_id)
qa_result = resource_score(resource, log)
log.info('Openness scoring: \n%r\n%r\n%r\n\n', qa_result, resource,
resource.url)
save_qa_result(resource.id, qa_result, log)
log.info('CKAN updated with openness score')
if toolkit.check_ckan_version(max_version='2.2.99'):
package = resource.resource_group.package
else:
package = resource.package
if package:
# Refresh the index for this dataset, so that it contains the latest
# qa info
_update_search_index(package.id, log)
else:
log.warning('Resource not connected to a package. Res: %r', resource)
return json.dumps(qa_result) #, cls=DateTimeJsonEncoder)
except Exception, e:
log.error('Exception occurred during QA update: %s: %s',
e.__class__.__name__, unicode(e))
raise
def update_resource_(resource_id):
from ckan import model
resource = model.Resource.get(resource_id)
if not resource:
raise QAError('Resource ID not found: %s' % resource_id)
qa_result = resource_score(resource)
log.info('Openness scoring: \n%r\n%r\n%r\n\n', qa_result, resource,
resource.url)
save_qa_result(resource, qa_result)
log.info('CKAN updated with openness score')
if toolkit.check_ckan_version(max_version='2.2.99'):
package = resource.resource_group.package
else:
package = resource.package
if package:
# Refresh the index for this dataset, so that it contains the latest
# qa info
_update_search_index(package.id)
else:
log.warning('Resource not connected to a package. Res: %r', resource)
return json.dumps(qa_result)
def test_user_show_no_additional_filters(self):
'''
Perform package_search with no additional filters should not include
showcases.
'''
if not toolkit.check_ckan_version(min_version='2.4'):
raise SkipTest('Filtering out showcases requires CKAN 2.4+ (ckan/ckan/issues/2380)')
user = factories.User()
factories.Dataset(user=user)
factories.Dataset(user=user)
factories.Dataset(user=user, type='showcase')
factories.Dataset(user=user, type='custom')
search_results = helpers.call_action('user_show', context={},
include_datasets=True,
id=user['name'])['datasets']
types = [result['type'] for result in search_results]
nosetools.assert_equal(len(search_results), 3)
nosetools.assert_true('showcase' not in types)
nosetools.assert_true('custom' in types)
def test_min_322_lt(self):
tk.ckan.__version__ = '1.5.1'
assert_equal(tk.check_ckan_version(min_version='1.5'), True)
"""
CKAN Issue Extension
"""
from logging import getLogger
log = getLogger(__name__)
import ckan.plugins as p
from ckan.plugins import implements, toolkit
# Imports are done in methods to speed up paster.
# Please don't move back up to here.
if toolkit.check_ckan_version(min_version='2.5'):
from ckan.lib.plugins import DefaultTranslation
class IssuesPluginBase(p.SingletonPlugin, DefaultTranslation):
p.implements(p.ITranslation, inherit=True)
else:
class IssuesPluginBase(p.SingletonPlugin):
pass
class IssuesPlugin(IssuesPluginBase):
"""
CKAN Issues Extension
"""
implements(p.IConfigurer, inherit=True)
implements(p.ITemplateHelpers, inherit=True)
implements(p.IRoutes, inherit=True)
implements(p.IActions)
implements(p.IAuthFunctions)
def update_config(self, config):
tk.add_template_directory(config, 'templates')
tk.add_public_directory(config, 'public')
if tk.check_ckan_version(min_version='2.4'):
tk.add_ckan_admin_tab(config, 'ckanext_showcase_admins',
'Showcase Config')
def __init__(self):
self.action = 'harvest_source_patch'
if toolkit.check_ckan_version(max_version='2.2.99'):
# harvest_source_patch only came in with ckan 2.3
raise SkipTest()
def harvest_source_clear(context,data_dict):
'''
Clears all datasets, jobs and objects related to a harvest source, but keeps the source itself.
This is useful to clean history of long running harvest sources to start again fresh.
:param id: the id of the harvest source to clear
:type id: string
'''
check_access('harvest_source_clear',context,data_dict)
harvest_source_id = data_dict.get('id',None)
source = HarvestSource.get(harvest_source_id)
if not source:
log.error('Harvest source %s does not exist', harvest_source_id)
raise NotFound('Harvest source %s does not exist' % harvest_source_id)
harvest_source_id = source.id
# Clear all datasets from this source from the index
harvest_source_index_clear(context, data_dict)
model = context['model']
sql = "select id from related where id in (select related_id from related_dataset where dataset_id in (select package_id from harvest_object where harvest_source_id = '{harvest_source_id}'));".format(harvest_source_id=harvest_source_id)
result = model.Session.execute(sql)
ids = []
for row in result:
ids.append(row[0])
related_ids = "('" + "','".join(ids) + "')"
sql = '''begin;
update package set state = 'to_delete' where id in (select package_id from harvest_object where harvest_source_id = '{harvest_source_id}');'''.format(
harvest_source_id=harvest_source_id)
# CKAN-2.3 or above: delete resource views, resource revisions & resources
if check_ckan_version(min_version='2.3'):
sql += '''
delete from resource_view where resource_id in (select id from resource where package_id in (select id from package where state = 'to_delete' ));
delete from resource_revision where package_id in (select id from package where state = 'to_delete' );
delete from resource where package_id in (select id from package where state = 'to_delete' );
'''
# Backwards-compatibility: support ResourceGroup (pre-CKAN-2.3)
else:
sql += '''
delete from resource_revision where resource_group_id in
(select id from resource_group where package_id in
(select id from package where state = 'to_delete'));
delete from resource where resource_group_id in
(select id from resource_group where package_id in
(select id from package where state = 'to_delete'));
delete from resource_group_revision where package_id in
(select id from package where state = 'to_delete');
delete from resource_group where package_id in
(select id from package where state = 'to_delete');
'''
sql += '''
delete from harvest_object_error where harvest_object_id in (select id from harvest_object where harvest_source_id = '{harvest_source_id}');
delete from harvest_object_extra where harvest_object_id in (select id from harvest_object where harvest_source_id = '{harvest_source_id}');
delete from harvest_object where harvest_source_id = '{harvest_source_id}';
delete from harvest_gather_error where harvest_job_id in (select id from harvest_job where source_id = '{harvest_source_id}');
delete from harvest_job where source_id = '{harvest_source_id}';
delete from package_role where package_id in (select id from package where state = 'to_delete' );
delete from user_object_role where id not in (select user_object_role_id from package_role) and context = 'Package';
delete from package_tag_revision where package_id in (select id from package where state = 'to_delete');
delete from member_revision where table_id in (select id from package where state = 'to_delete');
delete from package_extra_revision where package_id in (select id from package where state = 'to_delete');
delete from package_revision where id in (select id from package where state = 'to_delete');
delete from package_tag where package_id in (select id from package where state = 'to_delete');
delete from package_extra where package_id in (select id from package where state = 'to_delete');
delete from member where table_id in (select id from package where state = 'to_delete');
delete from related_dataset where dataset_id in (select id from package where state = 'to_delete');
delete from related where id in {related_ids};
delete from package where id in (select id from package where state = 'to_delete');
commit;
'''.format(
harvest_source_id=harvest_source_id, related_ids=related_ids)
model.Session.execute(sql)
# Refresh the index for this source to update the status object
get_action('harvest_source_reindex')(context, {'id': harvest_source_id})
return {'id': harvest_source_id}
import routes
from ckan.common import _
from ckan.lib import i18n
from ckan.plugins import toolkit
import ckan.lib.helpers as ckan_helpers
from sniff_format import sniff_file_format
import lib
from ckanext.archiver.model import Archival, Status
import logging
log = logging.getLogger(__name__)
if toolkit.check_ckan_version(max_version='2.6.99'):
from ckan.lib import celery_app
@celery_app.celery.task(name="qa.update_package")
def update_package_celery(*args, **kwargs):
update_package(*args, **kwargs)
@celery_app.celery.task(name="qa.update")
def update_celery(*args, **kwargs):
update(*args, **kwargs)
class QAError(Exception):
pass
def test_min_233_gt(self):
tk.ckan.__version__ = '2.2'
assert_equal(tk.check_ckan_version(min_version='2.2.1'), False)
def test_min_322_gt(self):
tk.ckan.__version__ = '1.5.1'
assert_equal(tk.check_ckan_version(min_version='1.6'), False)
def _distribution_format(self, distribution, normalize_ckan_format=True):
'''
Returns the Internet Media Type and format label for a distribution
Given a reference (URIRef or BNode) to a dcat:Distribution, it will
try to extract the media type (previously knowm as MIME type), eg
`text/csv`, and the format label, eg `CSV`
Values for the media type will be checked in the following order:
1. literal value of dcat:mediaType
2. literal value of dct:format if it contains a '/' character
3. value of dct:format if it is an instance of dct:IMT, eg:
<dct:format>
<dct:IMT rdf:value="text/html" rdfs:label="HTML"/>
</dct:format>
Values for the label will be checked in the following order:
1. literal value of dct:format if it not contains a '/' character
2. label of dct:format if it is an instance of dct:IMT (see above)
If `normalize_ckan_format` is True and using CKAN>=2.3, the label will
be tried to match against the standard list of formats that is included
with CKAN core
(https://github.com/ckan/ckan/blob/master/ckan/config/resource_formats.json)
This allows for instance to populate the CKAN resource format field
with a format that view plugins, etc will understand (`csv`, `xml`,
etc.)
Return a tuple with the media type and the label, both set to None if
they couldn't be found.
'''
imt = None
label = None
imt = self._object_value(distribution, DCAT.mediaType)
_format = self._object(distribution, DCT['format'])
if isinstance(_format, Literal):
if not imt and '/' in _format:
imt = unicode(_format)
else:
label = unicode(_format)
elif isinstance(_format, (BNode, URIRef)):
if self._object(_format, RDF.type) == DCT.IMT:
if not imt:
imt = unicode(self.g.value(_format, default=None))
label = unicode(self.g.label(_format, default=None))
if ((imt or label) and normalize_ckan_format and
toolkit.check_ckan_version(min_version='2.3')):
import ckan.config
from ckan.lib import helpers
format_registry = helpers.resource_formats()
if imt in format_registry:
label = format_registry[imt][1]
elif label in format_registry:
label = format_registry[label][1]
return imt, label
