class YtpDrupalPlugin(plugins.SingletonPlugin, DefaultTranslation):
plugins.implements(plugins.IConfigurable)
plugins.implements(plugins.ITemplateHelpers)
plugins.implements(plugins.IConfigurer, inherit=True)
plugins.implements(plugins.IAuthFunctions, inherit=True)
plugins.implements(plugins.IRoutes, inherit=True)
plugins.implements(plugins.ITranslation)
_config_template = "ckanext.ytp.drupal.%s"
_node_type = 'service_alert'
_node_status = '1' # published content has status 1, unpublised has status 0
_language_fallback_order = ['fi', 'en', 'sv']
cancel_url = None
def before_map(self, m):
""" Override delete page """
controller = 'ckanext.ytp_drupal.controller:YtpDrupalController'
m.connect('user_delete_me',
'/user/delete-me',
action='delete_me',
controller=controller)
return m
def configure(self, config):
connection_variable = self._config_template % "connection"
self.drupal_connection_url = config.get(connection_variable)
if not self.drupal_connection_url:
raise Exception(
'YtpDrupalPlugin: required configuration variable missing: %s'
% (connection_variable))
self.cancel_url = config.get(self._config_template % "cancel_url",
"/cancel-user.php")
self._node_type = config.get(self._config_template % "node_type",
self._node_type)
self._translations_disabled = asbool(
config.get(self._config_template % "translations_disabled",
"false"))
self.engine = sqlalchemy.create_engine(self.drupal_connection_url)
def update_config(self, config):
toolkit.add_template_directory(config, 'templates')
def _service_alerts(self):
""" Get service alerts from Drupal """
language = None if self._translations_disabled else helpers.lang()
return self.engine.execute(
"""SELECT nid, title FROM node WHERE type = %(type)s AND
language = %(language)s AND status = %(status)s""",
{
'type': self._node_type,
'language': language,
'status': self._node_status
})
def _fetch_drupal_content(self, identifier, language=None, fallback=True):
""" This helper fetches content from Drupal database using url alias identifier.
Return content as dictionary containing language, title, body, node_id and edit link. None if not found.
Tries to fallback to different language if fallback is True.
Not cached.
"""
if not language:
language = helpers.lang()
query = """SELECT url_alias.language, node.title, field_revision_body.body_value, node.nid from url_alias
INNER JOIN node ON node.nid = split_part(url_alias.source, '/', 2)::integer
INNER JOIN field_revision_body ON field_revision_body.entity_id = split_part(url_alias.source, '/', 2)::integer
WHERE url_alias.alias = %(identifier)s""" # noqa: E501
results = {}
for content_language, title, body, node_id in self.engine.execute(
query, {'identifier': identifier}):
results[content_language] = {
'language': content_language,
'title': title,
'body': body,
'node_id': node_id
}
result = results.get(language, None)
if not result and fallback and results:
for fallback_language in self._language_fallback_order:
result = results.get(fallback_language)
if result:
break
if not result:
result = results.itervalues().next()
if result:
result['edit'] = urllib.quote("/%s/node/%s/edit" %
(language, str(result['node_id'])))
result['body'] = literal(result['body'])
return result
def get_drupal_user_id(self, username):
result = self.engine.execute(
"SELECT uid FROM users_field_data WHERE name = %(name)s",
{'name': username})
for row in result:
return row[0]
raise NotFound
def get_drupal_session_cookie(self):
'''returns tuple of (cookie_name, cookie_value)'''
request_cookies = request.cookies
session_cookie = None
for cookie_key in request_cookies:
if cookie_key[:5] == 'SSESS':
session_cookie = (cookie_key, request_cookies[cookie_key])
return session_cookie
def get_drupal_session_token(self, domain, service, cookie_header=''):
'''return text of X-CSRF-Token)'''
token_url = 'https://' + domain + '/' + service + '/?q=services/session/token'
verify_cert = config.get('ckanext.drupal8.development_cert',
'') or True
token_request = requests.get(token_url,
headers={"Cookie": cookie_header},
verify=verify_cert)
token = token_request.text
return token
def get_helpers(self):
return {
'service_alerts': self._service_alerts,
'fetch_drupal_content': self._fetch_drupal_content
}
def get_auth_functions(self):
return {'user_delete_me': user_delete_me}
class SpatialMetadata(SingletonPlugin):
''' '''
implements(interfaces.IPackageController, inherit=True)
implements(interfaces.IConfigurable, inherit=True)
implements(interfaces.IConfigurer, inherit=True)
implements(interfaces.ITemplateHelpers, inherit=True)
def configure(self, config):
'''
:param config:
'''
from ckanext.spatial.model.package_extent import setup as setup_model
if not toolkit.asbool(config.get(u'ckan.spatial.testing', u'False')):
log.debug(u'Setting up the spatial model')
setup_model()
def update_config(self, config):
'''Set up the resource library, public directory and
template directory for all the spatial extensions
:param config:
'''
toolkit.add_public_directory(config, u'public')
toolkit.add_template_directory(config, u'templates')
toolkit.add_resource(u'public', u'ckanext-spatial')
# Add media types for common extensions not included in the mimetypes
# module
mimetypes.add_type(u'application/json', u'.geojson')
mimetypes.add_type(u'application/gml+xml', u'.gml')
def create(self, package):
'''
:param package:
'''
self.check_spatial_extra(package)
def edit(self, package):
'''
:param package:
'''
self.check_spatial_extra(package)
def check_spatial_extra(self, package):
'''For a given package, looks at the spatial extent (as given in the
extra "spatial" in GeoJSON format) and records it in PostGIS.
:param package:
'''
from ckanext.spatial.lib import save_package_extent
if not package.id:
log.warning(
u'Couldn\'t store spatial extent because no id was provided for the '
u'package')
return
# TODO: deleted extra
for extra in package.extras_list:
if extra.key == u'spatial':
if extra.state == u'active' and extra.value:
try:
log.debug(u'Received: %r' % extra.value)
geometry = json.loads(extra.value)
except ValueError, e:
error_dict = {
u'spatial':
[u'Error decoding JSON object: %s' % str(e)]
}
raise toolkit.ValidationError(
error_dict,
error_summary=package_error_summary(error_dict))
except TypeError, e:
error_dict = {
u'spatial':
[u'Error decoding JSON object: %s' % str(e)]
}
raise toolkit.ValidationError(
error_dict,
error_summary=package_error_summary(error_dict))
try:
save_package_extent(package.id, geometry)
except ValueError, e:
error_dict = {
u'spatial':
[u'Error creating geometry: %s' % str(e)]
}
raise toolkit.ValidationError(
error_dict,
error_summary=package_error_summary(error_dict))
except Exception, e:
if bool(os.getenv(u'DEBUG')):
raise
error_dict = {u'spatial': [u'Error: %s' % str(e)]}
raise toolkit.ValidationError(
error_dict,
error_summary=package_error_summary(error_dict))
class GoogleAnalyticsPlugin(p.SingletonPlugin):
p.implements(p.IConfigurable, inherit=True)
p.implements(p.IGenshiStreamFilter, inherit=True)
p.implements(p.IRoutes, inherit=True)
p.implements(p.IConfigurer, inherit=True)
p.implements(p.ITemplateHelpers)
def configure(self, config):
'''Load config settings for this extension from config file.
See IConfigurable.
'''
if 'googleanalytics.id' not in config:
msg = "Missing googleanalytics.id in config"
raise GoogleAnalyticsException(msg)
self.googleanalytics_id = config['googleanalytics.id']
self.googleanalytics_domain = config.get('googleanalytics.domain',
'auto')
self.googleanalytics_javascript_url = h.url_for_static(
'/scripts/ckanext-googleanalytics.js')
# If resource_prefix is not in config file then write the default value
# to the config dict, otherwise templates seem to get 'true' when they
# try to read resource_prefix from config.
if 'googleanalytics_resource_prefix' not in config:
config['googleanalytics_resource_prefix'] = (
commands.DEFAULT_RESOURCE_URL_TAG)
self.googleanalytics_resource_prefix = config[
'googleanalytics_resource_prefix']
self.show_downloads = converters.asbool(
config.get('googleanalytics.show_downloads', True))
self.track_events = converters.asbool(
config.get('googleanalytics.track_events', False))
if not converters.asbool(config.get('ckan.legacy_templates', 'false')):
p.toolkit.add_resource('fanstatic_library',
'ckanext-googleanalytics')
def update_config(self, config):
'''Change the CKAN (Pylons) environment configuration.
See IConfigurer.
'''
if converters.asbool(config.get('ckan.legacy_templates', 'false')):
p.toolkit.add_template_directory(config, 'legacy_templates')
p.toolkit.add_public_directory(config, 'legacy_public')
else:
p.toolkit.add_template_directory(config, 'templates')
def after_map(self, map):
'''Add new routes that this extension's controllers handle.
See IRoutes.
'''
map.redirect("/analytics/package/top", "/analytics/dataset/top")
map.connect(
'analytics',
'/analytics/dataset/top',
controller='ckanext.googleanalytics.controller:GAController',
action='view')
return map
def filter(self, stream):
'''Insert Google Analytics code into legacy Genshi templates.
This is called by CKAN whenever any page is rendered, _if_ using old
CKAN 1.x legacy templates. If using new CKAN 2.0 Jinja templates, the
template helper methods below are used instead.
See IGenshiStreamFilter.
'''
log.info("Inserting Google Analytics code into template")
# Add the Google Analytics tracking code into the page header.
header_code = genshi.HTML(
gasnippet.header_code %
(self.googleanalytics_id, self.googleanalytics_domain))
stream = stream | genshi.filters.Transformer('head').append(
header_code)
# Add the Google Analytics Event Tracking script into the page footer.
if self.track_events:
footer_code = genshi.HTML(gasnippet.footer_code %
self.googleanalytics_javascript_url)
stream = stream | genshi.filters.Transformer(
'body/div[@id="scripts"]').append(footer_code)
routes = pylons.request.environ.get('pylons.routes_dict')
action = routes.get('action')
controller = routes.get('controller')
if ((controller == 'package'
and action in ['search', 'read', 'resource_read'])
or (controller == 'group' and action == 'read')):
log.info("Tracking of resource downloads")
# add download tracking link
def js_attr(name, event):
attrs = event[1][1]
href = attrs.get('href').encode('utf-8')
link = '%s%s' % (self.googleanalytics_resource_prefix,
urllib.quote(href))
js = "javascript: _gaq.push(['_trackPageview', '%s']);" % link
return js
# add some stats
def download_adder(stream):
download_html = '''<span class="downloads-count">
[downloaded %s times]</span>'''
count = None
for mark, (kind, data, pos) in stream:
if mark and kind == genshi.core.START:
href = data[1].get('href')
if href:
count = dbutil.get_resource_visits_for_url(href)
if count and mark is genshi.filters.transform.EXIT:
# emit count
yield genshi.filters.transform.INSIDE, (
genshi.core.TEXT,
genshi.HTML(download_html % count), pos)
yield mark, (kind, data, pos)
# perform the stream transform
stream = stream | genshi.filters.Transformer(
'//a[contains(@class, "resource-url-analytics")]').attr(
'onclick', js_attr)
if (self.show_downloads and action == 'read'
and controller == 'package'):
stream = stream | genshi.filters.Transformer(
'//a[contains(@class, "resource-url-analytics")]').apply(
download_adder)
stream = stream | genshi.filters.Transformer('//head').append(
genshi.HTML(gasnippet.download_style))
return stream
def get_helpers(self):
'''Return the CKAN 2.0 template helper functions this plugin provides.
See ITemplateHelpers.
'''
return {'googleanalytics_header': self.googleanalytics_header}
def googleanalytics_header(self):
'''Render the googleanalytics_header snippet for CKAN 2.0 templates.
This is a template helper function that renders the
googleanalytics_header jinja snippet. To be called from the jinja
templates in this extension, see ITemplateHelpers.
'''
data = {
'googleanalytics_id': self.googleanalytics_id,
'googleanalytics_domain': self.googleanalytics_domain
}
return p.toolkit.render_snippet(
'googleanalytics/snippets/googleanalytics_header.html', data)
class UebPackagePlugins(p.SingletonPlugin):
# Set inherit=True so that we don't have to implement all functions of this interface
p.implements(p.IRoutes, inherit=True)
p.implements(p.IConfigurer)
p.implements(p.ITemplateHelpers, inherit=True)
#Ref: http://docs.ckan.org/sq/latest/resources.html#resources-within-extensions
p.toolkit.add_resource('public', 'uebresources')
# Update CKAN's config settings, see the IConfigurer plugin interface.
def update_config(self, config):
# Tell CKAN to use the template files in
# ckanext-uebpackage/ckanext/uebpackage/templates.
p.toolkit.add_template_directory(config, 'templates')
# add the extension's public dir path so that
# ckan can find any resources used from this path
# get the current dir path (here) for this plugin
here = os.path.dirname(__file__)
rootdir = os.path.dirname(os.path.dirname(here))
our_public_dir = os.path.join(rootdir, 'ckanext', 'uebpackage',
'public')
config['extra_public_paths'] = ','.join(
[our_public_dir,
config.get('extra_public_paths', '')])
# Update CKAN's map settings, see the IRoutes plugin interface.
def before_map(self, map):
# Here create route shortcuts to all your extension's controllers and their actions
map.connect(
'/uebpackage/createform',
controller=
'ckanext.uebpackage.controllers.packagecreate:PackagecreateController',
action='packagecreateform')
map.connect(
'/uebpackage/createformsubmit',
controller=
'ckanext.uebpackage.controllers.packagecreate:PackagecreateController',
action='submit')
map.connect(
'/uebpackage/selectpackagetoexecute',
controller=
'ckanext.uebpackage.controllers.uebexecute:UEBexecuteController',
action='select_model_package')
map.connect(
'/uebpackage/ueb_execute/{pkg_id}',
controller=
'ckanext.uebpackage.controllers.uebexecute:UEBexecuteController',
action='execute')
map.connect(
'/uebpackage/ueb_execute_status/{pkg_id}',
controller=
'ckanext.uebpackage.controllers.uebexecute:UEBexecuteController',
action='check_package_run_status')
map.connect(
'/uebpackage/check_package_build_status/{pkg_id}',
controller=
'ckanext.uebpackage.controllers.packagecreate:PackagecreateController',
action='check_package_build_status')
map.connect(
'/uebpackage/retrieve_input_package/{pkg_id}',
controller=
'ckanext.uebpackage.controllers.packagecreate:PackagecreateController',
action='retrieve_input_package')
map.connect(
'/uebpackage/retrieve_output_package/{pkg_id}',
controller=
'ckanext.uebpackage.controllers.uebexecute:UEBexecuteController',
action='retrieve_output_package')
return map
# ITemplateHelpers method implementation
def get_helpers(self):
# Template helper function names (e.g: 'uebpackage_build_nav_main')should begin with the name of the
# extension they belong to, to avoid clashing with functions from
# other extensions.
return {
'uebpackage_build_nav_main': uebpackage_build_main_navigation,
'get_package_extras': get_package_extras
}
class GoogleAnalyticsPlugin(p.SingletonPlugin):
p.implements(p.IConfigurable, inherit=True)
p.implements(p.IRoutes, inherit=True)
p.implements(p.IConfigurer, inherit=True)
p.implements(p.ITemplateHelpers)
analytics_queue = Queue.Queue()
def configure(self, config):
'''Load config settings for this extension from config file.
See IConfigurable.
'''
if 'googleanalytics.id' not in config:
msg = "Missing googleanalytics.id in config"
raise GoogleAnalyticsException(msg)
self.googleanalytics_id = config['googleanalytics.id']
self.googleanalytics_id2 = config.get('googleanalytics.id2')
self.googleanalytics_domain = config.get('googleanalytics.domain',
'auto')
self.googleanalytics_fields = ast.literal_eval(
config.get('googleanalytics.fields', '{}'))
googleanalytics_linked_domains = config.get(
'googleanalytics.linked_domains', '')
self.googleanalytics_linked_domains = [
x.strip() for x in googleanalytics_linked_domains.split(',') if x
]
if self.googleanalytics_linked_domains:
self.googleanalytics_fields['allowLinker'] = 'true'
self.googleanalytics_javascript_url = h.url_for_static(
'/scripts/ckanext-googleanalytics.js')
# If resource_prefix is not in config file then write the default value
# to the config dict, otherwise templates seem to get 'true' when they
# try to read resource_prefix from config.
if 'googleanalytics_resource_prefix' not in config:
config['googleanalytics_resource_prefix'] = (
commands.DEFAULT_RESOURCE_URL_TAG)
self.googleanalytics_resource_prefix = config[
'googleanalytics_resource_prefix']
self.show_downloads = converters.asbool(
config.get('googleanalytics.show_downloads', True))
self.track_events = converters.asbool(
config.get('googleanalytics.track_events', False))
if not converters.asbool(config.get('ckan.legacy_templates', 'false')):
p.toolkit.add_resource('fanstatic_library',
'ckanext-googleanalytics')
# spawn a pool of 5 threads, and pass them queue instance
for i in range(5):
t = AnalyticsPostThread(self.analytics_queue)
t.setDaemon(True)
t.start()
def update_config(self, config):
'''Change the CKAN (Pylons) environment configuration.
See IConfigurer.
'''
if converters.asbool(config.get('ckan.legacy_templates', 'false')):
p.toolkit.add_template_directory(config, 'legacy_templates')
p.toolkit.add_public_directory(config, 'legacy_public')
else:
p.toolkit.add_template_directory(config, 'templates')
def before_map(self, map):
'''Add new routes that this extension's controllers handle.
See IRoutes.
'''
# Helpers to reduce code clutter
GET = dict(method=['GET'])
PUT = dict(method=['PUT'])
POST = dict(method=['POST'])
DELETE = dict(method=['DELETE'])
GET_POST = dict(method=['GET', 'POST'])
# intercept API calls that we want to capture analytics on
register_list = [
'package', 'dataset', 'resource', 'tag', 'group', 'related',
'revision', 'licenses', 'rating', 'user', 'activity'
]
register_list_str = '|'.join(register_list)
# /api ver 3 or none
with SubMapper(
map,
controller='ckanext.googleanalytics.controller:GAApiController',
path_prefix='/api{ver:/3|}',
ver='/3') as m:
m.connect('/action/{logic_function}',
action='action',
conditions=GET_POST)
# /api ver 1, 2, 3 or none
with SubMapper(
map,
controller='ckanext.googleanalytics.controller:GAApiController',
path_prefix='/api{ver:/1|/2|/3|}',
ver='/1') as m:
m.connect('/search/{register}', action='search')
# /api/rest ver 1, 2 or none
with SubMapper(
map,
controller='ckanext.googleanalytics.controller:GAApiController',
path_prefix='/api{ver:/1|/2|}',
ver='/1',
requirements=dict(register=register_list_str)) as m:
m.connect('/rest/{register}', action='list', conditions=GET)
m.connect('/rest/{register}', action='create', conditions=POST)
m.connect('/rest/{register}/{id}', action='show', conditions=GET)
m.connect('/rest/{register}/{id}', action='update', conditions=PUT)
m.connect('/rest/{register}/{id}',
action='update',
conditions=POST)
m.connect('/rest/{register}/{id}',
action='delete',
conditions=DELETE)
with SubMapper(
map,
controller=
'ckanext.googleanalytics.controller:GADatastoreController'
) as m:
m.connect('/datastore/dump/{resource_id}', action='dump')
m.connect('/datastore/download/{resource_id}', action='dump')
return map
def after_map(self, map):
'''Add new routes that this extension's controllers handle.
See IRoutes.
'''
self.modify_resource_download_route(map)
map.redirect("/analytics/package/top", "/analytics/dataset/top")
map.connect(
'analytics',
'/analytics/dataset/top',
controller='ckanext.googleanalytics.controller:GAController',
action='view')
return map
def get_helpers(self):
'''Return the CKAN 2.0 template helper functions this plugin provides.
See ITemplateHelpers.
'''
return {'googleanalytics_header': self.googleanalytics_header}
def googleanalytics_header(self):
'''Render the googleanalytics_header snippet for CKAN 2.0 templates.
This is a template helper function that renders the
googleanalytics_header jinja snippet. To be called from the jinja
templates in this extension, see ITemplateHelpers.
'''
data = {
'googleanalytics_id': self.googleanalytics_id,
'googleanalytics_id2': self.googleanalytics_id2,
'googleanalytics_domain': self.googleanalytics_domain,
'googleanalytics_fields': str(self.googleanalytics_fields),
'googleanalytics_linked_domains':
self.googleanalytics_linked_domains
}
return p.toolkit.render_snippet(
'googleanalytics/snippets/googleanalytics_header.html', data)
def modify_resource_download_route(self, map):
'''Modifies resource_download method in related controller
to attach GA tracking code.
'''
if '_routenames' in map.__dict__:
if 'resource_download' in map.__dict__['_routenames']:
route_data = map.__dict__['_routenames'][
'resource_download'].__dict__
route_controller = route_data['defaults']['controller'].split(
':')
module = importlib.import_module(route_controller[0])
controller_class = getattr(module, route_controller[1])
controller_class.resource_download = post_analytics_decorator(
controller_class.resource_download)
else:
# If no custom uploader applied, use the default one
PackageController.resource_download = post_analytics_decorator(
PackageController.resource_download)
class SchemaPlugin(plugins.SingletonPlugin, tk.DefaultDatasetForm):
plugins.implements(plugins.IValidators)
plugins.implements(plugins.IDatasetForm)
plugins.implements(plugins.IPackageController, inherit=True)
# IValidators
def get_validators(self): # noqa
return {
'convert_list_to_string': converters.convert_list_to_string,
'convert_string_to_list': converters.convert_string_to_list,
'default_conversion': converters.default,
'single_value': validators.single_valued,
'multi_value': validators.multi_valued,
'is_string': validators.string,
'is_bool': validators.boolean,
'is_uri': validators.uri,
'is_date': validators.date,
'is_number': validators.number,
'controlled_vocabulary': validators.in_vocabulary,
'taxonomy': validators.in_taxonomy,
'determine_communities': validators.extract_communities,
'contact_point': validators.contact_point,
'temporal': validators.temporal,
'date_planned': validators.date_planned,
'legal_foundation': validators.legal_foundation,
'checksum': validators.checksum,
'rights': validators.rights,
'spatial': validators.spatial,
'epsg_28992': validators.epsg28992,
'postcode_huisnummer': validators.postcode_huisnummer
}
# IDatasetForm
def is_fallback(self): # noqa
return tk.asbool(
config.get('ckan.ckanext-dataoverheid.is_fallback', True))
def package_types(self): # noqa
return tk.aslist(
config.get('ckan.ckanext-dataoverheid.package_types', []))
def create_package_schema(self):
schema = super(SchemaPlugin, self).create_package_schema()
schema = dcat_ap_donl.create_schema(schema)
schema = dataoverheid.create_schema(schema)
return schema
def update_package_schema(self):
schema = super(SchemaPlugin, self).update_package_schema()
schema = dcat_ap_donl.update_schema(schema)
schema = dataoverheid.update_schema(schema)
return schema
def show_package_schema(self):
schema = super(SchemaPlugin, self).show_package_schema()
schema = dcat_ap_donl.show_schema(schema)
schema = dataoverheid.show_schema(schema)
return schema
# IPackageController
def before_index(self, data_dict): # noqa
return transformers.transform_multivalued_properties(data_dict)
def after_show(self, context, data_dict): # noqa
return context, transformers.remove_properties(data_dict)
class OLGeoView(GeoViewMixin, GeoViewBase):
p.implements(p.ITemplateHelpers)
GEOVIEW_FORMATS = [
"kml",
"geojson",
"gml",
"wms",
"wfs",
"esrigeojson",
"gft",
"arcgis_rest",
"wmts",
"esri rest",
]
# ITemplateHelpers
def get_helpers(self):
return {
"get_common_map_config_geoviews": utils.get_common_map_config,
"get_openlayers_viewer_config": utils.get_openlayers_viewer_config,
}
# IResourceView
def info(self):
return {
"name": "geo_view",
"title": "Map viewer (OpenLayers)",
"icon": "globe",
"iframed": True,
"default_title": toolkit._("Map viewer"),
"schema": {
"feature_hoveron": [ignore_empty, boolean_validator],
"feature_style": [ignore_empty],
},
}
def can_view(self, data_dict):
format_lower = data_dict["resource"].get("format", "").lower()
same_domain = on_same_domain(data_dict)
# Guess from file extension
if not format_lower and data_dict["resource"].get("url"):
format_lower = self._guess_format_from_extension(
data_dict["resource"]["url"])
if not format_lower:
return False
view_formats = toolkit.config.get("ckanext.geoview.ol_viewer.formats",
"")
if view_formats:
view_formats = view_formats.split(" ")
else:
view_formats = self.GEOVIEW_FORMATS
correct_format = format_lower in view_formats
can_preview_from_domain = self.proxy_enabled or same_domain
return correct_format and can_preview_from_domain
def view_template(self, context, data_dict):
return "dataviewer/openlayers.html"
def form_template(self, context, data_dict):
return "dataviewer/openlayers_form.html"
def _guess_format_from_extension(self, url):
try:
parsed_url = urlparse(url)
format_lower = (os.path.splitext(parsed_url.path)[1][1:].encode(
"ascii", "ignore").lower())
except ValueError as e:
log.error("Invalid URL: {0}, {1}".format(url, e))
format_lower = ""
return format_lower
def setup_template_variables(self, context, data_dict):
import ckanext.resourceproxy.plugin as proxy
same_domain = on_same_domain(data_dict)
if not data_dict["resource"].get("format"):
data_dict["resource"][
"format"] = self._guess_format_from_extension(
data_dict["resource"]["url"])
if self.proxy_enabled and not same_domain:
proxy_url = proxy.get_proxified_resource_url(data_dict)
proxy_service_url = utils.get_proxified_service_url(data_dict)
else:
proxy_url = data_dict["resource"]["url"]
proxy_service_url = data_dict["resource"]["url"]
gapi_key = toolkit.config.get("ckanext.geoview.gapi_key")
return {
"resource_view_json":
"resource_view" in data_dict
and json.dumps(data_dict["resource_view"]),
"proxy_service_url":
proxy_service_url,
"proxy_url":
proxy_url,
"gapi_key":
gapi_key,
"basemapsConfig":
self.basemapsConfig,
}
class GeoJSONView(GeoViewBase):
p.implements(p.ITemplateHelpers, inherit=True)
GeoJSON = ['gjson', 'geojson']
def update_config(self, config):
super(GeoJSONView, self).update_config(config)
mimetypes.add_type('application/geo+json', '.geojson')
# IResourceView (CKAN >=2.3)
def info(self):
return {
'name': 'geojson_view',
'title': 'GeoJSON',
'icon': 'map-marker',
'iframed': True,
'default_title': toolkit._('GeoJSON'),
}
def can_view(self, data_dict):
resource = data_dict['resource']
format_lower = resource.get('format', '').lower()
same_domain = on_same_domain(data_dict)
if format_lower in self.GeoJSON:
return same_domain or self.proxy_enabled
return False
def view_template(self, context, data_dict):
return 'dataviewer/geojson.html'
# IResourcePreview (CKAN < 2.3)
def can_preview(self, data_dict):
format_lower = data_dict['resource']['format'].lower()
correct_format = format_lower in self.GeoJSON
can_preview_from_domain = (self.proxy_enabled or
data_dict['resource'].get('on_same_domain'))
quality = 2
if toolkit.check_ckan_version('2.1'):
if correct_format:
if can_preview_from_domain:
return {'can_preview': True, 'quality': quality}
else:
return {
'can_preview': False,
'fixable': 'Enable resource_proxy',
'quality': quality
}
else:
return {'can_preview': False, 'quality': quality}
return correct_format and can_preview_from_domain
def preview_template(self, context, data_dict):
return 'dataviewer/geojson.html'
def setup_template_variables(self, context, data_dict):
import ckanext.resourceproxy.plugin as proxy
self.same_domain = data_dict['resource'].get('on_same_domain')
if self.proxy_enabled and not self.same_domain:
data_dict['resource']['original_url'] = \
data_dict['resource'].get('url')
data_dict['resource']['url'] = \
proxy.get_proxified_resource_url(data_dict)
# ITemplateHelpers
def get_helpers(self):
return {
'get_common_map_config_geojson': get_common_map_config,
'geojson_get_max_file_size': get_max_file_size,
}
class Cioos_HarvestPlugin(plugins.SingletonPlugin):
plugins.implements(plugins.IConfigurer)
plugins.implements(ISpatialHarvester, inherit=True)
plugins.implements(plugins.IOrganizationController, inherit=True)
# IOrganizationController
def read(self, entity):
pass
def create(self, entity):
if hasattr(entity, 'title_translated'):
if entity.title_translated == '{}' or not entity.title_translated:
toolkit.get_action('organization_patch')(data_dict={
'id':
entity.id,
'title':
entity.title,
'title_translated':
'{"en":"%s", "fr":"%s"}' % (entity.title, entity.title)
})
return entity
def edit(self, entity):
pass
def delete(self, entity):
pass
def before_view(self, pkg_dict):
return pkg_dict
# IConfigurer
def update_config(self, config_):
toolkit.add_template_directory(config_, 'templates')
toolkit.add_public_directory(config_, 'public')
toolkit.add_resource('fanstatic', 'cioos_harvest')
# ISpatialHarvester
def get_validators(self):
return [MyValidator]
def from_json(self, val):
try:
new_val = json.loads(val)
except Exception:
new_val = val
return new_val
def _get_object_extra(self, harvest_object, key):
'''
Helper function for retrieving the value from a harvest object extra,
given the key, copied from ckanext-spatial/ckanext/spatial/harvesters/base.py
'''
for extra in harvest_object.extras:
if extra.key == key:
return extra.value
return None
def trim_values(self, values):
if (isinstance(values, Number)):
return values
elif (isinstance(values, list)):
return [self.trim_values(x) for x in values]
elif (isinstance(values, dict)):
return {k.strip(): self.trim_values(v) for k, v in values.items()}
elif (isinstance(values, str)):
try:
json_object = json.loads(values)
except ValueError:
return values.strip()
else:
return json.dumps(self.trim_values(json_object))
return values
def cioos_guess_resource_format(self, url, use_mimetypes=True):
'''
Given a URL try to guess the best format to assign to the resource
This function does not replace the guess_resource_format() in the base
spatial harvester. In stead it adds some resource and file types that
are missing from that function.
Returns None if no format could be guessed.
'''
url = url.lower().strip()
resource_types = {
# ERDDAP
'ERDDAP': ('/erddap/', ),
}
for resource_type, parts in resource_types.items():
if any(part in url for part in parts):
return resource_type
file_types = {
'CSV': ('csv', ),
'PDF': ('pdf', ),
'TXT': ('txt', ),
'XML': ('xml', ),
'HTML': ('html', ),
'JSON': ('json', ),
}
for file_type, extensions in file_types.items():
if any(url.endswith(extension) for extension in extensions):
return file_type
return None
def get_package_dict(self, context, data_dict):
package_dict = data_dict['package_dict']
iso_values = data_dict['iso_values']
harvest_object = data_dict['harvest_object']
source_config = json.loads(data_dict['harvest_object'].source.config)
xml_location_url = self._get_object_extra(data_dict['harvest_object'],
'waf_location')
xml_modified_date = self._get_object_extra(data_dict['harvest_object'],
'waf_modified_date')
# convert extras key:value list to dictinary
extras = {x['key']: x['value'] for x in package_dict.get('extras', [])}
extras['xml_location_url'] = xml_location_url
if xml_modified_date:
extras['xml_modified_date'] = xml_modified_date
# copy some fields over from iso_values if they exist
if (iso_values.get('limitations-on-public-access')):
extras['limitations-on-public-access'] = iso_values.get(
'limitations-on-public-access')
if (iso_values.get('access-constraints')):
extras['access-constraints'] = iso_values.get('access-constraints')
if (iso_values.get('use-constraints')):
extras['use-constraints'] = iso_values.get('use-constraints')
if (iso_values.get('use-constraints-code')):
extras['use-constraints-code'] = iso_values.get(
'use-constraints-code')
if (iso_values.get('legal-constraints-reference-code')):
extras['legal-constraints-reference-code'] = iso_values.get(
'legal-constraints-reference-code')
if (iso_values.get('distributor')):
extras['distributor'] = iso_values.get('distributor')
# load remote xml content
package_dict = _extract_xml_from_harvest_object(
package_dict, harvest_object)
# Handle Scheming, Composit, and Fluent extensions
loaded_plugins = plugins.toolkit.config.get("ckan.plugins")
if 'scheming_datasets' in loaded_plugins:
# composite = 'composite' in loaded_plugins
fluent = 'fluent' in loaded_plugins
log.debug(
'#### Scheming, Composite, or Fluent extensions found, processing dictinary ####'
)
schema = plugins.toolkit.h.scheming_get_dataset_schema('dataset')
# Package name, default harvester uses title or guid in that order.
# we want to reverse that order, so guid or title. Also use english
# title only for name
title_as_name = self.from_json(package_dict.get(
'title', '{}')).get('en', package_dict['name'])
name = munge.munge_name(extras.get('guid', title_as_name)).lower()
package_dict['name'] = name
# populate license_id
package_dict['license_id'] = iso_values.get(
'legal-constraints-reference-code') or iso_values.get(
'use-constraints') or 'CC-BY-4.0'
# populate citation
package_dict['citation'] = iso_values.get('citation')
# populate trlanslation method for bilingual field
notes_translation_method = iso_values.get(
'abstract_translation_method')
title_translation_method = iso_values.get(
'title_translation_method')
if notes_translation_method:
extras['notes_translation_method'] = notes_translation_method
if title_translation_method:
extras['title_translation_method'] = title_translation_method
# iterate over schema fields and update package dictionary as needed
for field in schema['dataset_fields']:
handled_fields = []
self.handle_composite_harvest_dictinary(
field, iso_values, extras, package_dict, handled_fields)
if fluent:
self.handle_fluent_harvest_dictinary(
field, iso_values, package_dict, schema,
handled_fields, source_config)
self.handle_scheming_harvest_dictinary(field, iso_values,
extras, package_dict,
handled_fields)
# populate resource format if missing
for resource in package_dict.get('resources', []):
if not resource.get('format'):
if (resource.get('resource_locator_protocol').startswith(
'http') or resource.get('url').startswith('http')):
resource['format'] = 'text/html'
# set default values
package_dict['progress'] = extras.get('progress', 'onGoing')
package_dict['frequency-of-update'] = extras.get(
'frequency-of-update', 'asNeeded')
extras_as_list = []
for key, value in extras.items():
if package_dict.get(key, ''):
log.error('extras %s found in package dict: key:%s value:%s',
key, key, value)
if isinstance(value, (list, dict)):
extras_as_list.append({'key': key, 'value': json.dumps(value)})
else:
extras_as_list.append({'key': key, 'value': value})
package_dict['extras'] = extras_as_list
# update resource format
resources = package_dict.get('resources', [])
if len(resources):
for resource in resources:
url = resource.get('url', '').strip()
format = resource.get('format') or ''
if url:
format = self.cioos_guess_resource_format(url) or format
resource['format'] = format
package_dict['resources'] = resources
return self.trim_values(package_dict)
def handle_fluent_harvest_dictinary(self, field, iso_values, package_dict,
schema, handled_fields,
harvest_config):
field_name = field['field_name']
if field_name in handled_fields:
return
field_value = {}
if not field.get('preset', '').startswith(u'fluent'):
return
# set default language, default to english
default_language = iso_values.get('metadata-language', 'en')[0:2]
if not default_language:
default_language = 'en'
# handle tag fields
if field.get('preset', '') == u'fluent_tags':
fluent_tags = iso_values.get(field_name, [])
schema_languages = plugins.toolkit.h.fluent_form_languages(
schema=schema)
do_clean = toolkit.asbool(harvest_config.get('clean_tags', False))
# init language key
field_value = {sl: [] for sl in schema_languages}
# process fluent_tags by convert list of language dictionaries into
# a dictionary of language lists
for t in fluent_tags:
tobj = self.from_json(t.get('keyword', t))
if isinstance(tobj, Number):
tobj = str(tobj)
if isinstance(tobj, dict):
for key, value in tobj.items():
if key in schema_languages:
if do_clean:
if isinstance(value, list):
value = [
munge.munge_tag(kw) for kw in value
]
else:
value = munge.munge_tag(value)
field_value[key].append(value)
else:
if do_clean:
tobj = munge.munge_tag(tobj)
field_value[default_language].append(tobj)
package_dict[field_name] = field_value
# update tags with all values from fluent_tags
tag_list = [t['name'] for t in package_dict['tags']]
for item in field_value.get('en', []) + field_value.get('fr', []):
if item not in tag_list:
tag_list.append(item)
package_dict['tags'] = [{'name': t} for t in tag_list]
else:
# Populate translated fields from core. this could have been done in
# the spatial extensions. example 'title' -> 'title_translated'
# strip trailing _translated part of field name
if field_name.endswith(u'_translated'):
package_fn = field_name[:-11]
else:
package_fn = field_name
package_val = package_dict.get(package_fn, '')
field_value = self.from_json(package_val)
if isinstance(field_value,
dict): # assume bilingual values already in data
package_dict[field_name] = field_value
else:
# create bilingual dictionary. This will likely fail validation as it does not contain all the languages
package_dict[field_name] = {}
package_dict[field_name][default_language] = field_value
handled_fields.append(field_name)
def flatten_composite_keys(self, obj, new_obj={}, keys=[]):
for key, value in obj.items():
if isinstance(value, dict):
self.flatten_composite_keys(obj[key], new_obj, keys + [key])
else:
new_obj['_'.join(keys + [key])] = value
return new_obj
def handle_composite_harvest_dictinary(self, field, iso_values, extras,
package_dict, handled_fields):
sep = plugins.toolkit.h.scheming_composite_separator()
field_name = field['field_name']
if field_name in handled_fields:
return
field_value = iso_values.get(field_name, {})
# populate composite fields from multi-level dictionary
if field_value and field.get('simple_subfields'):
if isinstance(field_value, list):
field_value = field_value[0]
field_value = self.flatten_composite_keys(field_value, {}, [])
for key, value in field_value.items():
newKey = field_name + sep + key
package_dict[newKey] = value
# remove from extras so as not to duplicate fields
if extras.get(field_name):
del extras[field_name]
handled_fields.append(field_name)
# populate composite repeating fields
elif field_value and field.get('repeating_subfields'):
if isinstance(field_value, dict):
field_value[0] = field_value
for idx, subitem in enumerate(field_value):
# collapse subfields into one key value pair
subitem = self.flatten_composite_keys(subitem, {}, [])
for key, value in subitem.items():
newKey = field_name + sep + str(idx + 1) + sep + key
package_dict[newKey] = value
# remove from extras so as not to duplicate fields
if extras.get(field_name):
del extras[field_name]
handled_fields.append(field_name)
def handle_scheming_harvest_dictinary(self, field, iso_values, extras,
package_dict, handled_fields):
field_name = field['field_name']
if field_name in handled_fields:
return
iso_field_value = iso_values.get(field_name, {})
extra_field_value = extras.get(field_name, "")
# move schema fields, in extras, to package dictionary
if field_name in extras and not package_dict.get(field_name, ''):
package_dict[field_name] = extra_field_value
del extras[field_name]
handled_fields.append(field_name)
# move schema fields, in iso_values, to package dictionary
elif iso_field_value and not package_dict.get(field_name, ''):
# convert list to single value for select fields (not multi-select)
if field.get('preset', '') == 'select' and isinstance(
iso_field_value, list):
iso_field_value = iso_field_value[0]
package_dict[field_name] = iso_field_value
# remove from extras so as not to duplicate fields
if extras.get(field_name):
del extras[field_name]
handled_fields.append(field_name)
class ChartsPlugin(plugins.SingletonPlugin):
plugins.implements(plugins.IConfigurer, inherit=True)
plugins.implements(plugins.IResourceView, inherit=True)
# IConfigurer
def update_config(self, config):
toolkit.add_template_directory(config, 'templates')
toolkit.add_public_directory(config, 'public')
toolkit.add_resource('fanstatic', 'c3charts')
def info(self):
schema = {
'chart_type': [not_empty],
'key_fields': [not_empty],
'x_fields': [ignore_missing],
'color_scheme': [not_empty],
'header': [ignore_missing],
'measure_unit_x': [ignore_missing],
'measure_unit_y': [ignore_missing],
'text_chart_number_action': [not_empty],
'legend': [not_empty],
'rotated': [ignore_missing],
'data_labels': [ignore_missing],
'x_grid': [ignore_missing],
'y_grid': [ignore_missing],
'remap_key': [ignore_missing],
'aggregate': [ignore_missing]
}
return {
'name': 'Chart builder',
'icon': 'bar-chart-o',
'filterable': True,
'iframed': False,
'schema': schema
}
def can_view(self, data_dict):
return data_dict['resource'].get('datastore_active', False)
def setup_template_variables(self, context, data_dict):
resource = data_dict['resource']
resource_view = data_dict['resource_view']
fields = _get_fields_without_id(resource)
remap_keys = list(fields)
remap_keys.insert(0, {'value': ''})
logger.debug(remap_keys)
return {
'resource':
resource,
'resource_view':
resource_view,
'fields':
fields,
'remap_keys':
remap_keys,
'chart_types': [{
'value': 'Bar Chart'
}, {
'value': 'Stacked Bar Chart'
}, {
'value': 'Donut Chart'
}, {
'value': 'Line Chart'
}, {
'value': 'Pie Chart'
}, {
'value': 'Spline Chart'
}, {
'value': 'Table Chart'
}, {
'value': 'Simple Chart'
}],
'color_schemes': [{
'value':
'#B80000, #995522, #556677, #118888, #115588, '
'#4C3D3D, #2B2B2B, #660000, #221100',
'text':
'Saturated'
}, {
'value':
'#DDBBAA, #79E6F2, #88AA99, #00A864, #228899, '
'#3F797F, #775555, #118855, #008751, #3D4C46',
'text':
'Light'
}, {
'value':
'#ADC0D8, #79AFF2, #8899AA, #0EAAB2, #00A0A8, '
'#776655, #118888, #885511, #3F5C7F, #225599',
'text':
'Pastel'
}, {
'value':
'#ADB1D8, #8899AA, #7983F2, #777752, #887711, '
'#0070C0, #0062A8, #3F457F, #115588, #3D464C',
'text':
'Pastel 2'
}, {
'value':
'#AA9988, #A88600, #779922, #6C7F3F, #887711, '
'#555577, #665500, #665100, #4C493D, #2B2B2V',
'text':
'Contrast'
}],
'text_chart_number_actions': [{
'value': 'substract',
'text': 'Substract last two entries'
}, {
'value': 'average',
'text': 'Average'
}, {
'value': 'last',
'text': 'Show last'
}],
'legend_options': [{
'text': 'Hide',
'value': 'hide'
}, {
'text': 'Right',
'value': 'right'
}, {
'text': 'Bottom',
'value': 'bottom'
}]
}
def view_template(self, context, data_dict):
return 'charts_view.html'
def form_template(self, context, data_dict):
return 'charts_form.html'
class TagmanagerPlugin(plugins.SingletonPlugin):
plugins.implements(plugins.IConfigurer)
# IConfigurer
plugins.implements(plugins.IConfigurable)
plugins.implements(plugins.IRoutes, inherit=True)
#p.implements(p.IDomainObjectModification, inherit=True)
#p.implements(p.IResourceUrlChange)
plugins.implements(plugins.ITemplateHelpers)
def configure(self, config):
self.site_url = config.get('ckan.site_url')
def before_map(self, map):
tagmanager = 'ckanext.tagmanager.controller:TagmanagerController'
map.connect('/tagmanager',
'tagmanager',
controller=tagmanager,
action='index')
map.connect('/tagmanager/edit', controller=tagmanager, action='edit')
map.connect('/tagmanager/index_process_suggestions',
controller=tagmanager,
action='index_process_suggestions')
map.connect('/tagmanager/save_merge_suggestions',
controller=tagmanager,
action='save_merge_suggestions')
map.connect('/tagmanager/merge_0',
controller=tagmanager,
action='merge_0')
map.connect('/tagmanager/merge_1',
controller=tagmanager,
action='merge_1')
map.connect('/tagmanager/merge_2',
controller=tagmanager,
action='merge_2')
map.connect('/tagmanager/merge_form',
controller=tagmanager,
action='merge_form')
map.connect('/tagmanager', controller=tagmanager, action='index')
map.connect('/tagmanager/merge_confirm',
controller=tagmanager,
action='merge_confirm')
map.connect('/tagmanager/merge', controller=tagmanager, action='merge')
map.connect('/tagmanager/merge_do',
controller=tagmanager,
action='merge_do')
map.connect('/tagmanager/delete_confirm',
controller=tagmanager,
action='delete_confirm')
map.connect('/tagmanager/delete',
controller=tagmanager,
action='delete')
return map
def after_map(self, map):
return map
def update_config(self, config_):
toolkit.add_template_directory(config_, 'templates')
toolkit.add_public_directory(config_, 'public')
toolkit.add_resource('fanstatic', 'tagmanager')
def get_helpers(self):
return {
'tagmanager_list_tags': list_tags,
'tagmanager_tag_show': tag_show,
'tagmanager_tags_stats': tags_stats,
'tagmanager_tag_count': tag_count,
'tagmanager_has_suggestions': has_suggestions,
'tagmanager_get_suggestions': get_suggestions,
'tagmanager_get_name': get_name
}
class SchemaPlugin(plugins.SingletonPlugin):
plugins.implements(plugins.IConfigurer)
plugins.implements(plugins.IRoutes, inherit=True)
plugins.implements(plugins.ITemplateHelpers, inherit=False)
plugins.implements(plugins.IPackageController, inherit=True)
plugins.implements(plugins.IFacets, inherit=True)
plugins.implements(plugins.IActions, inherit=True)
plugins.implements(plugins.IAuthFunctions)
plugins.implements(plugins.IResourceController, inherit=True)
def get_helpers(self):
return {
"dataset_type": get_dataset_type,
"edc_tags": get_edc_tags,
"edc_orgs": get_organizations,
"edc_org_branches": get_organization_branches,
"edc_org_title": get_organization_title,
"edc_type_label": edc_type_label,
"edc_state_values": get_state_values,
"edc_username": get_username,
"get_sector": get_suborg_sector,
"get_user_orgs": get_user_orgs,
"get_user_orgs_id": get_user_orgs_id,
"get_user_toporgs": get_user_toporgs,
"get_suborg_sector": get_suborg_sector,
"get_user_dataset_num": get_user_dataset_num,
"get_edc_package": get_package_data,
"is_license_open": is_license_open,
"record_type_label": get_record_type_label,
"get_suborgs": get_suborgs,
"record_is_viewable": record_is_viewable,
"get_espg_id": get_espg_id,
"orgs_user_can_edit": get_orgs_user_can_edit,
"get_facets_selected": get_facets_selected,
"get_facets_unselected": get_facets_unselected,
"get_sectors_list": get_sectors_list,
"get_edc_org": get_edc_org,
"get_iso_topic_values": get_iso_topic_values,
"get_eas_login_url": get_eas_login_url,
"get_fqdn": get_fqdn,
"get_environment_name": get_environment_name,
"get_version": get_version,
"get_bcgov_commit_id": get_bcgov_commit_id,
"googleanalytics_resource_prefix": resource_prefix,
"get_parent_org": get_org_parent,
"size_or_link": size_or_link,
"display_pacific_time": display_pacific_time,
"sort_vocab_list": sort_vocab_list,
"debug_full_info_as_list": debug_full_info_as_list,
"remove_user_link": remove_user_link,
"get_dashboard_config": get_dashboard_config,
"get_ofi_config": get_ofi_config,
"get_ofi_resources": get_ofi_resources,
"get_non_ofi_resources": get_non_ofi_resources,
"can_view_resource": can_view_resource,
"get_pow_config": get_pow_config,
"get_package_tracking": get_package_tracking,
"get_resource_tracking": get_resource_tracking,
"log": log_this
}
def update_config(self, config):
toolkit.add_public_directory(config, 'public')
toolkit.add_template_directory(config, 'templates')
toolkit.add_resource('fanstatic', 'edc_resource')
toolkit.add_resource('public/scripts', 'theme_scripts')
# Customizing action mapping
def before_map(self, map):
from routes.mapper import SubMapper
package_controller = 'ckanext.bcgov.controllers.package:EDCPackageController'
user_controller = 'ckanext.bcgov.controllers.user:EDCUserController'
org_controller = 'ckanext.bcgov.controllers.organization:EDCOrganizationController'
site_map_controller = 'ckanext.bcgov.controllers.site_map:GsaSitemapController'
api_controller = 'ckanext.bcgov.controllers.api:EDCApiController'
ofi_controller = 'ckanext.bcgov.controllers.ofi:EDCOfiController'
GET_POST = dict(method=['GET', 'POST'])
map.connect('package_index',
'/',
controller=package_controller,
action='index')
with SubMapper(map, controller=package_controller) as m:
m.connect('/dataset/add', action='typeSelect')
m.connect('add dataset', '/dataset/new', action='new')
m.connect(
'new edc dataset',
'/{dataset_type}/new',
action='new',
requirements=dict(dataset_type='|'.join(
['Dataset', 'Geographic', 'WebService', 'Application'])))
m.connect('search',
'/dataset',
action='search',
highlight_actions='index search')
m.connect('dataset_read',
'/dataset/{id}',
action='read',
ckan_icon='sitemap')
m.connect(
'read edc dataset',
'/{dataset_type}/{id}',
action='read',
ckan_icon='sitemap',
requirements=dict(dataset_type='|'.join(
['Dataset', 'Geographic', 'WebService', 'Application'])))
m.connect('duplicate',
'/dataset/duplicate/{id}/{package_type}',
action='duplicate')
m.connect('/dataset/{id}/resource/{resource_id}',
action='resource_read')
m.connect('/dataset/{id}/resource_delete/{resource_id}',
action='resource_delete')
m.connect('/authorization-error', action='auth_error')
m.connect('resource_edit',
'/dataset/{id}/resource_edit/{resource_id}',
action='resource_edit',
ckan_icon='edit')
m.connect('new_resource',
'/dataset/new_resource/{id}',
action='new_resource')
m.connect('resources',
'/dataset/resources/{id}',
action='resources')
with SubMapper(map, controller=user_controller) as m:
m.connect('user_dashboard_unpublished',
'/dashboard/unpublished',
action='dashboard_unpublished',
ckan_icon='group')
m.connect('/user/edit', action='edit')
m.connect('/user/activity/{id}/{offset}', action='activity')
m.connect('user_activity_stream',
'/user/activity/{id}',
action='activity',
ckan_icon='time')
m.connect('user_dashboard',
'/dashboard',
action='dashboard',
ckan_icon='list')
m.connect('user_dashboard_datasets',
'/dashboard/datasets',
action='dashboard_datasets',
ckan_icon='sitemap')
m.connect('user_dashboard_organizations',
'/dashboard/organizations',
action='dashboard_organizations',
ckan_icon='building')
m.connect('/dashboard/{offset}', action='dashboard')
m.connect('user_follow', '/user/follow/{id}', action='follow')
m.connect('/user/unfollow/{id}', action='unfollow')
m.connect('user_followers',
'/user/followers/{id:.*}',
action='followers',
ckan_icon='group')
m.connect('user_edit',
'/user/edit/{id:.*}',
action='edit',
ckan_icon='cog')
m.connect('user_delete', '/user/delete/{id}', action='delete')
m.connect('/user/reset/{id:.*}', action='perform_reset')
m.connect('register', '/user/register', action='register')
m.connect('login', '/user/login', action='login')
m.connect('/user/_logout', action='logout')
m.connect('/user/logged_in', action='logged_in')
m.connect('/user/logged_out', action='logged_out')
m.connect('/user/logged_out_redirect', action='logged_out_page')
m.connect('/user/reset', action='request_reset')
m.connect('/user/me', action='me')
m.connect('/user/set_lang/{lang}', action='set_lang')
m.connect('user_datasets',
'/user/{id:.*}',
action='read',
ckan_icon='sitemap')
m.connect('user_index', '/user', action='index')
with SubMapper(map, controller=org_controller) as m:
m.connect('organizations_index', '/organization', action='index')
m.connect('/organization/list', action='list')
m.connect('/organization/new', action='new')
m.connect('/organization/{action}/{id}',
requirements=dict(action='|'.join([
'delete', 'admins', 'member_new', 'member_delete',
'history'
])))
m.connect('organization_activity',
'/organization/activity/{id}',
action='activity',
ckan_icon='time')
m.connect('organization_about',
'/organization/about/{id}',
action='about',
ckan_icon='info-sign')
m.connect('organization_edit',
'/organization/edit/{id}',
action='edit',
ckan_icon='edit')
m.connect('organization_members',
'/organization/members/{id}',
action='members',
ckan_icon='group')
m.connect('organization_bulk_process',
'/organization/bulk_process/{id}',
action='bulk_process',
ckan_icon='sitemap')
m.connect('organization_read',
'/organization/{id}',
action='read',
ckan_icon='sitemap')
map.connect('sitemap',
'/sitemap.html',
controller=site_map_controller,
action='view')
map.connect('sitemap',
'/sitemap.xml',
controller=site_map_controller,
action='read')
with SubMapper(map,
controller=api_controller,
path_prefix='/api{ver:/1|/2|/3|}',
ver='/1') as m:
m.connect('/i18n/{lang}', action='i18n_js_translations')
m.connect('/')
map.connect('ofi api',
'/api/ofi/{call_action}',
controller=ofi_controller,
action='action',
conditions=GET_POST)
map.connect('ofi resource',
'/api/ofi/{format}/{object_name}',
action='action')
m.connect('/action/organization_list_related',
action='organization_list_related',
conditions=GET_POST)
m.connect('/action/{logic_function}',
action='action',
conditions=GET_POST)
map.connect('/admin/trash', controller='admin', action='trash')
map.connect('ckanadmin_trash',
'/admin/trash',
controller='admin',
action='trash',
ckan_icon='trash')
return map
def after_map(self, map):
return map
def before_index(self, pkg_dict):
'''
Makes the sort by name case insensitive.
Note that the search index must be rebuild for the first time in order for the changes to take affect.
'''
title = pkg_dict['title']
if title:
#Assign title to title_string with all characters switched to lower case.
pkg_dict['title_string'] = title.lower()
res_format = pkg_dict.get('res_format', [])
if 'other' in res_format:
# custom download (other) supports a number of formats
res_format.remove('other')
res_format.extend(['shp', 'fgdb', 'e00'])
return pkg_dict
def before_search(self, search_params):
'''
Customizes package search and applies filters based on the dataset metadata-visibility
and user roles.
'''
#Change the default sort order when no query passed
if not search_params.get('q') and search_params.get('sort') in (
None, 'rank'):
search_params[
'sort'] = 'record_publish_date desc, metadata_modified desc'
#Change the query filter depending on the user
if 'fq' in search_params:
fq = search_params['fq']
else:
fq = ''
#need to append solr param q.op to force an AND query
if 'q' in search_params:
q = search_params['q']
else:
q = ''
try:
user_name = c.user or 'visitor'
# There are no restrictions for sysadmin
if c.userobj and c.userobj.sysadmin == True:
fq += ' '
fq = filter_query_regex.sub(r'+\1', fq)
else:
if user_name != 'visitor':
if 'edc_state' not in fq:
fq = filter_query_regex.sub(r'+\1', fq)
fq += ' +(edc_state:("PUBLISHED" OR "PENDING ARCHIVE")'
if 'owner_org' not in fq:
#IDIR users can also see private records of their organizations
user_id = c.userobj.id
#Get the list of orgs that the user is an admin or editor of
user_orgs = get_orgs_user_can_edit(
c.userobj
) #['"' + org + '"' for org in get_orgs_user_can_edit()]
#user_orgs = ['"' + org.get('id') + '"' for org in get_user_orgs(user_id, 'admin')]
#user_orgs += ['"' + org.get('id') + '"' for org in get_user_orgs(user_id, 'editor')]
if user_orgs != []:
fq += ' OR ' + 'owner_org:(' + ' OR '.join(
user_orgs) + ')'
fq += ')'
else:
if fq:
# make all fieds in Filter Query minditory with '+'
fq = filter_query_regex.sub(r'+\1', fq)
# Public user can only view public and published records
fq += ' +(edc_state:("PUBLISHED" OR "PENDING ARCHIVE") AND metadata_visibility:("Public"))'
except Exception:
if 'fq' in search_params:
fq = search_params['fq']
else:
fq = ''
fq += ' +edc_state:("PUBLISHED" OR "PENDING ARCHIVE") +metadata_visibility:("Public")'
search_params['fq'] = fq
return search_params
def before_view(self, pkg_dict):
# CITZEDC808
if not record_is_viewable(pkg_dict, c.userobj):
abort(401,
_('Unauthorized to read package %s') % pkg_dict.get("title"))
return pkg_dict
#def after_update(self, context, pkg_dict):
# If there are no resources added, redirect to the "add resource" page after saving
# if len(pkg_dict.get('resources', [])) == 0:
# toolkit.redirect_to(controller='package', action='new_resource', id=pkg_dict['id'])
def dataset_facets(self, facet_dict, package_type):
'''
Customizes search facet list.
'''
from collections import OrderedDict
facet_dict = OrderedDict()
#Add dataset types and organization sectors to the facet list
facet_dict['license_id'] = _('License')
facet_dict['sector'] = _('Sectors')
facet_dict['type'] = _('Dataset types')
facet_dict['res_format'] = _('Format')
facet_dict['organization'] = _('Organizations')
facet_dict['download_audience'] = _('Download permission')
if c.userobj and c.userobj.sysadmin:
facet_dict['edc_state'] = _('States')
return facet_dict
def group_facets(self, facet_dict, group_type, package_type):
'''
Use the same facets for filtering datasets within group pages
'''
return self.dataset_facets(facet_dict, package_type)
def get_actions(self):
import ckanext.bcgov.logic.action as edc_action
from ckanext.bcgov.logic.ofi import call_action as ofi
return {
'organization_list': edc_action.organization_list,
'edc_package_update': edc_action.edc_package_update,
'edc_package_update_bcgw': edc_action.edc_package_update_bcgw,
'package_update': edc_action.package_update,
'package_autocomplete': edc_action.package_autocomplete,
'check_object_name': ofi.check_object_name,
'file_formats': ofi.file_formats,
'crs_types': ofi.crs_types,
'populate_dataset_with_ofi': ofi.populate_dataset_with_ofi,
'geo_resource_form': ofi.geo_resource_form,
'remove_ofi_resources': ofi.remove_ofi_resources,
'edit_ofi_resources': ofi.edit_ofi_resources,
'get_max_aoi': ofi.get_max_aoi,
'ofi_create_order': ofi.ofi_create_order
}
def get_auth_functions(self):
from ckanext.bcgov.logic.auth import create as edc_auth_create
from ckanext.bcgov.logic.auth.ofi import call_action as ofi
return {
'package_create': edc_auth_create.package_create,
'check_object_name': ofi.check_object_name,
'file_formats': ofi.file_formats,
'crs_types': ofi.crs_types,
'populate_dataset_with_ofi': ofi.populate_dataset_with_ofi,
'geo_resource_form': ofi.geo_resource_form,
'remove_ofi_resources': ofi.remove_ofi_resources,
'edit_ofi_resources': ofi.edit_ofi_resources,
'get_max_aoi': ofi.get_max_aoi,
'ofi_create_order': ofi.ofi_create_order
}
# IResourceController
def before_create(self, context, resource):
# preventative fix for #386 - make sure facet format types are always lowercase;
resource['format'] = resource.get('format', '').lower()
class CollectionPlugin(plugins.SingletonPlugin, DefaultTranslation):
plugins.implements(plugins.IConfigurer)
plugins.implements(plugins.IRoutes, inherit=True)
plugins.implements(plugins.IPackageController, inherit=True)
if toolkit.check_ckan_version(min_version='2.5.0'):
plugins.implements(plugins.ITranslation, inherit=True)
plugins.implements(plugins.IActions, inherit=True)
plugins.implements(plugins.IFacets, inherit=True)
# IConfigurer
def update_config(self, config_):
toolkit.add_template_directory(config_, 'templates')
toolkit.add_public_directory(config_, 'public')
toolkit.add_resource('fanstatic', 'collection')
def update_config_schema(self, schema):
ignore_missing = toolkit.get_validator('ignore_missing')
schema.update({
'ckanext.collection.api_collection_name_or_id':
[ignore_missing, unicode],
})
return schema
# IRoutes
def before_map(self, map):
with SubMapper(
map,
controller='ckanext.collection.controller:CollectionController'
) as m:
m.connect('collection.index', '/collection', action='index')
m.connect('collection.new', '/collection/new', action='new')
m.connect('collection.read', '/collection/:id', action='read')
m.connect('collection.members',
'/collection/:id/members',
action='members')
m.connect('collection.edit', '/collection/edit/:id', action='edit')
m.connect('collection.delete',
'/collection/delete/:id',
action='delete')
m.connect('collection.about',
'/collection/about/:id',
action='about')
m.connect('dataset_collection_list',
'/dataset/collections/{id}',
action='dataset_collection_list',
ckan_icon='picture')
map.redirect('/collections', '/collection')
return map
def before_index(self, data_dict):
groups = json.loads(data_dict.get('data_dict', {})).get('groups', [])
data_dict['collections'] = [
group.get('name', '') for group in groups
if group.get('type', "") == 'collection'
]
groups_to_remove = [
group.get('name', '') for group in groups
if group.get('type', "") == 'collection'
]
data_dict['groups'] = [
group for group in data_dict['groups']
if group not in groups_to_remove
]
return data_dict
def after_search(self, search_results, search_params):
if search_results['search_facets'].get('collections'):
context = {'for_view': True, 'with_private': False}
data_dict = {
'all_fields': True,
'include_extras': True,
'type': 'collection'
}
collections_with_extras = get_action('group_list')(context,
data_dict)
for i, facet in enumerate(
search_results['search_facets']['collections'].get(
'items', [])):
for collection in collections_with_extras:
if facet['name'] == collection['name']:
search_results['search_facets']['collections'][
'items'][i]['title_translated'] = collection.get(
'title_translated')
if not collection.get('title_translated').get('en'):
search_results['search_facets']['collections'][
'items'][i]['title_translated'][
'en'] = collection.get('title')
if not collection.get('title_translated').get('sv'):
search_results['search_facets']['collections'][
'items'][i]['title_translated'][
'sv'] = collection.get('title')
return search_results
# IActions
def get_actions(self):
return {
'group_list_authz': action.group_list_authz,
'api_collection_show': action.api_collection_show
}
# IFacets
def group_facets(self, facets_dict, group_type, package_type):
if (group_type == 'collection'):
facets_dict = OrderedDict()
facets_dict.update({'res_format': _('Formats')})
facets_dict.update(
{'vocab_geographical_coverage': _('Geographical Coverage')})
facets_dict.update({'groups': _('Groups')})
facets_dict.update({'organization': _('Organizations')})
facets_dict.update({'collections': _('Collections')})
return facets_dict
try:
if g.facet_json_data:
print "global value is there..."
except AttributeError:
print "Facet json config is not available. Returning the default facets."
helpers.load_ngds_facets()
#print "search_results: ",search_results
return search_results
def before_view(self,pkg):
# pkg['title'] = "Muhaha"
# print pkg
# TODO - Use for rendering packages. Process resources to get more responsible party information from the email.
return pkg
implements(IFacets, inherit=True)
def dataset_facets(self, facets_dict, package_type):
if package_type == 'harvest':
return OrderedDict([('frequency', 'Frequency'), ('source_type', 'Type')])
ngds_facets = helpers.load_ngds_facets()
if ngds_facets:
facets_dict = ngds_facets
return facets_dict
def organization_facets(self, facets_dict, organization_type, package_type):
class PrivateDatasets(p.SingletonPlugin, tk.DefaultDatasetForm):
p.implements(p.IDatasetForm)
p.implements(p.IAuthFunctions)
p.implements(p.IConfigurer)
p.implements(p.IRoutes, inherit=True)
p.implements(p.IActions)
p.implements(p.IPackageController, inherit=True)
p.implements(p.ITemplateHelpers)
######################################################################
############################ DATASET FORM ############################
######################################################################
def __init__(self, name=None):
self.indexer = search.PackageSearchIndex()
def _modify_package_schema(self):
return {
# remove datasets_with_no_organization_cannot_be_private validator
'private': [
tk.get_validator('ignore_missing'),
tk.get_validator('boolean_validator')
],
constants.ALLOWED_USERS_STR: [
tk.get_validator('ignore_missing'),
conv_val.private_datasets_metadata_checker
],
constants.ALLOWED_USERS: [
conv_val.allowed_users_convert,
tk.get_validator('ignore_missing'),
conv_val.private_datasets_metadata_checker
],
constants.ACQUIRE_URL: [
tk.get_validator('ignore_missing'),
conv_val.private_datasets_metadata_checker,
conv_val.url_checker,
tk.get_converter('convert_to_extras')
],
constants.SEARCHABLE: [
tk.get_validator('ignore_missing'),
conv_val.private_datasets_metadata_checker,
tk.get_converter('convert_to_extras'),
tk.get_validator('boolean_validator')
]
}
def create_package_schema(self):
# grab the default schema in our plugin
schema = super(PrivateDatasets, self).create_package_schema()
schema.update(self._modify_package_schema())
# update resource
# Add our custom_resource_text metadata field to the schema
schema['resources'].update({
'allowed_users': [
tk.get_validator('ignore_missing'),
tk.get_converter('convert_to_extras')
]
})
return schema
def update_package_schema(self):
# grab the default schema in our plugin
schema = super(PrivateDatasets, self).update_package_schema()
schema.update(self._modify_package_schema())
# update resource
# Add our custom_resource_text metadata field to the schema
schema['resources'].update({
'allowed_users': [
tk.get_validator('ignore_missing'),
tk.get_converter('convert_to_extras')
]
})
return schema
def show_package_schema(self):
schema = super(PrivateDatasets, self).show_package_schema()
schema.update({
constants.ALLOWED_USERS:
[conv_val.get_allowed_users,
tk.get_validator('ignore_missing')],
constants.ACQUIRE_URL: [
tk.get_converter('convert_from_extras'),
tk.get_validator('ignore_missing')
],
constants.SEARCHABLE: [
tk.get_converter('convert_from_extras'),
tk.get_validator('ignore_missing')
]
})
# update resource
# Add our custom_resource_text metadata field to the schema
schema['resources'].update({
'allowed_users': [
tk.get_validator('ignore_missing'),
tk.get_converter('convert_from_extras')
]
})
return schema
def is_fallback(self):
# Return True to register this plugin as the default handler for
# package types not handled by any other IDatasetForm plugin.
return True
def package_types(self):
# This plugin doesn't handle any special package types, it just
# registers itself as the default (above).
return []
######################################################################
########################### AUTH FUNCTIONS ###########################
######################################################################
def get_auth_functions(self):
auth_functions = {
'package_show':
auth.package_show,
'package_update':
auth.package_update,
# 'resource_show': auth.resource_show,
constants.PACKAGE_ACQUIRED:
auth.package_acquired,
constants.ACQUISITIONS_LIST:
auth.acquisitions_list
}
# resource_show is not required in CKAN 2.3 because it delegates to
# package_show
if not tk.check_ckan_version(min_version='2.3'):
auth_functions['resource_show'] = auth.resource_show
return auth_functions
######################################################################
############################ ICONFIGURER #############################
######################################################################
def update_config(self, config):
# Add this plugin's templates dir to CKAN's extra_template_paths, so
# that CKAN will use this plugin's custom templates.
tk.add_template_directory(config, 'templates')
# Register this plugin's fanstatic directory with CKAN.
tk.add_resource('fanstatic', 'privatedatasets')
######################################################################
############################## IROUTES ###############################
######################################################################
def before_map(self, m):
# DataSet acquired notification
m.connect(
'user_acquired_datasets',
'/dashboard/acquired',
ckan_icon='shopping-cart',
controller=
'ckanext.privatedatasets.controllers.ui_controller:AcquiredDatasetsControllerUI',
action='user_acquired_datasets',
conditions=dict(method=['GET']))
return m
######################################################################
############################## IACTIONS ##############################
######################################################################
def get_actions(self):
return {
constants.PACKAGE_ACQUIRED: actions.package_acquired,
constants.ACQUISITIONS_LIST: actions.acquisitions_list
}
######################################################################
######################### IPACKAGECONTROLLER #########################
######################################################################
def _delete_pkg_atts(self, pkg_dict, attrs):
for attr in attrs:
if attr in pkg_dict:
del pkg_dict[attr]
def before_index(self, pkg_dict):
if 'extras_' + constants.SEARCHABLE in pkg_dict:
if pkg_dict['extras_searchable'] == 'False':
pkg_dict['capacity'] = 'private'
else:
pkg_dict['capacity'] = 'public'
return pkg_dict
def after_create(self, context, pkg_dict):
session = context['session']
update_cache = False
db.init_db(context['model'])
# Get the users and the package ID
if constants.ALLOWED_USERS in pkg_dict:
allowed_users = pkg_dict[constants.ALLOWED_USERS]
package_id = pkg_dict['id']
# Get current users
users = db.AllowedUser.get(package_id=package_id)
# Delete users and save the list of current users
current_users = []
for user in users:
current_users.append(user.user_name)
if user.user_name not in allowed_users:
session.delete(user)
update_cache = True
# Add non existing users
for user_name in allowed_users:
if user_name not in current_users:
out = db.AllowedUser()
out.package_id = package_id
out.user_name = user_name
out.save()
session.add(out)
update_cache = True
session.commit()
# The cache should be updated. Otherwise, the system may return
# outdated information in future requests
if update_cache:
new_pkg_dict = tk.get_action('package_show')(
{
'model': context['model'],
'ignore_auth': True,
'validate': False,
'use_cache': False
}, {
'id': package_id
})
# Prevent acquired datasets jumping to the first position
revision = tk.get_action('revision_show')(
{
'ignore_auth': True
}, {
'id': new_pkg_dict['revision_id']
})
new_pkg_dict['metadata_modified'] = revision.get(
'timestamp', '')
self.indexer.update_dict(new_pkg_dict)
return pkg_dict
def after_update(self, context, pkg_dict):
return self.after_create(context, pkg_dict)
def after_show(self, context, pkg_dict):
user_obj = context.get('auth_user_obj')
updating_via_api = context.get(constants.CONTEXT_CALLBACK, False)
# allowed_users and searchable fileds can be only viewed by (and only if the dataset is private):
# * the dataset creator
# * the sysadmin
# * users allowed to update the allowed_users list via the notification API
if pkg_dict.get('private') is False or not updating_via_api and (
not user_obj or (pkg_dict['creator_user_id'] != user_obj.id
and not user_obj.sysadmin)):
# The original list cannot be modified
attrs = list(HIDDEN_FIELDS)
self._delete_pkg_atts(pkg_dict, attrs)
return pkg_dict
def after_delete(self, context, pkg_dict):
session = context['session']
package_id = pkg_dict['id']
# Get current users
db.init_db(context['model'])
users = db.AllowedUser.get(package_id=package_id)
# Delete all the users
for user in users:
session.delete(user)
session.commit()
return pkg_dict
def after_search(self, search_results, search_params):
for result in search_results['results']:
# Extra fields should not be returned
# The original list cannot be modified
attrs = list(HIDDEN_FIELDS)
# Additionally, resources should not be included if the user is not allowed
# to show the resource
context = {
'model': model,
'session': model.Session,
'user': tk.c.user,
'user_obj': tk.c.userobj
}
try:
tk.check_access('package_show', context, result)
except tk.NotAuthorized:
# NotAuthorized exception is risen when the user is not allowed
# to read the package.
attrs.append('resources')
# Delete
self._delete_pkg_atts(result, attrs)
return search_results
######################################################################
######################### ITEMPLATESHELPER ###########################
######################################################################
def get_helpers(self):
return {
'is_dataset_acquired': helpers.is_dataset_acquired,
'get_allowed_users_str': helpers.get_allowed_users_str,
'is_owner': helpers.is_owner,
'can_read': helpers.can_read,
'show_acquire_url_on_create': helpers.show_acquire_url_on_create,
'show_acquire_url_on_edit': helpers.show_acquire_url_on_edit,
'acquire_button': helpers.acquire_button,
'is_dataresource_acquired': helpers.is_dataresource_acquired
}
class PluginObserverPlugin(MockSingletonPlugin):
implements(IPluginObserver)
class WPRDCPlugin(p.SingletonPlugin):
p.implements(p.IConfigurer, inherit=True)
p.implements(p.ITemplateHelpers, inherit=True)
p.implements(p.IRoutes, inherit=True)
p.implements(p.IPackageController, inherit=True)
# IConfigurer
def update_config(self, config):
p.toolkit.add_template_directory(config, 'templates')
p.toolkit.add_public_directory(config, 'public')
p.toolkit.add_resource('fanstatic', 'wprdc_theme')
# ITemplateHelpers
def get_helpers(self):
return {
'wprdc_user_terms': self.check_user_terms,
'wprdc_get_year': self.get_current_year,
'wprdc_wordpress_url': self.get_wordpress_url,
'wprdc_google_tracking': self.get_google_tracking,
'render_datetime': self.convert_to_local,
}
def check_user_terms(self):
if check_if_google():
return True
else:
if 'wprdc_user_terms' in request.cookies:
return True
else:
controller = 'ckanext.wprdc.controller:WPRDCController'
h.redirect_to(controller=controller,
action='view_terms',
came_from=request.url)
def get_current_year(self):
return datetime.date.today().year
def get_wordpress_url(self):
url = config.get('ckan.wordpress_url', 'http://www.wprdc.org')
return url
def get_google_tracking(self):
url = config.get('ckan.google_tracking', '')
return url
def convert_to_local(self, time, date_format=None, with_hours=False):
from_zone = tz.tzutc()
to_zone = tz.tzlocal()
utc = h._datestamp_to_datetime(str(time))
if not utc:
return ''
utc = utc.replace(tzinfo=from_zone)
time = utc.astimezone(to_zone)
# if date_format was supplied we use it
if date_format:
return time.strftime(date_format)
# if with_hours was supplied show them
if with_hours:
return time.strftime('%B %-d, %Y, %-I:%M %p')
else:
return time.strftime('%B %-d, %Y')
# IRoutes
def before_map(self, map):
controller = 'ckanext.wprdc.controller:WPRDCController'
map.redirect('/', '/dataset')
map.connect('terms',
'/terms-of-use',
controller=controller,
action='view_terms',
conditions=dict(method=['GET']))
map.connect('terms',
'/terms-of-use',
controller=controller,
action='submit_terms',
conditions=dict(method=['POST']))
return map
# IPackageController
def after_create(self, context, pkg_dict):
if 'group' in pkg_dict:
if pkg_dict['group']:
data = {
'id': pkg_dict['group'],
'object': pkg_dict['id'],
'object_type': 'package',
'capacity': 'public'
}
p.toolkit.get_action('member_create')(context, data)
def after_update(self, context, pkg_dict):
if 'group' in pkg_dict:
if pkg_dict['group']:
data = {
'id': pkg_dict['group'],
'object': pkg_dict['id'],
'object_type': 'package',
'capacity': 'public'
}
p.toolkit.get_action('member_create')(context, data)
self.remove_from_other_groups(context, pkg_dict['id'])
def remove_from_other_groups(self, context, package_id):
package = p.toolkit.get_action('package_show')(context, {
'id': package_id
})
for group in package['groups']:
if group['name'] != package['group']:
p.toolkit.get_action('member_delete')(context, {
'id': group['id'],
'object': package['id'],
'object_type': 'package'
})
class SHPView(GeoViewBase):
p.implements(p.ITemplateHelpers, inherit=True)
SHP = ['shp', 'shapefile']
# IResourceView (CKAN >=2.3)
def info(self):
return {
'name': 'shp_view',
'title': 'Shapefile',
'icon': 'map-marker',
'iframed': True,
'default_title': p.toolkit._('Shapefile'),
}
def can_view(self, data_dict):
resource = data_dict['resource']
format_lower = resource.get('format', '').lower()
if format_lower in self.SHP:
return self.same_domain or self.proxy_enabled
return False
def view_template(self, context, data_dict):
return 'dataviewer/shp.html'
# IResourcePreview (CKAN < 2.3)
def can_preview(self, data_dict):
format_lower = data_dict['resource']['format'].lower()
correct_format = format_lower in self.SHP
can_preview_from_domain = (self.proxy_enabled or
data_dict['resource'].get('on_same_domain'))
quality = 2
if p.toolkit.check_ckan_version('2.1'):
if correct_format:
if can_preview_from_domain:
return {'can_preview': True, 'quality': quality}
else:
return {
'can_preview': False,
'fixable': 'Enable resource_proxy',
'quality': quality
}
else:
return {'can_preview': False, 'quality': quality}
return correct_format and can_preview_from_domain
def preview_template(self, context, data_dict):
return 'dataviewer/shp.html'
def setup_template_variables(self, context, data_dict):
import ckanext.resourceproxy.plugin as proxy
self.same_domain = data_dict['resource'].get('on_same_domain')
if self.proxy_enabled and not self.same_domain:
data_dict['resource']['original_url'] = \
data_dict['resource'].get('url')
data_dict['resource']['url'] = \
proxy.get_proxified_resource_url(data_dict)
## ITemplateHelpers
def get_helpers(self):
return {
'get_common_map_config_shp': get_common_map_config,
'get_shapefile_viewer_config': get_shapefile_viewer_config,
}
class OAuth2Plugin(plugins.SingletonPlugin):
plugins.implements(plugins.IAuthenticator, inherit=True)
plugins.implements(plugins.IAuthFunctions, inherit=True)
plugins.implements(plugins.IBlueprint)
plugins.implements(plugins.IConfigurer)
def __init__(self, name=None):
"""Store the OAuth 2 client configuration"""
log.debug("Init OAuth2 extension")
self.verify_https = os.environ.get("OAUTHLIB_INSECURE_TRANSPORT",
"") == ""
if self.verify_https and os.environ.get("REQUESTS_CA_BUNDLE",
"").strip() != "":
self.verify_https = os.environ["REQUESTS_CA_BUNDLE"].strip()
self.jwt_enable = os.environ.get(
"CKAN_OAUTH2_JWT_ENABLE",
toolkit.config.get("ckan.oauth2.jwt.enable",
"")).strip().lower() in ("true", "1", "on")
self.authorization_endpoint = os.environ.get(
"CKAN_OAUTH2_AUTHORIZATION_ENDPOINT",
toolkit.config.get("ckan.oauth2.authorization_endpoint", ""),
).strip()
self.token_endpoint = os.environ.get(
"CKAN_OAUTH2_TOKEN_ENDPOINT",
toolkit.config.get("ckan.oauth2.token_endpoint", ""),
).strip()
self.profile_api_url = os.environ.get(
"CKAN_OAUTH2_PROFILE_API_URL",
toolkit.config.get("ckan.oauth2.profile_api_url", "")).strip()
self.client_id = os.environ.get(
"CKAN_OAUTH2_CLIENT_ID",
toolkit.config.get("ckan.oauth2.client_id", "")).strip()
self.client_secret = os.environ.get(
"CKAN_OAUTH2_CLIENT_SECRET",
toolkit.config.get("ckan.oauth2.client_secret", "")).strip()
self.scope = os.environ.get(
"CKAN_OAUTH2_SCOPE", toolkit.config.get("ckan.oauth2.scope",
"")).strip()
self.rememberer_name = os.environ.get(
"CKAN_OAUTH2_REMEMBER_NAME",
toolkit.config.get("ckan.oauth2.rememberer_name", "auth_tkt"),
).strip()
self.profile_api_user_field = os.environ.get(
"CKAN_OAUTH2_PROFILE_API_USER_FIELD",
toolkit.config.get("ckan.oauth2.profile_api_user_field", ""),
).strip()
self.profile_api_fullname_field = os.environ.get(
"CKAN_OAUTH2_PROFILE_API_FULLNAME_FIELD",
toolkit.config.get("ckan.oauth2.profile_api_fullname_field", ""),
).strip()
self.profile_api_mail_field = os.environ.get(
"CKAN_OAUTH2_PROFILE_API_MAIL_FIELD",
toolkit.config.get("ckan.oauth2.profile_api_mail_field", ""),
).strip()
self.profile_api_groupmembership_field = os.environ.get(
"CKAN_OAUTH2_PROFILE_API_GROUPMEMBERSHIP_FIELD",
toolkit.config.get("ckan.oauth2.profile_api_groupmembership_field",
""),
).strip()
self.sysadmin_group_name = os.environ.get(
"CKAN_OAUTH2_SYSADMIN_GROUP_NAME",
toolkit.config.get("ckan.oauth2.sysadmin_group_name", ""),
).strip()
self.redirect_uri = urljoin(
urljoin(
toolkit.config.get("ckan.site_url", "http://localhost:5000"),
toolkit.config.get("ckan.root_path"),
),
REDIRECT_URL,
)
# Init db
db.init_db(model)
missing = [
key for key in REQUIRED_CONF if getattr(self, key, "") == ""
]
if missing:
raise ValueError("Missing required oauth2 conf: %s" %
", ".join(missing))
elif self.scope == "":
self.scope = None
"""
# these still need to be added as rules to the blueprint
# Redirect the user to the OAuth service register page
if self.register_url:
redirect("/user/register", self.register_url)
# Redirect the user to the OAuth service reset page
if self.reset_url:
redirect("/user/reset", self.reset_url)
# Redirect the user to the OAuth service reset page
if self.edit_url:
redirect("/user/edit/{user}", self.edit_url)
"""
def get_blueprint(self):
"""Create Flask blueprint."""
blueprint = Blueprint("oauth2", self.__module__)
rules = [
("/user/login", "login", self.login),
("/oauth2/callback", "callback", self.callback),
]
for rule in rules:
blueprint.add_url_rule(*rule)
return blueprint
def get_auth_functions(self):
"""Prevent some actions from being authorized."""
return {
"user_create": user_create,
"user_update": user_update,
"user_reset": user_reset,
"request_reset": request_reset,
}
def update_config(self, config):
"""Update configuration."""
self.register_url = os.environ.get(
"CKAN_OAUTH2_REGISTER_URL",
config.get("ckan.oauth2.register_url", None))
self.reset_url = os.environ.get(
"CKAN_OAUTH2_RESET_URL", config.get("ckan.oauth2.reset_url", None))
self.edit_url = os.environ.get(
"CKAN_OAUTH2_EDIT_URL", config.get("ckan.oauth2.edit_url", None))
self.authorization_header = os.environ.get(
"CKAN_OAUTH2_AUTHORIZATION_HEADER",
config.get("ckan.oauth2.authorization_header", "Authorization"),
).lower()
# Add plugin's templates dir to CKAN's extra_template_paths, so CKAN will use them
plugins.toolkit.add_template_directory(config, "templates")
def login(self):
"""Start log in process."""
log.debug("login")
state = generate_state(get_previous_page())
oauth = OAuth2Session(self.client_id,
redirect_uri=self.redirect_uri,
scope=self.scope,
state=state)
auth_url, _ = oauth.authorization_url(self.authorization_endpoint)
log.debug(f"Challenge: Redirecting challenge to page {auth_url}")
return toolkit.redirect_to(auth_url)
def callback(self):
"""Resume login process from authorization service."""
log.debug("callback")
try:
token = self.get_token()
except InsecureTransportError as e:
log.warn(f"Error in getting token: {e}")
helpers.flash_error(
"Authentication error; please contact the administrator.")
return toolkit.redirect_to("/")
try:
user_name = self.authenticate(token)
except InsecureTransportError as e:
log.warn(f"Error authenticating user: {e}")
helpers.flash_error(
"Authentication error; please contact the administrator.")
return toolkit.redirect_to("/")
# create headers from remember token to be added to redirected response
remember_headers = self.remember(user_name)
self.update_token(user_name, token)
state = toolkit.request.params.get("state")
redirect = toolkit.redirect_to(get_came_from(state))
for header, value in remember_headers:
redirect.headers[header] = value
return redirect
def identify(self):
log.debug("identify")
def _refresh_and_save_token(user_name):
new_token = self.refresh_token(user_name)
if new_token:
toolkit.g.usertoken = new_token
environ = toolkit.request.environ
apikey = toolkit.request.headers.get(self.authorization_header, "")
user_name = None
if self.authorization_header == "authorization":
if apikey.startswith("Bearer "):
apikey = apikey[7:].strip()
else:
apikey = ""
# This API Key is not the one of CKAN, it's the one provided by the OAuth2 Service
if apikey:
try:
token = {"access_token": apikey}
user_name = self.authenticate(token)
except Exception:
pass
# If the authentication via API fails, we can still log in the user using session.
if user_name is None and "repoze.who.identity" in environ:
user_name = environ["repoze.who.identity"]["repoze.who.userid"]
log.info(f"User {user_name} logged using session")
# If we have been able to log in the user (via API or Session)
if user_name:
g.user = user_name
toolkit.g.user = user_name
toolkit.g.usertoken = self.get_stored_token(user_name)
toolkit.g.usertoken_refresh = partial(_refresh_and_save_token,
user_name)
else:
g.user = None
log.warn("The user is not currently logged in...")
def get_token(self):
"""Get token from authorization service."""
log.debug("get_token")
oauth = OAuth2Session(self.client_id,
redirect_uri=self.redirect_uri,
scope=self.scope)
try:
# NOTE: authorization_response/toolkit.request.url was using http instead of https
# hacked to replace http with https
authorization_response = toolkit.request.url
authorization_response = authorization_response.replace(
"http", "https")
token = oauth.fetch_token(
self.token_endpoint,
client_secret=self.client_secret,
authorization_response=authorization_response,
verify=self.verify_https,
)
except InsecureTransportError:
raise
return token
def authenticate(self, token):
log.debug("authenticate")
if self.jwt_enable:
access_token = token["access_token"]
user_data = jwt.decode(access_token, verify=False)
user = self.user_json(user_data)
else:
try:
oauth = OAuth2Session(self.client_id, token=token)
profile_response = oauth.get(self.profile_api_url,
verify=self.verify_https)
except InsecureTransportError:
raise
# Token can be invalid
if not profile_response.ok:
error = profile_response.json()
if error.get("error", "") == "invalid_token":
raise ValueError(error.get("error_description"))
else:
profile_response.raise_for_status()
else:
user_data = profile_response.json()
user = self.user_json(user_data)
# Save the user in the database
model.Session.add(user)
model.Session.commit()
model.Session.remove()
return user.name
def user_json(self, user_data):
email = user_data[self.profile_api_mail_field]
user_name = user_data[self.profile_api_user_field]
# In CKAN can exists more than one user associated with the same email
# Some providers, like Google and FIWARE only allows one account per email
user = None
users = model.User.by_email(email)
if len(users) == 1:
user = users[0]
# If the user does not exist, we have to create it...
if user is None:
user = model.User(email=email)
# Now we update his/her user_name with the one provided by the OAuth2 service
# In the future, users will be obtained based on this field
user.name = user_name
# Update fullname
if self.profile_api_fullname_field != "" and self.profile_api_fullname_field in user_data:
user.fullname = user_data[self.profile_api_fullname_field]
# Update sysadmin status
if (self.profile_api_groupmembership_field != ""
and self.profile_api_groupmembership_field in user_data):
user.sysadmin = (
self.sysadmin_group_name
in user_data[self.profile_api_groupmembership_field])
return user
def _get_rememberer(self, environ):
plugins = environ.get("repoze.who.plugins", {})
return plugins.get(self.rememberer_name)
def remember(self, user_name):
"""
Remember the authenticated identity.
This method simply delegates to another IIdentifier plugin if configured.
Return headers so they can be added to redirected response.
"""
log.debug("Repoze OAuth remember")
environ = toolkit.request.environ
rememberer = self._get_rememberer(environ)
identity = {"repoze.who.userid": user_name}
headers = rememberer.remember(environ, identity)
return headers
def get_stored_token(self, user_name):
user_token = db.UserToken.by_user_name(user_name=user_name)
if user_token:
return {
"access_token": user_token.access_token,
"refresh_token": user_token.refresh_token,
"expires_in": user_token.expires_in,
"token_type": user_token.token_type,
}
def update_token(self, user_name, token):
user_token = db.UserToken.by_user_name(user_name=user_name)
# Create the user if it does not exist
if not user_token:
user_token = db.UserToken()
user_token.user_name = user_name
# Save the new token
user_token.access_token = token["access_token"]
user_token.token_type = token["token_type"]
user_token.refresh_token = token.get("refresh_token")
if "expires_in" in token:
user_token.expires_in = token["expires_in"]
else:
access_token = jwt.decode(user_token.access_token, verify=False)
user_token.expires_in = access_token["exp"] - access_token["iat"]
model.Session.add(user_token)
model.Session.commit()
def refresh_token(self, user_name):
token = self.get_stored_token(user_name)
if token:
client = OAuth2Session(self.client_id,
token=token,
scope=self.scope)
try:
token = client.refresh_token(
self.token_endpoint,
client_secret=self.client_secret,
client_id=self.client_id,
verify=self.verify_https,
)
except InsecureTransportError:
raise
self.update_token(user_name, token)
log.info(f"Token for user {user_name} has been updated properly")
return token
else:
log.warn(f"User {user_name} has no refresh token")
class NapThemePlugin(plugins.SingletonPlugin, toolkit.DefaultDatasetForm):
plugins.implements(plugins.IConfigurer)
plugins.implements(plugins.ITemplateHelpers)
plugins.implements(plugins.IBlueprint)
plugins.implements(plugins.IPackageController, inherit=True)
plugins.implements(plugins.IValidators)
plugins.implements(plugins.IDatasetForm)
# IConfigurer
def update_config(self, config_):
toolkit.add_template_directory(config_, 'templates')
toolkit.add_public_directory(config_, 'fanstatic')
toolkit.add_resource('fanstatic', 'nap_theme')
# ITemplateHelpers
def get_helpers(self):
return {
'nap_theme_get_extra': get_extra,
'nap_theme_get_orgs': get_orgs,
'nap_theme_get_tags': get_tags,
'nap_theme_get_tag_names': get_tag_names,
'nap_theme_get_sorted_error_summary': get_sorted_error_summary,
'nap_theme_get_form_data': get_form_data,
'nap_theme_get_package_display_name': get_package_display_name,
'nap_theme_get_topics': get_topics,
'nap_theme_get_transport_modes': get_transport_modes,
'nap_theme_get_road_networks': get_road_networks,
'nap_theme_get_user_with_datasets': get_user_with_datasets,
'nap_theme_get_update_frequencies': get_update_frequencies,
'nap_theme_get_update_frequency_name': get_update_frequency_name,
'nap_theme_get_licences': get_licences,
'nap_theme_get_licence_name': get_licence_name,
'nap_theme_get_licence_type': get_licence_type,
'nap_theme_get_time_period': get_time_period,
'nap_theme_get_date_formatted': get_date_formatted,
'nap_theme_get_facets': get_facets,
}
# IBlueprint
def get_blueprint(self):
u'''Return a Flask Blueprint object to be registered by the app.'''
# Create Blueprint for plugin
blueprint = Blueprint(self.name, self.__module__)
blueprint.template_folder = u'templates'
# Add plugin url rules to Blueprint object
blueprint.add_url_rule(u'/cookies', u'cookies', cookies)
return blueprint
def _create_search_param_dict(self, filter_query):
from collections import defaultdict
# fix issue with spaces in params such as tags we need to differentiate spaces between search terms and spaces
filter_query = filter_query.replace('" ', '"|||')
# fix issue with : in params
filter_query = filter_query.replace(':"', '~~~"')
filter_params = filter_query.split('|||')
filter_dict = list(s.split('~~~') for s in filter_params)
new_dict = defaultdict(list)
for (key, value) in filter_dict:
new_dict[key].append(value)
return new_dict
# IPackageController
def before_search(self, search_params):
def make_filters_or(filter_dict):
filter_string = ""
for (key, value) in filter_dict.items():
string = f'{key}:({" OR ".join(value)})'
filter_string = filter_string + " " + string
return filter_string
if (search_params.get('fq', None)
and not '+owner_org' in search_params['fq']):
fq = make_filters_or(
self._create_search_param_dict(search_params['fq']))
search_params["fq"] = fq
extras = search_params.get('extras')
start_date = extras.get('ext_startdate')
end_date = extras.get('ext_enddate')
if not start_date or not end_date:
return search_params
start_date = datetime.strptime(
start_date, '%Y-%m-%d').strftime("%Y-%m-%dT%H:%M:%SZ")
end_date = datetime.strptime(end_date,
'%Y-%m-%d').strftime("%Y-%m-%dT%H:%M:%SZ")
fq = search_params["fq"]
fq = '{fq} (start_date:[* TO {end_date}] AND end_date:[{start_date} TO *])'.format(
fq=fq, start_date=start_date, end_date=end_date)
search_params["fq"] = fq
return search_params
# Schema Changes
def is_fallback(self):
# Return True to register this plugin as the default handler for
# package types not handled by any other IDatasetForm plugin.
return True
def package_types(self):
# This plugin doesn't handle any special package types, it just
# registers itself as the default (above).
return []
def create_package_schema(self):
schema = super(NapThemePlugin, self).create_package_schema()
schema = self._modify_package_schema(schema)
return schema
def update_package_schema(self):
schema = super(NapThemePlugin, self).update_package_schema()
schema = self._modify_package_schema(schema)
return schema
def _modify_package_schema(self, schema):
schema.update({
'url':
[custom_url_validator,
toolkit.get_converter('unicode_safe')],
'title': [
custom_description_validator,
toolkit.get_converter('unicode_safe')
],
'author_email': [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('unicode_safe'),
custom_author_email_validator,
custom_required_author_email_validator
],
'author': [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('unicode_safe'),
custom_required_author_name_validator
],
'name': [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('unicode_safe')
],
'location': [
custom_location_validator,
toolkit.get_converter('convert_to_extras')
],
'data_formats': [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_extras')
],
'update_frequency': [
custom_update_frequency_validator,
toolkit.get_converter('convert_to_extras')
],
'regularly_updated': [
custom_regularly_updated_validatior,
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_extras')
],
'date_range_earliest_day': [
custom_earliest_day_validator,
toolkit.get_converter('convert_to_extras')
],
'date_range_earliest_month': [
custom_earliest_month_validator,
toolkit.get_converter('convert_to_extras')
],
'date_range_earliest_year': [
custom_earliest_year_validator,
toolkit.get_converter('convert_to_extras')
],
'date_range_latest_day': [
custom_latest_day_validator,
toolkit.get_converter('convert_to_extras')
],
'date_range_latest_month': [
custom_latest_month_validator,
toolkit.get_converter('convert_to_extras')
],
'date_range_latest_year': [
custom_latest_year_validator,
toolkit.get_converter('convert_to_extras')
],
'date_range_earliest': [custom_date_range_earliest_validator],
'date_range_latest': [custom_date_range_latest_validator],
'regularly_updated_earliest_day': [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_extras')
],
'regularly_updated_earliest_month': [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_extras')
],
'regularly_updated_earliest_year': [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_extras')
],
'data_available': [
custom_data_available_validator,
toolkit.get_converter('convert_to_extras')
],
'topics': [
custom_topics_validator,
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_tags')('nap_topics')
],
'transport_modes': [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_tags')('nap_transport_modes')
],
'road_networks': [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_tags')('nap_road_networks')
],
'start_date': [
custom_date_range_start_converter,
toolkit.get_converter('convert_to_extras')
],
'end_date': [
custom_date_range_end_converter,
toolkit.get_converter('convert_to_extras')
]
})
return schema
def show_package_schema(self):
schema = super(NapThemePlugin, self).show_package_schema()
schema.update({
'url':
[toolkit.get_converter('unicode_safe'), custom_url_validator],
'title': [
toolkit.get_converter('unicode_safe'),
custom_description_validator
],
'author_email': [
toolkit.get_converter('unicode_safe'),
toolkit.get_validator('ignore_missing'),
custom_author_email_validator,
custom_required_author_email_validator
],
'author': [
toolkit.get_converter('unicode_safe'),
toolkit.get_validator('ignore_missing'),
custom_required_author_name_validator
],
'name': [
toolkit.get_converter('unicode_safe'),
toolkit.get_validator('ignore_missing')
],
'location': [
toolkit.get_converter('convert_from_extras'),
custom_location_validator
],
'data_formats': [
toolkit.get_converter('unicode_safe'),
toolkit.get_validator('convert_from_extras')
],
'update_frequency': [
toolkit.get_converter('unicode_safe'),
toolkit.get_validator('convert_from_extras')
],
'regularly_updated': [
toolkit.get_converter('unicode_safe'),
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_from_extras')
],
'date_range_earliest_day': [
toolkit.get_converter('unicode_safe'),
custom_earliest_day_validator,
toolkit.get_converter('convert_from_extras')
],
'date_range_earliest_month': [
toolkit.get_converter('unicode_safe'),
custom_earliest_month_validator,
toolkit.get_converter('convert_from_extras')
],
'date_range_earliest_year': [
toolkit.get_converter('unicode_safe'),
custom_earliest_year_validator,
toolkit.get_converter('convert_from_extras')
],
'date_range_latest_day': [
toolkit.get_converter('unicode_safe'),
custom_latest_day_validator,
toolkit.get_converter('convert_from_extras')
],
'date_range_latest_month': [
toolkit.get_converter('unicode_safe'),
custom_latest_month_validator,
toolkit.get_converter('convert_from_extras')
],
'date_range_latest_year': [
toolkit.get_converter('unicode_safe'),
custom_latest_year_validator,
toolkit.get_converter('convert_from_extras')
],
'regularly_updated_earliest_day': [
toolkit.get_converter('unicode_safe'),
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_from_extras')
],
'regularly_updated_earliest_month': [
toolkit.get_converter('unicode_safe'),
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_from_extras')
],
'regularly_updated_earliest_year': [
toolkit.get_converter('unicode_safe'),
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_from_extras')
],
'data_available': [
custom_data_available_validator,
toolkit.get_converter('unicode_safe'),
toolkit.get_converter('convert_from_extras')
],
'topics': [
custom_topics_validator,
toolkit.get_converter('convert_from_tags')('nap_topics'),
toolkit.get_validator('ignore_missing')
],
'transport_modes': [
toolkit.get_converter('convert_from_tags')(
'nap_transport_modes'),
toolkit.get_validator('ignore_missing')
],
'road_networks': [
toolkit.get_converter('convert_from_tags')(
'nap_road_networks'),
toolkit.get_validator('ignore_missing')
],
'start_date': [toolkit.get_converter('convert_from_extras')],
'end_date': [toolkit.get_converter('convert_from_extras')]
})
return schema
# IValidators
def get_validators(self):
return {
u'owner_org_validator': custom_owner_org_validator,
u'url_validator': custom_url_validator,
u'title_validator': custom_title_validator,
u'location_validator': custom_location_validator
}
class VectorStorer(SingletonPlugin):
STATE_DELETED = 'deleted'
resource_delete_action = None
resource_update_action = None
implements(IRoutes, inherit=True)
implements(IConfigurer, inherit=True)
implements(IConfigurable, inherit=True)
implements(IResourceUrlChange)
implements(ITemplateHelpers)
implements(IDomainObjectModification, inherit=True)
def get_helpers(self):
return {
'vectorstore_is_in_vectorstore': isInVectorStore,
'vectorstore_supported_format': supportedFormat
}
def configure(self, config):
''' Extend the resource_delete action in order to get notification of deleted resources'''
if self.resource_delete_action is None:
resource_delete = toolkit.get_action('resource_delete')
@logic.side_effect_free
def new_resource_delete(context, data_dict):
resource = ckan.model.Session.query(model.Resource).get(
data_dict['id'])
self.notify(resource,
model.domain_object.DomainObjectOperation.deleted)
res_delete = resource_delete(context, data_dict)
return res_delete
logic._actions['resource_delete'] = new_resource_delete
self.resource_delete_action = new_resource_delete
''' Extend the resource_update action in order to pass the extra keys to vectorstorer resources
when they are being updated'''
if self.resource_update_action is None:
resource_update = toolkit.get_action('resource_update')
@logic.side_effect_free
def new_resource_update(context, data_dict):
resource = ckan.model.Session.query(model.Resource).get(
data_dict['id']).as_dict()
if resource.has_key('vectorstorer_resource'):
if resource['format'].lower() == settings.WMS_FORMAT:
data_dict['parent_resource_id'] = resource[
'parent_resource_id']
data_dict['vectorstorer_resource'] = resource[
'vectorstorer_resource']
data_dict['wms_server'] = resource['wms_server']
data_dict['wms_layer'] = resource['wms_layer']
if resource['format'].lower() == settings.DB_TABLE_FORMAT:
data_dict['vectorstorer_resource'] = resource[
'vectorstorer_resource']
data_dict['parent_resource_id'] = resource[
'parent_resource_id']
data_dict['geometry'] = resource['geometry']
if not data_dict['url'] == resource['url']:
abort(
400,
_('You cant upload a file to a ' +
resource['format'] + ' resource.'))
res_update = resource_update(context, data_dict)
return res_update
logic._actions['resource_update'] = new_resource_update
self.resource_update_action = new_resource_update
def before_map(self, map):
map.connect(
'style',
'/dataset/{id}/resource/{resource_id}/style/{operation}',
controller='ckanext.vectorstorer.controllers.style:StyleController',
action='style',
operation='operation')
map.connect('export',
'/dataset/{id}/resource/{resource_id}/export/{operation}',
controller=
'ckanext.vectorstorer.controllers.export:ExportController',
action='export',
operation='{operation}')
map.connect('search_epsg',
'/api/search_epsg',
controller=
'ckanext.vectorstorer.controllers.export:ExportController',
action='search_epsg')
map.connect('publish',
'/api/vector/publish',
controller=
'ckanext.vectorstorer.controllers.vector:VectorController',
action='publish')
return map
def update_config(self, config):
toolkit.add_public_directory(config, 'public')
toolkit.add_template_directory(config, 'templates')
toolkit.add_resource('public', 'ckanext-vectorstorer')
def notify(self, entity, operation=None):
if isinstance(entity, model.resource.Resource):
if operation == model.domain_object.DomainObjectOperation.new and entity.format.lower(
) in settings.SUPPORTED_DATA_FORMATS:
#A new vector resource has been created
#resource_actions.create_vector_storer_task(entity)
resource_actions.identify_resource(entity)
#elif operation==model.domain_object.DomainObjectOperation.deleted:
##A vectorstorer resource has been deleted
#resource_actions.delete_vector_storer_task(entity.as_dict())
#elif operation is None:
##Resource Url has changed
#if entity.format.lower() in settings.SUPPORTED_DATA_FORMATS:
##Vector file was updated
#resource_actions.update_vector_storer_task(entity)
#else :
##Resource File updated but not in supported formats
#resource_actions.delete_vector_storer_task(entity.as_dict())
elif isinstance(entity, model.Package):
if entity.state == self.STATE_DELETED:
resource_actions.pkg_delete_vector_storer_task(
entity.as_dict())
class YtpCommentsPlugin(plugins.SingletonPlugin):
implements(plugins.IRoutes, inherit=True)
implements(plugins.IConfigurer, inherit=True)
implements(plugins.IPackageController, inherit=True)
implements(plugins.ITemplateHelpers, inherit=True)
implements(plugins.IActions, inherit=True)
implements(plugins.IAuthFunctions, inherit=True)
# IConfigurer
def configure(self, config):
log.debug("Configuring comments module")
def update_config(self, config):
toolkit.add_template_directory(config, "templates")
toolkit.add_public_directory(config, 'public')
toolkit.add_resource('public/javascript/', 'comments_js')
def get_helpers(self):
return {
'get_comment_thread': self._get_comment_thread,
'get_comment_count_for_dataset': self._get_comment_count_for_dataset
}
def get_actions(self):
from ckanext.ytp.comments.logic.action import get, create, delete, update
return {
"comment_create": create.comment_create,
"thread_show": get.thread_show,
"comment_update": update.comment_update,
"comment_show": get.comment_show,
"comment_delete": delete.comment_delete,
"comment_count": get.comment_count
}
def get_auth_functions(self):
from ckanext.ytp.comments.logic.auth import get, create, delete, update
return {
'comment_create': create.comment_create,
'comment_update': update.comment_update,
'comment_show': get.comment_show,
'comment_delete': delete.comment_delete,
"comment_count": get.comment_count
}
# IPackageController
def before_view(self, pkg_dict):
# TODO: append comments from model to pkg_dict
return pkg_dict
# IRoutes
def before_map(self, map):
"""
/dataset/NAME/comments/reply/PARENT_ID
/dataset/NAME/comments/add
"""
controller = 'ckanext.ytp.comments.controller:CommentController'
map.connect('/dataset/{dataset_id}/comments/add', controller=controller, action='add')
map.connect('/dataset/{dataset_id}/comments/{comment_id}/edit', controller=controller, action='edit')
map.connect('/dataset/{dataset_id}/comments/{parent_id}/reply', controller=controller, action='reply')
map.connect('/dataset/{dataset_id}/comments/{comment_id}/delete', controller=controller, action='delete')
return map
def _get_comment_thread(self, dataset_name):
import ckan.model as model
from ckan.logic import get_action
url = '/dataset/%s' % dataset_name
return get_action('thread_show')({'model': model, 'with_deleted': True}, {'url': url})
def _get_comment_count_for_dataset(self, dataset_name):
import ckan.model as model
from ckan.logic import get_action
url = '/dataset/%s' % dataset_name
count = get_action('comment_count')({'model': model}, {'url': url})
return count
class IntroExamplePlugin(p.SingletonPlugin):
p.implements(p.IConfigurer)
p.implements(p.IRoutes, inherit=True)
p.implements(p.IAuthFunctions)
p.implements(p.IActions)
## IConfigurer
def update_config(self, config):
'''
This method allows to access and modify the CKAN configuration object
'''
log.info('You are using the following plugins: {0}'
.format(config.get('ckan.plugins')))
# Check CKAN version
# To raise an exception instead, use:
# p.toolkit.require_ckan_version('2.1')
if not p.toolkit.check_ckan_version('2.1'):
log.warn('This extension has only been tested on CKAN 2.1!')
# Add the extension templates directory so it overrides the CKAN core
# one
p.toolkit.add_template_directory(config, 'theme/templates')
# Add the extension public directory so we can serve our own content
p.toolkit.add_public_directory(config, 'theme/public')
## IRoutes
def after_map(self, map):
controller = 'ckanext.intro.plugin:CustomController'
map.connect('/custom', controller=controller, action='custom_page')
map.connect('/csv', controller=controller, action='datasets_report')
# /changes obtiene los cambios recientes
map.connect('/changes', controller=controller, action='changes_recently')
return map
## IAuthFunctions
def get_auth_functions(self):
# Return a dict with the auth functions that we want to override
# or add
return {
'group_create': group_create,
'datasets_report_csv': datasets_report_csv_auth,
'changes_recently_csv' : changes_recently_auth,
}
## IActions
def get_actions(self):
# Return a dict with the action functions that we want to add
return {
'datasets_report_csv': datasets_report_csv,
'changes_recently_csv' : changes_recently_csv,
# asocia el action con el método
}
class LIREPlugin(p.SingletonPlugin):
p.implements(p.IConfigurer, inherit=True)
p.implements(p.IConfigurable)
p.implements(p.IRoutes, inherit=True)
p.implements(p.IDomainObjectModification, inherit=True)
p.implements(p.IResourceUrlChange)
p.implements(p.ITemplateHelpers)
def configure(self, config):
self.site_url = config.get('ckan.site_url')
def update_config(self, config):
# p.toolkit.add_resource('fanstatic', 'rem')
# check if new templates
if p.toolkit.check_ckan_version(min_version='2.0'):
if not p.toolkit.asbool(config.get('ckan.legacy_templates', False)):
# add the extend templates
p.toolkit.add_template_directory(config, 'templates_extend')
else:
# legacy templates
p.toolkit.add_template_directory(config, 'templates')
# templates for helper functions
p.toolkit.add_template_directory(config, 'templates_new')
else:
# FIXME we don't support ckan < 2.0
p.toolkit.add_template_directory(config, 'templates')
p.toolkit.add_public_directory(config, 'public')
def before_map(self, map):
lire = 'ckanext.lire.controller:LIREController'
rem = 'ckanext.lire.controllers.rem:REMController'
ace = 'ckanext.lire.controllers.ace:ACEController'
semre = 'ckanext.lire.controllers.semre:SEMREController'
map.connect('/lire', controller=lire, action='index')
map.connect('/lire/manager', controller=lire, action='manager')
map.connect('/lire/semantic', controller=semre, action='semantic')
map.connect('/lire/linksets.:id', controller=semre, action='linksets')
map.connect('/lire/organization/:org.:ext', controller=semre, action='org_rdf')
map.connect('/lire/examineDatasets', controller=rem, action='examineDatasets')
map.connect('/lire/storeRelationships', controller=ace, action='storeRelationships')
map.connect('/lire/checkDataset', controller=semre, action='checkDataset')
return map
#return random int necessary to create position of graphical element (dataset)
def randomNum(self,num):
randNum = random.randint(0,num)
return randNum
################
## LIRE SEMRE ##
################
# Use this function in custom template to get dataset relationships
# It will enable us to represent them semantically
def semre_dataset(self,datasetName):
semre = SEMREController()
# To get dataset relationships we call SEMRE to whom we forward the name of the dataset
datasetRelationships = semre.semre_create(datasetName)
return datasetRelationships
def get_helpers(self):
return {'randomNum': self.randomNum,'semre_dataset': self.semre_dataset}
class Dset_HarvesterPlugin(p.SingletonPlugin):
p.implements(ISpatialHarvester, inherit=True)
p.implements(p.IConfigurer)
p.implements(p.ITemplateHelpers)
# ISpatialHarvester
def get_package_dict(self, context, data_dict):
package_dict = data_dict['package_dict']
iso_values = data_dict['iso_values']
xml_tree = data_dict['xml_tree']
# Add ISO Topic Category from spatial harvester
package_dict['extras'].append(
{'key': 'topic-category', 'value': iso_values.get('topic-category')}
)
# Add author field
authorList = getNamesByRole(xml_tree, 'author', 'gmd:individualName/gco:CharacterString')
authorString = json.dumps(authorList)
package_dict['extras'].append(
{'key': 'harvest-author', 'value': authorString}
)
# Examine iso_values if there are authors, just to get a better idea of harvester data structures.
#if len(authorString) > 1:
#log.debug("START iso_values print:")
#log.debug(pprint.pformat(iso_values))
#log.debug("END iso_values print.")
# Add publisher field
publisherList = getNamesByRole(xml_tree, 'publisher', 'gmd:organisationName/gco:CharacterString')
publisherString = json.dumps(publisherList)
package_dict['extras'].append({'key': 'publisher', 'value': publisherString})
# Add Support Contact fields
# TODO: Resource Support Contact Object?
resourceSupportIsoPath = './/gmd:identificationInfo/gmd:MD_DataIdentification/gmd:pointOfContact/gmd:CI_ResponsibleParty'
resourceSupportName = getSupportContactName(xml_tree, resourceSupportIsoPath)
package_dict['extras'].append({'key': 'resource-support-name', 'value': resourceSupportName})
resourceSupportOrg = getSupportContactOrg(xml_tree, resourceSupportIsoPath)
package_dict['extras'].append({'key': 'resource-support-organization', 'value': resourceSupportOrg})
resourceSupportEmail = getSupportContactEmail(xml_tree, resourceSupportIsoPath)
package_dict['extras'].append({'key': 'resource-support-email', 'value': resourceSupportEmail})
# Metadata point of contact
# TODO: don't build string here. Esp check for null org.
pointOfContactIsoPath = './/gmd:contact/gmd:CI_ResponsibleParty'
pointOfContactName = getSupportContactName(xml_tree, pointOfContactIsoPath)
package_dict['extras'].append({'key': 'metadata-point-of-contact-name', 'value': pointOfContactName})
pointOfContactOrg = getSupportContactOrg(xml_tree, pointOfContactIsoPath)
package_dict['extras'].append({'key': 'metadata-point-of-contact-organization', 'value': pointOfContactOrg})
# Set CKAN Resource Type to first DataCite ResourceType keyword (CKAN only allows one keyword)
resourceTypeList = getDataCiteResourceTypes(xml_tree)
for extra in package_dict['extras']:
if extra['key'] == 'resource-type':
extra['value'] = resourceTypeList[0]
# Add Harvester-related values
harvest_object = data_dict['harvest_object']
package_dict['extras'].append({'key': 'harvested_object_id', 'value': harvest_object.id})
package_dict['extras'].append({'key': 'harvester_source_id', 'value': harvest_object.harvest_source_id})
package_dict['extras'].append({'key': 'harvester_source_title', 'value': harvest_object.source.title})
# Add Publication Date field
publication_date = getPublicationDate(iso_values['dataset-reference-date'])
package_dict['extras'].append({'key': 'publication-date', 'value': publication_date})
# Convert GCMD Keywords to split form
splitKeywords = getSplitKeywordsGCMD(iso_values.pop('tags'))
package_dict['tags'] = [{'name': t} for t in splitKeywords]
# Convert some harvested fields from lists to JSON (spatial harvester is inconsistent in its output)
for key in ('topic-category','access-constraints'):
for extra in package_dict['extras']:
if extra['key'] == key:
extra['value'] = json.dumps(extra['value'])
#log.debug("START data_dict print:")
#log.debug(pprint.pformat(data_dict))
#log.debug("END data_dict print.")
return package_dict
# IConfigurer
def update_config(self, config_):
toolkit.add_template_directory(config_, 'templates')
toolkit.add_public_directory(config_, 'public')
toolkit.add_resource('fanstatic', 'dset_harvester')
# ITemplateHelpers
def get_helpers(self):
function_names = (
'string_to_json',
'dset_sorted_extras',
'dset_render_datetime',
'dset_valid_temporal_extent',
)
return _get_module_functions(helpers, function_names)
class ExtendSearchPlugin(plugins.SingletonPlugin):
'''
Extends the Ckan dataset/package search
'''
print "loading ckanext-extend_search"
plugins.implements(plugins.IConfigurer)
plugins.implements(plugins.IPackageController, inherit=True)
def update_config(self, config):
toolkit.add_template_directory(config, 'templates')
toolkit.add_resource('fanstatic', 'ckanext-datesearch')
toolkit.add_resource('fanstatic', 'custodianpicker-module')
# Add the custom parameters to Solr's facet queries
def before_search(self, search_params):
extras = search_params.get('extras')
if not extras:
# There are no extras in the search params, so do nothing.
return search_params
start_date = extend_search_convert_local_to_utc_timestamp(
extras.get('ext_startdate'))
end_date = extend_search_convert_local_to_utc_timestamp(
extras.get('ext_enddate'))
cust_id = extras.get('ext_cust_id')
if not cust_id:
if not start_date or not end_date:
# The user didn't select any additional params, so do nothing.
return search_params
fq = search_params['fq']
if start_date and end_date:
# Add a date-range query with the selected start and end dates into the
# Solr facet queries.
fq = '{fq} +metadata_modified:[{start_date} TO {end_date}]'.format(
fq=fq, start_date=start_date, end_date=end_date)
#Add creator (user) id query to the Solr facet queries
if cust_id:
fq = '{fq} +maintainer:{cust_id}'.format(fq=fq, cust_id=cust_id)
#return modified facet queries
search_params['fq'] = fq
return search_params
def after_search(self, search_params, search_results):
context = {
'model': ckan.model,
'session': ckan.model.Session,
'user': pylons.c.user
}
#set permission level: read (default is edit)
data_dict = {'user': pylons.c.user, 'permission': 'read'}
#get list of organisations that the user is a member of
orgs = ckan.logic.get_action('organization_list_for_user')(context,
data_dict)
#user doesn't belong to an organisation
if not orgs:
print('User is not a member of any organisations!')
c.maintainers = []
return search_params
#get a distinct list of members who belong to the organisations
members = []
for org in orgs:
params = {'id': org['id'], 'object_type': 'user'}
member_list = ckan.logic.get_action('member_list')(context, params)
for m in member_list:
members.append(m)
memberset = set(members)
#need the user name to match with the maintainer field
current_user_name = None
member_names = []
for member in memberset:
user = User.get(member[0]) #user id
member_names.append(user.name)
#get all maintainers
maintainers = [
p[0] for p in meta.Session.query(distinct(Package.maintainer))
if p[0]
]
maintset = set(maintainers)
#filter maintainers by user-related organisation members
results = maintset.intersection(member_names)
c.maintainers = results
return search_params
class SchemingDatasetsPlugin(p.SingletonPlugin, DefaultDatasetForm,
_SchemingMixin):
p.implements(p.IConfigurer)
p.implements(p.ITemplateHelpers)
p.implements(p.IDatasetForm, inherit=True)
p.implements(p.IActions)
p.implements(p.IValidators)
SCHEMA_OPTION = 'scheming.dataset_schemas'
FALLBACK_OPTION = 'scheming.dataset_fallback'
SCHEMA_TYPE_FIELD = 'dataset_type'
@classmethod
def _store_instance(cls, self):
SchemingDatasetsPlugin.instance = self
def read_template(self):
return 'scheming/package/read.html'
def resource_template(self):
return 'scheming/package/resource_read.html'
def package_form(self):
return 'scheming/package/snippets/package_form.html'
def resource_form(self):
return 'scheming/package/snippets/resource_form.html'
def package_types(self):
return list(self._schemas)
def validate(self, context, data_dict, schema, action):
"""
Validate and convert for package_create, package_update and
package_show actions.
"""
thing, action_type = action.split('_')
t = data_dict.get('type')
if not t or t not in self._schemas:
return data_dict, {
'type': ["Unsupported dataset type: {t}".format(t=t)]
}
scheming_schema = self._expanded_schemas[t]
if action_type == 'show':
get_validators = _field_output_validators
elif action_type == 'create':
get_validators = _field_create_validators
else:
get_validators = _field_validators
for f in scheming_schema['dataset_fields']:
schema[f['field_name']] = get_validators(
f, scheming_schema, f['field_name'] not in schema)
resource_schema = schema['resources']
for f in scheming_schema.get('resource_fields', []):
resource_schema[f['field_name']] = get_validators(
f, scheming_schema, False)
return navl_validate(data_dict, schema, context)
def get_actions(self):
"""
publish dataset schemas
"""
return {
'scheming_dataset_schema_list': scheming_dataset_schema_list,
'scheming_dataset_schema_show': scheming_dataset_schema_show,
}
def setup_template_variables(self, context, data_dict):
super(SchemingDatasetsPlugin,
self).setup_template_variables(context, data_dict)
# do not override licenses if they were already added by some
# other extension. We just want to make sure, that licenses
# are not empty.
if not hasattr(c, 'licenses'):
c.licenses = [('', '')] + model.Package.get_license_options()
class SpatialQuery(SingletonPlugin):
''' '''
implements(interfaces.IRoutes, inherit=True)
implements(interfaces.IPackageController, inherit=True)
implements(interfaces.IConfigurable, inherit=True)
search_backend = None
def configure(self, config):
'''
:param config:
'''
self.search_backend = config.get(u'ckanext.spatial.search_backend',
u'postgis')
if self.search_backend != u'postgis' and not toolkit.check_ckan_version(
u'2.0.1'):
msg = u'The Solr backends for the spatial search require CKAN 2.0.1 or ' \
u'higher. ' + \
u'Please upgrade CKAN or select the \'postgis\' backend.'
raise toolkit.CkanVersionException(msg)
def before_map(self, map):
'''
:param map:
'''
map.connect(
u'api_spatial_query',
'/api/2/search/{register:dataset|package}/geo',
controller=u'ckanext.spatial.controllers.api:ApiController',
action=u'spatial_query')
return map
def before_index(self, pkg_dict):
'''
:param pkg_dict:
'''
import shapely.geometry
if pkg_dict.get(u'extras_spatial', None) and self.search_backend in (
u'solr', u'solr-spatial-field'):
try:
geometry = json.loads(pkg_dict[u'extras_spatial'])
except ValueError, e:
log.error(u'Geometry not valid GeoJSON, not indexing')
return pkg_dict
if self.search_backend == u'solr':
# Only bbox supported for this backend
if not (geometry[u'type'] == u'Polygon'
and len(geometry[u'coordinates']) == 1
and len(geometry[u'coordinates'][0]) == 5):
log.error(
u'Solr backend only supports bboxes (Polygons with 5 points), '
u'ignoring geometry {0}'.format(
pkg_dict[u'extras_spatial']))
return pkg_dict
coords = geometry[u'coordinates']
pkg_dict[u'maxy'] = max(coords[0][2][1], coords[0][0][1])
pkg_dict[u'miny'] = min(coords[0][2][1], coords[0][0][1])
pkg_dict[u'maxx'] = max(coords[0][2][0], coords[0][0][0])
pkg_dict[u'minx'] = min(coords[0][2][0], coords[0][0][0])
pkg_dict[u'bbox_area'] = (pkg_dict[u'maxx'] - pkg_dict[u'minx']) * \
(pkg_dict[u'maxy'] - pkg_dict[u'miny'])
elif self.search_backend == u'solr-spatial-field':
wkt = None
# Check potential problems with bboxes
if geometry[u'type'] == u'Polygon' \
and len(geometry[u'coordinates']) == 1 \
and len(geometry[u'coordinates'][0]) == 5:
# Check wrong bboxes (4 same points)
xs = [p[0] for p in geometry[u'coordinates'][0]]
ys = [p[1] for p in geometry[u'coordinates'][0]]
if xs.count(xs[0]) == 5 and ys.count(ys[0]) == 5:
wkt = u'POINT({x} {y})'.format(x=xs[0], y=ys[0])
else:
# Check if coordinates are defined counter-clockwise,
# otherwise we'll get wrong results from Solr
lr = shapely.geometry.polygon.LinearRing(
geometry[u'coordinates'][0])
if not lr.is_ccw:
lr.coords = list(lr.coords)[::-1]
polygon = shapely.geometry.polygon.Polygon(lr)
wkt = polygon.wkt
if not wkt:
shape = shapely.geometry.asShape(geometry)
if not shape.is_valid:
log.error(u'Wrong geometry, not indexing')
return pkg_dict
wkt = shape.wkt
pkg_dict[u'spatial_geom'] = wkt
return pkg_dict
class ExampleIAuthFunctionsPlugin(plugins.SingletonPlugin):
plugins.implements(plugins.IAuthFunctions)
def get_auth_functions(self):
return {'group_create': group_create}
# unfinished email trigger:
# self._email_on_change(context,resource,'privacy_contains_pii')
self._redirect_to_edit_on_change(resource, 'resource_type')
# reset monitored keys
for key in self.changed:
self.changed[key] = False
return
def before_delete(self, context, resource, resources):
return
def after_delete(self, context, resources):
return
def before_show(self, resource_dict):
return
p.implements(p.ITemplateHelpers)
def get_helpers(self):
return {'clean_select_multi': v.clean_select_multi,
'options_format': opts.format,
'options_storage_location': opts.storage_location,
'options_legal_authority_for_collection': opts.legal_authority_for_collection,
'options_privacy_pia_title': opts.privacy_pia_title,
'options_privacy_sorn_number': opts.privacy_sorn_number,
'tag_relevant_governing_documents': tag_relevant_governing_documents,
'options_relevant_governing_documents': opts.relevant_governing_documents,
'options_content_periodicity': opts.content_periodicity,
'options_update_frequency': opts.update_frequency,
'options_content_spatial': opts.content_spatial,
'options_pra_exclusion': opts.pra_exclusion,
'options_privacy_pia_notes': opts.privacy_pia_notes,
'options_transfer_method': opts.transfer_method,
class OLGeoView(GeoViewBase):
p.implements(p.IRoutes, inherit=True)
p.implements(p.ITemplateHelpers, inherit=True)
# IRoutes
def before_map(self, m):
controller = \
'ckanext.geoview.controllers.service_proxy:ServiceProxyController'
m.connect('/dataset/{id}/resource/{resource_id}/service_proxy',
controller=controller,
action='proxy_service')
m.connect('/basemap_service/{map_id}',
controller=controller,
action='proxy_service_url')
return m
# ITemplateHelpers
def get_helpers(self):
return {
'get_common_map_config_geoviews': get_common_map_config,
'get_openlayers_viewer_config': get_openlayers_viewer_config,
}
# IResourceView (CKAN >=2.3)
def info(self):
return {
'name': 'geo_view',
'title': 'Map viewer (OpenLayers)',
'icon': 'globe',
'iframed': True,
'default_title': toolkit._('Map viewer'),
'schema': {
'feature_hoveron': [ignore_empty, boolean_validator],
'feature_style': [ignore_empty]
},
}
def can_view(self, data_dict):
format_lower = data_dict['resource'].get('format', '').lower()
same_domain = on_same_domain(data_dict)
# Guess from file extension
if not format_lower and data_dict['resource'].get('url'):
format_lower = self._guess_format_from_extension(
data_dict['resource']['url'])
if not format_lower:
resource_locator_protocol = data_dict['resource'].get(
'resource_locator_protocol', '').lower()
if resource_locator_protocol and resource_locator_protocol.startswith(
'ogc:'):
format_lower = resource_locator_protocol.split(':')[1]
data_dict['resource']['format'] = format_lower
else:
return False
view_formats = toolkit.config.get('ckanext.geoview.ol_viewer.formats',
'')
if view_formats:
view_formats = view_formats.split(' ')
else:
view_formats = GEOVIEW_FORMATS
correct_format = format_lower in view_formats
can_preview_from_domain = self.proxy_enabled or same_domain
return correct_format and can_preview_from_domain
def view_template(self, context, data_dict):
return 'dataviewer/openlayers.html'
def form_template(self, context, data_dict):
return 'dataviewer/openlayers_form.html'
# IResourcePreview (CKAN < 2.3)
def can_preview(self, data_dict):
return self.can_view(data_dict)
def preview_template(self, context, data_dict):
return 'dataviewer/openlayers.html'
# Common for IResourceView and IResourcePreview
def _guess_format_from_extension(self, url):
try:
parsed_url = urlparse.urlparse(url)
format_lower = (os.path.splitext(parsed_url.path)[1][1:].encode(
'ascii', 'ignore').lower())
except ValueError, e:
log.error('Invalid URL: {0}, {1}'.format(url, e))
format_lower = ''
return format_lower
# upload the file into the dataset
resource = ckan.action.resource_create(
package_id=resource.get('package_id'),
name=resource.get('name'),
description=resource.get('description'),
format='CSV',
mimetype='CSV',
mimetype_inner='CSV',
url='dummy-value', # ignored, but required by ckan
upload=tmpfile
)
tmpfile.close()
p.implements(p.IResourceController, inherit=True)
def after_create(self, context, resource):
self.check_and_create_csv(context, resource)
def after_update(self, context, resource):
resource_url = resource.get('url')
self.check_and_create_csv(context, resource)
# TODO
# def after_delete(context, resource):
# pass
class MapperPlugin2(MapperPlugin):
implements(IMapper)