def update_config():
''' This code needs to be run when the config is changed to take those
changes into account. It is called whenever a plugin is loaded as the
plugin might have changed the config values (for instance it might
change ckan.site_url) '''
webassets_init()
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# Set whitelisted env vars on config object
# This is set up before globals are initialized
ckan_db = os.environ.get('CKAN_DB', None)
if ckan_db:
msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
log.warn(msg)
config['sqlalchemy.url'] = ckan_db
for option in CONFIG_FROM_ENV_VARS:
from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
if from_env:
config[option] = from_env
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
site_url = config.get('ckan.site_url', '')
if not site_url:
raise RuntimeError(
'ckan.site_url is not configured and it must have a value.'
' Please amend your .ini file.')
if not site_url.lower().startswith('http'):
raise RuntimeError(
'ckan.site_url should be a full URL, including the schema '
'(http or https)')
display_timezone = config.get('ckan.display_timezone', '')
if (display_timezone and display_timezone != 'server'
and display_timezone not in pytz.all_timezones):
raise CkanConfigurationException(
"ckan.display_timezone is not 'server' or a valid timezone")
# Remove backslash from site_url if present
config['ckan.site_url'] = config['ckan.site_url'].rstrip('/')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# ensure that a favicon has been set
favicon = config.get('ckan.favicon', '/base/images/ckan.ico')
config['ckan.favicon'] = favicon
# Init SOLR settings and check if the schema is compatible
# from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get('solr_url'), config.get('solr_user'),
config.get('solr_password'))
search.check_solr_schema_version()
if six.PY2:
routes_map = routing.make_map()
lib_plugins.reset_package_plugins()
lib_plugins.register_package_plugins()
lib_plugins.reset_group_plugins()
lib_plugins.register_group_plugins()
if six.PY2:
config['routes.map'] = routes_map
# The RoutesMiddleware needs its mapper updating if it exists
if 'routes.middleware' in config:
config['routes.middleware'].mapper = routes_map
# routes.named_routes is a CKAN thing
config['routes.named_routes'] = routing.named_routes
config['pylons.app_globals'] = app_globals.app_globals
# initialise the globals
app_globals.app_globals._init()
helpers.load_plugin_helpers()
config['pylons.h'] = helpers.helper_functions
# Templates and CSS loading from configuration
valid_base_templates_folder_names = ['templates']
templates = config.get('ckan.base_templates_folder', 'templates')
config['ckan.base_templates_folder'] = templates
if templates not in valid_base_templates_folder_names:
raise CkanConfigurationException(
'You provided an invalid value for ckan.base_templates_folder. '
'Possible values are: "templates".')
jinja2_templates_path = os.path.join(root, templates)
log.info('Loading templates from %s' % jinja2_templates_path)
template_paths = [jinja2_templates_path]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['computed_template_paths'] = template_paths
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
if six.PY2:
# Create Jinja2 environment
env = jinja_extensions.Environment(
**jinja_extensions.get_jinja_env_options())
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.policies['ext.i18n.trimmed'] = True
env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
config['pylons.app_globals'].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# Enable pessimistic disconnect handling (added in SQLAlchemy 1.2)
# to eliminate database errors due to stale pooled connections
config.setdefault('pool_pre_ping', True)
# Initialize SQLAlchemy
engine = sqlalchemy.engine_from_config(config)
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# reset the template cache - we do this here so that when we load the
# environment it is clean
render.reset_template_info_cache()
# clear other caches
logic.clear_actions_cache()
logic.clear_validators_cache()
authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# (ProgrammingError for Postgres, OperationalError for SQLite)
# The database is not initialised. This is a bit dirty. This occurs
# when running tests.
pass
except sqlalchemy.exc.InternalError:
# The database is not initialised. Travis hits this
pass
# Close current session and open database connections to ensure a clean
# clean environment even if an error occurs later on
model.Session.remove()
model.Session.bind.dispose()
def make_map():
"""Create, configure and return the routes Mapper"""
# import controllers here rather than at root level because
# pylons config is initialised by this point.
# Helpers to reduce code clutter
GET = dict(method=['GET'])
PUT = dict(method=['PUT'])
POST = dict(method=['POST'])
DELETE = dict(method=['DELETE'])
GET_POST = dict(method=['GET', 'POST'])
PUT_POST = dict(method=['PUT', 'POST'])
PUT_POST_DELETE = dict(method=['PUT', 'POST', 'DELETE'])
OPTIONS = dict(method=['OPTIONS'])
import ckan.lib.plugins as lib_plugins
lib_plugins.reset_package_plugins()
map = Mapper(directory=config['pylons.paths']['controllers'],
always_scan=config['debug'])
map.minimization = False
map.explicit = True
# CUSTOM ROUTES HERE
for plugin in p.PluginImplementations(p.IRoutes):
map = plugin.before_map(map)
# The ErrorController route (handles 404/500 error pages); it should
# likely stay at the top, ensuring it can always be resolved.
map.connect('/error/{action}', controller='error', ckan_core=True)
map.connect('/error/{action}/{id}', controller='error', ckan_core=True)
map.connect('*url',
controller='home',
action='cors_options',
conditions=OPTIONS,
ckan_core=True)
# Mark all routes added from extensions on the `before_map` extension point
# as non-core
for route in map.matchlist:
if not hasattr(route, '_ckan_core'):
route._ckan_core = False
map.connect('invite',
'/__invite__/',
controller='partyline',
action='join_party')
map.connect('home', '/', controller='home', action='index')
map.connect('about', '/about', controller='home', action='about')
# CKAN API versioned.
register_list = [
'package', 'dataset', 'resource', 'tag', 'group', 'revision',
'licenses', 'rating', 'user', 'activity'
]
register_list_str = '|'.join(register_list)
# /api ver 3 or none
with SubMapper(map,
controller='api',
path_prefix='/api{ver:/3|}',
ver='/3') as m:
m.connect('/action/{logic_function}',
action='action',
conditions=GET_POST)
# /api ver 1, 2, 3 or none
with SubMapper(map,
controller='api',
path_prefix='/api{ver:/1|/2|/3|}',
ver='/1') as m:
m.connect('/search/{register}', action='search')
# /api ver 1, 2 or none
with SubMapper(map,
controller='api',
path_prefix='/api{ver:/1|/2|}',
ver='/1') as m:
m.connect('/tag_counts', action='tag_counts')
m.connect('/rest', action='index')
m.connect('/qos/throughput/', action='throughput', conditions=GET)
# /api/rest ver 1, 2 or none
with SubMapper(map,
controller='api',
path_prefix='/api{ver:/1|/2|}',
ver='/1',
requirements=dict(register=register_list_str)) as m:
m.connect('/rest/{register}', action='list', conditions=GET)
m.connect('/rest/{register}', action='create', conditions=POST)
m.connect('/rest/{register}/{id}', action='show', conditions=GET)
m.connect('/rest/{register}/{id}', action='update', conditions=PUT)
m.connect('/rest/{register}/{id}', action='update', conditions=POST)
m.connect('/rest/{register}/{id}', action='delete', conditions=DELETE)
m.connect('/rest/{register}/{id}/:subregister',
action='list',
conditions=GET)
m.connect('/rest/{register}/{id}/:subregister',
action='create',
conditions=POST)
m.connect('/rest/{register}/{id}/:subregister/{id2}',
action='create',
conditions=POST)
m.connect('/rest/{register}/{id}/:subregister/{id2}',
action='show',
conditions=GET)
m.connect('/rest/{register}/{id}/:subregister/{id2}',
action='update',
conditions=PUT)
m.connect('/rest/{register}/{id}/:subregister/{id2}',
action='delete',
conditions=DELETE)
# /api/util ver 1, 2 or none
with SubMapper(map,
controller='api',
path_prefix='/api{ver:/1|/2|}',
ver='/1') as m:
m.connect('/util/user/autocomplete', action='user_autocomplete')
m.connect('/util/is_slug_valid',
action='is_slug_valid',
conditions=GET)
m.connect('/util/dataset/autocomplete',
action='dataset_autocomplete',
conditions=GET)
m.connect('/util/tag/autocomplete',
action='tag_autocomplete',
conditions=GET)
m.connect('/util/resource/format_autocomplete',
action='format_autocomplete',
conditions=GET)
m.connect('/util/resource/format_icon',
action='format_icon',
conditions=GET)
m.connect('/util/group/autocomplete', action='group_autocomplete')
m.connect('/util/organization/autocomplete',
action='organization_autocomplete',
conditions=GET)
m.connect('/util/markdown', action='markdown')
m.connect('/util/dataset/munge_name', action='munge_package_name')
m.connect('/util/dataset/munge_title_to_name',
action='munge_title_to_package_name')
m.connect('/util/tag/munge', action='munge_tag')
m.connect('/util/status', action='status')
m.connect('/util/snippet/{snippet_path:.*}', action='snippet')
m.connect('/i18n/{lang}', action='i18n_js_translations')
###########
## /END API
###########
map.redirect('/packages', '/dataset')
map.redirect('/packages/{url:.*}', '/dataset/{url}')
map.redirect('/package', '/dataset')
map.redirect('/package/{url:.*}', '/dataset/{url}')
with SubMapper(map, controller='package') as m:
m.connect('search',
'/dataset',
action='search',
highlight_actions='index search')
m.connect('add dataset', '/dataset/new', action='new')
m.connect('/dataset/{action}',
requirements=dict(
action='|'.join(['list', 'autocomplete', 'search'])))
m.connect('/dataset/{action}/{id}/{revision}',
action='read_ajax',
requirements=dict(action='|'.join([
'read',
'edit',
'history',
])))
m.connect('/dataset/{action}/{id}',
requirements=dict(action='|'.join([
'new_resource',
'history',
'read_ajax',
'history_ajax',
'follow',
'activity',
'groups',
'unfollow',
'delete',
'api_data',
])))
m.connect('dataset_edit',
'/dataset/edit/{id}',
action='edit',
ckan_icon='edit')
m.connect('dataset_followers',
'/dataset/followers/{id}',
action='followers',
ckan_icon='group')
m.connect('dataset_activity',
'/dataset/activity/{id}',
action='activity',
ckan_icon='time')
m.connect('/dataset/activity/{id}/{offset}', action='activity')
m.connect('dataset_groups',
'/dataset/groups/{id}',
action='groups',
ckan_icon='group')
m.connect('dataset_resources',
'/dataset/resources/{id}',
action='resources',
ckan_icon='reorder')
m.connect('dataset_read',
'/dataset/{id}',
action='read',
ckan_icon='sitemap')
m.connect('/dataset/{id}/resource/{resource_id}',
action='resource_read')
m.connect('/dataset/{id}/resource_delete/{resource_id}',
action='resource_delete')
m.connect('resource_edit',
'/dataset/{id}/resource_edit/{resource_id}',
action='resource_edit',
ckan_icon='edit')
m.connect('/dataset/{id}/resource/{resource_id}/download',
action='resource_download')
m.connect('/dataset/{id}/resource/{resource_id}/download/{filename}',
action='resource_download')
m.connect('/dataset/{id}/resource/{resource_id}/embed',
action='resource_embedded_dataviewer')
m.connect('/dataset/{id}/resource/{resource_id}/viewer',
action='resource_embedded_dataviewer',
width="960",
height="800")
m.connect('/dataset/{id}/resource/{resource_id}/preview',
action='resource_datapreview')
m.connect('views',
'/dataset/{id}/resource/{resource_id}/views',
action='resource_views',
ckan_icon='reorder')
m.connect('new_view',
'/dataset/{id}/resource/{resource_id}/new_view',
action='edit_view',
ckan_icon='edit')
m.connect('edit_view',
'/dataset/{id}/resource/{resource_id}/edit_view/{view_id}',
action='edit_view',
ckan_icon='edit')
m.connect('resource_view',
'/dataset/{id}/resource/{resource_id}/view/{view_id}',
action='resource_view')
m.connect('/dataset/{id}/resource/{resource_id}/view/',
action='resource_view')
# group
map.redirect('/groups', '/group')
map.redirect('/groups/{url:.*}', '/group/{url}')
# These named routes are used for custom group forms which will use the
# names below based on the group.type ('group' is the default type)
with SubMapper(map, controller='group') as m:
m.connect('group_index',
'/group',
action='index',
highlight_actions='index search')
m.connect('group_list', '/group/list', action='list')
m.connect('group_new', '/group/new', action='new')
m.connect('group_action',
'/group/{action}/{id}',
requirements=dict(action='|'.join([
'edit',
'delete',
'member_new',
'member_delete',
'history',
'followers',
'follow',
'unfollow',
'admins',
'activity',
])))
m.connect('group_about',
'/group/about/{id}',
action='about',
ckan_icon='info-sign'),
m.connect('group_edit',
'/group/edit/{id}',
action='edit',
ckan_icon='edit')
m.connect('group_members',
'/group/members/{id}',
action='members',
ckan_icon='group'),
m.connect('group_activity',
'/group/activity/{id}/{offset}',
action='activity',
ckan_icon='time'),
m.connect('group_read',
'/group/{id}',
action='read',
ckan_icon='sitemap')
# organizations these basically end up being the same as groups
with SubMapper(map, controller='organization') as m:
m.connect('organizations_index', '/organization', action='index')
m.connect('/organization/list', action='list')
m.connect('/organization/new', action='new')
m.connect(
'/organization/{action}/{id}',
requirements=dict(action='|'.join([
'delete', 'admins', 'member_new', 'member_delete', 'history'
])))
m.connect('organization_activity',
'/organization/activity/{id}/{offset}',
action='activity',
ckan_icon='time')
m.connect('organization_read', '/organization/{id}', action='read')
m.connect('organization_about',
'/organization/about/{id}',
action='about',
ckan_icon='info-sign')
m.connect('organization_read',
'/organization/{id}',
action='read',
ckan_icon='sitemap')
m.connect('organization_edit',
'/organization/edit/{id}',
action='edit',
ckan_icon='edit')
m.connect('organization_members',
'/organization/members/{id}',
action='members',
ckan_icon='group')
m.connect('organization_bulk_process',
'/organization/bulk_process/{id}',
action='bulk_process',
ckan_icon='sitemap')
lib_plugins.register_package_plugins(map)
lib_plugins.register_group_plugins(map)
# tags
map.redirect('/tags', '/tag')
map.redirect('/tags/{url:.*}', '/tag/{url}')
map.redirect('/tag/read/{url:.*}',
'/tag/{url}',
_redirect_code='301 Moved Permanently')
map.connect('/tag', controller='tag', action='index')
map.connect('/tag/{id}', controller='tag', action='read')
# users
map.redirect('/users/{url:.*}', '/user/{url}')
map.redirect('/user/', '/user')
with SubMapper(map, controller='user') as m:
m.connect('/user/edit', action='edit')
# Note: openid users have slashes in their ids, so need the wildcard
# in the route.
m.connect('user_generate_apikey',
'/user/generate_key/{id}',
action='generate_apikey')
m.connect('/user/activity/{id}/{offset}', action='activity')
m.connect('user_activity_stream',
'/user/activity/{id}',
action='activity',
ckan_icon='time')
m.connect('user_dashboard',
'/dashboard',
action='dashboard',
ckan_icon='list')
m.connect('user_dashboard_datasets',
'/dashboard/datasets',
action='dashboard_datasets',
ckan_icon='sitemap')
m.connect('user_dashboard_groups',
'/dashboard/groups',
action='dashboard_groups',
ckan_icon='group')
m.connect('user_dashboard_organizations',
'/dashboard/organizations',
action='dashboard_organizations',
ckan_icon='building')
m.connect('/dashboard/{offset}', action='dashboard')
m.connect('user_follow', '/user/follow/{id}', action='follow')
m.connect('/user/unfollow/{id}', action='unfollow')
m.connect('user_followers',
'/user/followers/{id:.*}',
action='followers',
ckan_icon='group')
m.connect('user_edit',
'/user/edit/{id:.*}',
action='edit',
ckan_icon='cog')
m.connect('user_delete', '/user/delete/{id}', action='delete')
m.connect('/user/reset/{id:.*}', action='perform_reset')
m.connect('register', '/user/register', action='register')
m.connect('login', '/user/login', action='login')
m.connect('/user/_logout', action='logout')
m.connect('/user/logged_in', action='logged_in')
m.connect('/user/logged_out', action='logged_out')
m.connect('/user/logged_out_redirect', action='logged_out_page')
m.connect('/user/reset', action='request_reset')
m.connect('/user/me', action='me')
m.connect('/user/set_lang/{lang}', action='set_lang')
m.connect('user_datasets',
'/user/{id:.*}',
action='read',
ckan_icon='sitemap')
m.connect('user_index', '/user', action='index')
with SubMapper(map, controller='revision') as m:
m.connect('/revision', action='index')
m.connect('/revision/edit/{id}', action='edit')
m.connect('/revision/diff/{id}', action='diff')
m.connect('/revision/list', action='list')
m.connect('/revision/{id}', action='read')
# feeds
with SubMapper(map, controller='feed') as m:
m.connect('/feeds/group/{id}.atom', action='group')
m.connect('/feeds/organization/{id}.atom', action='organization')
m.connect('/feeds/tag/{id}.atom', action='tag')
m.connect('/feeds/dataset.atom', action='general')
m.connect('/feeds/custom.atom', action='custom')
map.connect('ckanadmin_index',
'/ckan-admin',
controller='admin',
action='index',
ckan_icon='legal')
map.connect('ckanadmin_config',
'/ckan-admin/config',
controller='admin',
action='config',
ckan_icon='check')
map.connect('ckanadmin_trash',
'/ckan-admin/trash',
controller='admin',
action='trash',
ckan_icon='trash')
map.connect('ckanadmin', '/ckan-admin/{action}', controller='admin')
with SubMapper(
map, controller='ckan.controllers.storage:StorageController') as m:
m.connect('storage_file', '/storage/f/{label:.*}', action='file')
with SubMapper(map, controller='util') as m:
m.connect('/i18n/strings_{lang}.js', action='i18n_js_strings')
m.connect('/util/redirect', action='redirect')
m.connect('/testing/primer', action='primer')
m.connect('/testing/markup', action='markup')
# robots.txt
map.connect('/(robots.txt)', controller='template', action='view')
# Mark all unmarked routes added up until now as core routes
for route in map.matchlist:
if not hasattr(route, '_ckan_core'):
route._ckan_core = True
for plugin in p.PluginImplementations(p.IRoutes):
map = plugin.after_map(map)
# Mark all routes added from extensions on the `after_map` extension point
# as non-core
for route in map.matchlist:
if not hasattr(route, '_ckan_core'):
route._ckan_core = False
# sometimes we get requests for favicon.ico we should redirect to
# the real favicon location.
map.redirect('/favicon.ico', config.get('ckan.favicon'))
map.redirect('/*(url)/', '/{url}', _redirect_code='301 Moved Permanently')
map.connect('/*url', controller='template', action='view', ckan_core=True)
return map
def make_map():
"""Create, configure and return the routes Mapper"""
# import controllers here rather than at root level because
# pylons config is initialised by this point.
# Helpers to reduce code clutter
GET = dict(method=['GET'])
PUT = dict(method=['PUT'])
POST = dict(method=['POST'])
DELETE = dict(method=['DELETE'])
GET_POST = dict(method=['GET', 'POST'])
PUT_POST = dict(method=['PUT', 'POST'])
PUT_POST_DELETE = dict(method=['PUT', 'POST', 'DELETE'])
OPTIONS = dict(method=['OPTIONS'])
import ckan.lib.plugins as lib_plugins
lib_plugins.reset_package_plugins()
map = Mapper(directory=config['pylons.paths']['controllers'],
always_scan=config['debug'])
map.minimization = False
map.explicit = True
# CUSTOM ROUTES HERE
for plugin in p.PluginImplementations(p.IRoutes):
map = plugin.before_map(map)
# The ErrorController route (handles 404/500 error pages); it should
# likely stay at the top, ensuring it can always be resolved.
map.connect('/error/{action}', controller='error', ckan_core=True)
map.connect('/error/{action}/{id}', controller='error', ckan_core=True)
map.connect('*url', controller='home', action='cors_options',
conditions=OPTIONS, ckan_core=True)
# Mark all routes added from extensions on the `before_map` extension point
# as non-core
for route in map.matchlist:
if not hasattr(route, '_ckan_core'):
route._ckan_core = False
map.connect('invite', '/__invite__/', controller='partyline', action='join_party')
map.connect('home', '/', controller='home', action='index')
map.connect('about', '/about', controller='home', action='about')
# CKAN API versioned.
register_list = [
'package',
'dataset',
'resource',
'tag',
'group',
'revision',
'licenses',
'rating',
'user',
'activity'
]
register_list_str = '|'.join(register_list)
# /api ver 3 or none
with SubMapper(map, controller='api', path_prefix='/api{ver:/3|}',
ver='/3') as m:
m.connect('/action/{logic_function}', action='action',
conditions=GET_POST)
# /api ver 1, 2, 3 or none
with SubMapper(map, controller='api', path_prefix='/api{ver:/1|/2|/3|}',
ver='/1') as m:
m.connect('/search/{register}', action='search')
# /api ver 1, 2 or none
with SubMapper(map, controller='api', path_prefix='/api{ver:/1|/2|}',
ver='/1') as m:
m.connect('/tag_counts', action='tag_counts')
m.connect('/rest', action='index')
m.connect('/qos/throughput/', action='throughput', conditions=GET)
# /api/rest ver 1, 2 or none
with SubMapper(map, controller='api', path_prefix='/api{ver:/1|/2|}',
ver='/1', requirements=dict(register=register_list_str)
) as m:
m.connect('/rest/{register}', action='list', conditions=GET)
m.connect('/rest/{register}', action='create', conditions=POST)
m.connect('/rest/{register}/{id}', action='show', conditions=GET)
m.connect('/rest/{register}/{id}', action='update', conditions=PUT)
m.connect('/rest/{register}/{id}', action='update', conditions=POST)
m.connect('/rest/{register}/{id}', action='delete', conditions=DELETE)
m.connect('/rest/{register}/{id}/:subregister', action='list',
conditions=GET)
m.connect('/rest/{register}/{id}/:subregister', action='create',
conditions=POST)
m.connect('/rest/{register}/{id}/:subregister/{id2}', action='create',
conditions=POST)
m.connect('/rest/{register}/{id}/:subregister/{id2}', action='show',
conditions=GET)
m.connect('/rest/{register}/{id}/:subregister/{id2}', action='update',
conditions=PUT)
m.connect('/rest/{register}/{id}/:subregister/{id2}', action='delete',
conditions=DELETE)
# /api/util ver 1, 2 or none
with SubMapper(map, controller='api', path_prefix='/api{ver:/1|/2|}',
ver='/1') as m:
m.connect('/util/user/autocomplete', action='user_autocomplete')
m.connect('/util/is_slug_valid', action='is_slug_valid',
conditions=GET)
m.connect('/util/dataset/autocomplete', action='dataset_autocomplete',
conditions=GET)
m.connect('/util/tag/autocomplete', action='tag_autocomplete',
conditions=GET)
m.connect('/util/resource/format_autocomplete',
action='format_autocomplete', conditions=GET)
m.connect('/util/resource/format_icon',
action='format_icon', conditions=GET)
m.connect('/util/group/autocomplete', action='group_autocomplete')
m.connect('/util/organization/autocomplete', action='organization_autocomplete',
conditions=GET)
m.connect('/util/markdown', action='markdown')
m.connect('/util/dataset/munge_name', action='munge_package_name')
m.connect('/util/dataset/munge_title_to_name',
action='munge_title_to_package_name')
m.connect('/util/tag/munge', action='munge_tag')
m.connect('/util/status', action='status')
m.connect('/util/snippet/{snippet_path:.*}', action='snippet')
m.connect('/i18n/{lang}', action='i18n_js_translations')
###########
## /END API
###########
map.redirect('/packages', '/dataset')
map.redirect('/packages/{url:.*}', '/dataset/{url}')
map.redirect('/package', '/dataset')
map.redirect('/package/{url:.*}', '/dataset/{url}')
with SubMapper(map, controller='package') as m:
m.connect('search', '/dataset', action='search',
highlight_actions='index search')
m.connect('add dataset', '/dataset/new', action='new')
m.connect('/dataset/{action}',
requirements=dict(action='|'.join([
'list',
'autocomplete',
'search'
])))
m.connect('/dataset/{action}/{id}/{revision}', action='read_ajax',
requirements=dict(action='|'.join([
'read',
'edit',
'history',
])))
m.connect('/dataset/{action}/{id}',
requirements=dict(action='|'.join([
'new_resource',
'history',
'read_ajax',
'history_ajax',
'follow',
'activity',
'groups',
'unfollow',
'delete',
'api_data',
])))
m.connect('dataset_edit', '/dataset/edit/{id}', action='edit',
ckan_icon='pencil-square-o')
m.connect('dataset_followers', '/dataset/followers/{id}',
action='followers', ckan_icon='users')
m.connect('dataset_activity', '/dataset/activity/{id}',
action='activity', ckan_icon='clock-o')
m.connect('/dataset/activity/{id}/{offset}', action='activity')
m.connect('dataset_groups', '/dataset/groups/{id}',
action='groups', ckan_icon='users')
m.connect('dataset_resources', '/dataset/resources/{id}',
action='resources', ckan_icon='bars')
m.connect('dataset_read', '/dataset/{id}', action='read',
ckan_icon='sitemap')
m.connect('/dataset/{id}/resource/{resource_id}',
action='resource_read')
m.connect('/dataset/{id}/resource_delete/{resource_id}',
action='resource_delete')
m.connect('resource_edit', '/dataset/{id}/resource_edit/{resource_id}',
action='resource_edit', ckan_icon='pencil-square-o')
m.connect('/dataset/{id}/resource/{resource_id}/download',
action='resource_download')
m.connect('/dataset/{id}/resource/{resource_id}/download/{filename}',
action='resource_download')
m.connect('/dataset/{id}/resource/{resource_id}/embed',
action='resource_embedded_dataviewer')
m.connect('/dataset/{id}/resource/{resource_id}/viewer',
action='resource_embedded_dataviewer', width="960",
height="800")
m.connect('/dataset/{id}/resource/{resource_id}/preview',
action='resource_datapreview')
m.connect('views', '/dataset/{id}/resource/{resource_id}/views',
action='resource_views', ckan_icon='bars')
m.connect('new_view', '/dataset/{id}/resource/{resource_id}/new_view',
action='edit_view', ckan_icon='pencil-square-o')
m.connect('edit_view',
'/dataset/{id}/resource/{resource_id}/edit_view/{view_id}',
action='edit_view', ckan_icon='pencil-square-o')
m.connect('resource_view',
'/dataset/{id}/resource/{resource_id}/view/{view_id}',
action='resource_view')
m.connect('/dataset/{id}/resource/{resource_id}/view/',
action='resource_view')
# group
map.redirect('/groups', '/group')
map.redirect('/groups/{url:.*}', '/group/{url}')
# These named routes are used for custom group forms which will use the
# names below based on the group.type ('group' is the default type)
with SubMapper(map, controller='group') as m:
m.connect('group_index', '/group', action='index',
highlight_actions='index search')
m.connect('group_list', '/group/list', action='list')
m.connect('group_new', '/group/new', action='new')
m.connect('group_action', '/group/{action}/{id}',
requirements=dict(action='|'.join([
'edit',
'delete',
'member_new',
'member_delete',
'history',
'followers',
'follow',
'unfollow',
'admins',
'activity',
])))
m.connect('group_about', '/group/about/{id}', action='about',
ckan_icon='info-circle'),
m.connect('group_edit', '/group/edit/{id}', action='edit',
ckan_icon='pencil-square-o')
m.connect('group_members', '/group/members/{id}', action='members',
ckan_icon='users'),
m.connect('group_activity', '/group/activity/{id}/{offset}',
action='activity', ckan_icon='clock-o'),
m.connect('group_read', '/group/{id}', action='read',
ckan_icon='sitemap')
# organizations these basically end up being the same as groups
with SubMapper(map, controller='organization') as m:
m.connect('organizations_index', '/organization', action='index')
m.connect('/organization/list', action='list')
m.connect('/organization/new', action='new')
m.connect('/organization/{action}/{id}',
requirements=dict(action='|'.join([
'delete',
'admins',
'member_new',
'member_delete',
'history'
])))
m.connect('organization_activity', '/organization/activity/{id}/{offset}',
action='activity', ckan_icon='clock-o')
m.connect('organization_read', '/organization/{id}', action='read')
m.connect('organization_about', '/organization/about/{id}',
action='about', ckan_icon='info-circle')
m.connect('organization_read', '/organization/{id}', action='read',
ckan_icon='sitemap')
m.connect('organization_edit', '/organization/edit/{id}',
action='edit', ckan_icon='pencil-square-o')
m.connect('organization_members', '/organization/members/{id}',
action='members', ckan_icon='users')
m.connect('organization_bulk_process',
'/organization/bulk_process/{id}',
action='bulk_process', ckan_icon='sitemap')
lib_plugins.register_package_plugins(map)
lib_plugins.register_group_plugins(map)
# tags
map.redirect('/tags', '/tag')
map.redirect('/tags/{url:.*}', '/tag/{url}')
map.redirect('/tag/read/{url:.*}', '/tag/{url}',
_redirect_code='301 Moved Permanently')
map.connect('/tag', controller='tag', action='index')
map.connect('/tag/{id}', controller='tag', action='read')
# users
map.redirect('/users/{url:.*}', '/user/{url}')
map.redirect('/user/', '/user')
with SubMapper(map, controller='user') as m:
m.connect('/user/edit', action='edit')
# Note: openid users have slashes in their ids, so need the wildcard
# in the route.
m.connect('user_generate_apikey', '/user/generate_key/{id}', action='generate_apikey')
m.connect('/user/activity/{id}/{offset}', action='activity')
m.connect('user_activity_stream', '/user/activity/{id}',
action='activity', ckan_icon='clock-o')
m.connect('user_dashboard', '/dashboard', action='dashboard',
ckan_icon='list')
m.connect('user_dashboard_datasets', '/dashboard/datasets',
action='dashboard_datasets', ckan_icon='sitemap')
m.connect('user_dashboard_groups', '/dashboard/groups',
action='dashboard_groups', ckan_icon='users')
m.connect('user_dashboard_organizations', '/dashboard/organizations',
action='dashboard_organizations', ckan_icon='building-o')
m.connect('/dashboard/{offset}', action='dashboard')
m.connect('user_follow', '/user/follow/{id}', action='follow')
m.connect('/user/unfollow/{id}', action='unfollow')
m.connect('user_followers', '/user/followers/{id:.*}',
action='followers', ckan_icon='users')
m.connect('user_edit', '/user/edit/{id:.*}', action='edit',
ckan_icon='cog')
m.connect('user_delete', '/user/delete/{id}', action='delete')
m.connect('/user/reset/{id:.*}', action='perform_reset')
m.connect('register', '/user/register', action='register')
m.connect('login', '/user/login', action='login')
m.connect('/user/_logout', action='logout')
m.connect('/user/logged_in', action='logged_in')
m.connect('/user/logged_out', action='logged_out')
m.connect('/user/logged_out_redirect', action='logged_out_page')
m.connect('/user/reset', action='request_reset')
m.connect('/user/me', action='me')
m.connect('/user/set_lang/{lang}', action='set_lang')
m.connect('user_datasets', '/user/{id:.*}', action='read',
ckan_icon='sitemap')
m.connect('user_index', '/user', action='index')
with SubMapper(map, controller='revision') as m:
m.connect('/revision', action='index')
m.connect('/revision/edit/{id}', action='edit')
m.connect('/revision/diff/{id}', action='diff')
m.connect('/revision/list', action='list')
m.connect('/revision/{id}', action='read')
# feeds
with SubMapper(map, controller='feed') as m:
m.connect('/feeds/group/{id}.atom', action='group')
m.connect('/feeds/organization/{id}.atom', action='organization')
m.connect('/feeds/tag/{id}.atom', action='tag')
m.connect('/feeds/dataset.atom', action='general')
m.connect('/feeds/custom.atom', action='custom')
map.connect('ckanadmin_index', '/ckan-admin', controller='admin',
action='index', ckan_icon='gavel')
map.connect('ckanadmin_config', '/ckan-admin/config', controller='admin',
action='config', ckan_icon='check-square-o')
map.connect('ckanadmin_trash', '/ckan-admin/trash', controller='admin',
action='trash', ckan_icon='trash-o')
map.connect('ckanadmin', '/ckan-admin/{action}', controller='admin')
with SubMapper(map, controller='ckan.controllers.storage:StorageController') as m:
m.connect('storage_file', '/storage/f/{label:.*}',
action='file')
with SubMapper(map, controller='util') as m:
m.connect('/i18n/strings_{lang}.js', action='i18n_js_strings')
m.connect('/util/redirect', action='redirect')
m.connect('/testing/primer', action='primer')
m.connect('/testing/markup', action='markup')
# robots.txt
map.connect('/(robots.txt)', controller='template', action='view')
# Mark all unmarked routes added up until now as core routes
for route in map.matchlist:
if not hasattr(route, '_ckan_core'):
route._ckan_core = True
for plugin in p.PluginImplementations(p.IRoutes):
map = plugin.after_map(map)
# Mark all routes added from extensions on the `after_map` extension point
# as non-core
for route in map.matchlist:
if not hasattr(route, '_ckan_core'):
route._ckan_core = False
# sometimes we get requests for favicon.ico we should redirect to
# the real favicon location.
map.redirect('/favicon.ico', config.get('ckan.favicon'))
map.redirect('/*(url)/', '/{url}',
_redirect_code='301 Moved Permanently')
map.connect('/*url', controller='template', action='view', ckan_core=True)
return map
def update_config():
''' This code needs to be run when the config is changed to take those
changes into account. It is called whenever a plugin is loaded as the
plugin might have changed the config values (for instance it might
change ckan.site_url) '''
config_declaration.setup()
config_declaration.make_safe(config)
config_declaration.normalize(config)
webassets_init()
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# Set whitelisted env vars on config object
# This is set up before globals are initialized
ckan_db = os.environ.get('CKAN_DB', None)
if ckan_db:
msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
log.warn(msg)
config['sqlalchemy.url'] = ckan_db
for option in CONFIG_FROM_ENV_VARS:
from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
if from_env:
config[option] = from_env
if config.get_value("config.mode") == "strict":
_, errors = config_declaration.validate(config)
if errors:
msg = "\n".join("{}: {}".format(key, "; ".join(issues))
for key, issues in errors.items())
raise CkanConfigurationException(msg)
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
site_url = config.get_value('ckan.site_url')
if not site_url:
raise RuntimeError(
'ckan.site_url is not configured and it must have a value.'
' Please amend your .ini file.')
if not site_url.lower().startswith('http'):
raise RuntimeError(
'ckan.site_url should be a full URL, including the schema '
'(http or https)')
# Remove backslash from site_url if present
config['ckan.site_url'] = site_url.rstrip('/')
display_timezone = config.get_value('ckan.display_timezone')
if (display_timezone and display_timezone != 'server'
and display_timezone not in pytz.all_timezones):
raise CkanConfigurationException(
"ckan.display_timezone is not 'server' or a valid timezone")
# Init SOLR settings and check if the schema is compatible
# from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get_value('solr_url'),
config.get_value('solr_user'),
config.get_value('solr_password'))
search.check_solr_schema_version()
lib_plugins.reset_package_plugins()
lib_plugins.register_package_plugins()
lib_plugins.reset_group_plugins()
lib_plugins.register_group_plugins()
# initialise the globals
app_globals.app_globals._init()
helpers.load_plugin_helpers()
# Templates and CSS loading from configuration
valid_base_templates_folder_names = ['templates']
templates = config.get_value('ckan.base_templates_folder')
config['ckan.base_templates_folder'] = templates
if templates not in valid_base_templates_folder_names:
raise CkanConfigurationException(
'You provided an invalid value for ckan.base_templates_folder. '
'Possible values are: "templates".')
jinja2_templates_path = os.path.join(root, templates)
log.info('Loading templates from %s' % jinja2_templates_path)
template_paths = [jinja2_templates_path]
extra_template_paths = config.get_value('extra_template_paths')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['computed_template_paths'] = template_paths
# Enable pessimistic disconnect handling (added in SQLAlchemy 1.2)
# to eliminate database errors due to stale pooled connections
config.setdefault('sqlalchemy.pool_pre_ping', True)
# Initialize SQLAlchemy
engine = sqlalchemy.engine_from_config(config)
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# clear other caches
logic.clear_actions_cache()
logic.clear_validators_cache()
authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# The database is not yet initialised. It happens in `ckan db init`
pass
# Close current session and open database connections to ensure a clean
# clean environment even if an error occurs later on
model.Session.remove()
model.Session.bind.dispose()
def make_map():
"""Create, configure and return the routes Mapper"""
# import controllers here rather than at root level because
# pylons config is initialised by this point.
# Helpers to reduce code clutter
GET = dict(method=['GET'])
PUT = dict(method=['PUT'])
POST = dict(method=['POST'])
DELETE = dict(method=['DELETE'])
GET_POST = dict(method=['GET', 'POST'])
PUT_POST = dict(method=['PUT', 'POST'])
PUT_POST_DELETE = dict(method=['PUT', 'POST', 'DELETE'])
OPTIONS = dict(method=['OPTIONS'])
import ckan.lib.plugins as lib_plugins
lib_plugins.reset_package_plugins()
map = Mapper(directory=config['pylons.paths']['controllers'],
always_scan=config['debug'])
map.minimization = False
map.explicit = True
# CUSTOM ROUTES HERE
for plugin in p.PluginImplementations(p.IRoutes):
map = plugin.before_map(map)
# The ErrorController route (handles 404/500 error pages); it should
# likely stay at the top, ensuring it can always be resolved.
map.connect('/error/{action}', controller='error', ckan_core=True)
map.connect('/error/{action}/{id}', controller='error', ckan_core=True)
map.connect('*url', controller='home', action='cors_options',
conditions=OPTIONS, ckan_core=True)
# Mark all routes added from extensions on the `before_map` extension point
# as non-core
for route in map.matchlist:
if not hasattr(route, '_ckan_core'):
route._ckan_core = False
# CKAN API versioned.
register_list = [
'package',
'dataset',
'resource',
'tag',
'group',
'revision',
'licenses',
'rating',
'user',
'activity'
]
register_list_str = '|'.join(register_list)
# /api ver 1, 2, 3 or none
with SubMapper(map, controller='api', path_prefix='/api{ver:/1|/2|/3|}',
ver='/1') as m:
m.connect('/search/{register}', action='search')
# /api/util ver 1, 2 or none
with SubMapper(map, controller='api', path_prefix='/api{ver:/1|/2|}',
ver='/1') as m:
m.connect('/util/dataset/munge_name', action='munge_package_name')
m.connect('/util/dataset/munge_title_to_name',
action='munge_title_to_package_name')
m.connect('/util/tag/munge', action='munge_tag')
###########
## /END API
###########
map.redirect('/packages', '/dataset')
map.redirect('/packages/{url:.*}', '/dataset/{url}')
map.redirect('/package', '/dataset')
map.redirect('/package/{url:.*}', '/dataset/{url}')
with SubMapper(map, controller='package') as m:
m.connect('search', '/dataset', action='search',
highlight_actions='index search')
m.connect('dataset_new', '/dataset/new', action='new')
m.connect('/dataset/{action}',
requirements=dict(action='|'.join([
'list',
'autocomplete',
'search'
])))
m.connect('/dataset/{action}/{id}/{revision}', action='read_ajax',
requirements=dict(action='|'.join([
'read',
'edit',
'history',
])))
m.connect('/dataset/{action}/{id}',
requirements=dict(action='|'.join([
'new_resource',
'history',
'read_ajax',
'history_ajax',
'follow',
'activity',
'groups',
'unfollow',
'delete',
'api_data',
])))
m.connect('dataset_edit', '/dataset/edit/{id}', action='edit',
ckan_icon='pencil-square-o')
m.connect('dataset_followers', '/dataset/followers/{id}',
action='followers', ckan_icon='users')
m.connect('dataset_activity', '/dataset/activity/{id}',
action='activity', ckan_icon='clock-o')
m.connect('/dataset/activity/{id}/{offset}', action='activity')
m.connect('dataset_topics', '/dataset/topics/{id}',
action='groups', ckan_icon='users')
m.connect('dataset_resources', '/dataset/resources/{id}',
action='resources', ckan_icon='bars')
m.connect('dataset_read', '/dataset/{id}', action='read',
ckan_icon='sitemap')
m.connect('/dataset/{id}/resource/{resource_id}',
action='resource_read')
m.connect('/dataset/{id}/resource_delete/{resource_id}',
action='resource_delete')
m.connect('resource_edit', '/dataset/{id}/resource_edit/{resource_id}',
action='resource_edit', ckan_icon='pencil-square-o')
m.connect('/dataset/{id}/resource/{resource_id}/download',
action='resource_download')
m.connect('/dataset/{id}/resource/{resource_id}/download/{filename}',
action='resource_download')
m.connect('/dataset/{id}/resource/{resource_id}/embed',
action='resource_embedded_dataviewer')
m.connect('/dataset/{id}/resource/{resource_id}/viewer',
action='resource_embedded_dataviewer', width="960",
height="800")
m.connect('/dataset/{id}/resource/{resource_id}/preview',
action='resource_datapreview')
m.connect('views', '/dataset/{id}/resource/{resource_id}/views',
action='resource_views', ckan_icon='bars')
m.connect('new_view', '/dataset/{id}/resource/{resource_id}/new_view',
action='edit_view', ckan_icon='pencil-square-o')
m.connect('edit_view',
'/dataset/{id}/resource/{resource_id}/edit_view/{view_id}',
action='edit_view', ckan_icon='pencil-square-o')
m.connect('resource_view',
'/dataset/{id}/resource/{resource_id}/view/{view_id}',
action='resource_view')
m.connect('/dataset/{id}/resource/{resource_id}/view/',
action='resource_view')
# group
map.redirect('/topics', '/topic')
map.redirect('/topics/{url:.*}', '/topic/{url}')
# These named routes are used for custom group forms which will use the
# names below based on the group.type ('group' is the default type)
with SubMapper(map, controller='group') as m:
m.connect('group_index', '/topic', action='index',
highlight_actions='index search')
m.connect('group_list', '/topic/list', action='list')
m.connect('group_new', '/topic/new', action='new')
for action in [
'edit',
'delete',
'member_new',
'member_delete',
'history',
'followers',
'follow',
'unfollow',
'admins',
'activity',
]:
m.connect('group_' + action,
'/topic/' + action + '/{id}',
action=action)
m.connect('group_about', '/topic/about/{id}', action='about',
ckan_icon='info-circle'),
m.connect('group_edit', '/topic/edit/{id}', action='edit',
ckan_icon='pencil-square-o')
m.connect('group_members', '/topic/members/{id}', action='members',
ckan_icon='users'),
m.connect('group_activity', '/topic/activity/{id}/{offset}',
action='activity', ckan_icon='clock-o'),
m.connect('group_read', '/topic/{id}', action='read',
ckan_icon='sitemap')
# organizations these basically end up being the same as groups
with SubMapper(map, controller='organization') as m:
m.connect('organizations_index', '/organization', action='index')
m.connect('organization_index', '/organization', action='index')
m.connect('organization_new', '/organization/new', action='new')
for action in [
'delete',
'admins',
'member_new',
'member_delete',
'history']:
m.connect('organization_' + action,
'/organization/' + action + '/{id}',
action=action)
m.connect('organization_activity', '/organization/activity/{id}/{offset}',
action='activity', ckan_icon='clock-o')
m.connect('organization_read', '/organization/{id}', action='read')
m.connect('organization_about', '/organization/about/{id}',
action='about', ckan_icon='info-circle')
m.connect('organization_read', '/organization/{id}', action='read',
ckan_icon='sitemap')
m.connect('organization_edit', '/organization/edit/{id}',
action='edit', ckan_icon='pencil-square-o')
m.connect('organization_members', '/organization/members/{id}',
action='members', ckan_icon='users')
m.connect('organization_bulk_process',
'/organization/bulk_process/{id}',
action='bulk_process', ckan_icon='sitemap')
lib_plugins.register_package_plugins(map)
lib_plugins.register_group_plugins(map)
# tags
map.redirect('/tags', '/tag')
map.redirect('/tags/{url:.*}', '/tag/{url}')
map.redirect('/tag/read/{url:.*}', '/tag/{url}',
_redirect_code='301 Moved Permanently')
map.connect('/tag', controller='tag', action='index')
map.connect('/tag/{id}', controller='tag', action='read')
# users
map.redirect('/users/{url:.*}', '/user/{url}')
with SubMapper(map, controller='revision') as m:
m.connect('/revision', action='index')
m.connect('/revision/edit/{id}', action='edit')
m.connect('/revision/diff/{id}', action='diff')
m.connect('/revision/list', action='list')
m.connect('/revision/{id}', action='read')
with SubMapper(map, controller='ckan.controllers.storage:StorageController') as m:
m.connect('storage_file', '/storage/f/{label:.*}',
action='file')
with SubMapper(map, controller='util') as m:
m.connect('/i18n/strings_{lang}.js', action='i18n_js_strings')
m.connect('/util/redirect', action='redirect')
m.connect('/testing/primer', action='primer')
m.connect('/testing/markup', action='markup')
# robots.txt
map.connect('/(robots.txt)', controller='template', action='view')
# Mark all unmarked routes added up until now as core routes
for route in map.matchlist:
if not hasattr(route, '_ckan_core'):
route._ckan_core = True
for plugin in p.PluginImplementations(p.IRoutes):
map = plugin.after_map(map)
# Mark all routes added from extensions on the `after_map` extension point
# as non-core
for route in map.matchlist:
if not hasattr(route, '_ckan_core'):
route._ckan_core = False
# sometimes we get requests for favicon.ico we should redirect to
# the real favicon location.
map.redirect('/favicon.ico', config.get('ckan.favicon'))
map.redirect('/*(url)/', '/{url}',
_redirect_code='301 Moved Permanently')
map.connect('/*url', controller='template', action='view', ckan_core=True)
return map
def update_config():
''' This code needs to be run when the config is changed to take those
changes into account. It is called whenever a plugin is loaded as the
plugin might have changed the config values (for instance it might
change ckan.site_url) '''
webassets_init()
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# Set whitelisted env vars on config object
# This is set up before globals are initialized
ckan_db = os.environ.get('CKAN_DB', None)
if ckan_db:
msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
log.warn(msg)
config['sqlalchemy.url'] = ckan_db
for option in CONFIG_FROM_ENV_VARS:
from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
if from_env:
config[option] = from_env
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
site_url = config.get('ckan.site_url', '')
if not site_url:
raise RuntimeError(
'ckan.site_url is not configured and it must have a value.'
' Please amend your .ini file.')
if not site_url.lower().startswith('http'):
raise RuntimeError(
'ckan.site_url should be a full URL, including the schema '
'(http or https)')
display_timezone = config.get('ckan.display_timezone', '')
if (display_timezone and
display_timezone != 'server' and
display_timezone not in pytz.all_timezones):
raise CkanConfigurationException(
"ckan.display_timezone is not 'server' or a valid timezone"
)
# Remove backslash from site_url if present
config['ckan.site_url'] = config['ckan.site_url'].rstrip('/')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# ensure that a favicon has been set
favicon = config.get('ckan.favicon', '/base/images/ckan.ico')
config['ckan.favicon'] = favicon
# Init SOLR settings and check if the schema is compatible
# from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get('solr_url'),
config.get('solr_user'),
config.get('solr_password'))
search.check_solr_schema_version()
routes_map = routing.make_map()
lib_plugins.reset_package_plugins()
lib_plugins.register_package_plugins()
lib_plugins.reset_group_plugins()
lib_plugins.register_group_plugins()
config['routes.map'] = routes_map
# The RoutesMiddleware needs its mapper updating if it exists
if 'routes.middleware' in config:
config['routes.middleware'].mapper = routes_map
# routes.named_routes is a CKAN thing
config['routes.named_routes'] = routing.named_routes
config['pylons.app_globals'] = app_globals.app_globals
# initialise the globals
app_globals.app_globals._init()
helpers.load_plugin_helpers()
config['pylons.h'] = helpers.helper_functions
# Templates and CSS loading from configuration
valid_base_templates_folder_names = ['templates']
templates = config.get('ckan.base_templates_folder', 'templates')
config['ckan.base_templates_folder'] = templates
if templates not in valid_base_templates_folder_names:
raise CkanConfigurationException(
'You provided an invalid value for ckan.base_templates_folder. '
'Possible values are: "templates".'
)
jinja2_templates_path = os.path.join(root, templates)
log.info('Loading templates from %s' % jinja2_templates_path)
template_paths = [jinja2_templates_path]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['computed_template_paths'] = template_paths
# Set the default language for validation messages from formencode
# to what is set as the default locale in the config
default_lang = config.get('ckan.locale_default', 'en')
formencode.api.set_stdtranslation(domain="FormEncode",
languages=[default_lang])
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create Jinja2 environment
env = jinja_extensions.Environment(
**jinja_extensions.get_jinja_env_options())
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
config['pylons.app_globals'].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# Initialize SQLAlchemy
engine = sqlalchemy.engine_from_config(config)
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# reset the template cache - we do this here so that when we load the
# environment it is clean
render.reset_template_info_cache()
# clear other caches
logic.clear_actions_cache()
logic.clear_validators_cache()
authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# (ProgrammingError for Postgres, OperationalError for SQLite)
# The database is not initialised. This is a bit dirty. This occurs
# when running tests.
pass
except sqlalchemy.exc.InternalError:
# The database is not initialised. Travis hits this
pass
# Close current session and open database connections to ensure a clean
# clean environment even if an error occurs later on
model.Session.remove()
model.Session.bind.dispose()
def make_map():
"""Create, configure and return the routes Mapper"""
# import controllers here rather than at root level because
# pylons config is initialised by this point.
# Helpers to reduce code clutter
GET = dict(method=["GET"])
PUT = dict(method=["PUT"])
POST = dict(method=["POST"])
DELETE = dict(method=["DELETE"])
GET_POST = dict(method=["GET", "POST"])
PUT_POST = dict(method=["PUT", "POST"])
PUT_POST_DELETE = dict(method=["PUT", "POST", "DELETE"])
OPTIONS = dict(method=["OPTIONS"])
import ckan.lib.plugins as lib_plugins
lib_plugins.reset_package_plugins()
map = Mapper(directory=config["pylons.paths"]["controllers"], always_scan=config["debug"])
map.minimization = False
map.explicit = True
# The ErrorController route (handles 404/500 error pages); it should
# likely stay at the top, ensuring it can always be resolved.
map.connect("/error/{action}", controller="error")
map.connect("/error/{action}/{id}", controller="error")
map.connect("*url", controller="home", action="cors_options", conditions=OPTIONS)
# CUSTOM ROUTES HERE
for plugin in p.PluginImplementations(p.IRoutes):
map = plugin.before_map(map)
map.connect("home", "/", controller="home", action="index")
map.connect("about", "/about", controller="home", action="about")
# CKAN API versioned.
register_list = [
"package",
"dataset",
"resource",
"tag",
"group",
"related",
"revision",
"licenses",
"rating",
"user",
"activity",
]
register_list_str = "|".join(register_list)
# /api ver 3 or none
with SubMapper(map, controller="api", path_prefix="/api{ver:/3|}", ver="/3") as m:
m.connect("/action/{logic_function}", action="action", conditions=GET_POST)
# /api ver 1, 2, 3 or none
with SubMapper(map, controller="api", path_prefix="/api{ver:/1|/2|/3|}", ver="/1") as m:
m.connect("", action="get_api")
m.connect("/search/{register}", action="search")
# /api ver 1, 2 or none
with SubMapper(map, controller="api", path_prefix="/api{ver:/1|/2|}", ver="/1") as m:
m.connect("/tag_counts", action="tag_counts")
m.connect("/rest", action="index")
m.connect("/qos/throughput/", action="throughput", conditions=GET)
# /api/rest ver 1, 2 or none
with SubMapper(
map, controller="api", path_prefix="/api{ver:/1|/2|}", ver="/1", requirements=dict(register=register_list_str)
) as m:
m.connect("/rest/{register}", action="list", conditions=GET)
m.connect("/rest/{register}", action="create", conditions=POST)
m.connect("/rest/{register}/{id}", action="show", conditions=GET)
m.connect("/rest/{register}/{id}", action="update", conditions=PUT)
m.connect("/rest/{register}/{id}", action="update", conditions=POST)
m.connect("/rest/{register}/{id}", action="delete", conditions=DELETE)
m.connect("/rest/{register}/{id}/:subregister", action="list", conditions=GET)
m.connect("/rest/{register}/{id}/:subregister", action="create", conditions=POST)
m.connect("/rest/{register}/{id}/:subregister/{id2}", action="create", conditions=POST)
m.connect("/rest/{register}/{id}/:subregister/{id2}", action="show", conditions=GET)
m.connect("/rest/{register}/{id}/:subregister/{id2}", action="update", conditions=PUT)
m.connect("/rest/{register}/{id}/:subregister/{id2}", action="delete", conditions=DELETE)
# /api/util ver 1, 2 or none
with SubMapper(map, controller="api", path_prefix="/api{ver:/1|/2|}", ver="/1") as m:
m.connect("/util/user/autocomplete", action="user_autocomplete")
m.connect("/util/is_slug_valid", action="is_slug_valid", conditions=GET)
m.connect("/util/dataset/autocomplete", action="dataset_autocomplete", conditions=GET)
m.connect("/util/tag/autocomplete", action="tag_autocomplete", conditions=GET)
m.connect("/util/resource/format_autocomplete", action="format_autocomplete", conditions=GET)
m.connect("/util/resource/format_icon", action="format_icon", conditions=GET)
m.connect("/util/group/autocomplete", action="group_autocomplete")
m.connect("/util/organization/autocomplete", action="organization_autocomplete", conditions=GET)
m.connect("/util/markdown", action="markdown")
m.connect("/util/dataset/munge_name", action="munge_package_name")
m.connect("/util/dataset/munge_title_to_name", action="munge_title_to_package_name")
m.connect("/util/tag/munge", action="munge_tag")
m.connect("/util/status", action="status")
m.connect("/util/snippet/{snippet_path:.*}", action="snippet")
m.connect("/i18n/{lang}", action="i18n_js_translations")
###########
## /END API
###########
map.redirect("/packages", "/dataset")
map.redirect("/packages/{url:.*}", "/dataset/{url}")
map.redirect("/package", "/dataset")
map.redirect("/package/{url:.*}", "/dataset/{url}")
with SubMapper(map, controller="package") as m:
m.connect("search", "/dataset", action="search", highlight_actions="index search")
m.connect("add dataset", "/dataset/new", action="new")
m.connect("/dataset/{action}", requirements=dict(action="|".join(["list", "autocomplete", "search"])))
m.connect(
"/dataset/{action}/{id}/{revision}",
action="read_ajax",
requirements=dict(action="|".join(["read", "edit", "history"])),
)
m.connect(
"/dataset/{action}/{id}",
requirements=dict(
action="|".join(
[
"new_resource",
"history",
"read_ajax",
"history_ajax",
"follow",
"activity",
"groups",
"unfollow",
"delete",
"api_data",
]
)
),
)
m.connect("dataset_edit", "/dataset/edit/{id}", action="edit", ckan_icon="edit")
m.connect("dataset_followers", "/dataset/followers/{id}", action="followers", ckan_icon="group")
m.connect("dataset_activity", "/dataset/activity/{id}", action="activity", ckan_icon="time")
m.connect("/dataset/activity/{id}/{offset}", action="activity")
m.connect("dataset_groups", "/dataset/groups/{id}", action="groups", ckan_icon="group")
m.connect("/dataset/{id}.{format}", action="read")
m.connect("dataset_resources", "/dataset/resources/{id}", action="resources", ckan_icon="reorder")
m.connect("dataset_read", "/dataset/{id}", action="read", ckan_icon="sitemap")
m.connect("/dataset/{id}/resource/{resource_id}", action="resource_read")
m.connect("/dataset/{id}/resource_delete/{resource_id}", action="resource_delete")
m.connect(
"resource_edit", "/dataset/{id}/resource_edit/{resource_id}", action="resource_edit", ckan_icon="edit"
)
m.connect("/dataset/{id}/resource/{resource_id}/download", action="resource_download")
m.connect("/dataset/{id}/resource/{resource_id}/download/{filename}", action="resource_download")
m.connect("/dataset/{id}/resource/{resource_id}/embed", action="resource_embedded_dataviewer")
m.connect(
"/dataset/{id}/resource/{resource_id}/viewer",
action="resource_embedded_dataviewer",
width="960",
height="800",
)
m.connect("/dataset/{id}/resource/{resource_id}/preview", action="resource_datapreview")
m.connect("views", "/dataset/{id}/resource/{resource_id}/views", action="resource_views", ckan_icon="reorder")
m.connect("new_view", "/dataset/{id}/resource/{resource_id}/new_view", action="edit_view", ckan_icon="edit")
m.connect(
"edit_view",
"/dataset/{id}/resource/{resource_id}/edit_view/{view_id}",
action="edit_view",
ckan_icon="edit",
)
m.connect("resource_view", "/dataset/{id}/resource/{resource_id}/view/{view_id}", action="resource_view")
m.connect("/dataset/{id}/resource/{resource_id}/view/", action="resource_view")
# group
map.redirect("/groups", "/group")
map.redirect("/groups/{url:.*}", "/group/{url}")
# These named routes are used for custom group forms which will use the
# names below based on the group.type ('group' is the default type)
with SubMapper(map, controller="group") as m:
m.connect("group_index", "/group", action="index", highlight_actions="index search")
m.connect("group_list", "/group/list", action="list")
m.connect("group_new", "/group/new", action="new")
m.connect(
"group_action",
"/group/{action}/{id}",
requirements=dict(
action="|".join(
[
"edit",
"delete",
"member_new",
"member_delete",
"history",
"followers",
"follow",
"unfollow",
"admins",
"activity",
]
)
),
)
m.connect("group_about", "/group/about/{id}", action="about", ckan_icon="info-sign"),
m.connect("group_edit", "/group/edit/{id}", action="edit", ckan_icon="edit")
m.connect("group_members", "/group/members/{id}", action="members", ckan_icon="group"),
m.connect("group_activity", "/group/activity/{id}/{offset}", action="activity", ckan_icon="time"),
m.connect("group_read", "/group/{id}", action="read", ckan_icon="sitemap")
# organizations these basically end up being the same as groups
with SubMapper(map, controller="organization") as m:
m.connect("organizations_index", "/organization", action="index")
m.connect("/organization/list", action="list")
m.connect("/organization/new", action="new")
m.connect(
"/organization/{action}/{id}",
requirements=dict(action="|".join(["delete", "admins", "member_new", "member_delete", "history"])),
)
m.connect("organization_activity", "/organization/activity/{id}/{offset}", action="activity", ckan_icon="time")
m.connect("organization_read", "/organization/{id}", action="read")
m.connect("organization_about", "/organization/about/{id}", action="about", ckan_icon="info-sign")
m.connect("organization_read", "/organization/{id}", action="read", ckan_icon="sitemap")
m.connect("organization_edit", "/organization/edit/{id}", action="edit", ckan_icon="edit")
m.connect("organization_members", "/organization/members/{id}", action="members", ckan_icon="group")
m.connect(
"organization_bulk_process", "/organization/bulk_process/{id}", action="bulk_process", ckan_icon="sitemap"
)
lib_plugins.register_package_plugins(map)
lib_plugins.register_group_plugins(map)
# tags
map.redirect("/tags", "/tag")
map.redirect("/tags/{url:.*}", "/tag/{url}")
map.redirect("/tag/read/{url:.*}", "/tag/{url}", _redirect_code="301 Moved Permanently")
map.connect("/tag", controller="tag", action="index")
map.connect("/tag/{id}", controller="tag", action="read")
# users
map.redirect("/users/{url:.*}", "/user/{url}")
map.redirect("/user/", "/user")
with SubMapper(map, controller="user") as m:
m.connect("/user/edit", action="edit")
# Note: openid users have slashes in their ids, so need the wildcard
# in the route.
m.connect("user_generate_apikey", "/user/generate_key/{id}", action="generate_apikey")
m.connect("/user/activity/{id}/{offset}", action="activity")
m.connect("user_activity_stream", "/user/activity/{id}", action="activity", ckan_icon="time")
m.connect("user_dashboard", "/dashboard", action="dashboard", ckan_icon="list")
m.connect("user_dashboard_datasets", "/dashboard/datasets", action="dashboard_datasets", ckan_icon="sitemap")
m.connect("user_dashboard_groups", "/dashboard/groups", action="dashboard_groups", ckan_icon="group")
m.connect(
"user_dashboard_organizations",
"/dashboard/organizations",
action="dashboard_organizations",
ckan_icon="building",
)
m.connect("/dashboard/{offset}", action="dashboard")
m.connect("user_follow", "/user/follow/{id}", action="follow")
m.connect("/user/unfollow/{id}", action="unfollow")
m.connect("user_followers", "/user/followers/{id:.*}", action="followers", ckan_icon="group")
m.connect("user_edit", "/user/edit/{id:.*}", action="edit", ckan_icon="cog")
m.connect("user_delete", "/user/delete/{id}", action="delete")
m.connect("/user/reset/{id:.*}", action="perform_reset")
m.connect("register", "/user/register", action="register")
m.connect("login", "/user/login", action="login")
m.connect("/user/_logout", action="logout")
m.connect("/user/logged_in", action="logged_in")
m.connect("/user/logged_out", action="logged_out")
m.connect("/user/logged_out_redirect", action="logged_out_page")
m.connect("/user/reset", action="request_reset")
m.connect("/user/me", action="me")
m.connect("/user/set_lang/{lang}", action="set_lang")
m.connect("user_datasets", "/user/{id:.*}", action="read", ckan_icon="sitemap")
m.connect("user_index", "/user", action="index")
with SubMapper(map, controller="revision") as m:
m.connect("/revision", action="index")
m.connect("/revision/edit/{id}", action="edit")
m.connect("/revision/diff/{id}", action="diff")
m.connect("/revision/list", action="list")
m.connect("/revision/{id}", action="read")
# feeds
with SubMapper(map, controller="feed") as m:
m.connect("/feeds/group/{id}.atom", action="group")
m.connect("/feeds/organization/{id}.atom", action="organization")
m.connect("/feeds/tag/{id}.atom", action="tag")
m.connect("/feeds/dataset.atom", action="general")
m.connect("/feeds/custom.atom", action="custom")
map.connect("ckanadmin_index", "/ckan-admin", controller="admin", action="index", ckan_icon="legal")
map.connect("ckanadmin_config", "/ckan-admin/config", controller="admin", action="config", ckan_icon="check")
map.connect("ckanadmin_trash", "/ckan-admin/trash", controller="admin", action="trash", ckan_icon="trash")
map.connect("ckanadmin", "/ckan-admin/{action}", controller="admin")
with SubMapper(map, controller="ckan.controllers.storage:StorageController") as m:
m.connect("storage_file", "/storage/f/{label:.*}", action="file")
with SubMapper(map, controller="util") as m:
m.connect("/i18n/strings_{lang}.js", action="i18n_js_strings")
m.connect("/util/redirect", action="redirect")
m.connect("/testing/primer", action="primer")
m.connect("/testing/markup", action="markup")
for plugin in p.PluginImplementations(p.IRoutes):
map = plugin.after_map(map)
# sometimes we get requests for favicon.ico we should redirect to
# the real favicon location.
map.redirect("/favicon.ico", config.get("ckan.favicon"))
map.redirect("/*(url)/", "/{url}", _redirect_code="301 Moved Permanently")
map.connect("/*url", controller="template", action="view")
return map