def update_config():
''' This code needs to be run when the config is changed to take those
changes into account. It is called whenever a plugin is loaded as the
plugin might have changed the config values (for instance it might
change ckan.site_url) '''
webassets_init()
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# Set whitelisted env vars on config object
# This is set up before globals are initialized
ckan_db = os.environ.get('CKAN_DB', None)
if ckan_db:
msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
log.warn(msg)
config['sqlalchemy.url'] = ckan_db
for option in CONFIG_FROM_ENV_VARS:
from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
if from_env:
config[option] = from_env
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
site_url = config.get('ckan.site_url', '')
if not site_url:
raise RuntimeError(
'ckan.site_url is not configured and it must have a value.'
' Please amend your .ini file.')
if not site_url.lower().startswith('http'):
raise RuntimeError(
'ckan.site_url should be a full URL, including the schema '
'(http or https)')
display_timezone = config.get('ckan.display_timezone', '')
if (display_timezone and display_timezone != 'server'
and display_timezone not in pytz.all_timezones):
raise CkanConfigurationException(
"ckan.display_timezone is not 'server' or a valid timezone")
# Remove backslash from site_url if present
config['ckan.site_url'] = config['ckan.site_url'].rstrip('/')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# ensure that a favicon has been set
favicon = config.get('ckan.favicon', '/base/images/ckan.ico')
config['ckan.favicon'] = favicon
# Init SOLR settings and check if the schema is compatible
# from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get('solr_url'), config.get('solr_user'),
config.get('solr_password'))
search.check_solr_schema_version()
if six.PY2:
routes_map = routing.make_map()
lib_plugins.reset_package_plugins()
lib_plugins.register_package_plugins()
lib_plugins.reset_group_plugins()
lib_plugins.register_group_plugins()
if six.PY2:
config['routes.map'] = routes_map
# The RoutesMiddleware needs its mapper updating if it exists
if 'routes.middleware' in config:
config['routes.middleware'].mapper = routes_map
# routes.named_routes is a CKAN thing
config['routes.named_routes'] = routing.named_routes
config['pylons.app_globals'] = app_globals.app_globals
# initialise the globals
app_globals.app_globals._init()
helpers.load_plugin_helpers()
config['pylons.h'] = helpers.helper_functions
# Templates and CSS loading from configuration
valid_base_templates_folder_names = ['templates']
templates = config.get('ckan.base_templates_folder', 'templates')
config['ckan.base_templates_folder'] = templates
if templates not in valid_base_templates_folder_names:
raise CkanConfigurationException(
'You provided an invalid value for ckan.base_templates_folder. '
'Possible values are: "templates".')
jinja2_templates_path = os.path.join(root, templates)
log.info('Loading templates from %s' % jinja2_templates_path)
template_paths = [jinja2_templates_path]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['computed_template_paths'] = template_paths
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
if six.PY2:
# Create Jinja2 environment
env = jinja_extensions.Environment(
**jinja_extensions.get_jinja_env_options())
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.policies['ext.i18n.trimmed'] = True
env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
config['pylons.app_globals'].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# Enable pessimistic disconnect handling (added in SQLAlchemy 1.2)
# to eliminate database errors due to stale pooled connections
config.setdefault('pool_pre_ping', True)
# Initialize SQLAlchemy
engine = sqlalchemy.engine_from_config(config)
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# reset the template cache - we do this here so that when we load the
# environment it is clean
render.reset_template_info_cache()
# clear other caches
logic.clear_actions_cache()
logic.clear_validators_cache()
authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# (ProgrammingError for Postgres, OperationalError for SQLite)
# The database is not initialised. This is a bit dirty. This occurs
# when running tests.
pass
except sqlalchemy.exc.InternalError:
# The database is not initialised. Travis hits this
pass
# Close current session and open database connections to ensure a clean
# clean environment even if an error occurs later on
model.Session.remove()
model.Session.bind.dispose()
def make_map():
"""Create, configure and return the routes Mapper"""
# import controllers here rather than at root level because
# pylons config is initialised by this point.
# Helpers to reduce code clutter
GET = dict(method=['GET'])
PUT = dict(method=['PUT'])
POST = dict(method=['POST'])
DELETE = dict(method=['DELETE'])
GET_POST = dict(method=['GET', 'POST'])
PUT_POST = dict(method=['PUT', 'POST'])
PUT_POST_DELETE = dict(method=['PUT', 'POST', 'DELETE'])
OPTIONS = dict(method=['OPTIONS'])
import ckan.lib.plugins as lib_plugins
lib_plugins.reset_group_plugins()
map = Mapper(directory=config['pylons.paths']['controllers'],
always_scan=config['debug'])
map.minimization = False
map.explicit = True
# CUSTOM ROUTES HERE
for plugin in p.PluginImplementations(p.IRoutes):
map = plugin.before_map(map)
# The ErrorController route (handles 404/500 error pages); it should
# likely stay at the top, ensuring it can always be resolved.
map.connect('/error/{action}', controller='error', ckan_core=True)
map.connect('/error/{action}/{id}', controller='error', ckan_core=True)
map.connect('*url', controller='home', action='cors_options',
conditions=OPTIONS, ckan_core=True)
# Mark all routes added from extensions on the `before_map` extension point
# as non-core
for route in map.matchlist:
if not hasattr(route, '_ckan_core'):
route._ckan_core = False
# CKAN API versioned.
register_list = [
'package',
'dataset',
'resource',
'tag',
'group',
'revision',
'licenses',
'rating',
'user',
'activity'
]
register_list_str = '|'.join(register_list)
# /api ver 1, 2, 3 or none
with SubMapper(map, controller='api', path_prefix='/api{ver:/1|/2|/3|}',
ver='/1') as m:
m.connect('/search/{register}', action='search')
# /api/util ver 1, 2 or none
with SubMapper(map, controller='api', path_prefix='/api{ver:/1|/2|}',
ver='/1') as m:
m.connect('/util/dataset/munge_name', action='munge_package_name')
m.connect('/util/dataset/munge_title_to_name',
action='munge_title_to_package_name')
m.connect('/util/tag/munge', action='munge_tag')
###########
## /END API
###########
map.redirect('/packages', '/dataset')
map.redirect('/packages/{url:.*}', '/dataset/{url}')
map.redirect('/package', '/dataset')
map.redirect('/package/{url:.*}', '/dataset/{url}')
# group
map.redirect('/groups', '/group')
map.redirect('/groups/{url:.*}', '/group/{url}')
# These named routes are used for custom group forms which will use the
# names below based on the group.type ('group' is the default type)
with SubMapper(map, controller='group') as m:
m.connect('group_index', '/group', action='index',
highlight_actions='index search')
m.connect('group_list', '/group/list', action='list')
m.connect('group_new', '/group/new', action='new')
for action in [
'edit',
'delete',
'member_new',
'member_delete',
'history',
'followers',
'follow',
'unfollow',
'admins',
'activity',
]:
m.connect('group_' + action,
'/group/' + action + '/{id}',
action=action)
m.connect('group_about', '/group/about/{id}', action='about',
ckan_icon='info-circle'),
m.connect('group_edit', '/group/edit/{id}', action='edit',
ckan_icon='pencil-square-o')
m.connect('group_members', '/group/members/{id}', action='members',
ckan_icon='users'),
m.connect('group_activity', '/group/activity/{id}/{offset}',
action='activity', ckan_icon='clock-o'),
m.connect('group_read', '/group/{id}', action='read',
ckan_icon='sitemap')
# organizations these basically end up being the same as groups
with SubMapper(map, controller='organization') as m:
m.connect('organizations_index', '/organization', action='index')
m.connect('organization_index', '/organization', action='index')
m.connect('organization_new', '/organization/new', action='new')
for action in [
'delete',
'admins',
'member_new',
'member_delete',
'history']:
m.connect('organization_' + action,
'/organization/' + action + '/{id}',
action=action)
m.connect('organization_activity', '/organization/activity/{id}/{offset}',
action='activity', ckan_icon='clock-o')
m.connect('organization_read', '/organization/{id}', action='read')
m.connect('organization_about', '/organization/about/{id}',
action='about', ckan_icon='info-circle')
m.connect('organization_read', '/organization/{id}', action='read',
ckan_icon='sitemap')
m.connect('organization_edit', '/organization/edit/{id}',
action='edit', ckan_icon='pencil-square-o')
m.connect('organization_members', '/organization/members/{id}',
action='members', ckan_icon='users')
m.connect('organization_bulk_process',
'/organization/bulk_process/{id}',
action='bulk_process', ckan_icon='sitemap')
lib_plugins.register_group_plugins(map)
# tags
map.redirect('/tags', '/tag')
map.redirect('/tags/{url:.*}', '/tag/{url}')
map.redirect('/tag/read/{url:.*}', '/tag/{url}',
_redirect_code='301 Moved Permanently')
map.connect('/tag', controller='tag', action='index')
map.connect('/tag/{id}', controller='tag', action='read')
# users
map.redirect('/users/{url:.*}', '/user/{url}')
with SubMapper(map, controller='revision') as m:
m.connect('/revision', action='index')
m.connect('/revision/edit/{id}', action='edit')
m.connect('/revision/diff/{id}', action='diff')
m.connect('/revision/list', action='list')
m.connect('/revision/{id}', action='read')
with SubMapper(map, controller='util') as m:
m.connect('/i18n/strings_{lang}.js', action='i18n_js_strings')
m.connect('/util/redirect', action='redirect')
m.connect('/testing/primer', action='primer')
m.connect('/testing/markup', action='markup')
# robots.txt
map.connect('/(robots.txt)', controller='template', action='view')
# Mark all unmarked routes added up until now as core routes
for route in map.matchlist:
if not hasattr(route, '_ckan_core'):
route._ckan_core = True
for plugin in p.PluginImplementations(p.IRoutes):
map = plugin.after_map(map)
# Mark all routes added from extensions on the `after_map` extension point
# as non-core
for route in map.matchlist:
if not hasattr(route, '_ckan_core'):
route._ckan_core = False
# sometimes we get requests for favicon.ico we should redirect to
# the real favicon location.
map.redirect('/favicon.ico', config.get('ckan.favicon'))
map.redirect('/*(url)/', '/{url}',
_redirect_code='301 Moved Permanently')
map.connect('/*url', controller='template', action='view', ckan_core=True)
return map
def update_config():
''' This code needs to be run when the config is changed to take those
changes into account. It is called whenever a plugin is loaded as the
plugin might have changed the config values (for instance it might
change ckan.site_url) '''
config_declaration.setup()
config_declaration.make_safe(config)
config_declaration.normalize(config)
webassets_init()
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# Set whitelisted env vars on config object
# This is set up before globals are initialized
ckan_db = os.environ.get('CKAN_DB', None)
if ckan_db:
msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
log.warn(msg)
config['sqlalchemy.url'] = ckan_db
for option in CONFIG_FROM_ENV_VARS:
from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
if from_env:
config[option] = from_env
if config.get_value("config.mode") == "strict":
_, errors = config_declaration.validate(config)
if errors:
msg = "\n".join("{}: {}".format(key, "; ".join(issues))
for key, issues in errors.items())
raise CkanConfigurationException(msg)
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
site_url = config.get_value('ckan.site_url')
if not site_url:
raise RuntimeError(
'ckan.site_url is not configured and it must have a value.'
' Please amend your .ini file.')
if not site_url.lower().startswith('http'):
raise RuntimeError(
'ckan.site_url should be a full URL, including the schema '
'(http or https)')
# Remove backslash from site_url if present
config['ckan.site_url'] = site_url.rstrip('/')
display_timezone = config.get_value('ckan.display_timezone')
if (display_timezone and display_timezone != 'server'
and display_timezone not in pytz.all_timezones):
raise CkanConfigurationException(
"ckan.display_timezone is not 'server' or a valid timezone")
# Init SOLR settings and check if the schema is compatible
# from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get_value('solr_url'),
config.get_value('solr_user'),
config.get_value('solr_password'))
search.check_solr_schema_version()
lib_plugins.reset_package_plugins()
lib_plugins.register_package_plugins()
lib_plugins.reset_group_plugins()
lib_plugins.register_group_plugins()
# initialise the globals
app_globals.app_globals._init()
helpers.load_plugin_helpers()
# Templates and CSS loading from configuration
valid_base_templates_folder_names = ['templates']
templates = config.get_value('ckan.base_templates_folder')
config['ckan.base_templates_folder'] = templates
if templates not in valid_base_templates_folder_names:
raise CkanConfigurationException(
'You provided an invalid value for ckan.base_templates_folder. '
'Possible values are: "templates".')
jinja2_templates_path = os.path.join(root, templates)
log.info('Loading templates from %s' % jinja2_templates_path)
template_paths = [jinja2_templates_path]
extra_template_paths = config.get_value('extra_template_paths')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['computed_template_paths'] = template_paths
# Enable pessimistic disconnect handling (added in SQLAlchemy 1.2)
# to eliminate database errors due to stale pooled connections
config.setdefault('sqlalchemy.pool_pre_ping', True)
# Initialize SQLAlchemy
engine = sqlalchemy.engine_from_config(config)
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# clear other caches
logic.clear_actions_cache()
logic.clear_validators_cache()
authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# The database is not yet initialised. It happens in `ckan db init`
pass
# Close current session and open database connections to ensure a clean
# clean environment even if an error occurs later on
model.Session.remove()
model.Session.bind.dispose()
def update_config():
''' This code needs to be run when the config is changed to take those
changes into account. It is called whenever a plugin is loaded as the
plugin might have changed the config values (for instance it might
change ckan.site_url) '''
webassets_init()
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# Set whitelisted env vars on config object
# This is set up before globals are initialized
ckan_db = os.environ.get('CKAN_DB', None)
if ckan_db:
msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
log.warn(msg)
config['sqlalchemy.url'] = ckan_db
for option in CONFIG_FROM_ENV_VARS:
from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
if from_env:
config[option] = from_env
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
site_url = config.get('ckan.site_url', '')
if not site_url:
raise RuntimeError(
'ckan.site_url is not configured and it must have a value.'
' Please amend your .ini file.')
if not site_url.lower().startswith('http'):
raise RuntimeError(
'ckan.site_url should be a full URL, including the schema '
'(http or https)')
display_timezone = config.get('ckan.display_timezone', '')
if (display_timezone and
display_timezone != 'server' and
display_timezone not in pytz.all_timezones):
raise CkanConfigurationException(
"ckan.display_timezone is not 'server' or a valid timezone"
)
# Remove backslash from site_url if present
config['ckan.site_url'] = config['ckan.site_url'].rstrip('/')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# ensure that a favicon has been set
favicon = config.get('ckan.favicon', '/base/images/ckan.ico')
config['ckan.favicon'] = favicon
# Init SOLR settings and check if the schema is compatible
# from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get('solr_url'),
config.get('solr_user'),
config.get('solr_password'))
search.check_solr_schema_version()
routes_map = routing.make_map()
lib_plugins.reset_package_plugins()
lib_plugins.register_package_plugins()
lib_plugins.reset_group_plugins()
lib_plugins.register_group_plugins()
config['routes.map'] = routes_map
# The RoutesMiddleware needs its mapper updating if it exists
if 'routes.middleware' in config:
config['routes.middleware'].mapper = routes_map
# routes.named_routes is a CKAN thing
config['routes.named_routes'] = routing.named_routes
config['pylons.app_globals'] = app_globals.app_globals
# initialise the globals
app_globals.app_globals._init()
helpers.load_plugin_helpers()
config['pylons.h'] = helpers.helper_functions
# Templates and CSS loading from configuration
valid_base_templates_folder_names = ['templates']
templates = config.get('ckan.base_templates_folder', 'templates')
config['ckan.base_templates_folder'] = templates
if templates not in valid_base_templates_folder_names:
raise CkanConfigurationException(
'You provided an invalid value for ckan.base_templates_folder. '
'Possible values are: "templates".'
)
jinja2_templates_path = os.path.join(root, templates)
log.info('Loading templates from %s' % jinja2_templates_path)
template_paths = [jinja2_templates_path]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['computed_template_paths'] = template_paths
# Set the default language for validation messages from formencode
# to what is set as the default locale in the config
default_lang = config.get('ckan.locale_default', 'en')
formencode.api.set_stdtranslation(domain="FormEncode",
languages=[default_lang])
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create Jinja2 environment
env = jinja_extensions.Environment(
**jinja_extensions.get_jinja_env_options())
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
config['pylons.app_globals'].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# Initialize SQLAlchemy
engine = sqlalchemy.engine_from_config(config)
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# reset the template cache - we do this here so that when we load the
# environment it is clean
render.reset_template_info_cache()
# clear other caches
logic.clear_actions_cache()
logic.clear_validators_cache()
authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# (ProgrammingError for Postgres, OperationalError for SQLite)
# The database is not initialised. This is a bit dirty. This occurs
# when running tests.
pass
except sqlalchemy.exc.InternalError:
# The database is not initialised. Travis hits this
pass
# Close current session and open database connections to ensure a clean
# clean environment even if an error occurs later on
model.Session.remove()
model.Session.bind.dispose()
