def update_config():
''' This code needs to be run when the config is changed to take those
changes into account. It is called whenever a plugin is loaded as the
plugin might have changed the config values (for instance it might
change ckan.site_url) '''
webassets_init()
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# Set whitelisted env vars on config object
# This is set up before globals are initialized
ckan_db = os.environ.get('CKAN_DB', None)
if ckan_db:
msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
log.warn(msg)
config['sqlalchemy.url'] = ckan_db
for option in CONFIG_FROM_ENV_VARS:
from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
if from_env:
config[option] = from_env
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
site_url = config.get('ckan.site_url', '')
if not site_url:
raise RuntimeError(
'ckan.site_url is not configured and it must have a value.'
' Please amend your .ini file.')
if not site_url.lower().startswith('http'):
raise RuntimeError(
'ckan.site_url should be a full URL, including the schema '
'(http or https)')
display_timezone = config.get('ckan.display_timezone', '')
if (display_timezone and display_timezone != 'server'
and display_timezone not in pytz.all_timezones):
raise CkanConfigurationException(
"ckan.display_timezone is not 'server' or a valid timezone")
# Remove backslash from site_url if present
config['ckan.site_url'] = config['ckan.site_url'].rstrip('/')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# ensure that a favicon has been set
favicon = config.get('ckan.favicon', '/base/images/ckan.ico')
config['ckan.favicon'] = favicon
# Init SOLR settings and check if the schema is compatible
# from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get('solr_url'), config.get('solr_user'),
config.get('solr_password'))
search.check_solr_schema_version()
if six.PY2:
routes_map = routing.make_map()
lib_plugins.reset_package_plugins()
lib_plugins.register_package_plugins()
lib_plugins.reset_group_plugins()
lib_plugins.register_group_plugins()
if six.PY2:
config['routes.map'] = routes_map
# The RoutesMiddleware needs its mapper updating if it exists
if 'routes.middleware' in config:
config['routes.middleware'].mapper = routes_map
# routes.named_routes is a CKAN thing
config['routes.named_routes'] = routing.named_routes
config['pylons.app_globals'] = app_globals.app_globals
# initialise the globals
app_globals.app_globals._init()
helpers.load_plugin_helpers()
config['pylons.h'] = helpers.helper_functions
# Templates and CSS loading from configuration
valid_base_templates_folder_names = ['templates']
templates = config.get('ckan.base_templates_folder', 'templates')
config['ckan.base_templates_folder'] = templates
if templates not in valid_base_templates_folder_names:
raise CkanConfigurationException(
'You provided an invalid value for ckan.base_templates_folder. '
'Possible values are: "templates".')
jinja2_templates_path = os.path.join(root, templates)
log.info('Loading templates from %s' % jinja2_templates_path)
template_paths = [jinja2_templates_path]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['computed_template_paths'] = template_paths
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
if six.PY2:
# Create Jinja2 environment
env = jinja_extensions.Environment(
**jinja_extensions.get_jinja_env_options())
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.policies['ext.i18n.trimmed'] = True
env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
config['pylons.app_globals'].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# Enable pessimistic disconnect handling (added in SQLAlchemy 1.2)
# to eliminate database errors due to stale pooled connections
config.setdefault('pool_pre_ping', True)
# Initialize SQLAlchemy
engine = sqlalchemy.engine_from_config(config)
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# reset the template cache - we do this here so that when we load the
# environment it is clean
render.reset_template_info_cache()
# clear other caches
logic.clear_actions_cache()
logic.clear_validators_cache()
authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# (ProgrammingError for Postgres, OperationalError for SQLite)
# The database is not initialised. This is a bit dirty. This occurs
# when running tests.
pass
except sqlalchemy.exc.InternalError:
# The database is not initialised. Travis hits this
pass
# Close current session and open database connections to ensure a clean
# clean environment even if an error occurs later on
model.Session.remove()
model.Session.bind.dispose()
def update_config():
''' This code needs to be run when the config is changed to take those
changes into account. It is called whenever a plugin is loaded as the
plugin might have changed the config values (for instance it might
change ckan.site_url) '''
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# Set whitelisted env vars on config object
# This is set up before globals are initialized
ckan_db = os.environ.get('CKAN_DB', None)
if ckan_db:
msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
log.warn(msg)
config['sqlalchemy.url'] = ckan_db
for option in CONFIG_FROM_ENV_VARS:
from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
if from_env:
config[option] = from_env
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
site_url = config.get('ckan.site_url', '')
if not site_url:
raise RuntimeError(
'ckan.site_url is not configured and it must have a value.'
' Please amend your .ini file.')
if not site_url.lower().startswith('http'):
raise RuntimeError(
'ckan.site_url should be a full URL, including the schema '
'(http or https)')
display_timezone = config.get('ckan.display_timezone', '')
if (display_timezone and display_timezone != 'server'
and display_timezone not in pytz.all_timezones):
raise CkanConfigurationException(
"ckan.display_timezone is not 'server' or a valid timezone")
# Remove backslash from site_url if present
config['ckan.site_url'] = config['ckan.site_url'].rstrip('/')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# ensure that a favicon has been set
favicon = config.get('ckan.favicon', '/base/images/ckan.ico')
config['ckan.favicon'] = favicon
# Init SOLR settings and check if the schema is compatible
# from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get('solr_url'), config.get('solr_user'),
config.get('solr_password'))
search.check_solr_schema_version()
routes_map = routing.make_map()
config['routes.map'] = routes_map
# The RoutesMiddleware needs its mapper updating if it exists
if 'routes.middleware' in config:
config['routes.middleware'].mapper = routes_map
# routes.named_routes is a CKAN thing
config['routes.named_routes'] = routing.named_routes
config['pylons.app_globals'] = app_globals.app_globals
# initialise the globals
app_globals.app_globals._init()
helpers.load_plugin_helpers()
config['pylons.h'] = helpers.helper_functions
jinja2_templates_path = os.path.join(root, 'templates')
template_paths = [jinja2_templates_path]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['pylons.app_globals'].template_paths = template_paths
# Set the default language for validation messages from formencode
# to what is set as the default locale in the config
default_lang = config.get('ckan.locale_default', 'en')
formencode.api.set_stdtranslation(domain="FormEncode",
languages=[default_lang])
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create Jinja2 environment
env = jinja_extensions.Environment(
loader=jinja_extensions.CkanFileSystemLoader(template_paths),
autoescape=True,
extensions=[
'jinja2.ext.do', 'jinja2.ext.with_',
jinja_extensions.SnippetExtension, jinja_extensions.CkanExtend,
jinja_extensions.CkanInternationalizationExtension,
jinja_extensions.LinkForExtension,
jinja_extensions.ResourceExtension,
jinja_extensions.UrlForStaticExtension,
jinja_extensions.UrlForExtension
])
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
env.filters['truncate'] = jinja_extensions.truncate
config['pylons.app_globals'].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# Initialize SQLAlchemy
engine = sqlalchemy.engine_from_config(config, client_encoding='utf8')
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# reset the template cache - we do this here so that when we load the
# environment it is clean
render.reset_template_info_cache()
# clear other caches
logic.clear_actions_cache()
logic.clear_validators_cache()
authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# (ProgrammingError for Postgres, OperationalError for SQLite)
# The database is not initialised. This is a bit dirty. This occurs
# when running tests.
pass
except sqlalchemy.exc.InternalError:
# The database is not initialised. Travis hits this
pass
# if an extension or our code does not finish
# transaction properly db cli commands can fail
model.Session.remove()
def update_config():
''' This code needs to be run when the config is changed to take those
changes into account. It is called whenever a plugin is loaded as the
plugin might have changed the config values (for instance it might
change ckan.site_url) '''
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# Set whitelisted env vars on config object
# This is set up before globals are initialized
ckan_db = os.environ.get('CKAN_DB', None)
if ckan_db:
msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
log.warn(msg)
config['sqlalchemy.url'] = ckan_db
for option in CONFIG_FROM_ENV_VARS:
from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
if from_env:
config[option] = from_env
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
site_url = config.get('ckan.site_url', '')
if not site_url:
raise RuntimeError(
'ckan.site_url is not configured and it must have a value.'
' Please amend your .ini file.')
if not site_url.lower().startswith('http'):
raise RuntimeError(
'ckan.site_url should be a full URL, including the schema '
'(http or https)')
display_timezone = config.get('ckan.display_timezone', '')
if (display_timezone and
display_timezone != 'server' and
display_timezone not in pytz.all_timezones):
raise CkanConfigurationException(
"ckan.display_timezone is not 'server' or a valid timezone"
)
# Remove backslash from site_url if present
config['ckan.site_url'] = config['ckan.site_url'].rstrip('/')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# ensure that a favicon has been set
favicon = config.get('ckan.favicon', '/base/images/ckan.ico')
config['ckan.favicon'] = favicon
# Init SOLR settings and check if the schema is compatible
# from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get('solr_url'),
config.get('solr_user'),
config.get('solr_password'))
search.check_solr_schema_version()
routes_map = routing.make_map()
config['routes.map'] = routes_map
# The RoutesMiddleware needs its mapper updating if it exists
if 'routes.middleware' in config:
config['routes.middleware'].mapper = routes_map
# routes.named_routes is a CKAN thing
config['routes.named_routes'] = routing.named_routes
config['pylons.app_globals'] = app_globals.app_globals
# initialise the globals
app_globals.app_globals._init()
helpers.load_plugin_helpers()
config['pylons.h'] = helpers.helper_functions
# Templates and CSS loading from configuration
valid_base_templates_folder_names = ['templates', 'templates-bs2']
templates = config.get('ckan.base_templates_folder', 'templates')
config['ckan.base_templates_folder'] = templates
if templates not in valid_base_templates_folder_names:
raise CkanConfigurationException(
'You provided an invalid value for ckan.base_templates_folder. '
'Possible values are: "templates" and "templates-bs2".'
)
jinja2_templates_path = os.path.join(root, templates)
log.info('Loading templates from %s' % jinja2_templates_path)
template_paths = [jinja2_templates_path]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['computed_template_paths'] = template_paths
# Set the default language for validation messages from formencode
# to what is set as the default locale in the config
default_lang = config.get('ckan.locale_default', 'en')
formencode.api.set_stdtranslation(domain="FormEncode",
languages=[default_lang])
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create Jinja2 environment
env = jinja_extensions.Environment(
**jinja_extensions.get_jinja_env_options())
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
config['pylons.app_globals'].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# Initialize SQLAlchemy
engine = sqlalchemy.engine_from_config(config, client_encoding='utf8')
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# reset the template cache - we do this here so that when we load the
# environment it is clean
render.reset_template_info_cache()
# clear other caches
logic.clear_actions_cache()
logic.clear_validators_cache()
authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# (ProgrammingError for Postgres, OperationalError for SQLite)
# The database is not initialised. This is a bit dirty. This occurs
# when running tests.
pass
except sqlalchemy.exc.InternalError:
# The database is not initialised. Travis hits this
pass
# Close current session and open database connections to ensure a clean
# clean environment even if an error occurs later on
model.Session.remove()
model.Session.bind.dispose()
def update_config():
''' This code needs to be run when the config is changed to take those
changes into account. It is called whenever a plugin is loaded as the
plugin might have changed the config values (for instance it might
change ckan.site_url) '''
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# Set whitelisted env vars on config object
# This is set up before globals are initialized
ckan_db = os.environ.get('CKAN_DB', None)
if ckan_db:
msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
log.warn(msg)
config['sqlalchemy.url'] = ckan_db
database_user = os.environ.get("DATABASE_USER", None)
database_password = os.environ.get("DATABASE_PASSWORD", None)
database_host = os.environ.get("DATABASE_HOST", None)
database_port = os.environ.get("DATABASE_PORT", None)
if database_user is None or database_password is None or database_host is None:
print("Did not find either DATABASE_USER or DATABASE_PASSWORD or DATABASE_HOST")
else:
print("Setting the database url")
if database_port is None:
#use the default
database_port = "5432"
url = "postgres://"
url += database_user
url += ":"
url += database_password
url += "@"
url += database_host
url += ":"
url += database_port
url += "/datacatalogue"
config['sqlalchemy.url'] = url
print("Setting database " + database_host + ":" + database_port)
for option in CONFIG_FROM_ENV_VARS:
from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
if from_env:
if option == "AWS_ACCESS_KEY_ID" or option == "AWS_SECRET_ACCESS_KEY":
print(option)
print(from_env[0:10])
config[option] = from_env
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
site_url = config.get('ckan.site_url', '')
if not site_url:
raise RuntimeError(
'ckan.site_url is not configured and it must have a value.'
' Please amend your .ini file.')
if not site_url.lower().startswith('http'):
raise RuntimeError(
'ckan.site_url should be a full URL, including the schema '
'(http or https)')
# Remove backslash from site_url if present
config['ckan.site_url'] = config['ckan.site_url'].rstrip('/')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# ensure that a favicon has been set
favicon = config.get('ckan.favicon', '/images/icons/ckan.ico')
config['ckan.favicon'] = favicon
# Init SOLR settings and check if the schema is compatible
#from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get('solr_url'),
config.get('solr_user'),
config.get('solr_password'))
search.check_solr_schema_version()
routes_map = routing.make_map()
config['routes.map'] = routes_map
# The RoutesMiddleware needs its mapper updating if it exists
if 'routes.middleware' in config:
config['routes.middleware'].mapper = routes_map
config['routes.named_routes'] = routing.named_routes
config['pylons.app_globals'] = app_globals.app_globals
# initialise the globals
config['pylons.app_globals']._init()
# add helper functions
helpers = _Helpers(h)
config['pylons.h'] = helpers
## redo template setup to use genshi.search_path
## (so remove std template setup)
legacy_templates_path = os.path.join(root, 'templates_legacy')
jinja2_templates_path = os.path.join(root, 'templates')
if asbool(config.get('ckan.legacy_templates', 'no')):
# We want the new template path for extra snippets like the
# dataviewer and also for some testing stuff
msg = 'Support for Genshi templates is deprecated and will be removed'\
' in a future release'
log.warn(msg)
template_paths = [legacy_templates_path, jinja2_templates_path]
else:
template_paths = [jinja2_templates_path, legacy_templates_path]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['pylons.app_globals'].template_paths = template_paths
# Translator (i18n)
translator = Translator(pylons.translator)
def template_loaded(template):
translator.setup(template)
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create the Genshi TemplateLoader
config['pylons.app_globals'].genshi_loader = TemplateLoader(
template_paths, auto_reload=True, callback=template_loaded)
# Create Jinja2 environment
env = jinja_extensions.Environment(
loader=jinja_extensions.CkanFileSystemLoader(template_paths),
autoescape=True,
extensions=['jinja2.ext.do', 'jinja2.ext.with_',
jinja_extensions.SnippetExtension,
jinja_extensions.CkanExtend,
jinja_extensions.CkanInternationalizationExtension,
jinja_extensions.LinkForExtension,
jinja_extensions.ResourceExtension,
jinja_extensions.UrlForStaticExtension,
jinja_extensions.UrlForExtension]
)
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
env.filters['truncate'] = jinja_extensions.truncate
config['pylons.app_globals'].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# for postgresql we want to enforce utf-8
sqlalchemy_url = config.get('sqlalchemy.url', '')
if sqlalchemy_url.startswith('postgresql://'):
extras = {'client_encoding': 'utf8'}
else:
extras = {}
engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.', **extras)
if not model.meta.engine:
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# reset the template cache - we do this here so that when we load the
# environment it is clean
render.reset_template_info_cache()
# clear other caches
logic.clear_actions_cache()
logic.clear_validators_cache()
authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# (ProgrammingError for Postgres, OperationalError for SQLite)
# The database is not initialised. This is a bit dirty. This occurs
# when running tests.
pass
except sqlalchemy.exc.InternalError:
# The database is not initialised. Travis hits this
pass
# if an extension or our code does not finish
# transaction properly db cli commands can fail
model.Session.remove()
def update_config():
''' This code needs to be run when the config is changed to take those
changes into account. '''
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# This is set up before globals are initialized
site_id = os.environ.get('CKAN_SITE_ID')
if site_id:
config['ckan.site_id'] = site_id
site_url = config.get('ckan.site_url', '')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# ensure that a favicon has been set
favicon = config.get('ckan.favicon', '/images/icons/ckan.ico')
config['ckan.favicon'] = favicon
# Init SOLR settings and check if the schema is compatible
#from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get('solr_url'),
config.get('solr_user'),
config.get('solr_password'))
search.check_solr_schema_version()
routes_map = routing.make_map()
config['routes.map'] = routes_map
# The RoutesMiddleware needs its mapper updating if it exists
if 'routes.middleware' in config:
config['routes.middleware'].mapper = routes_map
config['routes.named_routes'] = routing.named_routes
config['pylons.app_globals'] = app_globals.app_globals
# initialise the globals
config['pylons.app_globals']._init()
# add helper functions
helpers = _Helpers(h)
config['pylons.h'] = helpers
## redo template setup to use genshi.search_path
## (so remove std template setup)
legacy_templates_path = os.path.join(root, 'templates_legacy')
jinja2_templates_path = os.path.join(root, 'templates')
if asbool(config.get('ckan.legacy_templates', 'no')):
# We want the new template path for extra snippets like the
# dataviewer and also for some testing stuff
msg = 'Support for Genshi templates is deprecated and will be removed'\
' in a future release'
log.warn(msg)
template_paths = [legacy_templates_path, jinja2_templates_path]
else:
template_paths = [jinja2_templates_path, legacy_templates_path]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['pylons.app_globals'].template_paths = template_paths
# Translator (i18n)
translator = Translator(pylons.translator)
def template_loaded(template):
translator.setup(template)
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create the Genshi TemplateLoader
config['pylons.app_globals'].genshi_loader = TemplateLoader(
template_paths, auto_reload=True, callback=template_loaded)
# Create Jinja2 environment
env = jinja_extensions.Environment(
loader=jinja_extensions.CkanFileSystemLoader(template_paths),
autoescape=True,
extensions=['jinja2.ext.do', 'jinja2.ext.with_',
jinja_extensions.SnippetExtension,
jinja_extensions.CkanExtend,
jinja_extensions.CkanInternationalizationExtension,
jinja_extensions.LinkForExtension,
jinja_extensions.ResourceExtension,
jinja_extensions.UrlForStaticExtension,
jinja_extensions.UrlForExtension]
)
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
env.filters['truncate'] = jinja_extensions.truncate
config['pylons.app_globals'].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
ckan_db = os.environ.get('CKAN_DB')
if ckan_db:
config['sqlalchemy.url'] = ckan_db
# for postgresql we want to enforce utf-8
sqlalchemy_url = config.get('sqlalchemy.url', '')
if sqlalchemy_url.startswith('postgresql://'):
extras = {'client_encoding': 'utf8'}
else:
extras = {}
engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.', **extras)
if not model.meta.engine:
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# reset the template cache - we do this here so that when we load the
# environment it is clean
render.reset_template_info_cache()
# clear other caches
logic.clear_actions_cache()
new_authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# (ProgrammingError for Postgres, OperationalError for SQLite)
# The database is not initialised. This is a bit dirty. This occurs
# when running tests.
pass
except sqlalchemy.exc.InternalError:
# The database is not initialised. Travis hits this
pass
# if an extension or our code does not finish
# transaction properly db cli commands can fail
model.Session.remove()
def update_config():
''' This code needs to be run when the config is changed to take those
changes into account. '''
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# This is set up before globals are initialized
site_id = os.environ.get('CKAN_SITE_ID')
if site_id:
config['ckan.site_id'] = site_id
site_url = config.get('ckan.site_url', '')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# ensure that a favicon has been set
favicon = config.get('ckan.favicon', '/images/icons/ckan.ico')
config['ckan.favicon'] = favicon
# Init SOLR settings and check if the schema is compatible
#from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get('solr_url'),
config.get('solr_user'),
config.get('solr_password'))
search.check_solr_schema_version()
routes_map = routing.make_map()
config['routes.map'] = routes_map
# The RoutesMiddleware needs its mapper updating if it exists
if 'routes.middleware' in config:
config['routes.middleware'].mapper = routes_map
config['routes.named_routes'] = routing.named_routes
config['pylons.app_globals'] = app_globals.app_globals
# initialise the globals
config['pylons.app_globals']._init()
# add helper functions
helpers = _Helpers(h)
config['pylons.h'] = helpers
## redo template setup to use genshi.search_path
## (so remove std template setup)
legacy_templates_path = os.path.join(root, 'templates_legacy')
jinja2_templates_path = os.path.join(root, 'templates')
if asbool(config.get('ckan.legacy_templates', 'no')):
# We want the new template path for extra snippets like the
# dataviewer and also for some testing stuff
msg = 'Support for Genshi templates is deprecated and will be removed'\
' in a future release'
log.warn(msg)
template_paths = [legacy_templates_path, jinja2_templates_path]
else:
template_paths = [jinja2_templates_path, legacy_templates_path]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['pylons.app_globals'].template_paths = template_paths
# Translator (i18n)
translator = Translator(pylons.translator)
def template_loaded(template):
translator.setup(template)
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create the Genshi TemplateLoader
config['pylons.app_globals'].genshi_loader = TemplateLoader(
template_paths, auto_reload=True, callback=template_loaded)
# Create Jinja2 environment
env = jinja_extensions.Environment(
loader=jinja_extensions.CkanFileSystemLoader(template_paths),
autoescape=True,
extensions=['jinja2.ext.do', 'jinja2.ext.with_',
jinja_extensions.SnippetExtension,
jinja_extensions.CkanExtend,
jinja_extensions.CkanInternationalizationExtension,
jinja_extensions.LinkForExtension,
jinja_extensions.ResourceExtension,
jinja_extensions.UrlForStaticExtension,
jinja_extensions.UrlForExtension]
)
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
env.filters['truncate'] = jinja_extensions.truncate
config['pylons.app_globals'].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
ckan_db = os.environ.get('CKAN_DB')
if ckan_db:
config['sqlalchemy.url'] = ckan_db
# for postgresql we want to enforce utf-8
sqlalchemy_url = config.get('sqlalchemy.url', '')
if sqlalchemy_url.startswith('postgresql://'):
extras = {'client_encoding': 'utf8'}
else:
extras = {}
engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.', **extras)
if not model.meta.engine:
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# reset the template cache - we do this here so that when we load the
# environment it is clean
render.reset_template_info_cache()
# clear other caches
logic.clear_actions_cache()
logic.clear_validators_cache()
new_authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# (ProgrammingError for Postgres, OperationalError for SQLite)
# The database is not initialised. This is a bit dirty. This occurs
# when running tests.
pass
except sqlalchemy.exc.InternalError:
# The database is not initialised. Travis hits this
pass
# if an extension or our code does not finish
# transaction properly db cli commands can fail
model.Session.remove()
def update_config():
""" This code needs to be run when the config is changed to take those
changes into account. It is called whenever a plugin is loaded as the
plugin might have changed the config values (for instance it might
change ckan.site_url) """
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# Set whitelisted env vars on config object
# This is set up before globals are initialized
ckan_db = os.environ.get("CKAN_DB", None)
if ckan_db:
msg = (
"Setting CKAN_DB as an env var is deprecated and will be"
" removed in a future release. Use CKAN_SQLALCHEMY_URL instead."
)
log.warn(msg)
config["sqlalchemy.url"] = ckan_db
for option in CONFIG_FROM_ENV_VARS:
from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
if from_env:
config[option] = from_env
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
site_url = config.get("ckan.site_url", "")
if not site_url:
raise RuntimeError("ckan.site_url is not configured and it must have a value." " Please amend your .ini file.")
if not site_url.lower().startswith("http"):
raise RuntimeError("ckan.site_url should be a full URL, including the schema " "(http or https)")
display_timezone = config.get("ckan.display_timezone", "")
if display_timezone and display_timezone != "server" and display_timezone not in pytz.all_timezones:
raise CkanConfigurationException("ckan.display_timezone is not 'server' or a valid timezone")
# Remove backslash from site_url if present
config["ckan.site_url"] = config["ckan.site_url"].rstrip("/")
ckan_host = config["ckan.host"] = urlparse(site_url).netloc
if config.get("ckan.site_id") is None:
if ":" in ckan_host:
ckan_host, port = ckan_host.split(":")
assert ckan_host, (
"You need to configure ckan.site_url or " "ckan.site_id for SOLR search-index rebuild to work."
)
config["ckan.site_id"] = ckan_host
# ensure that a favicon has been set
favicon = config.get("ckan.favicon", "/base/images/ckan.ico")
config["ckan.favicon"] = favicon
# Init SOLR settings and check if the schema is compatible
# from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get("solr_url"), config.get("solr_user"), config.get("solr_password"))
search.check_solr_schema_version()
routes_map = routing.make_map()
config["routes.map"] = routes_map
# The RoutesMiddleware needs its mapper updating if it exists
if "routes.middleware" in config:
config["routes.middleware"].mapper = routes_map
config["routes.named_routes"] = routing.named_routes
config["pylons.app_globals"] = app_globals.app_globals
# initialise the globals
config["pylons.app_globals"]._init()
# add helper functions
helpers = _Helpers(h)
config["pylons.h"] = helpers
jinja2_templates_path = os.path.join(root, "templates")
template_paths = [jinja2_templates_path]
extra_template_paths = config.get("extra_template_paths", "")
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(",") + template_paths
config["pylons.app_globals"].template_paths = template_paths
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create Jinja2 environment
env = jinja_extensions.Environment(
loader=jinja_extensions.CkanFileSystemLoader(template_paths),
autoescape=True,
extensions=[
"jinja2.ext.do",
"jinja2.ext.with_",
jinja_extensions.SnippetExtension,
jinja_extensions.CkanExtend,
jinja_extensions.CkanInternationalizationExtension,
jinja_extensions.LinkForExtension,
jinja_extensions.ResourceExtension,
jinja_extensions.UrlForStaticExtension,
jinja_extensions.UrlForExtension,
],
)
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.filters["empty_and_escape"] = jinja_extensions.empty_and_escape
env.filters["truncate"] = jinja_extensions.truncate
config["pylons.app_globals"].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# for postgresql we want to enforce utf-8
sqlalchemy_url = config.get("sqlalchemy.url", "")
if sqlalchemy_url.startswith("postgresql://"):
extras = {"client_encoding": "utf8"}
else:
extras = {}
engine = sqlalchemy.engine_from_config(config, "sqlalchemy.", **extras)
if not model.meta.engine:
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# reset the template cache - we do this here so that when we load the
# environment it is clean
render.reset_template_info_cache()
# clear other caches
logic.clear_actions_cache()
logic.clear_validators_cache()
authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action("get_site_user")({"ignore_auth": True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# (ProgrammingError for Postgres, OperationalError for SQLite)
# The database is not initialised. This is a bit dirty. This occurs
# when running tests.
pass
except sqlalchemy.exc.InternalError:
# The database is not initialised. Travis hits this
pass
# if an extension or our code does not finish
# transaction properly db cli commands can fail
model.Session.remove()
def update_config():
''' This code needs to be run when the config is changed to take those
changes into account. It is called whenever a plugin is loaded as the
plugin might have changed the config values (for instance it might
change ckan.site_url) '''
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# Set whitelisted env vars on config object
# This is set up before globals are initialized
ckan_db = os.environ.get('CKAN_DB', None)
if ckan_db:
msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
log.warn(msg)
config['sqlalchemy.url'] = ckan_db
database_user = os.environ.get("DATABASE_USER", None)
database_password = os.environ.get("DATABASE_PASSWORD", None)
database_host = os.environ.get("DATABASE_HOST", None)
database_port = os.environ.get("DATABASE_PORT", None)
if database_user is None or database_password is None or database_host is None:
print(
"Did not find either DATABASE_USER or DATABASE_PASSWORD or DATABASE_HOST"
)
else:
print("Setting the database url")
if database_port is None:
#use the default
database_port = "5432"
url = "postgres://"
url += database_user
url += ":"
url += database_password
url += "@"
url += database_host
url += ":"
url += database_port
url += "/datacatalogue"
config['sqlalchemy.url'] = url
print("Setting database " + database_host + ":" + database_port)
for option in CONFIG_FROM_ENV_VARS:
from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
if from_env:
if option == "AWS_ACCESS_KEY_ID" or option == "AWS_SECRET_ACCESS_KEY":
print(option)
print(from_env[0:10])
config[option] = from_env
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
site_url = config.get('ckan.site_url', '')
if not site_url:
raise RuntimeError(
'ckan.site_url is not configured and it must have a value.'
' Please amend your .ini file.')
if not site_url.lower().startswith('http'):
raise RuntimeError(
'ckan.site_url should be a full URL, including the schema '
'(http or https)')
# Remove backslash from site_url if present
config['ckan.site_url'] = config['ckan.site_url'].rstrip('/')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# ensure that a favicon has been set
favicon = config.get('ckan.favicon', '/images/icons/ckan.ico')
config['ckan.favicon'] = favicon
# Init SOLR settings and check if the schema is compatible
#from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get('solr_url'), config.get('solr_user'),
config.get('solr_password'))
search.check_solr_schema_version()
routes_map = routing.make_map()
config['routes.map'] = routes_map
# The RoutesMiddleware needs its mapper updating if it exists
if 'routes.middleware' in config:
config['routes.middleware'].mapper = routes_map
config['routes.named_routes'] = routing.named_routes
config['pylons.app_globals'] = app_globals.app_globals
# initialise the globals
config['pylons.app_globals']._init()
# add helper functions
helpers = _Helpers(h)
config['pylons.h'] = helpers
## redo template setup to use genshi.search_path
## (so remove std template setup)
legacy_templates_path = os.path.join(root, 'templates_legacy')
jinja2_templates_path = os.path.join(root, 'templates')
if asbool(config.get('ckan.legacy_templates', 'no')):
# We want the new template path for extra snippets like the
# dataviewer and also for some testing stuff
msg = 'Support for Genshi templates is deprecated and will be removed'\
' in a future release'
log.warn(msg)
template_paths = [legacy_templates_path, jinja2_templates_path]
else:
template_paths = [jinja2_templates_path, legacy_templates_path]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['pylons.app_globals'].template_paths = template_paths
# Translator (i18n)
translator = Translator(pylons.translator)
def template_loaded(template):
translator.setup(template)
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create the Genshi TemplateLoader
config['pylons.app_globals'].genshi_loader = TemplateLoader(
template_paths, auto_reload=True, callback=template_loaded)
# Create Jinja2 environment
env = jinja_extensions.Environment(
loader=jinja_extensions.CkanFileSystemLoader(template_paths),
autoescape=True,
extensions=[
'jinja2.ext.do', 'jinja2.ext.with_',
jinja_extensions.SnippetExtension, jinja_extensions.CkanExtend,
jinja_extensions.CkanInternationalizationExtension,
jinja_extensions.LinkForExtension,
jinja_extensions.ResourceExtension,
jinja_extensions.UrlForStaticExtension,
jinja_extensions.UrlForExtension
])
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
env.filters['truncate'] = jinja_extensions.truncate
config['pylons.app_globals'].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# for postgresql we want to enforce utf-8
sqlalchemy_url = config.get('sqlalchemy.url', '')
if sqlalchemy_url.startswith('postgresql://'):
extras = {'client_encoding': 'utf8'}
else:
extras = {}
engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.', **extras)
if not model.meta.engine:
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# reset the template cache - we do this here so that when we load the
# environment it is clean
render.reset_template_info_cache()
# clear other caches
logic.clear_actions_cache()
logic.clear_validators_cache()
authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# (ProgrammingError for Postgres, OperationalError for SQLite)
# The database is not initialised. This is a bit dirty. This occurs
# when running tests.
pass
except sqlalchemy.exc.InternalError:
# The database is not initialised. Travis hits this
pass
# if an extension or our code does not finish
# transaction properly db cli commands can fail
model.Session.remove()