def wrapper(*args, **kwargs): try: with mock.patch(auth_function_path) as mocked_auth: clear_auth_functions_cache() new_args = args + tuple([mocked_auth]) return_value = func(*new_args, **kwargs) finally: clear_auth_functions_cache() return return_value
def update_config(): ''' This code needs to be run when the config is changed to take those changes into account. It is called whenever a plugin is loaded as the plugin might have changed the config values (for instance it might change ckan.site_url) ''' webassets_init() for plugin in p.PluginImplementations(p.IConfigurer): # must do update in place as this does not work: # config = plugin.update_config(config) plugin.update_config(config) # Set whitelisted env vars on config object # This is set up before globals are initialized ckan_db = os.environ.get('CKAN_DB', None) if ckan_db: msg = 'Setting CKAN_DB as an env var is deprecated and will be' \ ' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.' log.warn(msg) config['sqlalchemy.url'] = ckan_db for option in CONFIG_FROM_ENV_VARS: from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None) if from_env: config[option] = from_env root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) site_url = config.get('ckan.site_url', '') if not site_url: raise RuntimeError( 'ckan.site_url is not configured and it must have a value.' ' Please amend your .ini file.') if not site_url.lower().startswith('http'): raise RuntimeError( 'ckan.site_url should be a full URL, including the schema ' '(http or https)') display_timezone = config.get('ckan.display_timezone', '') if (display_timezone and display_timezone != 'server' and display_timezone not in pytz.all_timezones): raise CkanConfigurationException( "ckan.display_timezone is not 'server' or a valid timezone") # Remove backslash from site_url if present config['ckan.site_url'] = config['ckan.site_url'].rstrip('/') ckan_host = config['ckan.host'] = urlparse(site_url).netloc if config.get('ckan.site_id') is None: if ':' in ckan_host: ckan_host, port = ckan_host.split(':') assert ckan_host, 'You need to configure ckan.site_url or ' \ 'ckan.site_id for SOLR search-index rebuild to work.' config['ckan.site_id'] = ckan_host # ensure that a favicon has been set favicon = config.get('ckan.favicon', '/base/images/ckan.ico') config['ckan.favicon'] = favicon # Init SOLR settings and check if the schema is compatible # from ckan.lib.search import SolrSettings, check_solr_schema_version # lib.search is imported here as we need the config enabled and parsed search.SolrSettings.init(config.get('solr_url'), config.get('solr_user'), config.get('solr_password')) search.check_solr_schema_version() if six.PY2: routes_map = routing.make_map() lib_plugins.reset_package_plugins() lib_plugins.register_package_plugins() lib_plugins.reset_group_plugins() lib_plugins.register_group_plugins() if six.PY2: config['routes.map'] = routes_map # The RoutesMiddleware needs its mapper updating if it exists if 'routes.middleware' in config: config['routes.middleware'].mapper = routes_map # routes.named_routes is a CKAN thing config['routes.named_routes'] = routing.named_routes config['pylons.app_globals'] = app_globals.app_globals # initialise the globals app_globals.app_globals._init() helpers.load_plugin_helpers() config['pylons.h'] = helpers.helper_functions # Templates and CSS loading from configuration valid_base_templates_folder_names = ['templates'] templates = config.get('ckan.base_templates_folder', 'templates') config['ckan.base_templates_folder'] = templates if templates not in valid_base_templates_folder_names: raise CkanConfigurationException( 'You provided an invalid value for ckan.base_templates_folder. ' 'Possible values are: "templates".') jinja2_templates_path = os.path.join(root, templates) log.info('Loading templates from %s' % jinja2_templates_path) template_paths = [jinja2_templates_path] extra_template_paths = config.get('extra_template_paths', '') if extra_template_paths: # must be first for them to override defaults template_paths = extra_template_paths.split(',') + template_paths config['computed_template_paths'] = template_paths # Markdown ignores the logger config, so to get rid of excessive # markdown debug messages in the log, set it to the level of the # root logger. logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level) if six.PY2: # Create Jinja2 environment env = jinja_extensions.Environment( **jinja_extensions.get_jinja_env_options()) env.install_gettext_callables(_, ungettext, newstyle=True) # custom filters env.policies['ext.i18n.trimmed'] = True env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape config['pylons.app_globals'].jinja_env = env # CONFIGURATION OPTIONS HERE (note: all config options will override # any Pylons config options) # Enable pessimistic disconnect handling (added in SQLAlchemy 1.2) # to eliminate database errors due to stale pooled connections config.setdefault('pool_pre_ping', True) # Initialize SQLAlchemy engine = sqlalchemy.engine_from_config(config) model.init_model(engine) for plugin in p.PluginImplementations(p.IConfigurable): plugin.configure(config) # reset the template cache - we do this here so that when we load the # environment it is clean render.reset_template_info_cache() # clear other caches logic.clear_actions_cache() logic.clear_validators_cache() authz.clear_auth_functions_cache() # Here we create the site user if they are not already in the database try: logic.get_action('get_site_user')({'ignore_auth': True}, None) except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError): # (ProgrammingError for Postgres, OperationalError for SQLite) # The database is not initialised. This is a bit dirty. This occurs # when running tests. pass except sqlalchemy.exc.InternalError: # The database is not initialised. Travis hits this pass # Close current session and open database connections to ensure a clean # clean environment even if an error occurs later on model.Session.remove() model.Session.bind.dispose()
def update_config(): ''' This code needs to be run when the config is changed to take those changes into account. It is called whenever a plugin is loaded as the plugin might have changed the config values (for instance it might change ckan.site_url) ''' for plugin in p.PluginImplementations(p.IConfigurer): # must do update in place as this does not work: # config = plugin.update_config(config) plugin.update_config(config) # Set whitelisted env vars on config object # This is set up before globals are initialized ckan_db = os.environ.get('CKAN_DB', None) if ckan_db: msg = 'Setting CKAN_DB as an env var is deprecated and will be' \ ' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.' log.warn(msg) config['sqlalchemy.url'] = ckan_db for option in CONFIG_FROM_ENV_VARS: from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None) if from_env: config[option] = from_env root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) site_url = config.get('ckan.site_url', '') if not site_url: raise RuntimeError( 'ckan.site_url is not configured and it must have a value.' ' Please amend your .ini file.') if not site_url.lower().startswith('http'): raise RuntimeError( 'ckan.site_url should be a full URL, including the schema ' '(http or https)') display_timezone = config.get('ckan.display_timezone', '') if (display_timezone and display_timezone != 'server' and display_timezone not in pytz.all_timezones): raise CkanConfigurationException( "ckan.display_timezone is not 'server' or a valid timezone" ) # Remove backslash from site_url if present config['ckan.site_url'] = config['ckan.site_url'].rstrip('/') ckan_host = config['ckan.host'] = urlparse(site_url).netloc if config.get('ckan.site_id') is None: if ':' in ckan_host: ckan_host, port = ckan_host.split(':') assert ckan_host, 'You need to configure ckan.site_url or ' \ 'ckan.site_id for SOLR search-index rebuild to work.' config['ckan.site_id'] = ckan_host # ensure that a favicon has been set favicon = config.get('ckan.favicon', '/base/images/ckan.ico') config['ckan.favicon'] = favicon # Init SOLR settings and check if the schema is compatible # from ckan.lib.search import SolrSettings, check_solr_schema_version # lib.search is imported here as we need the config enabled and parsed search.SolrSettings.init(config.get('solr_url'), config.get('solr_user'), config.get('solr_password')) search.check_solr_schema_version() routes_map = routing.make_map() config['routes.map'] = routes_map # The RoutesMiddleware needs its mapper updating if it exists if 'routes.middleware' in config: config['routes.middleware'].mapper = routes_map # routes.named_routes is a CKAN thing config['routes.named_routes'] = routing.named_routes config['pylons.app_globals'] = app_globals.app_globals # initialise the globals app_globals.app_globals._init() helpers.load_plugin_helpers() config['pylons.h'] = helpers.helper_functions # Templates and CSS loading from configuration valid_base_templates_folder_names = ['templates', 'templates-bs2'] templates = config.get('ckan.base_templates_folder', 'templates') config['ckan.base_templates_folder'] = templates if templates not in valid_base_templates_folder_names: raise CkanConfigurationException( 'You provided an invalid value for ckan.base_templates_folder. ' 'Possible values are: "templates" and "templates-bs2".' ) jinja2_templates_path = os.path.join(root, templates) log.info('Loading templates from %s' % jinja2_templates_path) template_paths = [jinja2_templates_path] extra_template_paths = config.get('extra_template_paths', '') if extra_template_paths: # must be first for them to override defaults template_paths = extra_template_paths.split(',') + template_paths config['computed_template_paths'] = template_paths # Set the default language for validation messages from formencode # to what is set as the default locale in the config default_lang = config.get('ckan.locale_default', 'en') formencode.api.set_stdtranslation(domain="FormEncode", languages=[default_lang]) # Markdown ignores the logger config, so to get rid of excessive # markdown debug messages in the log, set it to the level of the # root logger. logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level) # Create Jinja2 environment env = jinja_extensions.Environment( **jinja_extensions.get_jinja_env_options()) env.install_gettext_callables(_, ungettext, newstyle=True) # custom filters env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape config['pylons.app_globals'].jinja_env = env # CONFIGURATION OPTIONS HERE (note: all config options will override # any Pylons config options) # Initialize SQLAlchemy engine = sqlalchemy.engine_from_config(config, client_encoding='utf8') model.init_model(engine) for plugin in p.PluginImplementations(p.IConfigurable): plugin.configure(config) # reset the template cache - we do this here so that when we load the # environment it is clean render.reset_template_info_cache() # clear other caches logic.clear_actions_cache() logic.clear_validators_cache() authz.clear_auth_functions_cache() # Here we create the site user if they are not already in the database try: logic.get_action('get_site_user')({'ignore_auth': True}, None) except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError): # (ProgrammingError for Postgres, OperationalError for SQLite) # The database is not initialised. This is a bit dirty. This occurs # when running tests. pass except sqlalchemy.exc.InternalError: # The database is not initialised. Travis hits this pass # Close current session and open database connections to ensure a clean # clean environment even if an error occurs later on model.Session.remove() model.Session.bind.dispose()
def update_config(): ''' This code needs to be run when the config is changed to take those changes into account. It is called whenever a plugin is loaded as the plugin might have changed the config values (for instance it might change ckan.site_url) ''' for plugin in p.PluginImplementations(p.IConfigurer): # must do update in place as this does not work: # config = plugin.update_config(config) plugin.update_config(config) # Set whitelisted env vars on config object # This is set up before globals are initialized ckan_db = os.environ.get('CKAN_DB', None) if ckan_db: msg = 'Setting CKAN_DB as an env var is deprecated and will be' \ ' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.' log.warn(msg) config['sqlalchemy.url'] = ckan_db for option in CONFIG_FROM_ENV_VARS: from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None) if from_env: config[option] = from_env root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) site_url = config.get('ckan.site_url', '') if not site_url: raise RuntimeError( 'ckan.site_url is not configured and it must have a value.' ' Please amend your .ini file.') if not site_url.lower().startswith('http'): raise RuntimeError( 'ckan.site_url should be a full URL, including the schema ' '(http or https)') display_timezone = config.get('ckan.display_timezone', '') if (display_timezone and display_timezone != 'server' and display_timezone not in pytz.all_timezones): raise CkanConfigurationException( "ckan.display_timezone is not 'server' or a valid timezone") # Remove backslash from site_url if present config['ckan.site_url'] = config['ckan.site_url'].rstrip('/') ckan_host = config['ckan.host'] = urlparse(site_url).netloc if config.get('ckan.site_id') is None: if ':' in ckan_host: ckan_host, port = ckan_host.split(':') assert ckan_host, 'You need to configure ckan.site_url or ' \ 'ckan.site_id for SOLR search-index rebuild to work.' config['ckan.site_id'] = ckan_host # ensure that a favicon has been set favicon = config.get('ckan.favicon', '/base/images/ckan.ico') config['ckan.favicon'] = favicon # Init SOLR settings and check if the schema is compatible # from ckan.lib.search import SolrSettings, check_solr_schema_version # lib.search is imported here as we need the config enabled and parsed search.SolrSettings.init(config.get('solr_url'), config.get('solr_user'), config.get('solr_password')) search.check_solr_schema_version() routes_map = routing.make_map() config['routes.map'] = routes_map # The RoutesMiddleware needs its mapper updating if it exists if 'routes.middleware' in config: config['routes.middleware'].mapper = routes_map # routes.named_routes is a CKAN thing config['routes.named_routes'] = routing.named_routes config['pylons.app_globals'] = app_globals.app_globals # initialise the globals app_globals.app_globals._init() helpers.load_plugin_helpers() config['pylons.h'] = helpers.helper_functions jinja2_templates_path = os.path.join(root, 'templates') template_paths = [jinja2_templates_path] extra_template_paths = config.get('extra_template_paths', '') if extra_template_paths: # must be first for them to override defaults template_paths = extra_template_paths.split(',') + template_paths config['pylons.app_globals'].template_paths = template_paths # Set the default language for validation messages from formencode # to what is set as the default locale in the config default_lang = config.get('ckan.locale_default', 'en') formencode.api.set_stdtranslation(domain="FormEncode", languages=[default_lang]) # Markdown ignores the logger config, so to get rid of excessive # markdown debug messages in the log, set it to the level of the # root logger. logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level) # Create Jinja2 environment env = jinja_extensions.Environment( loader=jinja_extensions.CkanFileSystemLoader(template_paths), autoescape=True, extensions=[ 'jinja2.ext.do', 'jinja2.ext.with_', jinja_extensions.SnippetExtension, jinja_extensions.CkanExtend, jinja_extensions.CkanInternationalizationExtension, jinja_extensions.LinkForExtension, jinja_extensions.ResourceExtension, jinja_extensions.UrlForStaticExtension, jinja_extensions.UrlForExtension ]) env.install_gettext_callables(_, ungettext, newstyle=True) # custom filters env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape env.filters['truncate'] = jinja_extensions.truncate config['pylons.app_globals'].jinja_env = env # CONFIGURATION OPTIONS HERE (note: all config options will override # any Pylons config options) # Initialize SQLAlchemy engine = sqlalchemy.engine_from_config(config, client_encoding='utf8') model.init_model(engine) for plugin in p.PluginImplementations(p.IConfigurable): plugin.configure(config) # reset the template cache - we do this here so that when we load the # environment it is clean render.reset_template_info_cache() # clear other caches logic.clear_actions_cache() logic.clear_validators_cache() authz.clear_auth_functions_cache() # Here we create the site user if they are not already in the database try: logic.get_action('get_site_user')({'ignore_auth': True}, None) except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError): # (ProgrammingError for Postgres, OperationalError for SQLite) # The database is not initialised. This is a bit dirty. This occurs # when running tests. pass except sqlalchemy.exc.InternalError: # The database is not initialised. Travis hits this pass # if an extension or our code does not finish # transaction properly db cli commands can fail model.Session.remove()
def update_config(): ''' This code needs to be run when the config is changed to take those changes into account. It is called whenever a plugin is loaded as the plugin might have changed the config values (for instance it might change ckan.site_url) ''' for plugin in p.PluginImplementations(p.IConfigurer): # must do update in place as this does not work: # config = plugin.update_config(config) plugin.update_config(config) # Set whitelisted env vars on config object # This is set up before globals are initialized ckan_db = os.environ.get('CKAN_DB', None) if ckan_db: msg = 'Setting CKAN_DB as an env var is deprecated and will be' \ ' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.' log.warn(msg) config['sqlalchemy.url'] = ckan_db database_user = os.environ.get("DATABASE_USER", None) database_password = os.environ.get("DATABASE_PASSWORD", None) database_host = os.environ.get("DATABASE_HOST", None) database_port = os.environ.get("DATABASE_PORT", None) if database_user is None or database_password is None or database_host is None: print("Did not find either DATABASE_USER or DATABASE_PASSWORD or DATABASE_HOST") else: print("Setting the database url") if database_port is None: #use the default database_port = "5432" url = "postgres://" url += database_user url += ":" url += database_password url += "@" url += database_host url += ":" url += database_port url += "/datacatalogue" config['sqlalchemy.url'] = url print("Setting database " + database_host + ":" + database_port) for option in CONFIG_FROM_ENV_VARS: from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None) if from_env: if option == "AWS_ACCESS_KEY_ID" or option == "AWS_SECRET_ACCESS_KEY": print(option) print(from_env[0:10]) config[option] = from_env root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) site_url = config.get('ckan.site_url', '') if not site_url: raise RuntimeError( 'ckan.site_url is not configured and it must have a value.' ' Please amend your .ini file.') if not site_url.lower().startswith('http'): raise RuntimeError( 'ckan.site_url should be a full URL, including the schema ' '(http or https)') # Remove backslash from site_url if present config['ckan.site_url'] = config['ckan.site_url'].rstrip('/') ckan_host = config['ckan.host'] = urlparse(site_url).netloc if config.get('ckan.site_id') is None: if ':' in ckan_host: ckan_host, port = ckan_host.split(':') assert ckan_host, 'You need to configure ckan.site_url or ' \ 'ckan.site_id for SOLR search-index rebuild to work.' config['ckan.site_id'] = ckan_host # ensure that a favicon has been set favicon = config.get('ckan.favicon', '/images/icons/ckan.ico') config['ckan.favicon'] = favicon # Init SOLR settings and check if the schema is compatible #from ckan.lib.search import SolrSettings, check_solr_schema_version # lib.search is imported here as we need the config enabled and parsed search.SolrSettings.init(config.get('solr_url'), config.get('solr_user'), config.get('solr_password')) search.check_solr_schema_version() routes_map = routing.make_map() config['routes.map'] = routes_map # The RoutesMiddleware needs its mapper updating if it exists if 'routes.middleware' in config: config['routes.middleware'].mapper = routes_map config['routes.named_routes'] = routing.named_routes config['pylons.app_globals'] = app_globals.app_globals # initialise the globals config['pylons.app_globals']._init() # add helper functions helpers = _Helpers(h) config['pylons.h'] = helpers ## redo template setup to use genshi.search_path ## (so remove std template setup) legacy_templates_path = os.path.join(root, 'templates_legacy') jinja2_templates_path = os.path.join(root, 'templates') if asbool(config.get('ckan.legacy_templates', 'no')): # We want the new template path for extra snippets like the # dataviewer and also for some testing stuff msg = 'Support for Genshi templates is deprecated and will be removed'\ ' in a future release' log.warn(msg) template_paths = [legacy_templates_path, jinja2_templates_path] else: template_paths = [jinja2_templates_path, legacy_templates_path] extra_template_paths = config.get('extra_template_paths', '') if extra_template_paths: # must be first for them to override defaults template_paths = extra_template_paths.split(',') + template_paths config['pylons.app_globals'].template_paths = template_paths # Translator (i18n) translator = Translator(pylons.translator) def template_loaded(template): translator.setup(template) # Markdown ignores the logger config, so to get rid of excessive # markdown debug messages in the log, set it to the level of the # root logger. logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level) # Create the Genshi TemplateLoader config['pylons.app_globals'].genshi_loader = TemplateLoader( template_paths, auto_reload=True, callback=template_loaded) # Create Jinja2 environment env = jinja_extensions.Environment( loader=jinja_extensions.CkanFileSystemLoader(template_paths), autoescape=True, extensions=['jinja2.ext.do', 'jinja2.ext.with_', jinja_extensions.SnippetExtension, jinja_extensions.CkanExtend, jinja_extensions.CkanInternationalizationExtension, jinja_extensions.LinkForExtension, jinja_extensions.ResourceExtension, jinja_extensions.UrlForStaticExtension, jinja_extensions.UrlForExtension] ) env.install_gettext_callables(_, ungettext, newstyle=True) # custom filters env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape env.filters['truncate'] = jinja_extensions.truncate config['pylons.app_globals'].jinja_env = env # CONFIGURATION OPTIONS HERE (note: all config options will override # any Pylons config options) # for postgresql we want to enforce utf-8 sqlalchemy_url = config.get('sqlalchemy.url', '') if sqlalchemy_url.startswith('postgresql://'): extras = {'client_encoding': 'utf8'} else: extras = {} engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.', **extras) if not model.meta.engine: model.init_model(engine) for plugin in p.PluginImplementations(p.IConfigurable): plugin.configure(config) # reset the template cache - we do this here so that when we load the # environment it is clean render.reset_template_info_cache() # clear other caches logic.clear_actions_cache() logic.clear_validators_cache() authz.clear_auth_functions_cache() # Here we create the site user if they are not already in the database try: logic.get_action('get_site_user')({'ignore_auth': True}, None) except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError): # (ProgrammingError for Postgres, OperationalError for SQLite) # The database is not initialised. This is a bit dirty. This occurs # when running tests. pass except sqlalchemy.exc.InternalError: # The database is not initialised. Travis hits this pass # if an extension or our code does not finish # transaction properly db cli commands can fail model.Session.remove()
def update_config(): ''' This code needs to be run when the config is changed to take those changes into account. It is called whenever a plugin is loaded as the plugin might have changed the config values (for instance it might change ckan.site_url) ''' config_declaration.setup() config_declaration.make_safe(config) config_declaration.normalize(config) webassets_init() for plugin in p.PluginImplementations(p.IConfigurer): # must do update in place as this does not work: # config = plugin.update_config(config) plugin.update_config(config) # Set whitelisted env vars on config object # This is set up before globals are initialized ckan_db = os.environ.get('CKAN_DB', None) if ckan_db: msg = 'Setting CKAN_DB as an env var is deprecated and will be' \ ' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.' log.warn(msg) config['sqlalchemy.url'] = ckan_db for option in CONFIG_FROM_ENV_VARS: from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None) if from_env: config[option] = from_env if config.get_value("config.mode") == "strict": _, errors = config_declaration.validate(config) if errors: msg = "\n".join("{}: {}".format(key, "; ".join(issues)) for key, issues in errors.items()) raise CkanConfigurationException(msg) root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) site_url = config.get_value('ckan.site_url') if not site_url: raise RuntimeError( 'ckan.site_url is not configured and it must have a value.' ' Please amend your .ini file.') if not site_url.lower().startswith('http'): raise RuntimeError( 'ckan.site_url should be a full URL, including the schema ' '(http or https)') # Remove backslash from site_url if present config['ckan.site_url'] = site_url.rstrip('/') display_timezone = config.get_value('ckan.display_timezone') if (display_timezone and display_timezone != 'server' and display_timezone not in pytz.all_timezones): raise CkanConfigurationException( "ckan.display_timezone is not 'server' or a valid timezone") # Init SOLR settings and check if the schema is compatible # from ckan.lib.search import SolrSettings, check_solr_schema_version # lib.search is imported here as we need the config enabled and parsed search.SolrSettings.init(config.get_value('solr_url'), config.get_value('solr_user'), config.get_value('solr_password')) search.check_solr_schema_version() lib_plugins.reset_package_plugins() lib_plugins.register_package_plugins() lib_plugins.reset_group_plugins() lib_plugins.register_group_plugins() # initialise the globals app_globals.app_globals._init() helpers.load_plugin_helpers() # Templates and CSS loading from configuration valid_base_templates_folder_names = ['templates'] templates = config.get_value('ckan.base_templates_folder') config['ckan.base_templates_folder'] = templates if templates not in valid_base_templates_folder_names: raise CkanConfigurationException( 'You provided an invalid value for ckan.base_templates_folder. ' 'Possible values are: "templates".') jinja2_templates_path = os.path.join(root, templates) log.info('Loading templates from %s' % jinja2_templates_path) template_paths = [jinja2_templates_path] extra_template_paths = config.get_value('extra_template_paths') if extra_template_paths: # must be first for them to override defaults template_paths = extra_template_paths.split(',') + template_paths config['computed_template_paths'] = template_paths # Enable pessimistic disconnect handling (added in SQLAlchemy 1.2) # to eliminate database errors due to stale pooled connections config.setdefault('sqlalchemy.pool_pre_ping', True) # Initialize SQLAlchemy engine = sqlalchemy.engine_from_config(config) model.init_model(engine) for plugin in p.PluginImplementations(p.IConfigurable): plugin.configure(config) # clear other caches logic.clear_actions_cache() logic.clear_validators_cache() authz.clear_auth_functions_cache() # Here we create the site user if they are not already in the database try: logic.get_action('get_site_user')({'ignore_auth': True}, None) except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError): # The database is not yet initialised. It happens in `ckan db init` pass # Close current session and open database connections to ensure a clean # clean environment even if an error occurs later on model.Session.remove() model.Session.bind.dispose()
def update_config(): ''' This code needs to be run when the config is changed to take those changes into account. ''' for plugin in p.PluginImplementations(p.IConfigurer): # must do update in place as this does not work: # config = plugin.update_config(config) plugin.update_config(config) root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # This is set up before globals are initialized site_id = os.environ.get('CKAN_SITE_ID') if site_id: config['ckan.site_id'] = site_id site_url = config.get('ckan.site_url', '') ckan_host = config['ckan.host'] = urlparse(site_url).netloc if config.get('ckan.site_id') is None: if ':' in ckan_host: ckan_host, port = ckan_host.split(':') assert ckan_host, 'You need to configure ckan.site_url or ' \ 'ckan.site_id for SOLR search-index rebuild to work.' config['ckan.site_id'] = ckan_host # ensure that a favicon has been set favicon = config.get('ckan.favicon', '/images/icons/ckan.ico') config['ckan.favicon'] = favicon # Init SOLR settings and check if the schema is compatible #from ckan.lib.search import SolrSettings, check_solr_schema_version # lib.search is imported here as we need the config enabled and parsed search.SolrSettings.init(config.get('solr_url'), config.get('solr_user'), config.get('solr_password')) search.check_solr_schema_version() routes_map = routing.make_map() config['routes.map'] = routes_map # The RoutesMiddleware needs its mapper updating if it exists if 'routes.middleware' in config: config['routes.middleware'].mapper = routes_map config['routes.named_routes'] = routing.named_routes config['pylons.app_globals'] = app_globals.app_globals # initialise the globals config['pylons.app_globals']._init() # add helper functions helpers = _Helpers(h) config['pylons.h'] = helpers ## redo template setup to use genshi.search_path ## (so remove std template setup) legacy_templates_path = os.path.join(root, 'templates_legacy') jinja2_templates_path = os.path.join(root, 'templates') if asbool(config.get('ckan.legacy_templates', 'no')): # We want the new template path for extra snippets like the # dataviewer and also for some testing stuff msg = 'Support for Genshi templates is deprecated and will be removed'\ ' in a future release' log.warn(msg) template_paths = [legacy_templates_path, jinja2_templates_path] else: template_paths = [jinja2_templates_path, legacy_templates_path] extra_template_paths = config.get('extra_template_paths', '') if extra_template_paths: # must be first for them to override defaults template_paths = extra_template_paths.split(',') + template_paths config['pylons.app_globals'].template_paths = template_paths # Translator (i18n) translator = Translator(pylons.translator) def template_loaded(template): translator.setup(template) # Markdown ignores the logger config, so to get rid of excessive # markdown debug messages in the log, set it to the level of the # root logger. logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level) # Create the Genshi TemplateLoader config['pylons.app_globals'].genshi_loader = TemplateLoader( template_paths, auto_reload=True, callback=template_loaded) # Create Jinja2 environment env = jinja_extensions.Environment( loader=jinja_extensions.CkanFileSystemLoader(template_paths), autoescape=True, extensions=[ 'jinja2.ext.do', 'jinja2.ext.with_', jinja_extensions.SnippetExtension, jinja_extensions.CkanExtend, jinja_extensions.CkanInternationalizationExtension, jinja_extensions.LinkForExtension, jinja_extensions.ResourceExtension, jinja_extensions.UrlForStaticExtension, jinja_extensions.UrlForExtension ]) env.install_gettext_callables(_, ungettext, newstyle=True) # custom filters env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape env.filters['truncate'] = jinja_extensions.truncate config['pylons.app_globals'].jinja_env = env # CONFIGURATION OPTIONS HERE (note: all config options will override # any Pylons config options) ckan_db = os.environ.get('CKAN_DB') if ckan_db: config['sqlalchemy.url'] = ckan_db # for postgresql we want to enforce utf-8 sqlalchemy_url = config.get('sqlalchemy.url', '') if sqlalchemy_url.startswith('postgresql://'): extras = {'client_encoding': 'utf8'} else: extras = {} engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.', **extras) if not model.meta.engine: model.init_model(engine) for plugin in p.PluginImplementations(p.IConfigurable): plugin.configure(config) # reset the template cache - we do this here so that when we load the # environment it is clean render.reset_template_info_cache() # clear other caches logic.clear_actions_cache() logic.clear_validators_cache() authz.clear_auth_functions_cache() # Here we create the site user if they are not already in the database try: logic.get_action('get_site_user')({'ignore_auth': True}, None) except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError): # (ProgrammingError for Postgres, OperationalError for SQLite) # The database is not initialised. This is a bit dirty. This occurs # when running tests. pass except sqlalchemy.exc.InternalError: # The database is not initialised. Travis hits this pass # if an extension or our code does not finish # transaction properly db cli commands can fail model.Session.remove()
def update_config(): """ This code needs to be run when the config is changed to take those changes into account. It is called whenever a plugin is loaded as the plugin might have changed the config values (for instance it might change ckan.site_url) """ for plugin in p.PluginImplementations(p.IConfigurer): # must do update in place as this does not work: # config = plugin.update_config(config) plugin.update_config(config) # Set whitelisted env vars on config object # This is set up before globals are initialized ckan_db = os.environ.get("CKAN_DB", None) if ckan_db: msg = ( "Setting CKAN_DB as an env var is deprecated and will be" " removed in a future release. Use CKAN_SQLALCHEMY_URL instead." ) log.warn(msg) config["sqlalchemy.url"] = ckan_db for option in CONFIG_FROM_ENV_VARS: from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None) if from_env: config[option] = from_env root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) site_url = config.get("ckan.site_url", "") if not site_url: raise RuntimeError("ckan.site_url is not configured and it must have a value." " Please amend your .ini file.") if not site_url.lower().startswith("http"): raise RuntimeError("ckan.site_url should be a full URL, including the schema " "(http or https)") display_timezone = config.get("ckan.display_timezone", "") if display_timezone and display_timezone != "server" and display_timezone not in pytz.all_timezones: raise CkanConfigurationException("ckan.display_timezone is not 'server' or a valid timezone") # Remove backslash from site_url if present config["ckan.site_url"] = config["ckan.site_url"].rstrip("/") ckan_host = config["ckan.host"] = urlparse(site_url).netloc if config.get("ckan.site_id") is None: if ":" in ckan_host: ckan_host, port = ckan_host.split(":") assert ckan_host, ( "You need to configure ckan.site_url or " "ckan.site_id for SOLR search-index rebuild to work." ) config["ckan.site_id"] = ckan_host # ensure that a favicon has been set favicon = config.get("ckan.favicon", "/base/images/ckan.ico") config["ckan.favicon"] = favicon # Init SOLR settings and check if the schema is compatible # from ckan.lib.search import SolrSettings, check_solr_schema_version # lib.search is imported here as we need the config enabled and parsed search.SolrSettings.init(config.get("solr_url"), config.get("solr_user"), config.get("solr_password")) search.check_solr_schema_version() routes_map = routing.make_map() config["routes.map"] = routes_map # The RoutesMiddleware needs its mapper updating if it exists if "routes.middleware" in config: config["routes.middleware"].mapper = routes_map config["routes.named_routes"] = routing.named_routes config["pylons.app_globals"] = app_globals.app_globals # initialise the globals config["pylons.app_globals"]._init() # add helper functions helpers = _Helpers(h) config["pylons.h"] = helpers jinja2_templates_path = os.path.join(root, "templates") template_paths = [jinja2_templates_path] extra_template_paths = config.get("extra_template_paths", "") if extra_template_paths: # must be first for them to override defaults template_paths = extra_template_paths.split(",") + template_paths config["pylons.app_globals"].template_paths = template_paths # Markdown ignores the logger config, so to get rid of excessive # markdown debug messages in the log, set it to the level of the # root logger. logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level) # Create Jinja2 environment env = jinja_extensions.Environment( loader=jinja_extensions.CkanFileSystemLoader(template_paths), autoescape=True, extensions=[ "jinja2.ext.do", "jinja2.ext.with_", jinja_extensions.SnippetExtension, jinja_extensions.CkanExtend, jinja_extensions.CkanInternationalizationExtension, jinja_extensions.LinkForExtension, jinja_extensions.ResourceExtension, jinja_extensions.UrlForStaticExtension, jinja_extensions.UrlForExtension, ], ) env.install_gettext_callables(_, ungettext, newstyle=True) # custom filters env.filters["empty_and_escape"] = jinja_extensions.empty_and_escape env.filters["truncate"] = jinja_extensions.truncate config["pylons.app_globals"].jinja_env = env # CONFIGURATION OPTIONS HERE (note: all config options will override # any Pylons config options) # for postgresql we want to enforce utf-8 sqlalchemy_url = config.get("sqlalchemy.url", "") if sqlalchemy_url.startswith("postgresql://"): extras = {"client_encoding": "utf8"} else: extras = {} engine = sqlalchemy.engine_from_config(config, "sqlalchemy.", **extras) if not model.meta.engine: model.init_model(engine) for plugin in p.PluginImplementations(p.IConfigurable): plugin.configure(config) # reset the template cache - we do this here so that when we load the # environment it is clean render.reset_template_info_cache() # clear other caches logic.clear_actions_cache() logic.clear_validators_cache() authz.clear_auth_functions_cache() # Here we create the site user if they are not already in the database try: logic.get_action("get_site_user")({"ignore_auth": True}, None) except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError): # (ProgrammingError for Postgres, OperationalError for SQLite) # The database is not initialised. This is a bit dirty. This occurs # when running tests. pass except sqlalchemy.exc.InternalError: # The database is not initialised. Travis hits this pass # if an extension or our code does not finish # transaction properly db cli commands can fail model.Session.remove()
def update_config(): ''' This code needs to be run when the config is changed to take those changes into account. ''' for plugin in p.PluginImplementations(p.IConfigurer): # must do update in place as this does not work: # config = plugin.update_config(config) plugin.update_config(config) root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # This is set up before globals are initialized site_id = os.environ.get('CKAN_SITE_ID') if site_id: config['ckan.site_id'] = site_id site_url = config.get('ckan.site_url', '') ckan_host = config['ckan.host'] = urlparse(site_url).netloc if config.get('ckan.site_id') is None: if ':' in ckan_host: ckan_host, port = ckan_host.split(':') assert ckan_host, 'You need to configure ckan.site_url or ' \ 'ckan.site_id for SOLR search-index rebuild to work.' config['ckan.site_id'] = ckan_host # ensure that a favicon has been set favicon = config.get('ckan.favicon', '/images/icons/ckan.ico') config['ckan.favicon'] = favicon # Init SOLR settings and check if the schema is compatible #from ckan.lib.search import SolrSettings, check_solr_schema_version # lib.search is imported here as we need the config enabled and parsed search.SolrSettings.init(config.get('solr_url'), config.get('solr_user'), config.get('solr_password')) search.check_solr_schema_version() routes_map = routing.make_map() config['routes.map'] = routes_map # The RoutesMiddleware needs its mapper updating if it exists if 'routes.middleware' in config: config['routes.middleware'].mapper = routes_map config['routes.named_routes'] = routing.named_routes config['pylons.app_globals'] = app_globals.app_globals # initialise the globals config['pylons.app_globals']._init() # add helper functions helpers = _Helpers(h) config['pylons.h'] = helpers ## redo template setup to use genshi.search_path ## (so remove std template setup) legacy_templates_path = os.path.join(root, 'templates_legacy') jinja2_templates_path = os.path.join(root, 'templates') if asbool(config.get('ckan.legacy_templates', 'no')): # We want the new template path for extra snippets like the # dataviewer and also for some testing stuff msg = 'Support for Genshi templates is deprecated and will be removed'\ ' in a future release' log.warn(msg) template_paths = [legacy_templates_path, jinja2_templates_path] else: template_paths = [jinja2_templates_path, legacy_templates_path] extra_template_paths = config.get('extra_template_paths', '') if extra_template_paths: # must be first for them to override defaults template_paths = extra_template_paths.split(',') + template_paths config['pylons.app_globals'].template_paths = template_paths # Translator (i18n) translator = Translator(pylons.translator) def template_loaded(template): translator.setup(template) # Markdown ignores the logger config, so to get rid of excessive # markdown debug messages in the log, set it to the level of the # root logger. logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level) # Create the Genshi TemplateLoader config['pylons.app_globals'].genshi_loader = TemplateLoader( template_paths, auto_reload=True, callback=template_loaded) # Create Jinja2 environment env = jinja_extensions.Environment( loader=jinja_extensions.CkanFileSystemLoader(template_paths), autoescape=True, extensions=['jinja2.ext.do', 'jinja2.ext.with_', jinja_extensions.SnippetExtension, jinja_extensions.CkanExtend, jinja_extensions.CkanInternationalizationExtension, jinja_extensions.LinkForExtension, jinja_extensions.ResourceExtension, jinja_extensions.UrlForStaticExtension, jinja_extensions.UrlForExtension] ) env.install_gettext_callables(_, ungettext, newstyle=True) # custom filters env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape env.filters['truncate'] = jinja_extensions.truncate config['pylons.app_globals'].jinja_env = env # CONFIGURATION OPTIONS HERE (note: all config options will override # any Pylons config options) ckan_db = os.environ.get('CKAN_DB') if ckan_db: config['sqlalchemy.url'] = ckan_db # for postgresql we want to enforce utf-8 sqlalchemy_url = config.get('sqlalchemy.url', '') if sqlalchemy_url.startswith('postgresql://'): extras = {'client_encoding': 'utf8'} else: extras = {} engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.', **extras) if not model.meta.engine: model.init_model(engine) for plugin in p.PluginImplementations(p.IConfigurable): plugin.configure(config) # reset the template cache - we do this here so that when we load the # environment it is clean render.reset_template_info_cache() # clear other caches logic.clear_actions_cache() logic.clear_validators_cache() authz.clear_auth_functions_cache() # Here we create the site user if they are not already in the database try: logic.get_action('get_site_user')({'ignore_auth': True}, None) except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError): # (ProgrammingError for Postgres, OperationalError for SQLite) # The database is not initialised. This is a bit dirty. This occurs # when running tests. pass except sqlalchemy.exc.InternalError: # The database is not initialised. Travis hits this pass # if an extension or our code does not finish # transaction properly db cli commands can fail model.Session.remove()
def update_config(): ''' This code needs to be run when the config is changed to take those changes into account. It is called whenever a plugin is loaded as the plugin might have changed the config values (for instance it might change ckan.site_url) ''' for plugin in p.PluginImplementations(p.IConfigurer): # must do update in place as this does not work: # config = plugin.update_config(config) plugin.update_config(config) # Set whitelisted env vars on config object # This is set up before globals are initialized ckan_db = os.environ.get('CKAN_DB', None) if ckan_db: msg = 'Setting CKAN_DB as an env var is deprecated and will be' \ ' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.' log.warn(msg) config['sqlalchemy.url'] = ckan_db database_user = os.environ.get("DATABASE_USER", None) database_password = os.environ.get("DATABASE_PASSWORD", None) database_host = os.environ.get("DATABASE_HOST", None) database_port = os.environ.get("DATABASE_PORT", None) if database_user is None or database_password is None or database_host is None: print( "Did not find either DATABASE_USER or DATABASE_PASSWORD or DATABASE_HOST" ) else: print("Setting the database url") if database_port is None: #use the default database_port = "5432" url = "postgres://" url += database_user url += ":" url += database_password url += "@" url += database_host url += ":" url += database_port url += "/datacatalogue" config['sqlalchemy.url'] = url print("Setting database " + database_host + ":" + database_port) for option in CONFIG_FROM_ENV_VARS: from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None) if from_env: if option == "AWS_ACCESS_KEY_ID" or option == "AWS_SECRET_ACCESS_KEY": print(option) print(from_env[0:10]) config[option] = from_env root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) site_url = config.get('ckan.site_url', '') if not site_url: raise RuntimeError( 'ckan.site_url is not configured and it must have a value.' ' Please amend your .ini file.') if not site_url.lower().startswith('http'): raise RuntimeError( 'ckan.site_url should be a full URL, including the schema ' '(http or https)') # Remove backslash from site_url if present config['ckan.site_url'] = config['ckan.site_url'].rstrip('/') ckan_host = config['ckan.host'] = urlparse(site_url).netloc if config.get('ckan.site_id') is None: if ':' in ckan_host: ckan_host, port = ckan_host.split(':') assert ckan_host, 'You need to configure ckan.site_url or ' \ 'ckan.site_id for SOLR search-index rebuild to work.' config['ckan.site_id'] = ckan_host # ensure that a favicon has been set favicon = config.get('ckan.favicon', '/images/icons/ckan.ico') config['ckan.favicon'] = favicon # Init SOLR settings and check if the schema is compatible #from ckan.lib.search import SolrSettings, check_solr_schema_version # lib.search is imported here as we need the config enabled and parsed search.SolrSettings.init(config.get('solr_url'), config.get('solr_user'), config.get('solr_password')) search.check_solr_schema_version() routes_map = routing.make_map() config['routes.map'] = routes_map # The RoutesMiddleware needs its mapper updating if it exists if 'routes.middleware' in config: config['routes.middleware'].mapper = routes_map config['routes.named_routes'] = routing.named_routes config['pylons.app_globals'] = app_globals.app_globals # initialise the globals config['pylons.app_globals']._init() # add helper functions helpers = _Helpers(h) config['pylons.h'] = helpers ## redo template setup to use genshi.search_path ## (so remove std template setup) legacy_templates_path = os.path.join(root, 'templates_legacy') jinja2_templates_path = os.path.join(root, 'templates') if asbool(config.get('ckan.legacy_templates', 'no')): # We want the new template path for extra snippets like the # dataviewer and also for some testing stuff msg = 'Support for Genshi templates is deprecated and will be removed'\ ' in a future release' log.warn(msg) template_paths = [legacy_templates_path, jinja2_templates_path] else: template_paths = [jinja2_templates_path, legacy_templates_path] extra_template_paths = config.get('extra_template_paths', '') if extra_template_paths: # must be first for them to override defaults template_paths = extra_template_paths.split(',') + template_paths config['pylons.app_globals'].template_paths = template_paths # Translator (i18n) translator = Translator(pylons.translator) def template_loaded(template): translator.setup(template) # Markdown ignores the logger config, so to get rid of excessive # markdown debug messages in the log, set it to the level of the # root logger. logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level) # Create the Genshi TemplateLoader config['pylons.app_globals'].genshi_loader = TemplateLoader( template_paths, auto_reload=True, callback=template_loaded) # Create Jinja2 environment env = jinja_extensions.Environment( loader=jinja_extensions.CkanFileSystemLoader(template_paths), autoescape=True, extensions=[ 'jinja2.ext.do', 'jinja2.ext.with_', jinja_extensions.SnippetExtension, jinja_extensions.CkanExtend, jinja_extensions.CkanInternationalizationExtension, jinja_extensions.LinkForExtension, jinja_extensions.ResourceExtension, jinja_extensions.UrlForStaticExtension, jinja_extensions.UrlForExtension ]) env.install_gettext_callables(_, ungettext, newstyle=True) # custom filters env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape env.filters['truncate'] = jinja_extensions.truncate config['pylons.app_globals'].jinja_env = env # CONFIGURATION OPTIONS HERE (note: all config options will override # any Pylons config options) # for postgresql we want to enforce utf-8 sqlalchemy_url = config.get('sqlalchemy.url', '') if sqlalchemy_url.startswith('postgresql://'): extras = {'client_encoding': 'utf8'} else: extras = {} engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.', **extras) if not model.meta.engine: model.init_model(engine) for plugin in p.PluginImplementations(p.IConfigurable): plugin.configure(config) # reset the template cache - we do this here so that when we load the # environment it is clean render.reset_template_info_cache() # clear other caches logic.clear_actions_cache() logic.clear_validators_cache() authz.clear_auth_functions_cache() # Here we create the site user if they are not already in the database try: logic.get_action('get_site_user')({'ignore_auth': True}, None) except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError): # (ProgrammingError for Postgres, OperationalError for SQLite) # The database is not initialised. This is a bit dirty. This occurs # when running tests. pass except sqlalchemy.exc.InternalError: # The database is not initialised. Travis hits this pass # if an extension or our code does not finish # transaction properly db cli commands can fail model.Session.remove()