예제 #1
0
    def test_solr_schema_version_check(self):

        from ckan.lib.search import check_solr_schema_version, SearchError

        schema_file = self._get_current_schema()

        # Check that current schema version schema is supported
        assert check_solr_schema_version(schema_file)

        # An exception is thrown if version could not be extracted
        try:
            schema_file = os.path.join(self.root_dir,'solr','schema-no-version.xml')
            check_solr_schema_version(schema_file)

            #Should not happen
            assert False
        except SearchError as e:
            assert 'Could not extract version info' in str(e)

        # An exception is thrown if the schema version is not supported
        try:
            schema_file = os.path.join(self.root_dir,'solr','schema-wrong-version.xml')
            check_solr_schema_version(schema_file)

            #Should not happen
            assert False
        except SearchError as e:
            assert 'SOLR schema version not supported' in str(e)
예제 #2
0
    def test_solr_schema_version_check(self):

        from ckan.lib.search import check_solr_schema_version, SearchError

        schema_file = self._get_current_schema()

        # Check that current schema version schema is supported
        assert check_solr_schema_version(schema_file)

        # An exception is thrown if version could not be extracted
        try:
            schema_file = os.path.join(self.root_dir, 'solr',
                                       'schema-no-version.xml')
            check_solr_schema_version(schema_file)

            #Should not happen
            assert False
        except SearchError as e:
            assert 'Could not extract version info' in str(e)

        # An exception is thrown if the schema version is not supported
        try:
            schema_file = os.path.join(self.root_dir, 'solr',
                                       'schema-wrong-version.xml')
            check_solr_schema_version(schema_file)

            #Should not happen
            assert False
        except SearchError as e:
            assert 'SOLR schema version not supported' in str(e)
예제 #3
0
class TestSolrSchemaVersionCheck(TestController):

    @classmethod
    def setup_class(cls):

        cls.root_dir = os.path.dirname(os.path.realpath(__file__))

    def _get_current_schema(self):

        from ckan.lib.search import SUPPORTED_SCHEMA_VERSIONS

        current_version = sorted(SUPPORTED_SCHEMA_VERSIONS).pop()

        current_schema = os.path.join(self.root_dir,'..','..','config','solr','schema-%s.xml' % current_version)

        return current_schema

    def test_current_schema_exists(self):

        current_schema = self._get_current_schema()

        assert os.path.exists(current_schema)


    def test_solr_schema_version_check(self):

        from ckan.lib.search import check_solr_schema_version, SearchError

        schema_file = self._get_current_schema()

        # Check that current schema version schema is supported
        assert check_solr_schema_version(schema_file)

        # An exception is thrown if version could not be extracted
        try:
            schema_file = os.path.join(self.root_dir,'solr','schema-no-version.xml')
            check_solr_schema_version(schema_file)

            #Should not happen
            assert False
        except SearchError,e:
            assert 'Could not extract version info' in str(e)

        # An exception is thrown if the schema version is not supported
        try:
            schema_file = os.path.join(self.root_dir,'solr','schema-wrong-version.xml')
            check_solr_schema_version(schema_file)

            #Should not happen
            assert False
        except SearchError,e:
            assert 'SOLR schema version not supported' in str(e)
예제 #4
0
def load_environment(global_conf, app_conf):
    """Configure the Pylons environment via the ``pylons.config``
    object
    """

    ######  Pylons monkey-patch
    # this must be run at a time when the env is semi-setup, thus inlined here.
    # Required by the deliverance plugin and iATI
    from pylons.wsgiapp import PylonsApp
    import pkg_resources
    find_controller_generic = PylonsApp.find_controller

    # This is from pylons 1.0 source, will monkey-patch into 0.9.7
    def find_controller(self, controller):
        if controller in self.controller_classes:
            return self.controller_classes[controller]
        # Check to see if its a dotted name
        if '.' in controller or ':' in controller:
            mycontroller = pkg_resources \
                .EntryPoint \
                .parse('x=%s' % controller).load(False)
            self.controller_classes[controller] = mycontroller
            return mycontroller
        return find_controller_generic(self, controller)

    PylonsApp.find_controller = find_controller
    ###### END evil monkey-patch

    os.environ['CKAN_CONFIG'] = global_conf['__file__']

    # Pylons paths
    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    paths = dict(root=root,
                 controllers=os.path.join(root, 'controllers'),
                 static_files=os.path.join(root, 'public'),
                 templates=[])

    # Initialize config with the basic options

    config.init_app(global_conf, app_conf, package='ckan', paths=paths)

    # load all CKAN plugins
    p.load_all(config)

    # Load the synchronous search plugin, unless already loaded or
    # explicitly disabled
    if not 'synchronous_search' in config.get('ckan.plugins',[]) and \
            asbool(config.get('ckan.search.automatic_indexing', True)):
        log.debug('Loading the synchronous search plugin')
        p.load('synchronous_search')

    for plugin in p.PluginImplementations(p.IConfigurer):
        # must do update in place as this does not work:
        # config = plugin.update_config(config)
        plugin.update_config(config)

    # This is set up before globals are initialized
    site_id = os.environ.get('CKAN_SITE_ID')
    if site_id:
        config['ckan.site_id'] = site_id

    site_url = config.get('ckan.site_url', '')
    ckan_host = config['ckan.host'] = urlparse(site_url).netloc
    if config.get('ckan.site_id') is None:
        if ':' in ckan_host:
            ckan_host, port = ckan_host.split(':')
        assert ckan_host, 'You need to configure ckan.site_url or ' \
                          'ckan.site_id for SOLR search-index rebuild to work.'
        config['ckan.site_id'] = ckan_host

    # ensure that a favicon has been set
    favicon = config.get('ckan.favicon', '/images/icons/ckan.ico')
    config['ckan.favicon'] = favicon

    # Init SOLR settings and check if the schema is compatible
    #from ckan.lib.search import SolrSettings, check_solr_schema_version

    # lib.search is imported here as we need the config enabled and parsed
    import ckan.lib.search as search
    search.SolrSettings.init(config.get('solr_url'), config.get('solr_user'),
                             config.get('solr_password'))
    search.check_solr_schema_version()

    config['routes.map'] = routing.make_map()
    config['routes.named_routes'] = routing.named_routes
    config['pylons.app_globals'] = app_globals.app_globals
    # initialise the globals
    config['pylons.app_globals']._init()

    # add helper functions
    helpers = _Helpers(h)
    config['pylons.h'] = helpers

    ## redo template setup to use genshi.search_path
    ## (so remove std template setup)
    legacy_templates_path = os.path.join(root, 'templates_legacy')
    jinja2_templates_path = os.path.join(root, 'templates')
    if asbool(config.get('ckan.legacy_templates', 'no')):
        # We want the new template path for extra snippets like the
        # dataviewer and also for some testing stuff
        template_paths = [legacy_templates_path, jinja2_templates_path]
    else:
        template_paths = [jinja2_templates_path, legacy_templates_path]

    extra_template_paths = config.get('extra_template_paths', '')
    if extra_template_paths:
        # must be first for them to override defaults
        template_paths = extra_template_paths.split(',') + template_paths
    config['pylons.app_globals'].template_paths = template_paths

    # Translator (i18n)
    translator = Translator(pylons.translator)

    def template_loaded(template):
        translator.setup(template)

    # Markdown ignores the logger config, so to get rid of excessive
    # markdown debug messages in the log, set it to the level of the
    # root logger.
    logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)

    # Create the Genshi TemplateLoader
    config['pylons.app_globals'].genshi_loader = TemplateLoader(
        template_paths, auto_reload=True, callback=template_loaded)

    #################################################################
    #                                                               #
    #                   HORRIBLE GENSHI HACK                        #
    #                                                               #
    #################################################################
    #                                                               #
    # Genshi does strange things to get stuff out of the template   #
    # variables.  This stops it from handling properties in the     #
    # correct way as it returns the property rather than the actual #
    # value of the property.                                        #
    #                                                               #
    # By overriding lookup_attr() in the LookupBase class we are    #
    # able to get the required behaviour.  Using @property allows   #
    # us to move functionality out of templates whilst maintaining  #
    # backwards compatability.                                      #
    #                                                               #
    #################################################################
    '''
    This code is based on Genshi code

    Copyright © 2006-2012 Edgewall Software
    All rights reserved.

    Redistribution and use in source and binary forms, with or
    without modification, are permitted provided that the following
    conditions are met:

        Redistributions of source code must retain the above copyright
        notice, this list of conditions and the following disclaimer.

        Redistributions in binary form must reproduce the above
        copyright notice, this list of conditions and the following
        disclaimer in the documentation and/or other materials provided
        with the distribution.

        The name of the author may not be used to endorse or promote
        products derived from this software without specific prior
        written permission.

    THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR
    IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
    ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
    DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
    GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
    IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
    OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
    IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    '''
    from genshi.template.eval import LookupBase

    @classmethod
    def genshi_lookup_attr(cls, obj, key):
        __traceback_hide__ = True
        try:
            val = getattr(obj, key)
        except AttributeError:
            if hasattr(obj.__class__, key):
                raise
            else:
                try:
                    val = obj[key]
                except (KeyError, TypeError):
                    val = cls.undefined(key, owner=obj)
        if isinstance(val, property):
            val = val.fget()
        return val

    setattr(LookupBase, 'lookup_attr', genshi_lookup_attr)
    del genshi_lookup_attr
    del LookupBase

    #################################################################
    #                                                               #
    #                       END OF GENSHI HACK                      #
    #                                                               #
    #################################################################

    # Create Jinja2 environment
    env = lib.jinja_extensions.Environment(
        loader=lib.jinja_extensions.CkanFileSystemLoader(template_paths),
        autoescape=True,
        extensions=[
            'jinja2.ext.do', 'jinja2.ext.with_',
            lib.jinja_extensions.SnippetExtension,
            lib.jinja_extensions.CkanExtend,
            lib.jinja_extensions.CkanInternationalizationExtension,
            lib.jinja_extensions.LinkForExtension,
            lib.jinja_extensions.ResourceExtension,
            lib.jinja_extensions.UrlForStaticExtension,
            lib.jinja_extensions.UrlForExtension
        ])
    env.install_gettext_callables(_, ungettext, newstyle=True)
    # custom filters
    env.filters['empty_and_escape'] = lib.jinja_extensions.empty_and_escape
    env.filters['truncate'] = lib.jinja_extensions.truncate
    config['pylons.app_globals'].jinja_env = env

    # CONFIGURATION OPTIONS HERE (note: all config options will override
    # any Pylons config options)

    # Setup the SQLAlchemy database engine
    # Suppress a couple of sqlalchemy warnings
    msgs = [
        '^Unicode type received non-unicode bind param value',
        "^Did not recognize type 'BIGINT' of column 'size'",
        "^Did not recognize type 'tsvector' of column 'search_vector'"
    ]
    for msg in msgs:
        warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning)

    ckan_db = os.environ.get('CKAN_DB')

    if ckan_db:
        config['sqlalchemy.url'] = ckan_db

    # for postgresql we want to enforce utf-8
    sqlalchemy_url = config.get('sqlalchemy.url', '')
    if sqlalchemy_url.startswith('postgresql://'):
        extras = {'client_encoding': 'utf8'}
    else:
        extras = {}

    engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.', **extras)

    if not model.meta.engine:
        model.init_model(engine)

    for plugin in p.PluginImplementations(p.IConfigurable):
        plugin.configure(config)
예제 #5
0
def update_config():
    ''' This code needs to be run when the config is changed to take those
    changes into account. It is called whenever a plugin is loaded as the
    plugin might have changed the config values (for instance it might
    change ckan.site_url) '''

    for plugin in p.PluginImplementations(p.IConfigurer):
        # must do update in place as this does not work:
        # config = plugin.update_config(config)
        plugin.update_config(config)

    # Set whitelisted env vars on config object
    # This is set up before globals are initialized

    ckan_db = os.environ.get('CKAN_DB', None)
    if ckan_db:
        msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
            ' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
        log.warn(msg)
        config['sqlalchemy.url'] = ckan_db

    database_user = os.environ.get("DATABASE_USER", None)
    database_password = os.environ.get("DATABASE_PASSWORD", None)
    database_host = os.environ.get("DATABASE_HOST", None)
    database_port = os.environ.get("DATABASE_PORT", None)
    if database_user is None or database_password is None or database_host is None:
        print("Did not find either DATABASE_USER or DATABASE_PASSWORD or DATABASE_HOST")
    else: 
        print("Setting the database url")
        if database_port is None:
            #use the default
            database_port = "5432"
        url = "postgres://"
        url += database_user
        url += ":"
        url += database_password
        url += "@"
        url += database_host
        url += ":"
        url += database_port
        url += "/datacatalogue"

        config['sqlalchemy.url'] = url
        print("Setting database " + database_host + ":" + database_port)


    for option in CONFIG_FROM_ENV_VARS:
        from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
        if from_env:
            if option == "AWS_ACCESS_KEY_ID" or option == "AWS_SECRET_ACCESS_KEY":
                print(option)
                print(from_env[0:10])
            config[option] = from_env

    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


    site_url = config.get('ckan.site_url', '')
    if not site_url:
        raise RuntimeError(
            'ckan.site_url is not configured and it must have a value.'
            ' Please amend your .ini file.')
    if not site_url.lower().startswith('http'):
        raise RuntimeError(
            'ckan.site_url should be a full URL, including the schema '
            '(http or https)')

    # Remove backslash from site_url if present
    config['ckan.site_url'] = config['ckan.site_url'].rstrip('/')

    ckan_host = config['ckan.host'] = urlparse(site_url).netloc
    if config.get('ckan.site_id') is None:
        if ':' in ckan_host:
            ckan_host, port = ckan_host.split(':')
        assert ckan_host, 'You need to configure ckan.site_url or ' \
                          'ckan.site_id for SOLR search-index rebuild to work.'
        config['ckan.site_id'] = ckan_host

    # ensure that a favicon has been set
    favicon = config.get('ckan.favicon', '/images/icons/ckan.ico')
    config['ckan.favicon'] = favicon

    # Init SOLR settings and check if the schema is compatible
    #from ckan.lib.search import SolrSettings, check_solr_schema_version

    # lib.search is imported here as we need the config enabled and parsed
    search.SolrSettings.init(config.get('solr_url'),
                             config.get('solr_user'),
                             config.get('solr_password'))
    search.check_solr_schema_version()

    routes_map = routing.make_map()
    config['routes.map'] = routes_map
    # The RoutesMiddleware needs its mapper updating if it exists
    if 'routes.middleware' in config:
        config['routes.middleware'].mapper = routes_map
    config['routes.named_routes'] = routing.named_routes
    config['pylons.app_globals'] = app_globals.app_globals
    # initialise the globals
    config['pylons.app_globals']._init()

    # add helper functions
    helpers = _Helpers(h)
    config['pylons.h'] = helpers

    ## redo template setup to use genshi.search_path
    ## (so remove std template setup)
    legacy_templates_path = os.path.join(root, 'templates_legacy')
    jinja2_templates_path = os.path.join(root, 'templates')
    if asbool(config.get('ckan.legacy_templates', 'no')):
        # We want the new template path for extra snippets like the
        # dataviewer and also for some testing stuff
        msg = 'Support for Genshi templates is deprecated and will be removed'\
            ' in a future release'
        log.warn(msg)

        template_paths = [legacy_templates_path, jinja2_templates_path]
    else:
        template_paths = [jinja2_templates_path, legacy_templates_path]

    extra_template_paths = config.get('extra_template_paths', '')
    if extra_template_paths:
        # must be first for them to override defaults
        template_paths = extra_template_paths.split(',') + template_paths
    config['pylons.app_globals'].template_paths = template_paths

    # Translator (i18n)
    translator = Translator(pylons.translator)

    def template_loaded(template):
        translator.setup(template)

    # Markdown ignores the logger config, so to get rid of excessive
    # markdown debug messages in the log, set it to the level of the
    # root logger.
    logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)

    # Create the Genshi TemplateLoader
    config['pylons.app_globals'].genshi_loader = TemplateLoader(
        template_paths, auto_reload=True, callback=template_loaded)

    # Create Jinja2 environment
    env = jinja_extensions.Environment(
        loader=jinja_extensions.CkanFileSystemLoader(template_paths),
        autoescape=True,
        extensions=['jinja2.ext.do', 'jinja2.ext.with_',
                    jinja_extensions.SnippetExtension,
                    jinja_extensions.CkanExtend,
                    jinja_extensions.CkanInternationalizationExtension,
                    jinja_extensions.LinkForExtension,
                    jinja_extensions.ResourceExtension,
                    jinja_extensions.UrlForStaticExtension,
                    jinja_extensions.UrlForExtension]
    )
    env.install_gettext_callables(_, ungettext, newstyle=True)
    # custom filters
    env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
    env.filters['truncate'] = jinja_extensions.truncate
    config['pylons.app_globals'].jinja_env = env

    # CONFIGURATION OPTIONS HERE (note: all config options will override
    # any Pylons config options)

    # for postgresql we want to enforce utf-8
    sqlalchemy_url = config.get('sqlalchemy.url', '')
    if sqlalchemy_url.startswith('postgresql://'):
        extras = {'client_encoding': 'utf8'}
    else:
        extras = {}

    engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.', **extras)

    if not model.meta.engine:
        model.init_model(engine)

    for plugin in p.PluginImplementations(p.IConfigurable):
        plugin.configure(config)

    # reset the template cache - we do this here so that when we load the
    # environment it is clean
    render.reset_template_info_cache()

    # clear other caches
    logic.clear_actions_cache()
    logic.clear_validators_cache()
    authz.clear_auth_functions_cache()

    # Here we create the site user if they are not already in the database
    try:
        logic.get_action('get_site_user')({'ignore_auth': True}, None)
    except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
        # (ProgrammingError for Postgres, OperationalError for SQLite)
        # The database is not initialised.  This is a bit dirty.  This occurs
        # when running tests.
        pass
    except sqlalchemy.exc.InternalError:
        # The database is not initialised.  Travis hits this
        pass
    # if an extension or our code does not finish
    # transaction properly db cli commands can fail
    model.Session.remove()
예제 #6
0
파일: environment.py 프로젝트: EnxEng/ckan
def update_config():
    ''' This code needs to be run when the config is changed to take those
    changes into account. '''

    for plugin in p.PluginImplementations(p.IConfigurer):
        # must do update in place as this does not work:
        # config = plugin.update_config(config)
        plugin.update_config(config)

    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    # This is set up before globals are initialized
    site_id = os.environ.get('CKAN_SITE_ID')
    if site_id:
        config['ckan.site_id'] = site_id

    site_url = config.get('ckan.site_url', '')
    ckan_host = config['ckan.host'] = urlparse(site_url).netloc
    if config.get('ckan.site_id') is None:
        if ':' in ckan_host:
            ckan_host, port = ckan_host.split(':')
        assert ckan_host, 'You need to configure ckan.site_url or ' \
                          'ckan.site_id for SOLR search-index rebuild to work.'
        config['ckan.site_id'] = ckan_host

    # ensure that a favicon has been set
    favicon = config.get('ckan.favicon', '/images/icons/ckan.ico')
    config['ckan.favicon'] = favicon

    # Init SOLR settings and check if the schema is compatible
    #from ckan.lib.search import SolrSettings, check_solr_schema_version

    # lib.search is imported here as we need the config enabled and parsed
    search.SolrSettings.init(config.get('solr_url'),
                             config.get('solr_user'),
                             config.get('solr_password'))
    search.check_solr_schema_version()

    routes_map = routing.make_map()
    config['routes.map'] = routes_map
    # The RoutesMiddleware needs its mapper updating if it exists
    if 'routes.middleware' in config:
        config['routes.middleware'].mapper = routes_map
    config['routes.named_routes'] = routing.named_routes
    config['pylons.app_globals'] = app_globals.app_globals
    # initialise the globals
    config['pylons.app_globals']._init()

    # add helper functions
    helpers = _Helpers(h)
    config['pylons.h'] = helpers

    ## redo template setup to use genshi.search_path
    ## (so remove std template setup)
    legacy_templates_path = os.path.join(root, 'templates_legacy')
    jinja2_templates_path = os.path.join(root, 'templates')
    if asbool(config.get('ckan.legacy_templates', 'no')):
        # We want the new template path for extra snippets like the
        # dataviewer and also for some testing stuff
        msg = 'Support for Genshi templates is deprecated and will be removed'\
            ' in a future release'
        log.warn(msg)

        template_paths = [legacy_templates_path, jinja2_templates_path]
    else:
        template_paths = [jinja2_templates_path, legacy_templates_path]

    extra_template_paths = config.get('extra_template_paths', '')
    if extra_template_paths:
        # must be first for them to override defaults
        template_paths = extra_template_paths.split(',') + template_paths
    config['pylons.app_globals'].template_paths = template_paths

    # Translator (i18n)
    translator = Translator(pylons.translator)

    def template_loaded(template):
        translator.setup(template)

    # Markdown ignores the logger config, so to get rid of excessive
    # markdown debug messages in the log, set it to the level of the
    # root logger.
    logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)

    # Create the Genshi TemplateLoader
    config['pylons.app_globals'].genshi_loader = TemplateLoader(
        template_paths, auto_reload=True, callback=template_loaded)

    # Create Jinja2 environment
    env = jinja_extensions.Environment(
        loader=jinja_extensions.CkanFileSystemLoader(template_paths),
        autoescape=True,
        extensions=['jinja2.ext.do', 'jinja2.ext.with_',
                    jinja_extensions.SnippetExtension,
                    jinja_extensions.CkanExtend,
                    jinja_extensions.CkanInternationalizationExtension,
                    jinja_extensions.LinkForExtension,
                    jinja_extensions.ResourceExtension,
                    jinja_extensions.UrlForStaticExtension,
                    jinja_extensions.UrlForExtension]
    )
    env.install_gettext_callables(_, ungettext, newstyle=True)
    # custom filters
    env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
    env.filters['truncate'] = jinja_extensions.truncate
    config['pylons.app_globals'].jinja_env = env

    # CONFIGURATION OPTIONS HERE (note: all config options will override
    # any Pylons config options)

    ckan_db = os.environ.get('CKAN_DB')
    if ckan_db:
        config['sqlalchemy.url'] = ckan_db

    # for postgresql we want to enforce utf-8
    sqlalchemy_url = config.get('sqlalchemy.url', '')
    if sqlalchemy_url.startswith('postgresql://'):
        extras = {'client_encoding': 'utf8'}
    else:
        extras = {}

    engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.', **extras)

    if not model.meta.engine:
        model.init_model(engine)

    for plugin in p.PluginImplementations(p.IConfigurable):
        plugin.configure(config)

    # reset the template cache - we do this here so that when we load the
    # environment it is clean
    render.reset_template_info_cache()

    # clear other caches
    logic.clear_actions_cache()
    new_authz.clear_auth_functions_cache()

    # Here we create the site user if they are not already in the database
    try:
        logic.get_action('get_site_user')({'ignore_auth': True}, None)
    except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
        # (ProgrammingError for Postgres, OperationalError for SQLite)
        # The database is not initialised.  This is a bit dirty.  This occurs
        # when running tests.
        pass
    except sqlalchemy.exc.InternalError:
        # The database is not initialised.  Travis hits this
        pass
    # if an extension or our code does not finish
    # transaction properly db cli commands can fail
    model.Session.remove()
예제 #7
0
def update_config():
    ''' This code needs to be run when the config is changed to take those
    changes into account. It is called whenever a plugin is loaded as the
    plugin might have changed the config values (for instance it might
    change ckan.site_url) '''

    config_declaration.setup()
    config_declaration.make_safe(config)
    config_declaration.normalize(config)

    webassets_init()

    for plugin in p.PluginImplementations(p.IConfigurer):
        # must do update in place as this does not work:
        # config = plugin.update_config(config)
        plugin.update_config(config)

    # Set whitelisted env vars on config object
    # This is set up before globals are initialized

    ckan_db = os.environ.get('CKAN_DB', None)
    if ckan_db:
        msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
            ' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
        log.warn(msg)
        config['sqlalchemy.url'] = ckan_db

    for option in CONFIG_FROM_ENV_VARS:
        from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
        if from_env:
            config[option] = from_env

    if config.get_value("config.mode") == "strict":
        _, errors = config_declaration.validate(config)
        if errors:
            msg = "\n".join("{}: {}".format(key, "; ".join(issues))
                            for key, issues in errors.items())
            raise CkanConfigurationException(msg)

    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

    site_url = config.get_value('ckan.site_url')
    if not site_url:
        raise RuntimeError(
            'ckan.site_url is not configured and it must have a value.'
            ' Please amend your .ini file.')
    if not site_url.lower().startswith('http'):
        raise RuntimeError(
            'ckan.site_url should be a full URL, including the schema '
            '(http or https)')
    # Remove backslash from site_url if present
    config['ckan.site_url'] = site_url.rstrip('/')

    display_timezone = config.get_value('ckan.display_timezone')
    if (display_timezone and display_timezone != 'server'
            and display_timezone not in pytz.all_timezones):
        raise CkanConfigurationException(
            "ckan.display_timezone is not 'server' or a valid timezone")

    # Init SOLR settings and check if the schema is compatible
    # from ckan.lib.search import SolrSettings, check_solr_schema_version

    # lib.search is imported here as we need the config enabled and parsed
    search.SolrSettings.init(config.get_value('solr_url'),
                             config.get_value('solr_user'),
                             config.get_value('solr_password'))
    search.check_solr_schema_version()

    lib_plugins.reset_package_plugins()
    lib_plugins.register_package_plugins()
    lib_plugins.reset_group_plugins()
    lib_plugins.register_group_plugins()

    # initialise the globals
    app_globals.app_globals._init()

    helpers.load_plugin_helpers()

    # Templates and CSS loading from configuration
    valid_base_templates_folder_names = ['templates']
    templates = config.get_value('ckan.base_templates_folder')
    config['ckan.base_templates_folder'] = templates

    if templates not in valid_base_templates_folder_names:
        raise CkanConfigurationException(
            'You provided an invalid value for ckan.base_templates_folder. '
            'Possible values are: "templates".')

    jinja2_templates_path = os.path.join(root, templates)
    log.info('Loading templates from %s' % jinja2_templates_path)
    template_paths = [jinja2_templates_path]

    extra_template_paths = config.get_value('extra_template_paths')
    if extra_template_paths:
        # must be first for them to override defaults
        template_paths = extra_template_paths.split(',') + template_paths
    config['computed_template_paths'] = template_paths

    # Enable pessimistic disconnect handling (added in SQLAlchemy 1.2)
    # to eliminate database errors due to stale pooled connections
    config.setdefault('sqlalchemy.pool_pre_ping', True)
    # Initialize SQLAlchemy
    engine = sqlalchemy.engine_from_config(config)
    model.init_model(engine)

    for plugin in p.PluginImplementations(p.IConfigurable):
        plugin.configure(config)

    # clear other caches
    logic.clear_actions_cache()
    logic.clear_validators_cache()
    authz.clear_auth_functions_cache()

    # Here we create the site user if they are not already in the database
    try:
        logic.get_action('get_site_user')({'ignore_auth': True}, None)
    except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
        # The database is not yet initialised. It happens in `ckan db init`
        pass

    # Close current session and open database connections to ensure a clean
    # clean environment even if an error occurs later on
    model.Session.remove()
    model.Session.bind.dispose()
예제 #8
0
def test_check_valid_schema():
    schema_file = os.path.join(root_dir, "schema.xml")
    assert check_solr_schema_version(schema_file)
예제 #9
0
파일: environment.py 프로젝트: xiphl/ckan
def load_environment(conf):
    """
    Configure the Pylons environment via the ``pylons.config`` object. This
    code should only need to be run once.
    """
    if six.PY2:
        # this must be run at a time when the env is semi-setup, thus inlined
        # here. Required by the deliverance plugin and iATI
        from pylons.wsgiapp import PylonsApp
        import pkg_resources
        find_controller_generic = getattr(PylonsApp.find_controller,
                                          '_old_find_controller',
                                          PylonsApp.find_controller)

        # This is from pylons 1.0 source, will monkey-patch into 0.9.7
        def find_controller(self, controller):
            if controller in self.controller_classes:
                return self.controller_classes[controller]
            # Check to see if its a dotted name
            if '.' in controller or ':' in controller:
                ep = pkg_resources.EntryPoint.parse('x={0}'.format(controller))

                if hasattr(ep, 'resolve'):
                    # setuptools >= 10.2
                    mycontroller = ep.resolve()
                else:
                    # setuptools >= 11.3
                    mycontroller = ep.load(False)

                self.controller_classes[controller] = mycontroller
                return mycontroller
            return find_controller_generic(self, controller)

        find_controller._old_find_controller = find_controller_generic
        PylonsApp.find_controller = find_controller

    os.environ['CKAN_CONFIG'] = conf['__file__']

    # Pylons paths
    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

    valid_base_public_folder_names = ['public']
    static_files = conf.get('ckan.base_public_folder', 'public')
    conf['ckan.base_public_folder'] = static_files

    if static_files not in valid_base_public_folder_names:
        raise CkanConfigurationException(
            'You provided an invalid value for ckan.base_public_folder. '
            'Possible values are: "public".')

    log.info('Loading static files from %s' % static_files)
    paths = dict(root=root,
                 controllers=os.path.join(root, 'controllers'),
                 static_files=os.path.join(root, static_files),
                 templates=[])

    # Initialize main CKAN config object
    config.update(conf)

    if six.PY2:
        # Initialize Pylons own config object
        pylons_config.init_app(conf['global_conf'],
                               conf,
                               package='ckan',
                               paths=paths)

        # Update the main CKAN config object with the Pylons specific stuff,
        # as it is quite hard to keep them separated. This should be removed
        # once Pylons support is dropped
        config.update(pylons_config)

    # Setup the SQLAlchemy database engine
    # Suppress a couple of sqlalchemy warnings
    msgs = [
        '^Unicode type received non-unicode bind param value',
        "^Did not recognize type 'BIGINT' of column 'size'",
        "^Did not recognize type 'tsvector' of column 'search_vector'"
    ]
    for msg in msgs:
        warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning)

    # load all CKAN plugins
    p.load_all()

    if not p.plugin_loaded('managed_search_schema'):
        search.check_solr_schema_version()

    # Check Redis availability
    if not is_redis_available():
        log.critical('Could not connect to Redis.')

    app_globals.reset()

    # issue #3260: remove idle transaction
    # Session that was used for getting all config params nor committed,
    # neither removed and we have idle connection as result
    model.Session.commit()

    # Build JavaScript translations. Must be done after plugins have
    # been loaded.
    build_js_translations()
예제 #10
0
파일: environment.py 프로젝트: espona/ckan
def update_config():
    ''' This code needs to be run when the config is changed to take those
    changes into account. It is called whenever a plugin is loaded as the
    plugin might have changed the config values (for instance it might
    change ckan.site_url) '''

    for plugin in p.PluginImplementations(p.IConfigurer):
        # must do update in place as this does not work:
        # config = plugin.update_config(config)
        plugin.update_config(config)

    # Set whitelisted env vars on config object
    # This is set up before globals are initialized

    ckan_db = os.environ.get('CKAN_DB', None)
    if ckan_db:
        msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
            ' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
        log.warn(msg)
        config['sqlalchemy.url'] = ckan_db

    for option in CONFIG_FROM_ENV_VARS:
        from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
        if from_env:
            config[option] = from_env

    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

    site_url = config.get('ckan.site_url', '')
    if not site_url:
        raise RuntimeError(
            'ckan.site_url is not configured and it must have a value.'
            ' Please amend your .ini file.')
    if not site_url.lower().startswith('http'):
        raise RuntimeError(
            'ckan.site_url should be a full URL, including the schema '
            '(http or https)')

    display_timezone = config.get('ckan.display_timezone', '')
    if (display_timezone and
            display_timezone != 'server' and
            display_timezone not in pytz.all_timezones):
        raise CkanConfigurationException(
            "ckan.display_timezone is not 'server' or a valid timezone"
        )

    # Remove backslash from site_url if present
    config['ckan.site_url'] = config['ckan.site_url'].rstrip('/')

    ckan_host = config['ckan.host'] = urlparse(site_url).netloc
    if config.get('ckan.site_id') is None:
        if ':' in ckan_host:
            ckan_host, port = ckan_host.split(':')
        assert ckan_host, 'You need to configure ckan.site_url or ' \
                          'ckan.site_id for SOLR search-index rebuild to work.'
        config['ckan.site_id'] = ckan_host

    # ensure that a favicon has been set
    favicon = config.get('ckan.favicon', '/base/images/ckan.ico')
    config['ckan.favicon'] = favicon

    # Init SOLR settings and check if the schema is compatible
    # from ckan.lib.search import SolrSettings, check_solr_schema_version

    # lib.search is imported here as we need the config enabled and parsed
    search.SolrSettings.init(config.get('solr_url'),
                             config.get('solr_user'),
                             config.get('solr_password'))
    search.check_solr_schema_version()

    routes_map = routing.make_map()
    config['routes.map'] = routes_map
    # The RoutesMiddleware needs its mapper updating if it exists
    if 'routes.middleware' in config:
        config['routes.middleware'].mapper = routes_map
    # routes.named_routes is a CKAN thing
    config['routes.named_routes'] = routing.named_routes
    config['pylons.app_globals'] = app_globals.app_globals

    # initialise the globals
    app_globals.app_globals._init()

    helpers.load_plugin_helpers()
    config['pylons.h'] = helpers.helper_functions

    # Templates and CSS loading from configuration
    valid_base_templates_folder_names = ['templates', 'templates-bs2']
    templates = config.get('ckan.base_templates_folder', 'templates')
    config['ckan.base_templates_folder'] = templates

    if templates not in valid_base_templates_folder_names:
        raise CkanConfigurationException(
            'You provided an invalid value for ckan.base_templates_folder. '
            'Possible values are: "templates" and "templates-bs2".'
        )

    jinja2_templates_path = os.path.join(root, templates)
    log.info('Loading templates from %s' % jinja2_templates_path)
    template_paths = [jinja2_templates_path]

    extra_template_paths = config.get('extra_template_paths', '')
    if extra_template_paths:
        # must be first for them to override defaults
        template_paths = extra_template_paths.split(',') + template_paths
    config['computed_template_paths'] = template_paths

    # Set the default language for validation messages from formencode
    # to what is set as the default locale in the config
    default_lang = config.get('ckan.locale_default', 'en')
    formencode.api.set_stdtranslation(domain="FormEncode",
                                      languages=[default_lang])

    # Markdown ignores the logger config, so to get rid of excessive
    # markdown debug messages in the log, set it to the level of the
    # root logger.
    logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)

    # Create Jinja2 environment
    env = jinja_extensions.Environment(
        **jinja_extensions.get_jinja_env_options())
    env.install_gettext_callables(_, ungettext, newstyle=True)
    # custom filters
    env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
    config['pylons.app_globals'].jinja_env = env

    # CONFIGURATION OPTIONS HERE (note: all config options will override
    # any Pylons config options)

    # Initialize SQLAlchemy
    engine = sqlalchemy.engine_from_config(config, client_encoding='utf8')
    model.init_model(engine)

    for plugin in p.PluginImplementations(p.IConfigurable):
        plugin.configure(config)

    # reset the template cache - we do this here so that when we load the
    # environment it is clean
    render.reset_template_info_cache()

    # clear other caches
    logic.clear_actions_cache()
    logic.clear_validators_cache()
    authz.clear_auth_functions_cache()

    # Here we create the site user if they are not already in the database
    try:
        logic.get_action('get_site_user')({'ignore_auth': True}, None)
    except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
        # (ProgrammingError for Postgres, OperationalError for SQLite)
        # The database is not initialised.  This is a bit dirty.  This occurs
        # when running tests.
        pass
    except sqlalchemy.exc.InternalError:
        # The database is not initialised.  Travis hits this
        pass

    # Close current session and open database connections to ensure a clean
    # clean environment even if an error occurs later on
    model.Session.remove()
    model.Session.bind.dispose()
예제 #11
0
파일: environment.py 프로젝트: govro/ckan
def update_config():
    """ This code needs to be run when the config is changed to take those
    changes into account. It is called whenever a plugin is loaded as the
    plugin might have changed the config values (for instance it might
    change ckan.site_url) """

    for plugin in p.PluginImplementations(p.IConfigurer):
        # must do update in place as this does not work:
        # config = plugin.update_config(config)
        plugin.update_config(config)

    # Set whitelisted env vars on config object
    # This is set up before globals are initialized

    ckan_db = os.environ.get("CKAN_DB", None)
    if ckan_db:
        msg = (
            "Setting CKAN_DB as an env var is deprecated and will be"
            " removed in a future release. Use CKAN_SQLALCHEMY_URL instead."
        )
        log.warn(msg)
        config["sqlalchemy.url"] = ckan_db

    for option in CONFIG_FROM_ENV_VARS:
        from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
        if from_env:
            config[option] = from_env

    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

    site_url = config.get("ckan.site_url", "")
    if not site_url:
        raise RuntimeError("ckan.site_url is not configured and it must have a value." " Please amend your .ini file.")
    if not site_url.lower().startswith("http"):
        raise RuntimeError("ckan.site_url should be a full URL, including the schema " "(http or https)")

    display_timezone = config.get("ckan.display_timezone", "")
    if display_timezone and display_timezone != "server" and display_timezone not in pytz.all_timezones:
        raise CkanConfigurationException("ckan.display_timezone is not 'server' or a valid timezone")

    # Remove backslash from site_url if present
    config["ckan.site_url"] = config["ckan.site_url"].rstrip("/")

    ckan_host = config["ckan.host"] = urlparse(site_url).netloc
    if config.get("ckan.site_id") is None:
        if ":" in ckan_host:
            ckan_host, port = ckan_host.split(":")
        assert ckan_host, (
            "You need to configure ckan.site_url or " "ckan.site_id for SOLR search-index rebuild to work."
        )
        config["ckan.site_id"] = ckan_host

    # ensure that a favicon has been set
    favicon = config.get("ckan.favicon", "/base/images/ckan.ico")
    config["ckan.favicon"] = favicon

    # Init SOLR settings and check if the schema is compatible
    # from ckan.lib.search import SolrSettings, check_solr_schema_version

    # lib.search is imported here as we need the config enabled and parsed
    search.SolrSettings.init(config.get("solr_url"), config.get("solr_user"), config.get("solr_password"))
    search.check_solr_schema_version()

    routes_map = routing.make_map()
    config["routes.map"] = routes_map
    # The RoutesMiddleware needs its mapper updating if it exists
    if "routes.middleware" in config:
        config["routes.middleware"].mapper = routes_map
    config["routes.named_routes"] = routing.named_routes
    config["pylons.app_globals"] = app_globals.app_globals
    # initialise the globals
    config["pylons.app_globals"]._init()

    # add helper functions
    helpers = _Helpers(h)
    config["pylons.h"] = helpers

    jinja2_templates_path = os.path.join(root, "templates")
    template_paths = [jinja2_templates_path]

    extra_template_paths = config.get("extra_template_paths", "")
    if extra_template_paths:
        # must be first for them to override defaults
        template_paths = extra_template_paths.split(",") + template_paths
    config["pylons.app_globals"].template_paths = template_paths

    # Markdown ignores the logger config, so to get rid of excessive
    # markdown debug messages in the log, set it to the level of the
    # root logger.
    logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)

    # Create Jinja2 environment
    env = jinja_extensions.Environment(
        loader=jinja_extensions.CkanFileSystemLoader(template_paths),
        autoescape=True,
        extensions=[
            "jinja2.ext.do",
            "jinja2.ext.with_",
            jinja_extensions.SnippetExtension,
            jinja_extensions.CkanExtend,
            jinja_extensions.CkanInternationalizationExtension,
            jinja_extensions.LinkForExtension,
            jinja_extensions.ResourceExtension,
            jinja_extensions.UrlForStaticExtension,
            jinja_extensions.UrlForExtension,
        ],
    )
    env.install_gettext_callables(_, ungettext, newstyle=True)
    # custom filters
    env.filters["empty_and_escape"] = jinja_extensions.empty_and_escape
    env.filters["truncate"] = jinja_extensions.truncate
    config["pylons.app_globals"].jinja_env = env

    # CONFIGURATION OPTIONS HERE (note: all config options will override
    # any Pylons config options)

    # for postgresql we want to enforce utf-8
    sqlalchemy_url = config.get("sqlalchemy.url", "")
    if sqlalchemy_url.startswith("postgresql://"):
        extras = {"client_encoding": "utf8"}
    else:
        extras = {}

    engine = sqlalchemy.engine_from_config(config, "sqlalchemy.", **extras)

    if not model.meta.engine:
        model.init_model(engine)

    for plugin in p.PluginImplementations(p.IConfigurable):
        plugin.configure(config)

    # reset the template cache - we do this here so that when we load the
    # environment it is clean
    render.reset_template_info_cache()

    # clear other caches
    logic.clear_actions_cache()
    logic.clear_validators_cache()
    authz.clear_auth_functions_cache()

    # Here we create the site user if they are not already in the database
    try:
        logic.get_action("get_site_user")({"ignore_auth": True}, None)
    except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
        # (ProgrammingError for Postgres, OperationalError for SQLite)
        # The database is not initialised.  This is a bit dirty.  This occurs
        # when running tests.
        pass
    except sqlalchemy.exc.InternalError:
        # The database is not initialised.  Travis hits this
        pass
    # if an extension or our code does not finish
    # transaction properly db cli commands can fail
    model.Session.remove()
예제 #12
0
def test_check_schema_with_wrong_version():
    schema_file = os.path.join(data_dir, "schema-wrong-version.xml")
    with pytest.raises(SearchError) as e:
        check_solr_schema_version(schema_file)
    assert "SOLR schema version not supported" in str(e.value)
예제 #13
0
def test_check_invalid_schema():
    schema_file = os.path.join(data_dir, "schema-no-version.xml")
    with pytest.raises(SearchError) as e:
        check_solr_schema_version(schema_file)
    assert "Could not extract version info" in str(e.value)
예제 #14
0
def update_config():
    ''' This code needs to be run when the config is changed to take those
    changes into account. It is called whenever a plugin is loaded as the
    plugin might have changed the config values (for instance it might
    change ckan.site_url) '''

    for plugin in p.PluginImplementations(p.IConfigurer):
        # must do update in place as this does not work:
        # config = plugin.update_config(config)
        plugin.update_config(config)

    # Set whitelisted env vars on config object
    # This is set up before globals are initialized

    ckan_db = os.environ.get('CKAN_DB', None)
    if ckan_db:
        msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
            ' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
        log.warn(msg)
        config['sqlalchemy.url'] = ckan_db

    for option in CONFIG_FROM_ENV_VARS:
        from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
        if from_env:
            config[option] = from_env

    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

    site_url = config.get('ckan.site_url', '')
    if not site_url:
        raise RuntimeError(
            'ckan.site_url is not configured and it must have a value.'
            ' Please amend your .ini file.')
    if not site_url.lower().startswith('http'):
        raise RuntimeError(
            'ckan.site_url should be a full URL, including the schema '
            '(http or https)')

    display_timezone = config.get('ckan.display_timezone', '')
    if (display_timezone and display_timezone != 'server'
            and display_timezone not in pytz.all_timezones):
        raise CkanConfigurationException(
            "ckan.display_timezone is not 'server' or a valid timezone")

    # Remove backslash from site_url if present
    config['ckan.site_url'] = config['ckan.site_url'].rstrip('/')

    ckan_host = config['ckan.host'] = urlparse(site_url).netloc
    if config.get('ckan.site_id') is None:
        if ':' in ckan_host:
            ckan_host, port = ckan_host.split(':')
        assert ckan_host, 'You need to configure ckan.site_url or ' \
                          'ckan.site_id for SOLR search-index rebuild to work.'
        config['ckan.site_id'] = ckan_host

    # ensure that a favicon has been set
    favicon = config.get('ckan.favicon', '/base/images/ckan.ico')
    config['ckan.favicon'] = favicon

    # Init SOLR settings and check if the schema is compatible
    # from ckan.lib.search import SolrSettings, check_solr_schema_version

    # lib.search is imported here as we need the config enabled and parsed
    search.SolrSettings.init(config.get('solr_url'), config.get('solr_user'),
                             config.get('solr_password'))
    search.check_solr_schema_version()

    routes_map = routing.make_map()
    config['routes.map'] = routes_map
    # The RoutesMiddleware needs its mapper updating if it exists
    if 'routes.middleware' in config:
        config['routes.middleware'].mapper = routes_map
    # routes.named_routes is a CKAN thing
    config['routes.named_routes'] = routing.named_routes
    config['pylons.app_globals'] = app_globals.app_globals
    # initialise the globals
    app_globals.app_globals._init()

    helpers.load_plugin_helpers()
    config['pylons.h'] = helpers.helper_functions

    jinja2_templates_path = os.path.join(root, 'templates')
    template_paths = [jinja2_templates_path]

    extra_template_paths = config.get('extra_template_paths', '')
    if extra_template_paths:
        # must be first for them to override defaults
        template_paths = extra_template_paths.split(',') + template_paths
    config['pylons.app_globals'].template_paths = template_paths

    # Set the default language for validation messages from formencode
    # to what is set as the default locale in the config
    default_lang = config.get('ckan.locale_default', 'en')
    formencode.api.set_stdtranslation(domain="FormEncode",
                                      languages=[default_lang])

    # Markdown ignores the logger config, so to get rid of excessive
    # markdown debug messages in the log, set it to the level of the
    # root logger.
    logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)

    # Create Jinja2 environment
    env = jinja_extensions.Environment(
        loader=jinja_extensions.CkanFileSystemLoader(template_paths),
        autoescape=True,
        extensions=[
            'jinja2.ext.do', 'jinja2.ext.with_',
            jinja_extensions.SnippetExtension, jinja_extensions.CkanExtend,
            jinja_extensions.CkanInternationalizationExtension,
            jinja_extensions.LinkForExtension,
            jinja_extensions.ResourceExtension,
            jinja_extensions.UrlForStaticExtension,
            jinja_extensions.UrlForExtension
        ])
    env.install_gettext_callables(_, ungettext, newstyle=True)
    # custom filters
    env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
    env.filters['truncate'] = jinja_extensions.truncate
    config['pylons.app_globals'].jinja_env = env

    # CONFIGURATION OPTIONS HERE (note: all config options will override
    # any Pylons config options)

    # Initialize SQLAlchemy
    engine = sqlalchemy.engine_from_config(config, client_encoding='utf8')
    model.init_model(engine)

    for plugin in p.PluginImplementations(p.IConfigurable):
        plugin.configure(config)

    # reset the template cache - we do this here so that when we load the
    # environment it is clean
    render.reset_template_info_cache()

    # clear other caches
    logic.clear_actions_cache()
    logic.clear_validators_cache()
    authz.clear_auth_functions_cache()

    # Here we create the site user if they are not already in the database
    try:
        logic.get_action('get_site_user')({'ignore_auth': True}, None)
    except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
        # (ProgrammingError for Postgres, OperationalError for SQLite)
        # The database is not initialised.  This is a bit dirty.  This occurs
        # when running tests.
        pass
    except sqlalchemy.exc.InternalError:
        # The database is not initialised.  Travis hits this
        pass
    # if an extension or our code does not finish
    # transaction properly db cli commands can fail
    model.Session.remove()
예제 #15
0
파일: environment.py 프로젝트: pingali/ckan
def load_environment(global_conf, app_conf):
    """Configure the Pylons environment via the ``pylons.config``
    object
    """

    ######  Pylons monkey-patch
    # this must be run at a time when the env is semi-setup, thus inlined here.
    # Required by the deliverance plugin and iATI
    from pylons.wsgiapp import PylonsApp
    import pkg_resources
    find_controller_generic = PylonsApp.find_controller

    # This is from pylons 1.0 source, will monkey-patch into 0.9.7
    def find_controller(self, controller):
        if controller in self.controller_classes:
            return self.controller_classes[controller]
        # Check to see if its a dotted name
        if '.' in controller or ':' in controller:
            mycontroller = pkg_resources.EntryPoint.parse(
                'x=%s' % controller).load(False)
            self.controller_classes[controller] = mycontroller
            return mycontroller
        return find_controller_generic(self, controller)

    PylonsApp.find_controller = find_controller
    ###### END evil monkey-patch

    os.environ['CKAN_CONFIG'] = global_conf['__file__']

    # Pylons paths
    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    paths = dict(root=root,
                 controllers=os.path.join(root, 'controllers'),
                 static_files=os.path.join(root, 'public'),
                 templates=[os.path.join(root, 'templates')])

    # Initialize config with the basic options

    config.init_app(global_conf, app_conf, package='ckan', paths=paths)

    # load all CKAN plugins
    plugins.load_all(config)

    from ckan.plugins import PluginImplementations
    from ckan.plugins.interfaces import IConfigurer

    for plugin in PluginImplementations(IConfigurer):
        # must do update in place as this does not work:
        # config = plugin.update_config(config)
        plugin.update_config(config)

    # This is set up before globals are initialized
    site_url = config.get('ckan.site_url', '')
    ckan_host = config['ckan.host'] = urlparse(site_url).netloc
    if config.get('ckan.site_id') is None:
        if ':' in ckan_host:
            ckan_host, port = ckan_host.split(':')
        assert ckan_host, 'You need to configure ckan.site_url or ' \
                          'ckan.site_id for SOLR search-index rebuild to work.'
        config['ckan.site_id'] = ckan_host

    # Check if SOLR schema is compatible
    from ckan.lib.search import check_solr_schema_version
    check_solr_schema_version()

    config['routes.map'] = make_map()
    config['pylons.app_globals'] = app_globals.Globals()
    config['pylons.h'] = ckan.lib.helpers

    ## redo template setup to use genshi.search_path (so remove std template setup)
    template_paths = [paths['templates'][0]]
    extra_template_paths = config.get('extra_template_paths', '')
    if extra_template_paths:
        # must be first for them to override defaults
        template_paths = extra_template_paths.split(',') + template_paths

    # Translator (i18n)
    translator = Translator(pylons.translator)

    def template_loaded(template):
        translator.setup(template)

    # Markdown ignores the logger config, so to get rid of excessive
    # markdown debug messages in the log, set it to the level of the
    # root logger.
    logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)

    # Create the Genshi TemplateLoader
    # config['pylons.app_globals'].genshi_loader = TemplateLoader(
    #    paths['templates'], auto_reload=True)
    # tmpl_options["genshi.loader_callback"] = template_loaded
    config['pylons.app_globals'].genshi_loader = TemplateLoader(
        template_paths, auto_reload=True, callback=template_loaded)

    # CONFIGURATION OPTIONS HERE (note: all config options will override
    # any Pylons config options)

    # Setup the SQLAlchemy database engine
    engine = engine_from_config(config, 'sqlalchemy.')

    if not model.meta.engine:
        model.init_model(engine)

    from ckan.plugins import PluginImplementations
    from ckan.plugins.interfaces import IConfigurable

    for plugin in PluginImplementations(IConfigurable):
        plugin.configure(config)
예제 #16
0
def load_environment(global_conf, app_conf):
    """Configure the Pylons environment via the ``pylons.config``
    object
    """

    ######  Pylons monkey-patch
    # this must be run at a time when the env is semi-setup, thus inlined here.
    # Required by the deliverance plugin and iATI
    from pylons.wsgiapp import PylonsApp
    import pkg_resources
    find_controller_generic = PylonsApp.find_controller

    # This is from pylons 1.0 source, will monkey-patch into 0.9.7
    def find_controller(self, controller):
        if controller in self.controller_classes:
            return self.controller_classes[controller]
        # Check to see if its a dotted name
        if '.' in controller or ':' in controller:
            mycontroller = pkg_resources \
                .EntryPoint \
                .parse('x=%s' % controller).load(False)
            self.controller_classes[controller] = mycontroller
            return mycontroller
        return find_controller_generic(self, controller)
    PylonsApp.find_controller = find_controller
    ###### END evil monkey-patch

    os.environ['CKAN_CONFIG'] = global_conf['__file__']

    # Pylons paths
    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    paths = dict(root=root,
                 controllers=os.path.join(root, 'controllers'),
                 static_files=os.path.join(root, 'public'),
                 templates=[])

    # Initialize config with the basic options

    config.init_app(global_conf, app_conf, package='ckan', paths=paths)

    # load all CKAN plugins
    p.load_all(config)

    # Load the synchronous search plugin, unless already loaded or
    # explicitly disabled
    if not 'synchronous_search' in config.get('ckan.plugins',[]) and \
            asbool(config.get('ckan.search.automatic_indexing', True)):
        log.debug('Loading the synchronous search plugin')
        p.load('synchronous_search')

    for plugin in p.PluginImplementations(p.IConfigurer):
        # must do update in place as this does not work:
        # config = plugin.update_config(config)
        plugin.update_config(config)

    # This is set up before globals are initialized
    site_id = os.environ.get('CKAN_SITE_ID')
    if site_id:
        config['ckan.site_id'] = site_id

    site_url = config.get('ckan.site_url', '')
    ckan_host = config['ckan.host'] = urlparse(site_url).netloc
    if config.get('ckan.site_id') is None:
        if ':' in ckan_host:
            ckan_host, port = ckan_host.split(':')
        assert ckan_host, 'You need to configure ckan.site_url or ' \
                          'ckan.site_id for SOLR search-index rebuild to work.'
        config['ckan.site_id'] = ckan_host

    # ensure that a favicon has been set
    favicon = config.get('ckan.favicon', '/images/icons/ckan.ico')
    config['ckan.favicon'] = favicon

    # Init SOLR settings and check if the schema is compatible
    #from ckan.lib.search import SolrSettings, check_solr_schema_version

    # lib.search is imported here as we need the config enabled and parsed
    import ckan.lib.search as search
    search.SolrSettings.init(config.get('solr_url'),
                             config.get('solr_user'),
                             config.get('solr_password'))
    search.check_solr_schema_version()

    config['routes.map'] = routing.make_map()
    config['routes.named_routes'] = routing.named_routes
    config['pylons.app_globals'] = app_globals.app_globals
    # initialise the globals
    config['pylons.app_globals']._init()

    # add helper functions
    helpers = _Helpers(h)
    config['pylons.h'] = helpers

    ## redo template setup to use genshi.search_path
    ## (so remove std template setup)
    legacy_templates_path = os.path.join(root, 'templates_legacy')
    jinja2_templates_path = os.path.join(root, 'templates')
    if asbool(config.get('ckan.legacy_templates', 'no')):
        # We want the new template path for extra snippets like the
        # dataviewer and also for some testing stuff
        template_paths = [legacy_templates_path, jinja2_templates_path]
    else:
        template_paths = [jinja2_templates_path, legacy_templates_path]

    extra_template_paths = config.get('extra_template_paths', '')
    if extra_template_paths:
        # must be first for them to override defaults
        template_paths = extra_template_paths.split(',') + template_paths
    config['pylons.app_globals'].template_paths = template_paths

    # Translator (i18n)
    translator = Translator(pylons.translator)

    def template_loaded(template):
        translator.setup(template)

    # Markdown ignores the logger config, so to get rid of excessive
    # markdown debug messages in the log, set it to the level of the
    # root logger.
    logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)

    # Create the Genshi TemplateLoader
    config['pylons.app_globals'].genshi_loader = TemplateLoader(
        template_paths, auto_reload=True, callback=template_loaded)

    #################################################################
    #                                                               #
    #                   HORRIBLE GENSHI HACK                        #
    #                                                               #
    #################################################################
    #                                                               #
    # Genshi does strange things to get stuff out of the template   #
    # variables.  This stops it from handling properties in the     #
    # correct way as it returns the property rather than the actual #
    # value of the property.                                        #
    #                                                               #
    # By overriding lookup_attr() in the LookupBase class we are    #
    # able to get the required behaviour.  Using @property allows   #
    # us to move functionality out of templates whilst maintaining  #
    # backwards compatability.                                      #
    #                                                               #
    #################################################################

    '''
    This code is based on Genshi code

    Copyright © 2006-2012 Edgewall Software
    All rights reserved.

    Redistribution and use in source and binary forms, with or
    without modification, are permitted provided that the following
    conditions are met:

        Redistributions of source code must retain the above copyright
        notice, this list of conditions and the following disclaimer.

        Redistributions in binary form must reproduce the above
        copyright notice, this list of conditions and the following
        disclaimer in the documentation and/or other materials provided
        with the distribution.

        The name of the author may not be used to endorse or promote
        products derived from this software without specific prior
        written permission.

    THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR
    IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
    ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
    DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
    GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
    IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
    OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
    IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    '''
    from genshi.template.eval import LookupBase

    @classmethod
    def genshi_lookup_attr(cls, obj, key):
        __traceback_hide__ = True
        try:
            val = getattr(obj, key)
        except AttributeError:
            if hasattr(obj.__class__, key):
                raise
            else:
                try:
                    val = obj[key]
                except (KeyError, TypeError):
                    val = cls.undefined(key, owner=obj)
        if isinstance(val, property):
            val = val.fget()
        return val

    setattr(LookupBase, 'lookup_attr', genshi_lookup_attr)
    del genshi_lookup_attr
    del LookupBase

    #################################################################
    #                                                               #
    #                       END OF GENSHI HACK                      #
    #                                                               #
    #################################################################


    # Create Jinja2 environment
    env = jinja_extensions.Environment(
        loader=jinja_extensions.CkanFileSystemLoader(template_paths),
        autoescape=True,
        extensions=['jinja2.ext.do', 'jinja2.ext.with_',
                    jinja_extensions.SnippetExtension,
                    jinja_extensions.CkanExtend,
                    jinja_extensions.CkanInternationalizationExtension,
                    jinja_extensions.LinkForExtension,
                    jinja_extensions.ResourceExtension,
                    jinja_extensions.UrlForStaticExtension,
                    jinja_extensions.UrlForExtension]
    )
    env.install_gettext_callables(_, ungettext, newstyle=True)
    # custom filters
    env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
    env.filters['truncate'] = jinja_extensions.truncate
    config['pylons.app_globals'].jinja_env = env

    # CONFIGURATION OPTIONS HERE (note: all config options will override
    # any Pylons config options)

    # Setup the SQLAlchemy database engine
    # Suppress a couple of sqlalchemy warnings
    msgs = ['^Unicode type received non-unicode bind param value',
            "^Did not recognize type 'BIGINT' of column 'size'",
            "^Did not recognize type 'tsvector' of column 'search_vector'"
            ]
    for msg in msgs:
        warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning)

    ckan_db = os.environ.get('CKAN_DB')

    if ckan_db:
        config['sqlalchemy.url'] = ckan_db

    # for postgresql we want to enforce utf-8
    sqlalchemy_url = config.get('sqlalchemy.url', '')
    if sqlalchemy_url.startswith('postgresql://'):
        extras = {'client_encoding': 'utf8'}
    else:
        extras = {}

    engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.', **extras)

    if not model.meta.engine:
        model.init_model(engine)


    for plugin in p.PluginImplementations(p.IConfigurable):
        plugin.configure(config)
예제 #17
0
def update_config():
    ''' This code needs to be run when the config is changed to take those
    changes into account. '''

    for plugin in p.PluginImplementations(p.IConfigurer):
        # must do update in place as this does not work:
        # config = plugin.update_config(config)
        plugin.update_config(config)

    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    # This is set up before globals are initialized
    site_id = os.environ.get('CKAN_SITE_ID')
    if site_id:
        config['ckan.site_id'] = site_id

    site_url = config.get('ckan.site_url', '')
    ckan_host = config['ckan.host'] = urlparse(site_url).netloc
    if config.get('ckan.site_id') is None:
        if ':' in ckan_host:
            ckan_host, port = ckan_host.split(':')
        assert ckan_host, 'You need to configure ckan.site_url or ' \
                          'ckan.site_id for SOLR search-index rebuild to work.'
        config['ckan.site_id'] = ckan_host

    # ensure that a favicon has been set
    favicon = config.get('ckan.favicon', '/images/icons/ckan.ico')
    config['ckan.favicon'] = favicon

    # Init SOLR settings and check if the schema is compatible
    #from ckan.lib.search import SolrSettings, check_solr_schema_version

    # lib.search is imported here as we need the config enabled and parsed
    search.SolrSettings.init(config.get('solr_url'),
                             config.get('solr_user'),
                             config.get('solr_password'))
    search.check_solr_schema_version()

    routes_map = routing.make_map()
    config['routes.map'] = routes_map
    # The RoutesMiddleware needs its mapper updating if it exists
    if 'routes.middleware' in config:
        config['routes.middleware'].mapper = routes_map
    config['routes.named_routes'] = routing.named_routes
    config['pylons.app_globals'] = app_globals.app_globals
    # initialise the globals
    config['pylons.app_globals']._init()

    # add helper functions
    helpers = _Helpers(h)
    config['pylons.h'] = helpers

    ## redo template setup to use genshi.search_path
    ## (so remove std template setup)
    legacy_templates_path = os.path.join(root, 'templates_legacy')
    jinja2_templates_path = os.path.join(root, 'templates')
    if asbool(config.get('ckan.legacy_templates', 'no')):
        # We want the new template path for extra snippets like the
        # dataviewer and also for some testing stuff
        msg = 'Support for Genshi templates is deprecated and will be removed'\
            ' in a future release'
        log.warn(msg)

        template_paths = [legacy_templates_path, jinja2_templates_path]
    else:
        template_paths = [jinja2_templates_path, legacy_templates_path]

    extra_template_paths = config.get('extra_template_paths', '')
    if extra_template_paths:
        # must be first for them to override defaults
        template_paths = extra_template_paths.split(',') + template_paths
    config['pylons.app_globals'].template_paths = template_paths

    # Translator (i18n)
    translator = Translator(pylons.translator)

    def template_loaded(template):
        translator.setup(template)

    # Markdown ignores the logger config, so to get rid of excessive
    # markdown debug messages in the log, set it to the level of the
    # root logger.
    logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)

    # Create the Genshi TemplateLoader
    config['pylons.app_globals'].genshi_loader = TemplateLoader(
        template_paths, auto_reload=True, callback=template_loaded)

    # Create Jinja2 environment
    env = jinja_extensions.Environment(
        loader=jinja_extensions.CkanFileSystemLoader(template_paths),
        autoescape=True,
        extensions=['jinja2.ext.do', 'jinja2.ext.with_',
                    jinja_extensions.SnippetExtension,
                    jinja_extensions.CkanExtend,
                    jinja_extensions.CkanInternationalizationExtension,
                    jinja_extensions.LinkForExtension,
                    jinja_extensions.ResourceExtension,
                    jinja_extensions.UrlForStaticExtension,
                    jinja_extensions.UrlForExtension]
    )
    env.install_gettext_callables(_, ungettext, newstyle=True)
    # custom filters
    env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
    env.filters['truncate'] = jinja_extensions.truncate
    config['pylons.app_globals'].jinja_env = env

    # CONFIGURATION OPTIONS HERE (note: all config options will override
    # any Pylons config options)

    ckan_db = os.environ.get('CKAN_DB')
    if ckan_db:
        config['sqlalchemy.url'] = ckan_db

    # for postgresql we want to enforce utf-8
    sqlalchemy_url = config.get('sqlalchemy.url', '')
    if sqlalchemy_url.startswith('postgresql://'):
        extras = {'client_encoding': 'utf8'}
    else:
        extras = {}

    engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.', **extras)

    if not model.meta.engine:
        model.init_model(engine)

    for plugin in p.PluginImplementations(p.IConfigurable):
        plugin.configure(config)

    # reset the template cache - we do this here so that when we load the
    # environment it is clean
    render.reset_template_info_cache()

    # clear other caches
    logic.clear_actions_cache()
    logic.clear_validators_cache()
    new_authz.clear_auth_functions_cache()

    # Here we create the site user if they are not already in the database
    try:
        logic.get_action('get_site_user')({'ignore_auth': True}, None)
    except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
        # (ProgrammingError for Postgres, OperationalError for SQLite)
        # The database is not initialised.  This is a bit dirty.  This occurs
        # when running tests.
        pass
    except sqlalchemy.exc.InternalError:
        # The database is not initialised.  Travis hits this
        pass
    # if an extension or our code does not finish
    # transaction properly db cli commands can fail
    model.Session.remove()
예제 #18
0
파일: environment.py 프로젝트: xiphl/ckan
def update_config():
    ''' This code needs to be run when the config is changed to take those
    changes into account. It is called whenever a plugin is loaded as the
    plugin might have changed the config values (for instance it might
    change ckan.site_url) '''

    webassets_init()

    for plugin in p.PluginImplementations(p.IConfigurer):
        # must do update in place as this does not work:
        # config = plugin.update_config(config)
        plugin.update_config(config)

    # Set whitelisted env vars on config object
    # This is set up before globals are initialized

    ckan_db = os.environ.get('CKAN_DB', None)
    if ckan_db:
        msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
            ' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
        log.warn(msg)
        config['sqlalchemy.url'] = ckan_db

    for option in CONFIG_FROM_ENV_VARS:
        from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
        if from_env:
            config[option] = from_env

    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

    site_url = config.get('ckan.site_url', '')
    if not site_url:
        raise RuntimeError(
            'ckan.site_url is not configured and it must have a value.'
            ' Please amend your .ini file.')
    if not site_url.lower().startswith('http'):
        raise RuntimeError(
            'ckan.site_url should be a full URL, including the schema '
            '(http or https)')

    display_timezone = config.get('ckan.display_timezone', '')
    if (display_timezone and display_timezone != 'server'
            and display_timezone not in pytz.all_timezones):
        raise CkanConfigurationException(
            "ckan.display_timezone is not 'server' or a valid timezone")

    # Remove backslash from site_url if present
    config['ckan.site_url'] = config['ckan.site_url'].rstrip('/')

    ckan_host = config['ckan.host'] = urlparse(site_url).netloc
    if config.get('ckan.site_id') is None:
        if ':' in ckan_host:
            ckan_host, port = ckan_host.split(':')
        assert ckan_host, 'You need to configure ckan.site_url or ' \
                          'ckan.site_id for SOLR search-index rebuild to work.'
        config['ckan.site_id'] = ckan_host

    # ensure that a favicon has been set
    favicon = config.get('ckan.favicon', '/base/images/ckan.ico')
    config['ckan.favicon'] = favicon

    # Init SOLR settings and check if the schema is compatible
    # from ckan.lib.search import SolrSettings, check_solr_schema_version

    # lib.search is imported here as we need the config enabled and parsed
    search.SolrSettings.init(config.get('solr_url'), config.get('solr_user'),
                             config.get('solr_password'))
    search.check_solr_schema_version()

    if six.PY2:
        routes_map = routing.make_map()

    lib_plugins.reset_package_plugins()
    lib_plugins.register_package_plugins()
    lib_plugins.reset_group_plugins()
    lib_plugins.register_group_plugins()

    if six.PY2:
        config['routes.map'] = routes_map
        # The RoutesMiddleware needs its mapper updating if it exists
        if 'routes.middleware' in config:
            config['routes.middleware'].mapper = routes_map
        # routes.named_routes is a CKAN thing
        config['routes.named_routes'] = routing.named_routes
        config['pylons.app_globals'] = app_globals.app_globals

    # initialise the globals
    app_globals.app_globals._init()

    helpers.load_plugin_helpers()
    config['pylons.h'] = helpers.helper_functions

    # Templates and CSS loading from configuration
    valid_base_templates_folder_names = ['templates']
    templates = config.get('ckan.base_templates_folder', 'templates')
    config['ckan.base_templates_folder'] = templates

    if templates not in valid_base_templates_folder_names:
        raise CkanConfigurationException(
            'You provided an invalid value for ckan.base_templates_folder. '
            'Possible values are: "templates".')

    jinja2_templates_path = os.path.join(root, templates)
    log.info('Loading templates from %s' % jinja2_templates_path)
    template_paths = [jinja2_templates_path]

    extra_template_paths = config.get('extra_template_paths', '')
    if extra_template_paths:
        # must be first for them to override defaults
        template_paths = extra_template_paths.split(',') + template_paths
    config['computed_template_paths'] = template_paths

    # Markdown ignores the logger config, so to get rid of excessive
    # markdown debug messages in the log, set it to the level of the
    # root logger.
    logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)

    if six.PY2:
        # Create Jinja2 environment
        env = jinja_extensions.Environment(
            **jinja_extensions.get_jinja_env_options())
        env.install_gettext_callables(_, ungettext, newstyle=True)
        # custom filters
        env.policies['ext.i18n.trimmed'] = True
        env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
        config['pylons.app_globals'].jinja_env = env

    # CONFIGURATION OPTIONS HERE (note: all config options will override
    # any Pylons config options)

    # Enable pessimistic disconnect handling (added in SQLAlchemy 1.2)
    # to eliminate database errors due to stale pooled connections
    config.setdefault('pool_pre_ping', True)

    # Initialize SQLAlchemy
    engine = sqlalchemy.engine_from_config(config)
    model.init_model(engine)

    for plugin in p.PluginImplementations(p.IConfigurable):
        plugin.configure(config)

    # reset the template cache - we do this here so that when we load the
    # environment it is clean
    render.reset_template_info_cache()

    # clear other caches
    logic.clear_actions_cache()
    logic.clear_validators_cache()
    authz.clear_auth_functions_cache()

    # Here we create the site user if they are not already in the database
    try:
        logic.get_action('get_site_user')({'ignore_auth': True}, None)
    except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
        # (ProgrammingError for Postgres, OperationalError for SQLite)
        # The database is not initialised.  This is a bit dirty.  This occurs
        # when running tests.
        pass
    except sqlalchemy.exc.InternalError:
        # The database is not initialised.  Travis hits this
        pass

    # Close current session and open database connections to ensure a clean
    # clean environment even if an error occurs later on
    model.Session.remove()
    model.Session.bind.dispose()
예제 #19
0
def load_environment(global_conf, app_conf):
    """Configure the Pylons environment via the ``pylons.config``
    object
    """

    ######  Pylons monkey-patch
    # this must be run at a time when the env is semi-setup, thus inlined here.
    # Required by the deliverance plugin and iATI
    from pylons.wsgiapp import PylonsApp
    import pkg_resources

    find_controller_generic = PylonsApp.find_controller

    # This is from pylons 1.0 source, will monkey-patch into 0.9.7
    def find_controller(self, controller):
        if controller in self.controller_classes:
            return self.controller_classes[controller]
        # Check to see if its a dotted name
        if "." in controller or ":" in controller:
            mycontroller = pkg_resources.EntryPoint.parse("x=%s" % controller).load(False)
            self.controller_classes[controller] = mycontroller
            return mycontroller
        return find_controller_generic(self, controller)

    PylonsApp.find_controller = find_controller
    ###### END evil monkey-patch

    os.environ["CKAN_CONFIG"] = global_conf["__file__"]

    # Pylons paths
    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    paths = dict(
        root=root,
        controllers=os.path.join(root, "controllers"),
        static_files=os.path.join(root, "public"),
        templates=[os.path.join(root, "templates")],
    )

    # Initialize config with the basic options

    config.init_app(global_conf, app_conf, package="ckan", paths=paths)

    # load all CKAN plugins
    p.load_all(config)

    # Load the synchronous search plugin, unless already loaded or
    # explicitly disabled
    if not "synchronous_search" in config.get("ckan.plugins", []) and asbool(
        config.get("ckan.search.automatic_indexing", True)
    ):
        log.debug("Loading the synchronous search plugin")
        p.load("synchronous_search")

    for plugin in p.PluginImplementations(p.IConfigurer):
        # must do update in place as this does not work:
        # config = plugin.update_config(config)
        plugin.update_config(config)

    # This is set up before globals are initialized
    site_id = os.environ.get("CKAN_SITE_ID")
    if site_id:
        config["ckan.site_id"] = site_id

    site_url = config.get("ckan.site_url", "")
    ckan_host = config["ckan.host"] = urlparse(site_url).netloc
    if config.get("ckan.site_id") is None:
        if ":" in ckan_host:
            ckan_host, port = ckan_host.split(":")
        assert ckan_host, (
            "You need to configure ckan.site_url or " "ckan.site_id for SOLR search-index rebuild to work."
        )
        config["ckan.site_id"] = ckan_host

    # Init SOLR settings and check if the schema is compatible
    # from ckan.lib.search import SolrSettings, check_solr_schema_version
    search.SolrSettings.init(config.get("solr_url"), config.get("solr_user"), config.get("solr_password"))
    search.check_solr_schema_version()

    config["routes.map"] = routing.make_map()
    config["pylons.app_globals"] = app_globals.Globals()

    # add helper functions
    restrict_helpers = asbool(config.get("ckan.restrict_template_vars", "true"))
    helpers = _Helpers(h, restrict_helpers)
    config["pylons.h"] = helpers

    # Redo template setup to use genshi.search_path
    # (so remove std template setup)
    template_paths = [paths["templates"][0]]
    extra_template_paths = config.get("extra_template_paths", "")
    if extra_template_paths:
        # must be first for them to override defaults
        template_paths = extra_template_paths.split(",") + template_paths

    # Translator (i18n)
    translator = Translator(pylons.translator)

    def template_loaded(template):
        translator.setup(template)

    # Markdown ignores the logger config, so to get rid of excessive
    # markdown debug messages in the log, set it to the level of the
    # root logger.
    logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)

    # Create the Genshi TemplateLoader
    config["pylons.app_globals"].genshi_loader = TemplateLoader(
        template_paths, auto_reload=True, callback=template_loaded
    )

    #################################################################
    #                                                               #
    #                   HORRIBLE GENSHI HACK                        #
    #                                                               #
    #################################################################
    #                                                               #
    # Genshi does strange things to get stuff out of the template   #
    # variables.  This stops it from handling properties in the     #
    # correct way as it returns the property rather than the actual #
    # value of the property.                                        #
    #                                                               #
    # By overriding lookup_attr() in the LookupBase class we are    #
    # able to get the required behaviour.  Using @property allows   #
    # us to move functionality out of templates whilst maintaining  #
    # backwards compatability.                                      #
    #                                                               #
    #################################################################

    """
    This code is based on Genshi code

    Copyright © 2006-2012 Edgewall Software
    All rights reserved.

    Redistribution and use in source and binary forms, with or
    without modification, are permitted provided that the following
    conditions are met:

        Redistributions of source code must retain the above copyright
        notice, this list of conditions and the following disclaimer.

        Redistributions in binary form must reproduce the above
        copyright notice, this list of conditions and the following
        disclaimer in the documentation and/or other materials provided
        with the distribution.

        The name of the author may not be used to endorse or promote
        products derived from this software without specific prior
        written permission.

    THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR
    IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
    ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
    DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
    GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
    IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
    OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
    IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    """
    from genshi.template.eval import LookupBase

    @classmethod
    def genshi_lookup_attr(cls, obj, key):
        __traceback_hide__ = True
        try:
            val = getattr(obj, key)
        except AttributeError:
            if hasattr(obj.__class__, key):
                raise
            else:
                try:
                    val = obj[key]
                except (KeyError, TypeError):
                    val = cls.undefined(key, owner=obj)
        if isinstance(val, property):
            val = val.fget()
        return val

    setattr(LookupBase, "lookup_attr", genshi_lookup_attr)
    del genshi_lookup_attr
    del LookupBase

    #################################################################
    #                                                               #
    #                       END OF GENSHI HACK                      #
    #                                                               #
    #################################################################

    # CONFIGURATION OPTIONS HERE (note: all config options will override
    # any Pylons config options)

    # Setup the SQLAlchemy database engine
    # Suppress a couple of sqlalchemy warnings
    msgs = [
        "^Unicode type received non-unicode bind param value",
        "^Did not recognize type 'BIGINT' of column 'size'",
        "^Did not recognize type 'tsvector' of column 'search_vector'",
    ]
    for msg in msgs:
        warnings.filterwarnings("ignore", msg, sqlalchemy.exc.SAWarning)

    ckan_db = os.environ.get("CKAN_DB")

    if ckan_db:
        config["sqlalchemy.url"] = ckan_db

    # for postgresql we want to enforce utf-8
    sqlalchemy_url = config.get("sqlalchemy.url", "")
    if sqlalchemy_url.startswith("postgresql://"):
        extras = {"client_encoding": "utf8"}
    else:
        extras = {}

    engine = sqlalchemy.engine_from_config(config, "sqlalchemy.", **extras)

    if not model.meta.engine:
        model.init_model(engine)

    for plugin in p.PluginImplementations(p.IConfigurable):
        plugin.configure(config)
예제 #20
0
def load_environment(global_conf, app_conf):
    """Configure the Pylons environment via the ``pylons.config``
    object
    """
    
    ######  Pylons monkey-patch
    # this must be run at a time when the env is semi-setup, thus inlined here. 
    # Required by the deliverance plugin and iATI
    from pylons.wsgiapp import PylonsApp
    import pkg_resources
    find_controller_generic = PylonsApp.find_controller
    # This is from pylons 1.0 source, will monkey-patch into 0.9.7
    def find_controller(self, controller):
        if controller in self.controller_classes:
            return self.controller_classes[controller]
        # Check to see if its a dotted name
        if '.' in controller or ':' in controller:
            mycontroller = pkg_resources.EntryPoint.parse('x=%s' % controller).load(False)
            self.controller_classes[controller] = mycontroller
            return mycontroller
        return find_controller_generic(self, controller)
    PylonsApp.find_controller = find_controller
    ###### END evil monkey-patch 

    os.environ['CKAN_CONFIG'] = global_conf['__file__']

    # Pylons paths
    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    paths = dict(root=root,
                 controllers=os.path.join(root, 'controllers'),
                 static_files=os.path.join(root, 'public'),
                 templates=[os.path.join(root, 'templates')])

    # Initialize config with the basic options
    
    config.init_app(global_conf, app_conf, package='ckan', paths=paths)
    
    # load all CKAN plugins
    plugins.load_all(config)

    from ckan.plugins import PluginImplementations
    from ckan.plugins.interfaces import IConfigurer
    
    for plugin in PluginImplementations(IConfigurer):
        # must do update in place as this does not work:
        # config = plugin.update_config(config)
        plugin.update_config(config)
    
    # This is set up before globals are initialized
    site_url = config.get('ckan.site_url', '')
    ckan_host = config['ckan.host'] = urlparse(site_url).netloc
    if config.get('ckan.site_id') is None:
        if ':' in ckan_host:
            ckan_host, port = ckan_host.split(':')
        assert ckan_host, 'You need to configure ckan.site_url or ' \
                          'ckan.site_id for SOLR search-index rebuild to work.'
        config['ckan.site_id'] = ckan_host

    # Init SOLR settings and check if the schema is compatible
    from ckan.lib.search import SolrSettings, check_solr_schema_version
    SolrSettings.init(config.get('solr_url'),
                      config.get('solr_user'),
                      config.get('solr_password'))
    check_solr_schema_version()

    config['routes.map'] = make_map()
    config['pylons.app_globals'] = app_globals.Globals()
    config['pylons.h'] = ckan.lib.helpers
        
    ## redo template setup to use genshi.search_path (so remove std template setup)
    template_paths = [paths['templates'][0]]
    extra_template_paths = config.get('extra_template_paths', '')
    if extra_template_paths:
        # must be first for them to override defaults
        template_paths = extra_template_paths.split(',') + template_paths

    # Translator (i18n)
    translator = Translator(pylons.translator)
    def template_loaded(template):
        translator.setup(template)

    # Markdown ignores the logger config, so to get rid of excessive
    # markdown debug messages in the log, set it to the level of the
    # root logger.
    logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)

    # Create the Genshi TemplateLoader
    # config['pylons.app_globals'].genshi_loader = TemplateLoader(
    #    paths['templates'], auto_reload=True)
    # tmpl_options["genshi.loader_callback"] = template_loaded
    config['pylons.app_globals'].genshi_loader = TemplateLoader(
        template_paths, auto_reload=True, callback=template_loaded)

    # CONFIGURATION OPTIONS HERE (note: all config options will override
    # any Pylons config options)    

    # Setup the SQLAlchemy database engine
    engine = engine_from_config(config, 'sqlalchemy.')

    if not model.meta.engine:
        model.init_model(engine)
    
    from ckan.plugins import PluginImplementations
    from ckan.plugins.interfaces import IConfigurable
    
    for plugin in PluginImplementations(IConfigurable):
        plugin.configure(config)
def update_config():
    ''' This code needs to be run when the config is changed to take those
    changes into account. It is called whenever a plugin is loaded as the
    plugin might have changed the config values (for instance it might
    change ckan.site_url) '''

    for plugin in p.PluginImplementations(p.IConfigurer):
        # must do update in place as this does not work:
        # config = plugin.update_config(config)
        plugin.update_config(config)

    # Set whitelisted env vars on config object
    # This is set up before globals are initialized

    ckan_db = os.environ.get('CKAN_DB', None)
    if ckan_db:
        msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
            ' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
        log.warn(msg)
        config['sqlalchemy.url'] = ckan_db

    database_user = os.environ.get("DATABASE_USER", None)
    database_password = os.environ.get("DATABASE_PASSWORD", None)
    database_host = os.environ.get("DATABASE_HOST", None)
    database_port = os.environ.get("DATABASE_PORT", None)
    if database_user is None or database_password is None or database_host is None:
        print(
            "Did not find either DATABASE_USER or DATABASE_PASSWORD or DATABASE_HOST"
        )
    else:
        print("Setting the database url")
        if database_port is None:
            #use the default
            database_port = "5432"
        url = "postgres://"
        url += database_user
        url += ":"
        url += database_password
        url += "@"
        url += database_host
        url += ":"
        url += database_port
        url += "/datacatalogue"

        config['sqlalchemy.url'] = url
        print("Setting database " + database_host + ":" + database_port)

    for option in CONFIG_FROM_ENV_VARS:
        from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
        if from_env:
            if option == "AWS_ACCESS_KEY_ID" or option == "AWS_SECRET_ACCESS_KEY":
                print(option)
                print(from_env[0:10])
            config[option] = from_env

    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

    site_url = config.get('ckan.site_url', '')
    if not site_url:
        raise RuntimeError(
            'ckan.site_url is not configured and it must have a value.'
            ' Please amend your .ini file.')
    if not site_url.lower().startswith('http'):
        raise RuntimeError(
            'ckan.site_url should be a full URL, including the schema '
            '(http or https)')

    # Remove backslash from site_url if present
    config['ckan.site_url'] = config['ckan.site_url'].rstrip('/')

    ckan_host = config['ckan.host'] = urlparse(site_url).netloc
    if config.get('ckan.site_id') is None:
        if ':' in ckan_host:
            ckan_host, port = ckan_host.split(':')
        assert ckan_host, 'You need to configure ckan.site_url or ' \
                          'ckan.site_id for SOLR search-index rebuild to work.'
        config['ckan.site_id'] = ckan_host

    # ensure that a favicon has been set
    favicon = config.get('ckan.favicon', '/images/icons/ckan.ico')
    config['ckan.favicon'] = favicon

    # Init SOLR settings and check if the schema is compatible
    #from ckan.lib.search import SolrSettings, check_solr_schema_version

    # lib.search is imported here as we need the config enabled and parsed
    search.SolrSettings.init(config.get('solr_url'), config.get('solr_user'),
                             config.get('solr_password'))
    search.check_solr_schema_version()

    routes_map = routing.make_map()
    config['routes.map'] = routes_map
    # The RoutesMiddleware needs its mapper updating if it exists
    if 'routes.middleware' in config:
        config['routes.middleware'].mapper = routes_map
    config['routes.named_routes'] = routing.named_routes
    config['pylons.app_globals'] = app_globals.app_globals
    # initialise the globals
    config['pylons.app_globals']._init()

    # add helper functions
    helpers = _Helpers(h)
    config['pylons.h'] = helpers

    ## redo template setup to use genshi.search_path
    ## (so remove std template setup)
    legacy_templates_path = os.path.join(root, 'templates_legacy')
    jinja2_templates_path = os.path.join(root, 'templates')
    if asbool(config.get('ckan.legacy_templates', 'no')):
        # We want the new template path for extra snippets like the
        # dataviewer and also for some testing stuff
        msg = 'Support for Genshi templates is deprecated and will be removed'\
            ' in a future release'
        log.warn(msg)

        template_paths = [legacy_templates_path, jinja2_templates_path]
    else:
        template_paths = [jinja2_templates_path, legacy_templates_path]

    extra_template_paths = config.get('extra_template_paths', '')
    if extra_template_paths:
        # must be first for them to override defaults
        template_paths = extra_template_paths.split(',') + template_paths
    config['pylons.app_globals'].template_paths = template_paths

    # Translator (i18n)
    translator = Translator(pylons.translator)

    def template_loaded(template):
        translator.setup(template)

    # Markdown ignores the logger config, so to get rid of excessive
    # markdown debug messages in the log, set it to the level of the
    # root logger.
    logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)

    # Create the Genshi TemplateLoader
    config['pylons.app_globals'].genshi_loader = TemplateLoader(
        template_paths, auto_reload=True, callback=template_loaded)

    # Create Jinja2 environment
    env = jinja_extensions.Environment(
        loader=jinja_extensions.CkanFileSystemLoader(template_paths),
        autoescape=True,
        extensions=[
            'jinja2.ext.do', 'jinja2.ext.with_',
            jinja_extensions.SnippetExtension, jinja_extensions.CkanExtend,
            jinja_extensions.CkanInternationalizationExtension,
            jinja_extensions.LinkForExtension,
            jinja_extensions.ResourceExtension,
            jinja_extensions.UrlForStaticExtension,
            jinja_extensions.UrlForExtension
        ])
    env.install_gettext_callables(_, ungettext, newstyle=True)
    # custom filters
    env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
    env.filters['truncate'] = jinja_extensions.truncate
    config['pylons.app_globals'].jinja_env = env

    # CONFIGURATION OPTIONS HERE (note: all config options will override
    # any Pylons config options)

    # for postgresql we want to enforce utf-8
    sqlalchemy_url = config.get('sqlalchemy.url', '')
    if sqlalchemy_url.startswith('postgresql://'):
        extras = {'client_encoding': 'utf8'}
    else:
        extras = {}

    engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.', **extras)

    if not model.meta.engine:
        model.init_model(engine)

    for plugin in p.PluginImplementations(p.IConfigurable):
        plugin.configure(config)

    # reset the template cache - we do this here so that when we load the
    # environment it is clean
    render.reset_template_info_cache()

    # clear other caches
    logic.clear_actions_cache()
    logic.clear_validators_cache()
    authz.clear_auth_functions_cache()

    # Here we create the site user if they are not already in the database
    try:
        logic.get_action('get_site_user')({'ignore_auth': True}, None)
    except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
        # (ProgrammingError for Postgres, OperationalError for SQLite)
        # The database is not initialised.  This is a bit dirty.  This occurs
        # when running tests.
        pass
    except sqlalchemy.exc.InternalError:
        # The database is not initialised.  Travis hits this
        pass
    # if an extension or our code does not finish
    # transaction properly db cli commands can fail
    model.Session.remove()