def command(cls, config_ini):
config_ini_filepath = os.path.abspath(config_ini)
cls.load_config(config_ini_filepath)
engine = engine_from_config(config, 'sqlalchemy.')
from ckan import model
from ckanext.dgu.drupalclient import DrupalClient
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
global global_log
global_log = log
model.init_model(engine)
model.repo.new_revision()
cls.drupal_client = DrupalClient({'xmlrpc_domain': 'data.gov.uk',
'xmlrpc_username': '******',
'xmlrpc_password': config.get('dgu.xmlrpc_password')})
publisher_dicts = cls.drupal_client.get_organisation_list()
for publisher_dict in publisher_dicts:
if not (publisher_dict['status'] == '1' or \
publisher_dict['nid'] == '16248'):
# Make an exception for 16248 - Met Office under BIS is correct
log.info('Ignoring unpublished publisher with status %r: %r',
publisher_dict['status'], publisher_dict)
continue
cls.add_publisher(publisher_dict['nid'])
all_groups = model.Session.query(model.Group).\
filter(model.Group.type == 'organization').order_by('title').all()
log.info('Total number of groups: %i', len(all_groups))
log.info('Warnings: %r', warnings)
def command(cls, config_ini):
config_ini_filepath = os.path.abspath(config_ini)
cls.load_config(config_ini_filepath)
engine = engine_from_config(config, 'sqlalchemy.')
from ckan import model
from ckanext.dgu.drupalclient import DrupalClient
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
global global_log
global_log = log
model.init_model(engine)
model.repo.new_revision()
cls.drupal_client = DrupalClient({'xmlrpc_domain': 'data.gov.uk',
'xmlrpc_username': '******',
'xmlrpc_password': config.get('dgu.xmlrpc_password')})
publisher_dicts = cls.drupal_client.get_organisation_list()
for publisher_dict in publisher_dicts:
if not (publisher_dict['status'] == '1' or \
publisher_dict['nid'] == '16248'):
# Make an exception for 16248 - Met Office under BIS is correct
log.info('Ignoring unpublished publisher with status %r: %r',
publisher_dict['status'], publisher_dict)
continue
cls.add_publisher(publisher_dict['nid'])
all_groups = model.Session.query(model.Group).\
filter(model.Group.type == 'publisher').order_by('title').all()
log.info('Total number of groups: %i', len(all_groups))
log.info('Warnings: %r', warnings)
def command(cls, config_ini):
config_ini_filepath = os.path.abspath(config_ini)
cls.load_config(config_ini_filepath)
engine = engine_from_config(config, 'sqlalchemy.')
from ckan import model
from ckanext.dgu.drupalclient import DrupalClient, DrupalRequestError
import ckanext.dgu.drupalclient
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
global global_log
global_log = log
model.init_model(engine)
model.repo.new_revision()
# disable xmlrpc logs
ckanext.dgu.drupalclient.log.disabled = True
cls.drupal_client = DrupalClient({
'xmlrpc_domain':
'data.gov.uk',
'xmlrpc_username':
'******',
'xmlrpc_password':
config.get('dgu.xmlrpc_password')
})
f = open('users.csv', 'wb')
users = csv.writer(f, quoting=csv.QUOTE_ALL)
rows = []
for nid in range(28, 35000):
try:
user = cls.drupal_client.get_user_properties(nid)
except DrupalRequestError, e:
if '404' in str(e):
# node not a user
continue
else:
raise
publishers = user['publishers']
if len(publishers) > 1:
log.info('Multiple publishers for user %s [%s]!: %r',
user['name'], user['uid'],
repr(publishers)[:100])
if len(publishers) > 100:
warn('Ignoring user %s [%s] with %i publishers!', user['name'],
user['uid'], len(publishers))
continue
for publisher in publishers:
row = [user['uid'], user['name'], user['mail'], publisher]
rows.append(row)
log.info('User: %r', row)
users.writerow(row)
f.flush()
def command(cls, config_ini):
config_ini_filepath = os.path.abspath(config_ini)
cls.load_config(config_ini_filepath)
engine = engine_from_config(config, 'sqlalchemy.')
from ckan import model
from ckanext.dgu.drupalclient import DrupalClient, DrupalRequestError
import ckanext.dgu.drupalclient
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
global global_log
global_log = log
model.init_model(engine)
model.repo.new_revision()
# disable xmlrpc logs
ckanext.dgu.drupalclient.log.disabled = True
cls.drupal_client = DrupalClient({'xmlrpc_domain': 'data.gov.uk',
'xmlrpc_username': '******',
'xmlrpc_password': config.get('dgu.xmlrpc_password')})
f = open('users.csv', 'wb')
users = csv.writer(f, quoting=csv.QUOTE_ALL)
rows = []
for nid in range(28, 35000):
try:
user = cls.drupal_client.get_user_properties(nid)
except DrupalRequestError, e:
if '404' in str(e):
# node not a user
continue
else:
raise
publishers = user['publishers']
if len(publishers) > 1:
log.info('Multiple publishers for user %s [%s]!: %r',
user['name'], user['uid'], repr(publishers)[:100])
if len(publishers) > 100:
warn('Ignoring user %s [%s] with %i publishers!',
user['name'], user['uid'], len(publishers))
continue
for publisher in publishers:
row = [user['uid'], user['name'], user['mail'], publisher]
rows.append(row)
log.info('User: %r', row)
users.writerow(row)
f.flush()
def command(config_ini):
config_ini_filepath = os.path.abspath(config_ini)
load_config(config_ini_filepath)
engine = engine_from_config(config, 'sqlalchemy.')
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
global global_log
global_log = log
from ckan import model
model.init_model(engine)
generate_stats()
def command(config_ini):
config_ini_filepath = os.path.abspath(config_ini)
load_config(config_ini_filepath)
engine = engine_from_config(config, 'sqlalchemy.')
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
global global_log
global_log = log
from ckan import model
model.init_model(engine)
generate_stats()
def command(config_ini, commit):
config_ini_filepath = os.path.abspath(config_ini)
load_config(config_ini_filepath)
engine = engine_from_config(config, 'sqlalchemy.')
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
global global_log
global_log = log
from ckan import model
model.init_model(engine)
model.repo.new_revision()
guess_theme(commit)
def command(config_ini, commit):
config_ini_filepath = os.path.abspath(config_ini)
load_config(config_ini_filepath)
engine = engine_from_config(config, 'sqlalchemy.')
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
global global_log
global_log = log
from ckan import model
model.init_model(engine)
model.repo.new_revision()
guess_theme(commit)
def command(cls, config_ini):
config_ini_filepath = os.path.abspath(config_ini)
cls.load_config(config_ini_filepath)
engine = engine_from_config(config, "sqlalchemy.")
from ckan import model
from ckanext.dgu.drupalclient import DrupalClient, log as drupal_client_log
drupal_client_log.disabled = True
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
global global_log
global_log = log
cls.status = status.Status()
model.init_model(engine)
model.repo.new_revision()
cls.drupal_client = DrupalClient(
{
"xmlrpc_domain": "data.gov.uk",
"xmlrpc_username": "******",
"xmlrpc_password": config.get("dgu.xmlrpc_password"),
}
)
publisher_dicts = cls.drupal_client.get_organisation_list()
for publisher_dict in publisher_dicts:
if not (publisher_dict["status"] == "1" or publisher_dict["nid"] == "16248"):
# Make an exception for 16248 - Met Office under BIS is correct
cls.status.record("Unpublished in Drupal", publisher_dict["title"], do_print=False)
log.info("Ignoring unpublished publisher with status %r: %r", publisher_dict["status"], publisher_dict)
continue
publisher_nid = publisher_dict["nid"]
if int(publisher_nid) in ignore_publishers:
cls.status.record('On "ignore" list', publisher_dict["title"], do_print=False)
global_log.info("Publisher ignored: %s", publisher_nid)
continue
cls.do_publisher(publisher_nid)
all_groups = model.Session.query(model.Group).filter(model.Group.type == "publisher").order_by("title").all()
log.info("Total number of groups: %i", len(all_groups))
log.info("Warnings: %r", warnings)
print cls.status
def command(self):
if not all(os.environ.get(i) for i in ('CKAN_TEST_SYSADMIN_NAME', 'CKAN_INI')):
print('CKAN_TEST_SYSADMIN_NAME or CKAN_INI env var not set')
return
print('====== Removing DGU test data')
self._load_config()
engine = sqlalchemy.create_engine(config.get('sqlalchemy.url'))
model.init_model(engine)
command = "paster --plugin=ckanext-harvest harvester clearsource example-harvest-1 -c $CKAN_INI"
run_command(command)
command = "paster --plugin=ckanext-harvest harvester rmsource example-harvest-1 -c $CKAN_INI"
run_command(command)
sql = '''
DELETE FROM package_extra_revision WHERE package_id in (SELECT id FROM package WHERE name='example-harvest-1');
DELETE FROM package_extra WHERE package_id IN (SELECT id FROM package WHERE name='example-harvest-1');
DELETE FROM package_revision WHERE name = 'example-harvest-1';
DELETE FROM package WHERE name = 'example-harvest-1';
DELETE FROM harvest_object WHERE harvest_source_id IN (SELECT id FROM harvest_source WHERE title = 'Example Harvest #1');
DELETE FROM harvest_job WHERE source_id IN (SELECT id FROM harvest_source WHERE title = 'Example Harvest #1');
DELETE FROM harvest_source WHERE title = 'Example Harvest #1';
DELETE FROM member_revision WHERE group_id IN (SELECT id FROM "group" WHERE name = 'example-publisher-1');
DELETE FROM member WHERE group_id IN (SELECT id FROM "group" WHERE name = 'example-publisher-1');
DELETE FROM group_extra_revision WHERE group_id IN (SELECT id FROM "group" WHERE name = 'example-publisher-1');
DELETE FROM group_extra WHERE group_id IN (SELECT id FROM "group" WHERE name = 'example-publisher-1');
DELETE FROM group_revision WHERE name = 'example-publisher-1';
DELETE FROM "group" WHERE name = 'example-publisher-1';
DELETE FROM group_revision WHERE name = 'example-publisher-2';
DELETE FROM "group" WHERE name = 'example-publisher-2';
DELETE FROM "user" WHERE name = :testadmin_name;
'''
model.Session.execute(sql, {"testadmin_name": os.environ.get('CKAN_TEST_SYSADMIN_NAME')})
model.repo.commit_and_remove()
command = 'paster --plugin=ckan search-index clear example-publisher-1 -c $CKAN_INI'
run_command(command)
command = 'paster --plugin=ckan search-index clear example-publisher-2 -c $CKAN_INI'
run_command(command)
print("====== DGU test data removed")
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
connectable = engine_from_config(config.get_section(
config.config_ini_section),
prefix=u'sqlalchemy.',
poolclass=pool.NullPool)
connection = connectable.connect()
init_model(connectable)
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
def command(config_ini, nodepublisher_csv):
config_ini_filepath = os.path.abspath(config_ini)
load_config(config_ini_filepath)
engine = engine_from_config(config,'sqlalchemy.')
from ckan import model
from ckan.lib.munge import munge_title_to_name
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
global global_log
global_log = log
model.init_model(engine)
# Register a translator in this thread so that
# the _() functions in logic layer can work
from ckan.lib.cli import MockTranslator
registry=Registry()
registry.prepare()
translator_obj=MockTranslator()
registry.register(translator, translator_obj)
model.repo.new_revision()
log.info('Reading %s', nodepublisher_csv)
with open(nodepublisher_csv, 'rU') as f:
reader = csv.reader( f)
for row in reader:
nid, title = row
publishers[ int(nid) ] = munge_title_to_name(title)
# Mappings where we are getting rid of duplicate publishers
publishers[16268] = publishers[11408] # UKSA -> ONS
publishers[11606] = publishers[11408] # ONS
publishers[20054] = publishers[16248] # Met Office
publishers[33036] = publishers[15255] # Windsor & Maidenhead
publishers[32619] = publishers[33245] # Monmouthshire
publishers[12662] = publishers[11567] # NHS
update_datasets()
generate_harvest_publishers()
log.info('Warnings: %r', warnings)
def command(config_ini, nodepublisher_csv):
config_ini_filepath = os.path.abspath(config_ini)
load_config(config_ini_filepath)
engine = engine_from_config(config,'sqlalchemy.')
from ckan import model
from ckan.lib.munge import munge_title_to_name
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
global global_log
global_log = log
model.init_model(engine)
# Register a translator in this thread so that
# the _() functions in logic layer can work
from ckan.lib.cli import MockTranslator
registry=Registry()
registry.prepare()
translator_obj=MockTranslator()
registry.register(translator, translator_obj)
model.repo.new_revision()
log.info('Reading %s', nodepublisher_csv)
with open(nodepublisher_csv, 'rU') as f:
reader = csv.reader( f)
for row in reader:
nid, title = row
publishers[ int(nid) ] = munge_title_to_name(title)
# Mappings where we are getting rid of duplicate publishers
publishers[16268] = publishers[11408] # UKSA -> ONS
publishers[11606] = publishers[11408] # ONS
publishers[20054] = publishers[16248] # Met Office
publishers[33036] = publishers[15255] # Windsor & Maidenhead
publishers[32619] = publishers[33245] # Monmouthshire
publishers[12662] = publishers[11567] # NHS
update_datasets()
generate_harvest_publishers()
log.info('Warnings: %r', warnings)
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix=u'sqlalchemy.',
poolclass=pool.NullPool
)
connection = connectable.connect()
init_model(connection)
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
def update_config():
''' This code needs to be run when the config is changed to take those
changes into account. It is called whenever a plugin is loaded as the
plugin might have changed the config values (for instance it might
change ckan.site_url) '''
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# Set whitelisted env vars on config object
# This is set up before globals are initialized
ckan_db = os.environ.get('CKAN_DB', None)
if ckan_db:
msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
log.warn(msg)
config['sqlalchemy.url'] = ckan_db
database_user = os.environ.get("DATABASE_USER", None)
database_password = os.environ.get("DATABASE_PASSWORD", None)
database_host = os.environ.get("DATABASE_HOST", None)
database_port = os.environ.get("DATABASE_PORT", None)
if database_user is None or database_password is None or database_host is None:
print("Did not find either DATABASE_USER or DATABASE_PASSWORD or DATABASE_HOST")
else:
print("Setting the database url")
if database_port is None:
#use the default
database_port = "5432"
url = "postgres://"
url += database_user
url += ":"
url += database_password
url += "@"
url += database_host
url += ":"
url += database_port
url += "/datacatalogue"
config['sqlalchemy.url'] = url
print("Setting database " + database_host + ":" + database_port)
for option in CONFIG_FROM_ENV_VARS:
from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
if from_env:
if option == "AWS_ACCESS_KEY_ID" or option == "AWS_SECRET_ACCESS_KEY":
print(option)
print(from_env[0:10])
config[option] = from_env
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
site_url = config.get('ckan.site_url', '')
if not site_url:
raise RuntimeError(
'ckan.site_url is not configured and it must have a value.'
' Please amend your .ini file.')
if not site_url.lower().startswith('http'):
raise RuntimeError(
'ckan.site_url should be a full URL, including the schema '
'(http or https)')
# Remove backslash from site_url if present
config['ckan.site_url'] = config['ckan.site_url'].rstrip('/')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# ensure that a favicon has been set
favicon = config.get('ckan.favicon', '/images/icons/ckan.ico')
config['ckan.favicon'] = favicon
# Init SOLR settings and check if the schema is compatible
#from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get('solr_url'),
config.get('solr_user'),
config.get('solr_password'))
search.check_solr_schema_version()
routes_map = routing.make_map()
config['routes.map'] = routes_map
# The RoutesMiddleware needs its mapper updating if it exists
if 'routes.middleware' in config:
config['routes.middleware'].mapper = routes_map
config['routes.named_routes'] = routing.named_routes
config['pylons.app_globals'] = app_globals.app_globals
# initialise the globals
config['pylons.app_globals']._init()
# add helper functions
helpers = _Helpers(h)
config['pylons.h'] = helpers
## redo template setup to use genshi.search_path
## (so remove std template setup)
legacy_templates_path = os.path.join(root, 'templates_legacy')
jinja2_templates_path = os.path.join(root, 'templates')
if asbool(config.get('ckan.legacy_templates', 'no')):
# We want the new template path for extra snippets like the
# dataviewer and also for some testing stuff
msg = 'Support for Genshi templates is deprecated and will be removed'\
' in a future release'
log.warn(msg)
template_paths = [legacy_templates_path, jinja2_templates_path]
else:
template_paths = [jinja2_templates_path, legacy_templates_path]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['pylons.app_globals'].template_paths = template_paths
# Translator (i18n)
translator = Translator(pylons.translator)
def template_loaded(template):
translator.setup(template)
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create the Genshi TemplateLoader
config['pylons.app_globals'].genshi_loader = TemplateLoader(
template_paths, auto_reload=True, callback=template_loaded)
# Create Jinja2 environment
env = jinja_extensions.Environment(
loader=jinja_extensions.CkanFileSystemLoader(template_paths),
autoescape=True,
extensions=['jinja2.ext.do', 'jinja2.ext.with_',
jinja_extensions.SnippetExtension,
jinja_extensions.CkanExtend,
jinja_extensions.CkanInternationalizationExtension,
jinja_extensions.LinkForExtension,
jinja_extensions.ResourceExtension,
jinja_extensions.UrlForStaticExtension,
jinja_extensions.UrlForExtension]
)
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
env.filters['truncate'] = jinja_extensions.truncate
config['pylons.app_globals'].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# for postgresql we want to enforce utf-8
sqlalchemy_url = config.get('sqlalchemy.url', '')
if sqlalchemy_url.startswith('postgresql://'):
extras = {'client_encoding': 'utf8'}
else:
extras = {}
engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.', **extras)
if not model.meta.engine:
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# reset the template cache - we do this here so that when we load the
# environment it is clean
render.reset_template_info_cache()
# clear other caches
logic.clear_actions_cache()
logic.clear_validators_cache()
authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# (ProgrammingError for Postgres, OperationalError for SQLite)
# The database is not initialised. This is a bit dirty. This occurs
# when running tests.
pass
except sqlalchemy.exc.InternalError:
# The database is not initialised. Travis hits this
pass
# if an extension or our code does not finish
# transaction properly db cli commands can fail
model.Session.remove()
def update_config():
''' This code needs to be run when the config is changed to take those
changes into account. '''
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# This is set up before globals are initialized
site_id = os.environ.get('CKAN_SITE_ID')
if site_id:
config['ckan.site_id'] = site_id
site_url = config.get('ckan.site_url', '')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# ensure that a favicon has been set
favicon = config.get('ckan.favicon', '/images/icons/ckan.ico')
config['ckan.favicon'] = favicon
# Init SOLR settings and check if the schema is compatible
#from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get('solr_url'),
config.get('solr_user'),
config.get('solr_password'))
search.check_solr_schema_version()
routes_map = routing.make_map()
config['routes.map'] = routes_map
# The RoutesMiddleware needs its mapper updating if it exists
if 'routes.middleware' in config:
config['routes.middleware'].mapper = routes_map
config['routes.named_routes'] = routing.named_routes
config['pylons.app_globals'] = app_globals.app_globals
# initialise the globals
config['pylons.app_globals']._init()
# add helper functions
helpers = _Helpers(h)
config['pylons.h'] = helpers
## redo template setup to use genshi.search_path
## (so remove std template setup)
legacy_templates_path = os.path.join(root, 'templates_legacy')
jinja2_templates_path = os.path.join(root, 'templates')
if asbool(config.get('ckan.legacy_templates', 'no')):
# We want the new template path for extra snippets like the
# dataviewer and also for some testing stuff
msg = 'Support for Genshi templates is deprecated and will be removed'\
' in a future release'
log.warn(msg)
template_paths = [legacy_templates_path, jinja2_templates_path]
else:
template_paths = [jinja2_templates_path, legacy_templates_path]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['pylons.app_globals'].template_paths = template_paths
# Translator (i18n)
translator = Translator(pylons.translator)
def template_loaded(template):
translator.setup(template)
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create the Genshi TemplateLoader
config['pylons.app_globals'].genshi_loader = TemplateLoader(
template_paths, auto_reload=True, callback=template_loaded)
# Create Jinja2 environment
env = jinja_extensions.Environment(
loader=jinja_extensions.CkanFileSystemLoader(template_paths),
autoescape=True,
extensions=['jinja2.ext.do', 'jinja2.ext.with_',
jinja_extensions.SnippetExtension,
jinja_extensions.CkanExtend,
jinja_extensions.CkanInternationalizationExtension,
jinja_extensions.LinkForExtension,
jinja_extensions.ResourceExtension,
jinja_extensions.UrlForStaticExtension,
jinja_extensions.UrlForExtension]
)
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
env.filters['truncate'] = jinja_extensions.truncate
config['pylons.app_globals'].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
ckan_db = os.environ.get('CKAN_DB')
if ckan_db:
config['sqlalchemy.url'] = ckan_db
# for postgresql we want to enforce utf-8
sqlalchemy_url = config.get('sqlalchemy.url', '')
if sqlalchemy_url.startswith('postgresql://'):
extras = {'client_encoding': 'utf8'}
else:
extras = {}
engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.', **extras)
if not model.meta.engine:
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# reset the template cache - we do this here so that when we load the
# environment it is clean
render.reset_template_info_cache()
# clear other caches
logic.clear_actions_cache()
new_authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# (ProgrammingError for Postgres, OperationalError for SQLite)
# The database is not initialised. This is a bit dirty. This occurs
# when running tests.
pass
except sqlalchemy.exc.InternalError:
# The database is not initialised. Travis hits this
pass
# if an extension or our code does not finish
# transaction properly db cli commands can fail
model.Session.remove()
def update_config():
''' This code needs to be run when the config is changed to take those
changes into account. It is called whenever a plugin is loaded as the
plugin might have changed the config values (for instance it might
change ckan.site_url) '''
config_declaration.setup()
config_declaration.make_safe(config)
config_declaration.normalize(config)
webassets_init()
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# Set whitelisted env vars on config object
# This is set up before globals are initialized
ckan_db = os.environ.get('CKAN_DB', None)
if ckan_db:
msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
log.warn(msg)
config['sqlalchemy.url'] = ckan_db
for option in CONFIG_FROM_ENV_VARS:
from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
if from_env:
config[option] = from_env
if config.get_value("config.mode") == "strict":
_, errors = config_declaration.validate(config)
if errors:
msg = "\n".join("{}: {}".format(key, "; ".join(issues))
for key, issues in errors.items())
raise CkanConfigurationException(msg)
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
site_url = config.get_value('ckan.site_url')
if not site_url:
raise RuntimeError(
'ckan.site_url is not configured and it must have a value.'
' Please amend your .ini file.')
if not site_url.lower().startswith('http'):
raise RuntimeError(
'ckan.site_url should be a full URL, including the schema '
'(http or https)')
# Remove backslash from site_url if present
config['ckan.site_url'] = site_url.rstrip('/')
display_timezone = config.get_value('ckan.display_timezone')
if (display_timezone and display_timezone != 'server'
and display_timezone not in pytz.all_timezones):
raise CkanConfigurationException(
"ckan.display_timezone is not 'server' or a valid timezone")
# Init SOLR settings and check if the schema is compatible
# from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get_value('solr_url'),
config.get_value('solr_user'),
config.get_value('solr_password'))
search.check_solr_schema_version()
lib_plugins.reset_package_plugins()
lib_plugins.register_package_plugins()
lib_plugins.reset_group_plugins()
lib_plugins.register_group_plugins()
# initialise the globals
app_globals.app_globals._init()
helpers.load_plugin_helpers()
# Templates and CSS loading from configuration
valid_base_templates_folder_names = ['templates']
templates = config.get_value('ckan.base_templates_folder')
config['ckan.base_templates_folder'] = templates
if templates not in valid_base_templates_folder_names:
raise CkanConfigurationException(
'You provided an invalid value for ckan.base_templates_folder. '
'Possible values are: "templates".')
jinja2_templates_path = os.path.join(root, templates)
log.info('Loading templates from %s' % jinja2_templates_path)
template_paths = [jinja2_templates_path]
extra_template_paths = config.get_value('extra_template_paths')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['computed_template_paths'] = template_paths
# Enable pessimistic disconnect handling (added in SQLAlchemy 1.2)
# to eliminate database errors due to stale pooled connections
config.setdefault('sqlalchemy.pool_pre_ping', True)
# Initialize SQLAlchemy
engine = sqlalchemy.engine_from_config(config)
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# clear other caches
logic.clear_actions_cache()
logic.clear_validators_cache()
authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# The database is not yet initialised. It happens in `ckan db init`
pass
# Close current session and open database connections to ensure a clean
# clean environment even if an error occurs later on
model.Session.remove()
model.Session.bind.dispose()
def command(input_csv, config_ini, commit=False):
config_ini_filepath = os.path.abspath(config_ini)
load_config(config_ini_filepath)
engine = engine_from_config(config, 'sqlalchemy.')
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
from ckan import model
from ckan.logic import get_action
from ckan.lib.cli import MockTranslator
model.init_model(engine)
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
ctx = {
'model':
model,
'session':
model.Session,
'user':
get_action('get_site_user')({
'model': model,
'ignore_auth': True
}, {})['name']
}
if commit:
rev = model.repo.new_revision()
packages_to_check = set()
reader = csv.reader(open(input_csv, 'r'))
for row in reader:
# For each URL in the csv, get the list of resources referencing
# that URL
resources = model.Session.query(model.Resource)\
.filter(model.Resource.state=='active')\
.filter(model.Resource.url==row[0]).all()
for resource in resources:
# For each resource, add the package to the list
packages_to_check.add(resource.get_package_id())
# Delete the resource
resource.state = 'deleted'
model.Session.add(resource)
if commit:
model.Session.commit()
print "Deleted resource: {0}".format(resource.id)
stats.increment("Deleted resource")
for pid in packages_to_check:
# for each package we need to check, see if it has any
# resources left, it not, delete it.
pkg = model.Package.get(pid)
if len(pkg.resources) == 0:
pkg.state = 'deleted'
model.Session.add(pkg)
if commit:
model.Session.commit()
stats.increment('Deleted packages')
print "Deleted package: {0}".format(pkg.name)
if commit:
model.repo.commit_and_remove()
else:
print ""
print '*' * 60
print "DON'T PANIC, this was a dry run, nothing was committed"
print '*' * 60
print ''
print '*' * 60, 'Deletion Report'
print stats.report(order_by_title=True)
def command(input_csv, config_ini, commit=False):
config_ini_filepath = os.path.abspath(config_ini)
load_config(config_ini_filepath)
engine = engine_from_config(config, 'sqlalchemy.')
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
from ckan import model
from ckan.logic import get_action
from ckan.lib.cli import MockTranslator
model.init_model(engine)
registry=Registry()
registry.prepare()
translator_obj=MockTranslator()
registry.register(translator, translator_obj)
ctx = {
'model': model, 'session': model.Session,
'user': get_action('get_site_user')({'model': model,'ignore_auth': True}, {})['name']
}
if commit:
rev = model.repo.new_revision()
packages_to_check = set()
reader = csv.reader(open(input_csv, 'r'))
for row in reader:
# For each URL in the csv, get the list of resources referencing
# that URL
resources = model.Session.query(model.Resource)\
.filter(model.Resource.state=='active')\
.filter(model.Resource.url==row[0]).all()
for resource in resources:
# For each resource, add the package to the list
packages_to_check.add(resource.get_package_id())
# Delete the resource
resource.state = 'deleted'
model.Session.add(resource)
if commit:
model.Session.commit()
print "Deleted resource: {0}".format(resource.id)
stats.increment("Deleted resource")
for pid in packages_to_check:
# for each package we need to check, see if it has any
# resources left, it not, delete it.
pkg = model.Package.get(pid)
if len(pkg.resources) == 0:
pkg.state = 'deleted'
model.Session.add(pkg)
if commit:
model.Session.commit()
stats.increment('Deleted packages')
print "Deleted package: {0}".format(pkg.name)
if commit:
model.repo.commit_and_remove()
else:
print ""
print '*' * 60
print "DON'T PANIC, this was a dry run, nothing was committed"
print '*' * 60
print ''
print '*' * 60, 'Deletion Report'
print stats.report(order_by_title=True)
def update_config():
''' This code needs to be run when the config is changed to take those
changes into account. It is called whenever a plugin is loaded as the
plugin might have changed the config values (for instance it might
change ckan.site_url) '''
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# Set whitelisted env vars on config object
# This is set up before globals are initialized
ckan_db = os.environ.get('CKAN_DB', None)
if ckan_db:
msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
log.warn(msg)
config['sqlalchemy.url'] = ckan_db
for option in CONFIG_FROM_ENV_VARS:
from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
if from_env:
config[option] = from_env
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
site_url = config.get('ckan.site_url', '')
if not site_url:
raise RuntimeError(
'ckan.site_url is not configured and it must have a value.'
' Please amend your .ini file.')
if not site_url.lower().startswith('http'):
raise RuntimeError(
'ckan.site_url should be a full URL, including the schema '
'(http or https)')
display_timezone = config.get('ckan.display_timezone', '')
if (display_timezone and
display_timezone != 'server' and
display_timezone not in pytz.all_timezones):
raise CkanConfigurationException(
"ckan.display_timezone is not 'server' or a valid timezone"
)
# Remove backslash from site_url if present
config['ckan.site_url'] = config['ckan.site_url'].rstrip('/')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# ensure that a favicon has been set
favicon = config.get('ckan.favicon', '/base/images/ckan.ico')
config['ckan.favicon'] = favicon
# Init SOLR settings and check if the schema is compatible
# from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get('solr_url'),
config.get('solr_user'),
config.get('solr_password'))
search.check_solr_schema_version()
routes_map = routing.make_map()
config['routes.map'] = routes_map
# The RoutesMiddleware needs its mapper updating if it exists
if 'routes.middleware' in config:
config['routes.middleware'].mapper = routes_map
# routes.named_routes is a CKAN thing
config['routes.named_routes'] = routing.named_routes
config['pylons.app_globals'] = app_globals.app_globals
# initialise the globals
app_globals.app_globals._init()
helpers.load_plugin_helpers()
config['pylons.h'] = helpers.helper_functions
# Templates and CSS loading from configuration
valid_base_templates_folder_names = ['templates', 'templates-bs2']
templates = config.get('ckan.base_templates_folder', 'templates')
config['ckan.base_templates_folder'] = templates
if templates not in valid_base_templates_folder_names:
raise CkanConfigurationException(
'You provided an invalid value for ckan.base_templates_folder. '
'Possible values are: "templates" and "templates-bs2".'
)
jinja2_templates_path = os.path.join(root, templates)
log.info('Loading templates from %s' % jinja2_templates_path)
template_paths = [jinja2_templates_path]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['computed_template_paths'] = template_paths
# Set the default language for validation messages from formencode
# to what is set as the default locale in the config
default_lang = config.get('ckan.locale_default', 'en')
formencode.api.set_stdtranslation(domain="FormEncode",
languages=[default_lang])
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create Jinja2 environment
env = jinja_extensions.Environment(
**jinja_extensions.get_jinja_env_options())
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
config['pylons.app_globals'].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# Initialize SQLAlchemy
engine = sqlalchemy.engine_from_config(config, client_encoding='utf8')
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# reset the template cache - we do this here so that when we load the
# environment it is clean
render.reset_template_info_cache()
# clear other caches
logic.clear_actions_cache()
logic.clear_validators_cache()
authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# (ProgrammingError for Postgres, OperationalError for SQLite)
# The database is not initialised. This is a bit dirty. This occurs
# when running tests.
pass
except sqlalchemy.exc.InternalError:
# The database is not initialised. Travis hits this
pass
# Close current session and open database connections to ensure a clean
# clean environment even if an error occurs later on
model.Session.remove()
model.Session.bind.dispose()
def command(config_ini, nodepublisher_csv, users_csv):
config_ini_filepath = os.path.abspath(config_ini)
load_config(config_ini_filepath)
engine = engine_from_config(config, 'sqlalchemy.')
from ckan import model
from ckan.lib.munge import munge_title_to_name
import ckanext.dgu.plugin
logging.config.fileConfig(config_ini_filepath)
global log
log = logging.getLogger(os.path.basename(__file__))
model.init_model(engine)
# Register a translator in this thread so that
# the _() functions in logic layer can work
from ckan.lib.cli import MockTranslator
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
with open(nodepublisher_csv, 'rU') as f:
reader = csv.reader(f)
for row in reader:
publishers[int(row[0])] = munge_title_to_name(row[1])
log.info('Opened list of %i publishers', reader.line_num)
# get rid of flash message warnings
warnings_.filterwarnings('ignore', '.*flash message.*')
ckanext.dgu.plugin.log.disabled = True
with open(users_csv, 'rU') as f:
reader = csv.reader(f)
for row in reader:
model.repo.new_revision()
node_id, name, email, publisher_id = row
# create a new user
uname, user_id = _add_new_user(int(node_id), name, email)
if not user_id:
# validation error. warning already printed
continue
# Find the publisher and add them as editor
if node_id:
publisher_name = publishers[int(publisher_id)]
publisher = model.Group.by_name(publisher_name)
if not publisher:
warn(
'Could not find publisher %r so skipping making %r editor for it.',
publisher_name, name)
continue
capacity = 'editor'
# Check for Member where table_name is u['id']
res = model.Session.query(model.Member).\
from_statement(MEMBER_LOOKUP).\
params(userid=user_id, groupid=publisher.id).all()
if len(res) == 0:
m = model.Member(group_id=publisher.id,
table_id=user_id,
table_name='user',
capacity=capacity)
model.Session.add(m)
log.info('Made %r editor for %r', name, publisher_name)
else:
log.info('%r already editor for %r', name, publisher_name)
# Update harvest_source user_id field to new user id.
model.Session.execute(HARVEST_QUERY,
params={
'uid': user_id,
'node_id': str(node_id)
})
model.Session.commit()
log.info('Processed list of %i users', reader.line_num)
log.info('Warnings (%i): %r', len(warnings), warnings)
warnings.append(msg % params)
global_log.warn(msg, *params)
if __name__ == '__main__':
usage = '''usage: %prog [options]
''' # NB Options are automatically listed
parser = OptionParser(usage=usage)
parser.add_option('-c', '--config', dest='config', help='Config filepath', default='development.ini')
parser.add_option('-d', '--dry-run', dest='dry_run', help='Dry run',
action='store_true', default=False)
(options, args) = parser.parse_args()
if len(args) > 0:
parser.print_help()
sys.exit(1)
if options.config:
config_path = os.path.abspath(options.config)
if not os.path.exists(config_path):
print 'Config file does not exist: %s' % config_path
sys.exit(1)
load_config(config_path)
engine = engine_from_config(config, 'sqlalchemy.')
from ckan import model
model.init_model(engine)
logging.config.fileConfig(config_path)
global_log = logging.getLogger(os.path.basename(__file__))
command(dry_run=options.dry_run)
def command(config_ini, nodepublisher_csv, users_csv):
config_ini_filepath = os.path.abspath(config_ini)
load_config(config_ini_filepath)
engine = engine_from_config(config, 'sqlalchemy.')
from ckan import model
from ckan.lib.munge import munge_title_to_name
import ckanext.dgu.plugin
logging.config.fileConfig(config_ini_filepath)
global log
log = logging.getLogger(os.path.basename(__file__))
model.init_model(engine)
# Register a translator in this thread so that
# the _() functions in logic layer can work
from ckan.lib.cli import MockTranslator
registry=Registry()
registry.prepare()
translator_obj=MockTranslator()
registry.register(translator, translator_obj)
with open(nodepublisher_csv, 'rU') as f:
reader = csv.reader( f)
for row in reader:
publishers[ int(row[0]) ] = munge_title_to_name(row[1])
log.info('Opened list of %i publishers', reader.line_num)
# get rid of flash message warnings
warnings_.filterwarnings('ignore', '.*flash message.*')
ckanext.dgu.plugin.log.disabled = True
with open(users_csv, 'rU') as f:
reader = csv.reader(f)
for row in reader:
model.repo.new_revision()
node_id, name, email, publisher_id = row
# create a new user
uname, user_id = _add_new_user(int(node_id), name, email)
if not user_id:
# validation error. warning already printed
continue
# Find the publisher and add them as editor
if node_id:
publisher_name = publishers[int(publisher_id)]
publisher = model.Group.by_name(publisher_name)
if not publisher:
warn('Could not find publisher %r so skipping making %r editor for it.',
publisher_name, name)
continue
capacity = 'editor'
# Check for Member where table_name is u['id']
res = model.Session.query(model.Member).\
from_statement(MEMBER_LOOKUP).\
params(userid=user_id, groupid=publisher.id).all()
if len(res) == 0:
m = model.Member(group_id=publisher.id, table_id=user_id,
table_name='user', capacity=capacity)
model.Session.add(m)
log.info('Made %r editor for %r', name, publisher_name)
else:
log.info('%r already editor for %r', name, publisher_name)
# Update harvest_source user_id field to new user id.
model.Session.execute(HARVEST_QUERY,params={'uid':user_id, 'node_id': str(node_id)})
model.Session.commit()
log.info('Processed list of %i users', reader.line_num)
log.info('Warnings (%i): %r', len(warnings), warnings)
def load_environment(global_conf, app_conf):
"""Configure the Pylons environment via the ``pylons.config``
object
"""
###### Pylons monkey-patch
# this must be run at a time when the env is semi-setup, thus inlined here.
# Required by the deliverance plugin and iATI
from pylons.wsgiapp import PylonsApp
import pkg_resources
find_controller_generic = PylonsApp.find_controller
# This is from pylons 1.0 source, will monkey-patch into 0.9.7
def find_controller(self, controller):
if controller in self.controller_classes:
return self.controller_classes[controller]
# Check to see if its a dotted name
if '.' in controller or ':' in controller:
mycontroller = pkg_resources.EntryPoint.parse(
'x=%s' % controller).load(False)
self.controller_classes[controller] = mycontroller
return mycontroller
return find_controller_generic(self, controller)
PylonsApp.find_controller = find_controller
###### END evil monkey-patch
os.environ['CKAN_CONFIG'] = global_conf['__file__']
# Pylons paths
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
paths = dict(root=root,
controllers=os.path.join(root, 'controllers'),
static_files=os.path.join(root, 'public'),
templates=[os.path.join(root, 'templates')])
# Initialize config with the basic options
config.init_app(global_conf, app_conf, package='ckan', paths=paths)
# load all CKAN plugins
plugins.load_all(config)
from ckan.plugins import PluginImplementations
from ckan.plugins.interfaces import IConfigurer
for plugin in PluginImplementations(IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# This is set up before globals are initialized
site_url = config.get('ckan.site_url', '')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# Check if SOLR schema is compatible
from ckan.lib.search import check_solr_schema_version
check_solr_schema_version()
config['routes.map'] = make_map()
config['pylons.app_globals'] = app_globals.Globals()
config['pylons.h'] = ckan.lib.helpers
## redo template setup to use genshi.search_path (so remove std template setup)
template_paths = [paths['templates'][0]]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
# Translator (i18n)
translator = Translator(pylons.translator)
def template_loaded(template):
translator.setup(template)
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create the Genshi TemplateLoader
# config['pylons.app_globals'].genshi_loader = TemplateLoader(
# paths['templates'], auto_reload=True)
# tmpl_options["genshi.loader_callback"] = template_loaded
config['pylons.app_globals'].genshi_loader = TemplateLoader(
template_paths, auto_reload=True, callback=template_loaded)
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# Setup the SQLAlchemy database engine
engine = engine_from_config(config, 'sqlalchemy.')
if not model.meta.engine:
model.init_model(engine)
from ckan.plugins import PluginImplementations
from ckan.plugins.interfaces import IConfigurable
for plugin in PluginImplementations(IConfigurable):
plugin.configure(config)
def update_config():
""" This code needs to be run when the config is changed to take those
changes into account. It is called whenever a plugin is loaded as the
plugin might have changed the config values (for instance it might
change ckan.site_url) """
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# Set whitelisted env vars on config object
# This is set up before globals are initialized
ckan_db = os.environ.get("CKAN_DB", None)
if ckan_db:
msg = (
"Setting CKAN_DB as an env var is deprecated and will be"
" removed in a future release. Use CKAN_SQLALCHEMY_URL instead."
)
log.warn(msg)
config["sqlalchemy.url"] = ckan_db
for option in CONFIG_FROM_ENV_VARS:
from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
if from_env:
config[option] = from_env
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
site_url = config.get("ckan.site_url", "")
if not site_url:
raise RuntimeError("ckan.site_url is not configured and it must have a value." " Please amend your .ini file.")
if not site_url.lower().startswith("http"):
raise RuntimeError("ckan.site_url should be a full URL, including the schema " "(http or https)")
display_timezone = config.get("ckan.display_timezone", "")
if display_timezone and display_timezone != "server" and display_timezone not in pytz.all_timezones:
raise CkanConfigurationException("ckan.display_timezone is not 'server' or a valid timezone")
# Remove backslash from site_url if present
config["ckan.site_url"] = config["ckan.site_url"].rstrip("/")
ckan_host = config["ckan.host"] = urlparse(site_url).netloc
if config.get("ckan.site_id") is None:
if ":" in ckan_host:
ckan_host, port = ckan_host.split(":")
assert ckan_host, (
"You need to configure ckan.site_url or " "ckan.site_id for SOLR search-index rebuild to work."
)
config["ckan.site_id"] = ckan_host
# ensure that a favicon has been set
favicon = config.get("ckan.favicon", "/base/images/ckan.ico")
config["ckan.favicon"] = favicon
# Init SOLR settings and check if the schema is compatible
# from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get("solr_url"), config.get("solr_user"), config.get("solr_password"))
search.check_solr_schema_version()
routes_map = routing.make_map()
config["routes.map"] = routes_map
# The RoutesMiddleware needs its mapper updating if it exists
if "routes.middleware" in config:
config["routes.middleware"].mapper = routes_map
config["routes.named_routes"] = routing.named_routes
config["pylons.app_globals"] = app_globals.app_globals
# initialise the globals
config["pylons.app_globals"]._init()
# add helper functions
helpers = _Helpers(h)
config["pylons.h"] = helpers
jinja2_templates_path = os.path.join(root, "templates")
template_paths = [jinja2_templates_path]
extra_template_paths = config.get("extra_template_paths", "")
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(",") + template_paths
config["pylons.app_globals"].template_paths = template_paths
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create Jinja2 environment
env = jinja_extensions.Environment(
loader=jinja_extensions.CkanFileSystemLoader(template_paths),
autoescape=True,
extensions=[
"jinja2.ext.do",
"jinja2.ext.with_",
jinja_extensions.SnippetExtension,
jinja_extensions.CkanExtend,
jinja_extensions.CkanInternationalizationExtension,
jinja_extensions.LinkForExtension,
jinja_extensions.ResourceExtension,
jinja_extensions.UrlForStaticExtension,
jinja_extensions.UrlForExtension,
],
)
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.filters["empty_and_escape"] = jinja_extensions.empty_and_escape
env.filters["truncate"] = jinja_extensions.truncate
config["pylons.app_globals"].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# for postgresql we want to enforce utf-8
sqlalchemy_url = config.get("sqlalchemy.url", "")
if sqlalchemy_url.startswith("postgresql://"):
extras = {"client_encoding": "utf8"}
else:
extras = {}
engine = sqlalchemy.engine_from_config(config, "sqlalchemy.", **extras)
if not model.meta.engine:
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# reset the template cache - we do this here so that when we load the
# environment it is clean
render.reset_template_info_cache()
# clear other caches
logic.clear_actions_cache()
logic.clear_validators_cache()
authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action("get_site_user")({"ignore_auth": True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# (ProgrammingError for Postgres, OperationalError for SQLite)
# The database is not initialised. This is a bit dirty. This occurs
# when running tests.
pass
except sqlalchemy.exc.InternalError:
# The database is not initialised. Travis hits this
pass
# if an extension or our code does not finish
# transaction properly db cli commands can fail
model.Session.remove()
def load_environment(global_conf, app_conf):
"""Configure the Pylons environment via the ``pylons.config``
object
"""
###### Pylons monkey-patch
# this must be run at a time when the env is semi-setup, thus inlined here.
# Required by the deliverance plugin and iATI
from pylons.wsgiapp import PylonsApp
import pkg_resources
find_controller_generic = PylonsApp.find_controller
# This is from pylons 1.0 source, will monkey-patch into 0.9.7
def find_controller(self, controller):
if controller in self.controller_classes:
return self.controller_classes[controller]
# Check to see if its a dotted name
if '.' in controller or ':' in controller:
mycontroller = pkg_resources \
.EntryPoint \
.parse('x=%s' % controller).load(False)
self.controller_classes[controller] = mycontroller
return mycontroller
return find_controller_generic(self, controller)
PylonsApp.find_controller = find_controller
###### END evil monkey-patch
os.environ['CKAN_CONFIG'] = global_conf['__file__']
# Pylons paths
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
paths = dict(root=root,
controllers=os.path.join(root, 'controllers'),
static_files=os.path.join(root, 'public'),
templates=[])
# Initialize config with the basic options
config.init_app(global_conf, app_conf, package='ckan', paths=paths)
# load all CKAN plugins
p.load_all(config)
# Load the synchronous search plugin, unless already loaded or
# explicitly disabled
if not 'synchronous_search' in config.get('ckan.plugins',[]) and \
asbool(config.get('ckan.search.automatic_indexing', True)):
log.debug('Loading the synchronous search plugin')
p.load('synchronous_search')
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# This is set up before globals are initialized
site_id = os.environ.get('CKAN_SITE_ID')
if site_id:
config['ckan.site_id'] = site_id
site_url = config.get('ckan.site_url', '')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# ensure that a favicon has been set
favicon = config.get('ckan.favicon', '/images/icons/ckan.ico')
config['ckan.favicon'] = favicon
# Init SOLR settings and check if the schema is compatible
#from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
import ckan.lib.search as search
search.SolrSettings.init(config.get('solr_url'), config.get('solr_user'),
config.get('solr_password'))
search.check_solr_schema_version()
config['routes.map'] = routing.make_map()
config['routes.named_routes'] = routing.named_routes
config['pylons.app_globals'] = app_globals.app_globals
# initialise the globals
config['pylons.app_globals']._init()
# add helper functions
helpers = _Helpers(h)
config['pylons.h'] = helpers
## redo template setup to use genshi.search_path
## (so remove std template setup)
legacy_templates_path = os.path.join(root, 'templates_legacy')
jinja2_templates_path = os.path.join(root, 'templates')
if asbool(config.get('ckan.legacy_templates', 'no')):
# We want the new template path for extra snippets like the
# dataviewer and also for some testing stuff
template_paths = [legacy_templates_path, jinja2_templates_path]
else:
template_paths = [jinja2_templates_path, legacy_templates_path]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['pylons.app_globals'].template_paths = template_paths
# Translator (i18n)
translator = Translator(pylons.translator)
def template_loaded(template):
translator.setup(template)
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create the Genshi TemplateLoader
config['pylons.app_globals'].genshi_loader = TemplateLoader(
template_paths, auto_reload=True, callback=template_loaded)
#################################################################
# #
# HORRIBLE GENSHI HACK #
# #
#################################################################
# #
# Genshi does strange things to get stuff out of the template #
# variables. This stops it from handling properties in the #
# correct way as it returns the property rather than the actual #
# value of the property. #
# #
# By overriding lookup_attr() in the LookupBase class we are #
# able to get the required behaviour. Using @property allows #
# us to move functionality out of templates whilst maintaining #
# backwards compatability. #
# #
#################################################################
'''
This code is based on Genshi code
Copyright © 2006-2012 Edgewall Software
All rights reserved.
Redistribution and use in source and binary forms, with or
without modification, are permitted provided that the following
conditions are met:
Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following
disclaimer in the documentation and/or other materials provided
with the distribution.
The name of the author may not be used to endorse or promote
products derived from this software without specific prior
written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
'''
from genshi.template.eval import LookupBase
@classmethod
def genshi_lookup_attr(cls, obj, key):
__traceback_hide__ = True
try:
val = getattr(obj, key)
except AttributeError:
if hasattr(obj.__class__, key):
raise
else:
try:
val = obj[key]
except (KeyError, TypeError):
val = cls.undefined(key, owner=obj)
if isinstance(val, property):
val = val.fget()
return val
setattr(LookupBase, 'lookup_attr', genshi_lookup_attr)
del genshi_lookup_attr
del LookupBase
#################################################################
# #
# END OF GENSHI HACK #
# #
#################################################################
# Create Jinja2 environment
env = lib.jinja_extensions.Environment(
loader=lib.jinja_extensions.CkanFileSystemLoader(template_paths),
autoescape=True,
extensions=[
'jinja2.ext.do', 'jinja2.ext.with_',
lib.jinja_extensions.SnippetExtension,
lib.jinja_extensions.CkanExtend,
lib.jinja_extensions.CkanInternationalizationExtension,
lib.jinja_extensions.LinkForExtension,
lib.jinja_extensions.ResourceExtension,
lib.jinja_extensions.UrlForStaticExtension,
lib.jinja_extensions.UrlForExtension
])
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.filters['empty_and_escape'] = lib.jinja_extensions.empty_and_escape
env.filters['truncate'] = lib.jinja_extensions.truncate
config['pylons.app_globals'].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# Setup the SQLAlchemy database engine
# Suppress a couple of sqlalchemy warnings
msgs = [
'^Unicode type received non-unicode bind param value',
"^Did not recognize type 'BIGINT' of column 'size'",
"^Did not recognize type 'tsvector' of column 'search_vector'"
]
for msg in msgs:
warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning)
ckan_db = os.environ.get('CKAN_DB')
if ckan_db:
config['sqlalchemy.url'] = ckan_db
# for postgresql we want to enforce utf-8
sqlalchemy_url = config.get('sqlalchemy.url', '')
if sqlalchemy_url.startswith('postgresql://'):
extras = {'client_encoding': 'utf8'}
else:
extras = {}
engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.', **extras)
if not model.meta.engine:
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
def update_config():
''' This code needs to be run when the config is changed to take those
changes into account. It is called whenever a plugin is loaded as the
plugin might have changed the config values (for instance it might
change ckan.site_url) '''
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# Set whitelisted env vars on config object
# This is set up before globals are initialized
ckan_db = os.environ.get('CKAN_DB', None)
if ckan_db:
msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
log.warn(msg)
config['sqlalchemy.url'] = ckan_db
for option in CONFIG_FROM_ENV_VARS:
from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
if from_env:
config[option] = from_env
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
site_url = config.get('ckan.site_url', '')
if not site_url:
raise RuntimeError(
'ckan.site_url is not configured and it must have a value.'
' Please amend your .ini file.')
if not site_url.lower().startswith('http'):
raise RuntimeError(
'ckan.site_url should be a full URL, including the schema '
'(http or https)')
display_timezone = config.get('ckan.display_timezone', '')
if (display_timezone and display_timezone != 'server'
and display_timezone not in pytz.all_timezones):
raise CkanConfigurationException(
"ckan.display_timezone is not 'server' or a valid timezone")
# Remove backslash from site_url if present
config['ckan.site_url'] = config['ckan.site_url'].rstrip('/')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# ensure that a favicon has been set
favicon = config.get('ckan.favicon', '/base/images/ckan.ico')
config['ckan.favicon'] = favicon
# Init SOLR settings and check if the schema is compatible
# from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get('solr_url'), config.get('solr_user'),
config.get('solr_password'))
search.check_solr_schema_version()
routes_map = routing.make_map()
config['routes.map'] = routes_map
# The RoutesMiddleware needs its mapper updating if it exists
if 'routes.middleware' in config:
config['routes.middleware'].mapper = routes_map
# routes.named_routes is a CKAN thing
config['routes.named_routes'] = routing.named_routes
config['pylons.app_globals'] = app_globals.app_globals
# initialise the globals
app_globals.app_globals._init()
helpers.load_plugin_helpers()
config['pylons.h'] = helpers.helper_functions
jinja2_templates_path = os.path.join(root, 'templates')
template_paths = [jinja2_templates_path]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['pylons.app_globals'].template_paths = template_paths
# Set the default language for validation messages from formencode
# to what is set as the default locale in the config
default_lang = config.get('ckan.locale_default', 'en')
formencode.api.set_stdtranslation(domain="FormEncode",
languages=[default_lang])
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create Jinja2 environment
env = jinja_extensions.Environment(
loader=jinja_extensions.CkanFileSystemLoader(template_paths),
autoescape=True,
extensions=[
'jinja2.ext.do', 'jinja2.ext.with_',
jinja_extensions.SnippetExtension, jinja_extensions.CkanExtend,
jinja_extensions.CkanInternationalizationExtension,
jinja_extensions.LinkForExtension,
jinja_extensions.ResourceExtension,
jinja_extensions.UrlForStaticExtension,
jinja_extensions.UrlForExtension
])
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
env.filters['truncate'] = jinja_extensions.truncate
config['pylons.app_globals'].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# Initialize SQLAlchemy
engine = sqlalchemy.engine_from_config(config, client_encoding='utf8')
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# reset the template cache - we do this here so that when we load the
# environment it is clean
render.reset_template_info_cache()
# clear other caches
logic.clear_actions_cache()
logic.clear_validators_cache()
authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# (ProgrammingError for Postgres, OperationalError for SQLite)
# The database is not initialised. This is a bit dirty. This occurs
# when running tests.
pass
except sqlalchemy.exc.InternalError:
# The database is not initialised. Travis hits this
pass
# if an extension or our code does not finish
# transaction properly db cli commands can fail
model.Session.remove()
def load_environment(global_conf, app_conf):
"""Configure the Pylons environment via the ``pylons.config``
object
"""
###### Pylons monkey-patch
# this must be run at a time when the env is semi-setup, thus inlined here.
# Required by the deliverance plugin and iATI
from pylons.wsgiapp import PylonsApp
import pkg_resources
find_controller_generic = PylonsApp.find_controller
# This is from pylons 1.0 source, will monkey-patch into 0.9.7
def find_controller(self, controller):
if controller in self.controller_classes:
return self.controller_classes[controller]
# Check to see if its a dotted name
if '.' in controller or ':' in controller:
mycontroller = pkg_resources.EntryPoint.parse('x=%s' % controller).load(False)
self.controller_classes[controller] = mycontroller
return mycontroller
return find_controller_generic(self, controller)
PylonsApp.find_controller = find_controller
###### END evil monkey-patch
os.environ['CKAN_CONFIG'] = global_conf['__file__']
# Pylons paths
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
paths = dict(root=root,
controllers=os.path.join(root, 'controllers'),
static_files=os.path.join(root, 'public'),
templates=[os.path.join(root, 'templates')])
# Initialize config with the basic options
config.init_app(global_conf, app_conf, package='ckan', paths=paths)
# load all CKAN plugins
plugins.load_all(config)
from ckan.plugins import PluginImplementations
from ckan.plugins.interfaces import IConfigurer
for plugin in PluginImplementations(IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# This is set up before globals are initialized
site_url = config.get('ckan.site_url', '')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# Init SOLR settings and check if the schema is compatible
from ckan.lib.search import SolrSettings, check_solr_schema_version
SolrSettings.init(config.get('solr_url'),
config.get('solr_user'),
config.get('solr_password'))
check_solr_schema_version()
config['routes.map'] = make_map()
config['pylons.app_globals'] = app_globals.Globals()
config['pylons.h'] = ckan.lib.helpers
## redo template setup to use genshi.search_path (so remove std template setup)
template_paths = [paths['templates'][0]]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
# Translator (i18n)
translator = Translator(pylons.translator)
def template_loaded(template):
translator.setup(template)
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create the Genshi TemplateLoader
# config['pylons.app_globals'].genshi_loader = TemplateLoader(
# paths['templates'], auto_reload=True)
# tmpl_options["genshi.loader_callback"] = template_loaded
config['pylons.app_globals'].genshi_loader = TemplateLoader(
template_paths, auto_reload=True, callback=template_loaded)
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# Setup the SQLAlchemy database engine
engine = engine_from_config(config, 'sqlalchemy.')
if not model.meta.engine:
model.init_model(engine)
from ckan.plugins import PluginImplementations
from ckan.plugins.interfaces import IConfigurable
for plugin in PluginImplementations(IConfigurable):
plugin.configure(config)
def load_environment(global_conf, app_conf):
"""Configure the Pylons environment via the ``pylons.config``
object
"""
###### Pylons monkey-patch
# this must be run at a time when the env is semi-setup, thus inlined here.
# Required by the deliverance plugin and iATI
from pylons.wsgiapp import PylonsApp
import pkg_resources
find_controller_generic = PylonsApp.find_controller
# This is from pylons 1.0 source, will monkey-patch into 0.9.7
def find_controller(self, controller):
if controller in self.controller_classes:
return self.controller_classes[controller]
# Check to see if its a dotted name
if '.' in controller or ':' in controller:
mycontroller = pkg_resources \
.EntryPoint \
.parse('x=%s' % controller).load(False)
self.controller_classes[controller] = mycontroller
return mycontroller
return find_controller_generic(self, controller)
PylonsApp.find_controller = find_controller
###### END evil monkey-patch
os.environ['CKAN_CONFIG'] = global_conf['__file__']
# Pylons paths
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
paths = dict(root=root,
controllers=os.path.join(root, 'controllers'),
static_files=os.path.join(root, 'public'),
templates=[])
# Initialize config with the basic options
config.init_app(global_conf, app_conf, package='ckan', paths=paths)
# load all CKAN plugins
p.load_all(config)
# Load the synchronous search plugin, unless already loaded or
# explicitly disabled
if not 'synchronous_search' in config.get('ckan.plugins',[]) and \
asbool(config.get('ckan.search.automatic_indexing', True)):
log.debug('Loading the synchronous search plugin')
p.load('synchronous_search')
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# This is set up before globals are initialized
site_id = os.environ.get('CKAN_SITE_ID')
if site_id:
config['ckan.site_id'] = site_id
site_url = config.get('ckan.site_url', '')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# ensure that a favicon has been set
favicon = config.get('ckan.favicon', '/images/icons/ckan.ico')
config['ckan.favicon'] = favicon
# Init SOLR settings and check if the schema is compatible
#from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
import ckan.lib.search as search
search.SolrSettings.init(config.get('solr_url'),
config.get('solr_user'),
config.get('solr_password'))
search.check_solr_schema_version()
config['routes.map'] = routing.make_map()
config['routes.named_routes'] = routing.named_routes
config['pylons.app_globals'] = app_globals.app_globals
# initialise the globals
config['pylons.app_globals']._init()
# add helper functions
helpers = _Helpers(h)
config['pylons.h'] = helpers
## redo template setup to use genshi.search_path
## (so remove std template setup)
legacy_templates_path = os.path.join(root, 'templates_legacy')
jinja2_templates_path = os.path.join(root, 'templates')
if asbool(config.get('ckan.legacy_templates', 'no')):
# We want the new template path for extra snippets like the
# dataviewer and also for some testing stuff
template_paths = [legacy_templates_path, jinja2_templates_path]
else:
template_paths = [jinja2_templates_path, legacy_templates_path]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['pylons.app_globals'].template_paths = template_paths
# Translator (i18n)
translator = Translator(pylons.translator)
def template_loaded(template):
translator.setup(template)
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create the Genshi TemplateLoader
config['pylons.app_globals'].genshi_loader = TemplateLoader(
template_paths, auto_reload=True, callback=template_loaded)
#################################################################
# #
# HORRIBLE GENSHI HACK #
# #
#################################################################
# #
# Genshi does strange things to get stuff out of the template #
# variables. This stops it from handling properties in the #
# correct way as it returns the property rather than the actual #
# value of the property. #
# #
# By overriding lookup_attr() in the LookupBase class we are #
# able to get the required behaviour. Using @property allows #
# us to move functionality out of templates whilst maintaining #
# backwards compatability. #
# #
#################################################################
'''
This code is based on Genshi code
Copyright © 2006-2012 Edgewall Software
All rights reserved.
Redistribution and use in source and binary forms, with or
without modification, are permitted provided that the following
conditions are met:
Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following
disclaimer in the documentation and/or other materials provided
with the distribution.
The name of the author may not be used to endorse or promote
products derived from this software without specific prior
written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
'''
from genshi.template.eval import LookupBase
@classmethod
def genshi_lookup_attr(cls, obj, key):
__traceback_hide__ = True
try:
val = getattr(obj, key)
except AttributeError:
if hasattr(obj.__class__, key):
raise
else:
try:
val = obj[key]
except (KeyError, TypeError):
val = cls.undefined(key, owner=obj)
if isinstance(val, property):
val = val.fget()
return val
setattr(LookupBase, 'lookup_attr', genshi_lookup_attr)
del genshi_lookup_attr
del LookupBase
#################################################################
# #
# END OF GENSHI HACK #
# #
#################################################################
# Create Jinja2 environment
env = jinja_extensions.Environment(
loader=jinja_extensions.CkanFileSystemLoader(template_paths),
autoescape=True,
extensions=['jinja2.ext.do', 'jinja2.ext.with_',
jinja_extensions.SnippetExtension,
jinja_extensions.CkanExtend,
jinja_extensions.CkanInternationalizationExtension,
jinja_extensions.LinkForExtension,
jinja_extensions.ResourceExtension,
jinja_extensions.UrlForStaticExtension,
jinja_extensions.UrlForExtension]
)
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
env.filters['truncate'] = jinja_extensions.truncate
config['pylons.app_globals'].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# Setup the SQLAlchemy database engine
# Suppress a couple of sqlalchemy warnings
msgs = ['^Unicode type received non-unicode bind param value',
"^Did not recognize type 'BIGINT' of column 'size'",
"^Did not recognize type 'tsvector' of column 'search_vector'"
]
for msg in msgs:
warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning)
ckan_db = os.environ.get('CKAN_DB')
if ckan_db:
config['sqlalchemy.url'] = ckan_db
# for postgresql we want to enforce utf-8
sqlalchemy_url = config.get('sqlalchemy.url', '')
if sqlalchemy_url.startswith('postgresql://'):
extras = {'client_encoding': 'utf8'}
else:
extras = {}
engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.', **extras)
if not model.meta.engine:
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
def update_config():
''' This code needs to be run when the config is changed to take those
changes into account. It is called whenever a plugin is loaded as the
plugin might have changed the config values (for instance it might
change ckan.site_url) '''
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# Set whitelisted env vars on config object
# This is set up before globals are initialized
ckan_db = os.environ.get('CKAN_DB', None)
if ckan_db:
msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
log.warn(msg)
config['sqlalchemy.url'] = ckan_db
database_user = os.environ.get("DATABASE_USER", None)
database_password = os.environ.get("DATABASE_PASSWORD", None)
database_host = os.environ.get("DATABASE_HOST", None)
database_port = os.environ.get("DATABASE_PORT", None)
if database_user is None or database_password is None or database_host is None:
print(
"Did not find either DATABASE_USER or DATABASE_PASSWORD or DATABASE_HOST"
)
else:
print("Setting the database url")
if database_port is None:
#use the default
database_port = "5432"
url = "postgres://"
url += database_user
url += ":"
url += database_password
url += "@"
url += database_host
url += ":"
url += database_port
url += "/datacatalogue"
config['sqlalchemy.url'] = url
print("Setting database " + database_host + ":" + database_port)
for option in CONFIG_FROM_ENV_VARS:
from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
if from_env:
if option == "AWS_ACCESS_KEY_ID" or option == "AWS_SECRET_ACCESS_KEY":
print(option)
print(from_env[0:10])
config[option] = from_env
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
site_url = config.get('ckan.site_url', '')
if not site_url:
raise RuntimeError(
'ckan.site_url is not configured and it must have a value.'
' Please amend your .ini file.')
if not site_url.lower().startswith('http'):
raise RuntimeError(
'ckan.site_url should be a full URL, including the schema '
'(http or https)')
# Remove backslash from site_url if present
config['ckan.site_url'] = config['ckan.site_url'].rstrip('/')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# ensure that a favicon has been set
favicon = config.get('ckan.favicon', '/images/icons/ckan.ico')
config['ckan.favicon'] = favicon
# Init SOLR settings and check if the schema is compatible
#from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get('solr_url'), config.get('solr_user'),
config.get('solr_password'))
search.check_solr_schema_version()
routes_map = routing.make_map()
config['routes.map'] = routes_map
# The RoutesMiddleware needs its mapper updating if it exists
if 'routes.middleware' in config:
config['routes.middleware'].mapper = routes_map
config['routes.named_routes'] = routing.named_routes
config['pylons.app_globals'] = app_globals.app_globals
# initialise the globals
config['pylons.app_globals']._init()
# add helper functions
helpers = _Helpers(h)
config['pylons.h'] = helpers
## redo template setup to use genshi.search_path
## (so remove std template setup)
legacy_templates_path = os.path.join(root, 'templates_legacy')
jinja2_templates_path = os.path.join(root, 'templates')
if asbool(config.get('ckan.legacy_templates', 'no')):
# We want the new template path for extra snippets like the
# dataviewer and also for some testing stuff
msg = 'Support for Genshi templates is deprecated and will be removed'\
' in a future release'
log.warn(msg)
template_paths = [legacy_templates_path, jinja2_templates_path]
else:
template_paths = [jinja2_templates_path, legacy_templates_path]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['pylons.app_globals'].template_paths = template_paths
# Translator (i18n)
translator = Translator(pylons.translator)
def template_loaded(template):
translator.setup(template)
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create the Genshi TemplateLoader
config['pylons.app_globals'].genshi_loader = TemplateLoader(
template_paths, auto_reload=True, callback=template_loaded)
# Create Jinja2 environment
env = jinja_extensions.Environment(
loader=jinja_extensions.CkanFileSystemLoader(template_paths),
autoescape=True,
extensions=[
'jinja2.ext.do', 'jinja2.ext.with_',
jinja_extensions.SnippetExtension, jinja_extensions.CkanExtend,
jinja_extensions.CkanInternationalizationExtension,
jinja_extensions.LinkForExtension,
jinja_extensions.ResourceExtension,
jinja_extensions.UrlForStaticExtension,
jinja_extensions.UrlForExtension
])
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
env.filters['truncate'] = jinja_extensions.truncate
config['pylons.app_globals'].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# for postgresql we want to enforce utf-8
sqlalchemy_url = config.get('sqlalchemy.url', '')
if sqlalchemy_url.startswith('postgresql://'):
extras = {'client_encoding': 'utf8'}
else:
extras = {}
engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.', **extras)
if not model.meta.engine:
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# reset the template cache - we do this here so that when we load the
# environment it is clean
render.reset_template_info_cache()
# clear other caches
logic.clear_actions_cache()
logic.clear_validators_cache()
authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# (ProgrammingError for Postgres, OperationalError for SQLite)
# The database is not initialised. This is a bit dirty. This occurs
# when running tests.
pass
except sqlalchemy.exc.InternalError:
# The database is not initialised. Travis hits this
pass
# if an extension or our code does not finish
# transaction properly db cli commands can fail
model.Session.remove()
def update_config():
''' This code needs to be run when the config is changed to take those
changes into account. It is called whenever a plugin is loaded as the
plugin might have changed the config values (for instance it might
change ckan.site_url) '''
webassets_init()
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# Set whitelisted env vars on config object
# This is set up before globals are initialized
ckan_db = os.environ.get('CKAN_DB', None)
if ckan_db:
msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
log.warn(msg)
config['sqlalchemy.url'] = ckan_db
for option in CONFIG_FROM_ENV_VARS:
from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
if from_env:
config[option] = from_env
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
site_url = config.get('ckan.site_url', '')
if not site_url:
raise RuntimeError(
'ckan.site_url is not configured and it must have a value.'
' Please amend your .ini file.')
if not site_url.lower().startswith('http'):
raise RuntimeError(
'ckan.site_url should be a full URL, including the schema '
'(http or https)')
display_timezone = config.get('ckan.display_timezone', '')
if (display_timezone and display_timezone != 'server'
and display_timezone not in pytz.all_timezones):
raise CkanConfigurationException(
"ckan.display_timezone is not 'server' or a valid timezone")
# Remove backslash from site_url if present
config['ckan.site_url'] = config['ckan.site_url'].rstrip('/')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# ensure that a favicon has been set
favicon = config.get('ckan.favicon', '/base/images/ckan.ico')
config['ckan.favicon'] = favicon
# Init SOLR settings and check if the schema is compatible
# from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get('solr_url'), config.get('solr_user'),
config.get('solr_password'))
search.check_solr_schema_version()
if six.PY2:
routes_map = routing.make_map()
lib_plugins.reset_package_plugins()
lib_plugins.register_package_plugins()
lib_plugins.reset_group_plugins()
lib_plugins.register_group_plugins()
if six.PY2:
config['routes.map'] = routes_map
# The RoutesMiddleware needs its mapper updating if it exists
if 'routes.middleware' in config:
config['routes.middleware'].mapper = routes_map
# routes.named_routes is a CKAN thing
config['routes.named_routes'] = routing.named_routes
config['pylons.app_globals'] = app_globals.app_globals
# initialise the globals
app_globals.app_globals._init()
helpers.load_plugin_helpers()
config['pylons.h'] = helpers.helper_functions
# Templates and CSS loading from configuration
valid_base_templates_folder_names = ['templates']
templates = config.get('ckan.base_templates_folder', 'templates')
config['ckan.base_templates_folder'] = templates
if templates not in valid_base_templates_folder_names:
raise CkanConfigurationException(
'You provided an invalid value for ckan.base_templates_folder. '
'Possible values are: "templates".')
jinja2_templates_path = os.path.join(root, templates)
log.info('Loading templates from %s' % jinja2_templates_path)
template_paths = [jinja2_templates_path]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['computed_template_paths'] = template_paths
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
if six.PY2:
# Create Jinja2 environment
env = jinja_extensions.Environment(
**jinja_extensions.get_jinja_env_options())
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.policies['ext.i18n.trimmed'] = True
env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
config['pylons.app_globals'].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# Enable pessimistic disconnect handling (added in SQLAlchemy 1.2)
# to eliminate database errors due to stale pooled connections
config.setdefault('pool_pre_ping', True)
# Initialize SQLAlchemy
engine = sqlalchemy.engine_from_config(config)
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# reset the template cache - we do this here so that when we load the
# environment it is clean
render.reset_template_info_cache()
# clear other caches
logic.clear_actions_cache()
logic.clear_validators_cache()
authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# (ProgrammingError for Postgres, OperationalError for SQLite)
# The database is not initialised. This is a bit dirty. This occurs
# when running tests.
pass
except sqlalchemy.exc.InternalError:
# The database is not initialised. Travis hits this
pass
# Close current session and open database connections to ensure a clean
# clean environment even if an error occurs later on
model.Session.remove()
model.Session.bind.dispose()
def command(self):
if not all(os.environ.get(i) for i in ('CKAN_TEST_SYSADMIN_NAME', 'CKAN_TEST_SYSADMIN_PASSWORD', 'CKAN_INI')):
print('One of these env vars not set: CKAN_INI, CKAN_TEST_SYSADMIN_NAME or CKAN_TEST_SYSADMIN_PASSWORD')
return
print('====== Creating DGU test data')
self._load_config()
engine = sqlalchemy.create_engine(config.get('sqlalchemy.url'))
model.init_model(engine)
sysadmin_user = model.User.get(os.environ.get('CKAN_TEST_SYSADMIN_NAME'))
if not sysadmin_user:
print('=== Creating test sysadmin')
sysadmin_user = model.User(
name=os.environ.get('CKAN_TEST_SYSADMIN_NAME'),
password=os.environ.get('CKAN_TEST_SYSADMIN_PASSWORD')
)
sysadmin_user.sysadmin = True
model.Session.add(sysadmin_user)
model.repo.commit_and_remove()
publisher = model.Group.get('Example Publisher #1')
if not publisher:
print('=== Creating example publisher 1')
model.Session.flush()
rev = model.repo.new_revision()
rev.author = u"DGU test admin"
rev.message = u"Creating Example Publisher #1."
publisher = model.Group(
name=u"example-publisher-1",
title=u"Example Publisher #1",
type=u"organization"
)
publisher.is_organization = True
model.Session.add(publisher)
model.repo.commit_and_remove()
rev = model.repo.new_revision()
rev.author = u"DGU test admin"
rev.message = u"Adding charity-ngo category for example publisher 1."
category = model.GroupExtra(group_id=publisher.id, key="category", value="charity-ngo")
model.Session.add(category)
model.repo.commit_and_remove()
if not model.Package.by_name(u"example-harvest-1"):
print('=== Creating harvest source')
version = ''
if "ckan@db" in config.get('sqlalchemy.url'):
if '5001' in config.get('ckan.site_url'):
version = '-2.8'
elif '5002' in config.get('ckan.site_url'):
verions = '-2.9'
source_dict = {
'title': 'Example Harvest #1',
'name': 'example-harvest-1',
'url': "http://static-mock-harvest-source{}:11088/".format(version)\
if "ckan@db" in config.get('sqlalchemy.url') else \
"https://ckan-static-mock-harvest-source.cloudapps.digital/",
'source_type': 'ckan',
'owner_org': publisher.id,
'notes': 'An example harvest source',
'frequency': "MANUAL",
'active': True,
"config": None
}
context = {
"model": model,
"session": model.Session,
"user": sysadmin_user.name,
"ignore_auth": True,
"schema": harvest_source_schema(),
"message": "Create DGU example harvest source",
"return_id_only": True
}
harvest_source_id = harvest_source_create(context, source_dict)
if harvest_source_id:
print("=== Creating harvest job")
harvest_job_create(context, {"source_id": harvest_source_id, "run": False})
print("=== Running harvest job")
command = "paster --plugin=ckanext-harvest harvester run_test example-harvest-1 -c $CKAN_INI"
run_command(command)
model.Session.flush()
print("=== Updating the example dataset to be in line with how DGU processes it")
rev = model.repo.new_revision()
rev.author = u"DGU test admin"
rev.message = u"Updating example-data-number-one for CKAN functional tests"
dataset = model.Package.get("example-dataset-number-one")
contact_name = model.PackageExtra(package_id=dataset.id, key="contact-name", value="Example User")
model.Session.add(contact_name)
empty_fields = [
"contact-email",
"contact-phone",
"schema-vocabulary",
"codelist",
"licence-custom",
"foi-web",
"foi-name",
"foi-email",
"foi-phone",
"theme-primary"
]
for key in empty_fields:
field = model.PackageExtra(package_id=dataset.id, key=key, value="")
model.Session.add(field)
delete_fields = [
"guid", "responsible-party", "taxonomy_url"
]
for key in delete_fields:
field = model.Session.query(model.PackageExtra).filter(
model.PackageExtra.package_id == dataset.id, model.PackageExtra.key == key
).first()
if field:
field.delete()
model.repo.commit_and_remove()
print("=== Running search index rebuild")
command = 'paster --plugin=ckan search-index rebuild %s -c $CKAN_INI' % dataset.name
run_command(command)
publisher2 = model.Group.get('Example Publisher #2')
if not publisher2:
print('=== Creating example publisher 2')
model.Session.flush()
rev = model.repo.new_revision()
rev.author = u"DGU test admin"
rev.message = u'''Creating Example Publisher #2.'''
publisher2 = model.Group(
name=u"example-publisher-2",
title=u"Example Publisher #2",
type="organization"
)
publisher2.is_organization = True
model.Session.add(publisher2)
model.repo.commit_and_remove()
print("=== To use with CKAN functional tests in ckan-vars.conf set OWNER_ORG=%s" % publisher.id)
print("====== DGU test data created")
def update_config():
''' This code needs to be run when the config is changed to take those
changes into account. '''
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# This is set up before globals are initialized
site_id = os.environ.get('CKAN_SITE_ID')
if site_id:
config['ckan.site_id'] = site_id
site_url = config.get('ckan.site_url', '')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# ensure that a favicon has been set
favicon = config.get('ckan.favicon', '/images/icons/ckan.ico')
config['ckan.favicon'] = favicon
# Init SOLR settings and check if the schema is compatible
#from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get('solr_url'),
config.get('solr_user'),
config.get('solr_password'))
search.check_solr_schema_version()
routes_map = routing.make_map()
config['routes.map'] = routes_map
# The RoutesMiddleware needs its mapper updating if it exists
if 'routes.middleware' in config:
config['routes.middleware'].mapper = routes_map
config['routes.named_routes'] = routing.named_routes
config['pylons.app_globals'] = app_globals.app_globals
# initialise the globals
config['pylons.app_globals']._init()
# add helper functions
helpers = _Helpers(h)
config['pylons.h'] = helpers
## redo template setup to use genshi.search_path
## (so remove std template setup)
legacy_templates_path = os.path.join(root, 'templates_legacy')
jinja2_templates_path = os.path.join(root, 'templates')
if asbool(config.get('ckan.legacy_templates', 'no')):
# We want the new template path for extra snippets like the
# dataviewer and also for some testing stuff
msg = 'Support for Genshi templates is deprecated and will be removed'\
' in a future release'
log.warn(msg)
template_paths = [legacy_templates_path, jinja2_templates_path]
else:
template_paths = [jinja2_templates_path, legacy_templates_path]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['pylons.app_globals'].template_paths = template_paths
# Translator (i18n)
translator = Translator(pylons.translator)
def template_loaded(template):
translator.setup(template)
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create the Genshi TemplateLoader
config['pylons.app_globals'].genshi_loader = TemplateLoader(
template_paths, auto_reload=True, callback=template_loaded)
# Create Jinja2 environment
env = jinja_extensions.Environment(
loader=jinja_extensions.CkanFileSystemLoader(template_paths),
autoescape=True,
extensions=['jinja2.ext.do', 'jinja2.ext.with_',
jinja_extensions.SnippetExtension,
jinja_extensions.CkanExtend,
jinja_extensions.CkanInternationalizationExtension,
jinja_extensions.LinkForExtension,
jinja_extensions.ResourceExtension,
jinja_extensions.UrlForStaticExtension,
jinja_extensions.UrlForExtension]
)
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
env.filters['truncate'] = jinja_extensions.truncate
config['pylons.app_globals'].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
ckan_db = os.environ.get('CKAN_DB')
if ckan_db:
config['sqlalchemy.url'] = ckan_db
# for postgresql we want to enforce utf-8
sqlalchemy_url = config.get('sqlalchemy.url', '')
if sqlalchemy_url.startswith('postgresql://'):
extras = {'client_encoding': 'utf8'}
else:
extras = {}
engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.', **extras)
if not model.meta.engine:
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# reset the template cache - we do this here so that when we load the
# environment it is clean
render.reset_template_info_cache()
# clear other caches
logic.clear_actions_cache()
logic.clear_validators_cache()
new_authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# (ProgrammingError for Postgres, OperationalError for SQLite)
# The database is not initialised. This is a bit dirty. This occurs
# when running tests.
pass
except sqlalchemy.exc.InternalError:
# The database is not initialised. Travis hits this
pass
# if an extension or our code does not finish
# transaction properly db cli commands can fail
model.Session.remove()
def load_environment(global_conf, app_conf):
"""Configure the Pylons environment via the ``pylons.config``
object
"""
###### Pylons monkey-patch
# this must be run at a time when the env is semi-setup, thus inlined here.
# Required by the deliverance plugin and iATI
from pylons.wsgiapp import PylonsApp
import pkg_resources
find_controller_generic = PylonsApp.find_controller
# This is from pylons 1.0 source, will monkey-patch into 0.9.7
def find_controller(self, controller):
if controller in self.controller_classes:
return self.controller_classes[controller]
# Check to see if its a dotted name
if "." in controller or ":" in controller:
mycontroller = pkg_resources.EntryPoint.parse("x=%s" % controller).load(False)
self.controller_classes[controller] = mycontroller
return mycontroller
return find_controller_generic(self, controller)
PylonsApp.find_controller = find_controller
###### END evil monkey-patch
os.environ["CKAN_CONFIG"] = global_conf["__file__"]
# Pylons paths
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
paths = dict(
root=root,
controllers=os.path.join(root, "controllers"),
static_files=os.path.join(root, "public"),
templates=[os.path.join(root, "templates")],
)
# Initialize config with the basic options
config.init_app(global_conf, app_conf, package="ckan", paths=paths)
# load all CKAN plugins
p.load_all(config)
# Load the synchronous search plugin, unless already loaded or
# explicitly disabled
if not "synchronous_search" in config.get("ckan.plugins", []) and asbool(
config.get("ckan.search.automatic_indexing", True)
):
log.debug("Loading the synchronous search plugin")
p.load("synchronous_search")
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# This is set up before globals are initialized
site_id = os.environ.get("CKAN_SITE_ID")
if site_id:
config["ckan.site_id"] = site_id
site_url = config.get("ckan.site_url", "")
ckan_host = config["ckan.host"] = urlparse(site_url).netloc
if config.get("ckan.site_id") is None:
if ":" in ckan_host:
ckan_host, port = ckan_host.split(":")
assert ckan_host, (
"You need to configure ckan.site_url or " "ckan.site_id for SOLR search-index rebuild to work."
)
config["ckan.site_id"] = ckan_host
# Init SOLR settings and check if the schema is compatible
# from ckan.lib.search import SolrSettings, check_solr_schema_version
search.SolrSettings.init(config.get("solr_url"), config.get("solr_user"), config.get("solr_password"))
search.check_solr_schema_version()
config["routes.map"] = routing.make_map()
config["pylons.app_globals"] = app_globals.Globals()
# add helper functions
restrict_helpers = asbool(config.get("ckan.restrict_template_vars", "true"))
helpers = _Helpers(h, restrict_helpers)
config["pylons.h"] = helpers
# Redo template setup to use genshi.search_path
# (so remove std template setup)
template_paths = [paths["templates"][0]]
extra_template_paths = config.get("extra_template_paths", "")
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(",") + template_paths
# Translator (i18n)
translator = Translator(pylons.translator)
def template_loaded(template):
translator.setup(template)
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create the Genshi TemplateLoader
config["pylons.app_globals"].genshi_loader = TemplateLoader(
template_paths, auto_reload=True, callback=template_loaded
)
#################################################################
# #
# HORRIBLE GENSHI HACK #
# #
#################################################################
# #
# Genshi does strange things to get stuff out of the template #
# variables. This stops it from handling properties in the #
# correct way as it returns the property rather than the actual #
# value of the property. #
# #
# By overriding lookup_attr() in the LookupBase class we are #
# able to get the required behaviour. Using @property allows #
# us to move functionality out of templates whilst maintaining #
# backwards compatability. #
# #
#################################################################
"""
This code is based on Genshi code
Copyright © 2006-2012 Edgewall Software
All rights reserved.
Redistribution and use in source and binary forms, with or
without modification, are permitted provided that the following
conditions are met:
Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following
disclaimer in the documentation and/or other materials provided
with the distribution.
The name of the author may not be used to endorse or promote
products derived from this software without specific prior
written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
from genshi.template.eval import LookupBase
@classmethod
def genshi_lookup_attr(cls, obj, key):
__traceback_hide__ = True
try:
val = getattr(obj, key)
except AttributeError:
if hasattr(obj.__class__, key):
raise
else:
try:
val = obj[key]
except (KeyError, TypeError):
val = cls.undefined(key, owner=obj)
if isinstance(val, property):
val = val.fget()
return val
setattr(LookupBase, "lookup_attr", genshi_lookup_attr)
del genshi_lookup_attr
del LookupBase
#################################################################
# #
# END OF GENSHI HACK #
# #
#################################################################
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# Setup the SQLAlchemy database engine
# Suppress a couple of sqlalchemy warnings
msgs = [
"^Unicode type received non-unicode bind param value",
"^Did not recognize type 'BIGINT' of column 'size'",
"^Did not recognize type 'tsvector' of column 'search_vector'",
]
for msg in msgs:
warnings.filterwarnings("ignore", msg, sqlalchemy.exc.SAWarning)
ckan_db = os.environ.get("CKAN_DB")
if ckan_db:
config["sqlalchemy.url"] = ckan_db
# for postgresql we want to enforce utf-8
sqlalchemy_url = config.get("sqlalchemy.url", "")
if sqlalchemy_url.startswith("postgresql://"):
extras = {"client_encoding": "utf8"}
else:
extras = {}
engine = sqlalchemy.engine_from_config(config, "sqlalchemy.", **extras)
if not model.meta.engine:
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)