def test_interpolate_exception():
try:
appconfig('config:test_error.ini', relative_to=config_path)
except Exception:
e = sys.exc_info()[1]
expected = "Error in file %s" % os.path.join(config_path, 'test_error.ini')
eq_(str(e).split(':')[0], expected)
else:
assert False, 'Should have raised an exception'
def test_interpolate_exception():
try:
appconfig('config:test_error.ini', relative_to=config_path)
except Exception:
e = sys.exc_info()[1]
expected = "Error in file %s" % os.path.join(config_path,
'test_error.ini')
assert str(e).split(':')[0] == expected
else:
assert False, 'Should have raised an exception'
def add_settings_to_appconfig(ini_file="development.ini"):
"""
Configure app config to set correct ini-file. Defaults to development.ini for testing purposes.
If D-BAS runs inside a docker container and no ini-file is provided, then load the docker.ini.
:param ini_file: name of ini-file
:return: config with loaded ini-file
:rtype: dict
"""
if os.path.isfile("/.dockerenv") and not ini_file:
return appconfig("config:" + path_to_settings("docker.ini"))
return appconfig("config:" + path_to_settings(ini_file))
def _app_config(self):
"""
This is the first part of CkanCommand._load_config()
"""
from paste.deploy import appconfig
if not self.options.config:
msg = 'No config file supplied'
raise self.BadCommand(msg)
self.filename = os.path.abspath(self.options.config)
if not os.path.exists(self.filename):
raise AssertionError('Config filename %r does not exist.' %
self.filename)
fileConfig(self.filename)
appconfig('config:' + self.filename)
def _app_config(self):
"""
This is the first part of CkanCommand._load_config()
"""
from paste.deploy import appconfig
if not self.options.config:
msg = 'No config file supplied'
raise self.BadCommand(msg)
self.filename = os.path.abspath(self.options.config)
if not os.path.exists(self.filename):
raise AssertionError(
'Config filename %r does not exist.' % self.filename
)
fileConfig(self.filename)
appconfig('config:' + self.filename)
def _load_app(self):
config_file = self.options.config_file
if not os.path.isfile(config_file):
raise BadCommand('Error: CONFIG_FILE not found at: %s\nPlease specify a CONFIG_FILE' % config_file)
config_name = 'config:%s' % config_file
here_dir = os.getcwd()
if self.options.verbose:
# Configure logging from the config file
self.logging_file_config(config_file)
self.config = appconfig(config_name, relative_to=here_dir)
self.config.update({'app_conf': self.config.local_conf,
'global_conf': self.config.global_conf})
paste.deploy.config.CONFIG.push_thread_config(self.config)
# Load locals and populate with objects for use in shell
sys.path.insert(0, here_dir)
# Load the wsgi app first so that everything is initialized right
self.wsgiapp = loadapp(config_name, relative_to=here_dir)
self.app = paste.fixture.TestApp(self.wsgiapp)
# Query the test app to setup the environment
tresponse = self.app.get('/_test_vars')
request_id = int(tresponse.body)
# Disable restoration during app requests
self.app.pre_request_hook = lambda self: paste.registry.restorer.restoration_end()
self.app.post_request_hook = lambda self: paste.registry.restorer.restoration_begin(request_id)
paste.registry.restorer.restoration_begin(request_id)
def __init__(self, config_filename):
conf = appconfig('config:%s' % config_filename, relative_to='.')
load_environment(conf.global_conf, conf.local_conf)
self.config = config
self.Session = Session
self.model = model
self.h = h
def load_config(filename):
filename = os.path.normpath(os.path.abspath(filename))
config = appconfig('config:' + filename)
# Load the logging options
# (must be done before environment is loaded or sqlalchemy won't log)
logging_config.fileConfig(filename)
return config
def setup_config(command, filename, section, vars):
"""Place any commands to setup sssweb here"""
conf = appconfig('config:' + filename)
load_environment(conf.global_conf, conf.local_conf)
# Create the tables if they don't already exist
meta.metadata.create_all(bind=meta.engine)
def __init__(self, *args, **kwargs):
'''
initialize the test class
'''
TestCase.__init__(self, *args, **kwargs)
LOG.debug("ConfigFile: %s " % config['__file__'])
conffile = config['__file__']
if pylons.test.pylonsapp:
wsgiapp = pylons.test.pylonsapp
else:
wsgiapp = loadapp('config: %s' % config['__file__'])
self.app = TestApp(wsgiapp)
conf = None
if conffile.startswith('/'):
conf = appconfig('config:%s' % config['__file__'], relative_to=None)
else:
raise Exception('dont know how to load the application relatively')
#conf = appconfig('config: %s' % config['__file__'], relative_to=rel)
load_environment(conf.global_conf, conf.local_conf)
self.appconf = conf
url._push_object(URLGenerator(config['routes.map'], environ))
self.isSelfTest = False
if env.has_key("linotp.selfTest"):
self.isSelfTest = True
return
def configure(self, config_file):
config_url = 'config:' + config_file
cwd = os.getcwd()
self.config = appconfig(config_url, 'MAGMaWeb', relative_to=cwd)
engine = engine_from_config(self.config)
init_user_db(engine)
self.job_factory = make_job_factory(self.config)
def __init__(self, uploader_conf, plugin_name, regex=None, cutoff=None):
super(LogUploader, self).__init__(uploader_conf)
log_name = '%s-log-uploader' % plugin_name
self.logger = utils.get_logger(uploader_conf,
log_name,
log_route=plugin_name)
self.log_dir = uploader_conf.get('log_dir', '/var/log/swift/')
self.swift_account = uploader_conf['swift_account']
self.container_name = uploader_conf['container_name']
proxy_server_conf_loc = uploader_conf.get(
'proxy_server_conf', '/etc/swift/proxy-server.conf')
proxy_server_conf = appconfig('config:%s' % proxy_server_conf_loc,
name='proxy-server')
self.internal_proxy = InternalProxy(proxy_server_conf)
self.new_log_cutoff = int(
cutoff or uploader_conf.get('new_log_cutoff', '7200'))
self.unlink_log = uploader_conf.get('unlink_log', 'true').lower() in \
utils.TRUE_VALUES
self.filename_pattern = regex or \
uploader_conf.get('source_filename_pattern',
'''
^%s-
(?P<year>[0-9]{4})
(?P<month>[0-1][0-9])
(?P<day>[0-3][0-9])
(?P<hour>[0-2][0-9])
.*$''' % plugin_name)
self.content_type = uploader_conf.get('content_type')
def load_config(ini_file, log_from_ini=False, do_logging=True):
from paste.deploy import appconfig
from desio.config.environment import load_environment
ini_file = os.path.abspath(ini_file)
# load config / pylons environment
# note: appconfig (as opposed to loadapp) doesn't load wsgi
app_config = appconfig("config:%s" % (ini_file,))
config = load_environment(app_config.global_conf, app_config.local_conf)
from desio import model
engine = config["pylons.app_globals"].sa_default_engine
model.init_model(engine)
pylons.config.push_process_config(config)
if not do_logging:
return
if log_from_ini:
try:
# this fails because I want it to fail.
logging.config.fileConfig(ini_file)
except Exception, e:
import traceback
sys.stderr.write(traceback.format_exc())
else:
return
def main():
""" The main MokshaHub method """
setup_logger('-v' in sys.argv or '--verbose' in sys.argv)
config_path = get_moksha_config_path()
if not config_path:
print """
Cannot find Moksha configuration! Place a development.ini or production.ini
in /etc/moksha or in the current directory.
"""
return
cfg = appconfig('config:' + config_path)
config.update(cfg)
hub = CentralMokshaHub()
global _hub
_hub = hub
def handle_signal(signum, stackframe):
from moksha.hub.reactor import reactor
if signum in [signal.SIGHUP, signal.SIGINT]:
hub.stop()
try:
reactor.stop()
except ReactorNotRunning:
pass
signal.signal(signal.SIGHUP, handle_signal)
signal.signal(signal.SIGINT, handle_signal)
log.info("Running the MokshaHub reactor")
reactor.run(installSignalHandlers=False)
log.info("MokshaHub reactor stopped")
def _setup(self):
"""
Set up logging, import pylons/paste/debexpo modules, parse config file, create config
class and chdir to the incoming directory.
"""
# Look for ini file
if not os.path.isfile(self.ini_file):
self._fail('Cannot find ini file')
self._setup_logging()
# Import debexpo root directory
sys.path.append(os.path.dirname(self.ini_file))
# Initialize Pylons app
conf = appconfig('config:' + self.ini_file)
pylons.config = load_environment(conf.global_conf, conf.local_conf)
# Change into the incoming directory
incoming_dir = pylons.config['debexpo.upload.incoming']
logging.info("Changing dir to %s", incoming_dir)
os.chdir(incoming_dir)
# Look for the changes file
if not os.path.isfile(self.changes_file):
self._fail('Cannot find changes file')
def test_config(self):
config = appconfig("config:test.ini", relative_to=conf_dir)
config.local_conf["ckan.plugins"] = "googleanalytics"
config.local_conf["googleanalytics.id"] = ""
command = LoadAnalytics("loadanalytics")
command.CONFIG = config.local_conf
self.assertRaises(Exception, command.run, [])
def setup_class(cls):
config = appconfig('config:test.ini', relative_to=tests.conf_dir)
config.local_conf['ckan.plugins'] = 'recline_preview'
wsgiapp = make_app(config.global_conf, **config.local_conf)
cls.app = paste.fixture.TestApp(wsgiapp)
cls.p = previewplugin.ReclinePreview()
# create test resource
CreateTestData.create()
context = {
'model': model,
'session': model.Session,
'user': model.User.get('testsysadmin').name
}
cls.package = model.Package.get('annakarenina')
cls.resource = logic.get_action('resource_show')(
context, {
'id': cls.package.resources[1].id
})
cls.resource['url'] = pylons.config.get('ckan.site_url',
'//localhost:5000')
cls.resource['format'] = 'csv'
logic.action.update.resource_update(context, cls.resource)
def basic_setup(self):
global log, M
if self.args[0]:
# Probably being called from the command line - load the config file
self.config = conf = appconfig('config:%s' % self.args[0],
relative_to=os.getcwd())
# Configure logging
try:
# ... logging does not understand section#subsection syntax
logging_config = self.args[0].split('#')[0]
logging.config.fileConfig(logging_config,
disable_existing_loggers=False)
except Exception: # pragma no cover
print >> sys.stderr, (
'Could not configure logging with config file %s' %
self.args[0])
log = logging.getLogger('allura.command')
log.info('Initialize command with config %r', self.args[0])
load_environment(conf.global_conf, conf.local_conf)
self.setup_globals()
from allura import model
M = model
ming.configure(**conf)
pylons.c.user = M.User.anonymous()
else:
# Probably being called from another script (websetup, perhaps?)
log = logging.getLogger('allura.command')
conf = pylons.config
self.tools = pylons.g.entry_points['tool'].values()
for ep in iter_entry_points('allura.command_init'):
log.info('Running command_init for %s', ep.name)
ep.load()(conf)
log.info('Loaded tools')
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('config', help='path of configuration file')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(
level=logging.INFO if args.verbose else logging.WARNING,
stream=sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
# model.repo.new_revision()
# for resource in model.Session.query(model.Resource).filter(
# model.Resource.url.like('%etalab2.fr%'),
# ):
# url = resource.url
# if url.startswith('http://ckan.etalab2.fr/'):
# resource.url = resource.url.replace('http://ckan.etalab2.fr/', 'http://www.data.gouv.fr/fr/')
# elif url.startswith('http://ckan-hetic.etalab2.fr/'):
# resource.url = resource.url.replace('http://ckan-hetic.etalab2.fr/', 'http://www.data.gouv.fr/fr/')
# elif url.startswith('http://www.etalab2.fr/'):
# resource.url = resource.url.replace('http://www.etalab2.fr/', 'http://www.data.gouv.fr/')
# else:
# print resource.url
# model.repo.commit_and_remove()
# model.repo.new_revision()
# for resource in model.Session.query(model.Resource).filter(
# model.Resource.url.like('%www.data.gouv.fr%'),
# ):
# if resource.url.startswith('http://www.data.gouv.fr/') and not resource.url.startswith('http://www.data.gouv.fr/var'):
# resource.url = resource.url.replace('http://www.data.gouv.fr/', 'http://new.data.gouv.fr/')
# model.repo.commit_and_remove()
model.repo.new_revision()
for resource in model.Session.query(model.Resource).filter(
model.Resource.url.like('http://new.data.gouv.fr/%'), ):
resource.url = resource.url.replace('http://new.data.gouv.fr/',
'https://www.data.gouv.fr/')
model.repo.commit_and_remove()
model.repo.new_revision()
for resource in model.Session.query(model.Resource).filter(
model.Resource.url.like('https://new.data.gouv.fr/%'), ):
resource.url = resource.url.replace('https://new.data.gouv.fr/',
'https://www.data.gouv.fr/')
model.repo.commit_and_remove()
return 0
def test_routes_plugin_fired(self):
local_config = appconfig('config:%s' % config['__file__'], relative_to=conf_dir)
local_config.local_conf['ckan.plugins'] = 'routes_plugin'
app = make_app(local_config.global_conf, **local_config.local_conf)
routes_plugin = PluginGlobals.env().plugin_registry['RoutesPlugin'].__instance__
assert routes_plugin.calls_made == ['before_map', 'after_map'], \
routes_plugin.calls_made
def load_config(ini_path=None):
from paste.deploy import appconfig
if ini_path:
if ini_path.startswith(u'~'):
ini_path = os.path.expanduser(ini_path)
filename = os.path.abspath(ini_path)
config_source = u'-c parameter'
elif os.environ.get(u'CKAN_INI'):
filename = os.environ.get(u'CKAN_INI')
config_source = u'$CKAN_INI'
else:
default_filename = u'development.ini'
filename = os.path.join(os.getcwd(), default_filename)
if not os.path.exists(filename):
# give really clear error message for this common situation
msg = u'ERROR: You need to specify the CKAN config (.ini) '\
u'file path.'\
u'\nUse the --config parameter or set environment ' \
u'variable CKAN_INI or have {}\nin the current directory.' \
.format(default_filename)
exit(msg)
if not os.path.exists(filename):
msg = u'Config file not found: %s' % filename
msg += u'\n(Given by: %s)' % config_source
exit(msg)
loggingFileConfig(filename)
log.info(u'Using configuration file {}'.format(filename))
return appconfig(u'config:' + filename)
def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('config', help = 'path of configuration file')
parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
for package_extra in model.Session.query(model.PackageExtra).filter(
model.PackageExtra.key == 'territorial_coverage_granularity',
model.PackageExtra.value == 'france',
):
package = package_extra.package
if package.private or package.state != 'active':
log.warning(u'Territorial coverage granularity of package {} must be manually corrected'.format(
package.name))
continue
package_extra.value = 'pays'
model.repo.commit_and_remove()
return 0
def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('config', help = 'path of configuration file')
parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
for package_extra in model.Session.query(model.PackageExtra).filter(
model.PackageExtra.key == 'supplier_id',
model.PackageExtra.package_id.in_(
model.Session.query(model.Package.id).filter(model.Package.name.like('%-fork-%'))),
):
model.Session.delete(package_extra)
model.repo.commit_and_remove()
return 0
def take_action(self, opts):
config_spec = opts.config_file
section = opts.section_name
if section is None:
if '#' in config_spec:
config_spec, section = config_spec.split('#', 1)
else:
section = 'main'
if not ':' in section:
plain_section = section
section = 'app:'+section
else:
plain_section = section.split(':', 1)[0]
if not config_spec.startswith('config:'):
config_spec = 'config:' + config_spec
if plain_section != 'main':
config_spec += '#' + plain_section
config_file = config_spec[len('config:'):].split('#', 1)[0]
config_file = os.path.join(os.getcwd(), config_file)
conf = appconfig(config_spec, relative_to=os.getcwd())
ep_name = conf.context.entry_point_name
ep_group = conf.context.protocol
dist = conf.context.distribution
if dist is None:
raise RuntimeError("The section %r is not the application (probably a filter). You should add #section_name, where section_name is the section that configures your application" % plain_section)
self._setup_config(dist, config_file, section, {}, verbosity=self.app.options.verbose_level)
def __init__(self, *args, **kwargs):
'''
initialize the test class
'''
TestCase.__init__(self, *args, **kwargs)
LOG.error("ConfigFile: %s " % config['__file__'])
conffile = config['__file__']
if pylons.test.pylonsapp:
wsgiapp = pylons.test.pylonsapp
else:
wsgiapp = loadapp('config: %s' % config['__file__'])
self.app = TestApp(wsgiapp)
conf = None
if conffile.startswith('/'):
conf = appconfig('config:%s' % config['__file__'], relative_to=None)
else:
raise Exception('dont know how to load the application relatively')
#conf = appconfig('config: %s' % config['__file__'], relative_to=rel)
load_environment(conf.global_conf, conf.local_conf)
self.appconf = conf
url._push_object(URLGenerator(config['routes.map'], environ))
self.isSelfTest = False
if env.has_key("privacyidea.selfTest"):
self.isSelfTest = True
self.license = 'CE'
return
def __init__(self, uploader_conf, plugin_name):
super(LogUploader, self).__init__(uploader_conf)
log_name = '%s-log-uploader' % plugin_name
self.logger = utils.get_logger(uploader_conf,
log_name,
log_route=plugin_name)
self.log_dir = uploader_conf.get('log_dir', '/var/log/swift/')
self.swift_account = uploader_conf['swift_account']
self.container_name = uploader_conf['container_name']
proxy_server_conf_loc = uploader_conf.get(
'proxy_server_conf', '/etc/swift/proxy-server.conf')
proxy_server_conf = appconfig('config:%s' % proxy_server_conf_loc,
name='proxy-server')
self.internal_proxy = InternalProxy(proxy_server_conf)
self.new_log_cutoff = int(uploader_conf.get('new_log_cutoff', '7200'))
self.unlink_log = uploader_conf.get('unlink_log', 'True').lower() in \
utils.TRUE_VALUES
# source_filename_format is deprecated
source_filename_format = uploader_conf.get('source_filename_format')
source_filename_pattern = uploader_conf.get('source_filename_pattern')
if source_filename_format and not source_filename_pattern:
self.logger.warning(
_('source_filename_format is unreliable and '
'deprecated; use source_filename_pattern'))
self.pattern = self.convert_glob_to_regex(source_filename_format)
else:
self.pattern = source_filename_pattern or '%Y%m%d%H'
def load_paste_config(app_name, options, args):
"""
Looks for a config file to use for an app and returns the
config file path and a configuration mapping from a paste config file.
We search for the paste config file in the following order:
* If --config-file option is used, use that
* If args[0] is a file, use that
* Search for quantum.conf in standard directories:
* .
* ~.quantum/
* ~
* /etc/quantum
* /etc
:param app_name: Name of the application to load config for, or None.
None signifies to only load the [DEFAULT] section of
the config file.
:param options: Set of typed options returned from parse_options()
:param args: Command line arguments from argv[1:]
:retval Tuple of (conf_file, conf)
:raises RuntimeError when config file cannot be located or there was a
problem loading the configuration file.
"""
conf_file = find_config_file(options, args)
if not conf_file:
raise RuntimeError("Unable to locate any configuration file. "
"Cannot load application %s" % app_name)
try:
conf = deploy.appconfig("config:%s" % conf_file, name=app_name)
return conf_file, conf
except Exception, e:
raise RuntimeError("Error trying to load config %s: %s" %
(conf_file, e))
def main():
parser = OptionParser(usage="%prog -u FILE -i FILE")
parser.add_option(
'-u',
'--user-json-path',
dest='user_json_path',
help=
'Path to JSON file with user data to import (/dev/stdin is permissible)',
metavar='FILE',
default=None)
parser.add_option('-i',
'--ini',
dest='ini',
help='Path to application ini file',
metavar='FILE',
default=None)
(options, args) = parser.parse_args()
if not options.user_json_path:
parser.print_help()
sys.exit(1)
# Initialize Pylons app
conf = appconfig('config:' + os.path.abspath(options.ini))
pylons.config = load_environment(conf.global_conf, conf.local_conf)
list_of_dicts = simplejson.load(open(options.user_json_path))
import_users(list_of_dicts)
return 0
def load_config(ini_file, log_from_ini=False, do_logging=True):
from paste.deploy import appconfig
from desio.config.environment import load_environment
ini_file = os.path.abspath(ini_file)
# load config / pylons environment
# note: appconfig (as opposed to loadapp) doesn't load wsgi
app_config = appconfig("config:%s" % (ini_file, ))
config = load_environment(app_config.global_conf, app_config.local_conf)
from desio import model
engine = config['pylons.app_globals'].sa_default_engine
model.init_model(engine)
pylons.config.push_process_config(config)
if not do_logging:
return
if log_from_ini:
try:
# this fails because I want it to fail.
logging.config.fileConfig(ini_file)
except Exception, e:
import traceback
sys.stderr.write(traceback.format_exc())
else:
return
def __init__(self, config_filename):
conf = appconfig('config:%s' % config_filename, relative_to='.')
load_environment(conf.global_conf, conf.local_conf)
self.config = config
self.Session = Session
self.model = model
self.h = h
def main():
""" The main MokshaHub method """
setup_logger('-v' in sys.argv or '--verbose' in sys.argv)
config_path = get_moksha_config_path()
if not config_path:
print """
Cannot find Moksha configuration! Place a development.ini or production.ini
in /etc/moksha or in the current directory.
"""
return
cfg = appconfig('config:' + config_path)
config.update(cfg)
hub = CentralMokshaHub()
global _hub
_hub = hub
def handle_signal(signum, stackframe):
from moksha.hub.reactor import reactor
if signum in [signal.SIGHUP, signal.SIGINT]:
hub.stop()
try:
reactor.stop()
except ReactorNotRunning:
pass
signal.signal(signal.SIGHUP, handle_signal)
signal.signal(signal.SIGINT, handle_signal)
log.info("Running the MokshaHub reactor")
reactor.run(installSignalHandlers=False)
log.info("MokshaHub reactor stopped")
def main(argv=sys.argv):
if len(argv) != 2:
usage(argv)
config_uri = argv[1]
path, section = _getpathsec(config_uri, "pyramid")
config_name = 'config:%s' % path
here_dir = os.getcwd()
global_conf = None
if 'OPENSHIFT_APP_NAME' in os.environ:
if 'OPENSHIFT_MYSQL_DB_URL' in os.environ:
template = '{OPENSHIFT_MYSQL_DB_URL}{OPENSHIFT_APP_NAME}'
elif 'OPENSHIFT_POSTGRESQL_DB_URL' in os.environ:
template = '{OPENSHIFT_POSTGRESQL_DB_URL}{OPENSHIFT_APP_NAME}'
global_conf = {
'sqlalchemy.url': template.format(**os.environ)
}
settings = appconfig(config_name, name=section, relative_to=here_dir,
global_conf=global_conf)
engine = engine_from_config(settings, 'sqlalchemy.')
DBSession.configure(bind=engine)
DeclarativeBase.metadata.create_all(engine)
return # Skip all the following... just leaving it for posterity
with transaction.manager:
issuer = Issuer(
name="Ralph Bean",
origin="http://badges.threebean.org",
org="threebean.org",
contact="*****@*****.**",
)
DBSession.add(issuer)
badge = Badge(
name="Plus One!",
image="/pngs/threebean-plus-one.png",
description="""
Got a recommendation from threebean for being awesome.
""".strip(),
criteria="/badges/plus-one", # TODO -- how should this work?
issuer=issuer,
)
DBSession.add(badge)
person = Person(
email="*****@*****.**",
)
DBSession.add(person)
assertion = Assertion(
badge=badge,
person=person,
issued_on=datetime.datetime.now(),
)
DBSession.add(assertion)
pprint.pprint(assertion.__json__())
def test_config(self):
config = appconfig('config:test.ini', relative_to=conf_dir)
config.local_conf['ckan.plugins'] = 'googleanalytics'
config.local_conf['googleanalytics.id'] = ''
command = LoadAnalytics("loadanalytics")
command.CONFIG = config.local_conf
self.assertRaises(Exception, command.run, [])
def take_action(self, opts):
config_spec = opts.config_file
section = opts.section_name
if section is None:
if '#' in config_spec:
config_spec, section = config_spec.split('#', 1)
else:
section = 'main'
if not ':' in section:
plain_section = section
section = 'app:' + section
else:
plain_section = section.split(':', 1)[0]
if not config_spec.startswith('config:'):
config_spec = 'config:' + config_spec
if plain_section != 'main':
config_spec += '#' + plain_section
config_file = config_spec[len('config:'):].split('#', 1)[0]
config_file = os.path.join(os.getcwd(), config_file)
conf = appconfig(config_spec, relative_to=os.getcwd())
ep_name = conf.context.entry_point_name
ep_group = conf.context.protocol
dist = conf.context.distribution
if dist is None:
raise RuntimeError(
"The section %r is not the application (probably a filter). You should add #section_name, where section_name is the section that configures your application"
% plain_section)
self._setup_config(dist,
config_file,
section, {},
verbosity=self.app.options.verbose_level)
def get_expiring_certs():
"""
Most of this function is from find_expiring_certs.py
"""
# load turbogears config, required for vhosts.connect to work
config = appconfig('config:' + os.path.abspath('development.ini'))
load_environment(config.global_conf, config.local_conf)
now = datetime.utcnow()
vhosts.connect()
res = vhosts.conn.search_s(
"ou=VirtualHosts,dc=scripts,dc=mit,dc=edu",
ldap.SCOPE_ONELEVEL,
"(&(objectClass=scriptsVhost)(scriptsVhostCertificate=*))",
[
"scriptsVhostName", "scriptsVhostAlias", "uid",
"scriptsVhostCertificate"
],
)
expiring = []
for _, attrs in res:
vhost, = attrs["scriptsVhostName"]
aliases = attrs.get("scriptsVhostAlias", [])
uid, = attrs["uid"]
scripts, = attrs["scriptsVhostCertificate"]
chain = cert.scripts_to_chain(scripts)
expires = cert.chain_notAfter(chain) - now
if expires < timedelta(days=14):
expiring.append((expires, uid, [vhost] + aliases))
expiring.sort()
return expiring
def get_config(path: str) -> dict:
try:
config = appconfig('config:%s' % path, relative_to=os.getcwd())
except Exception:
raise Exception(CONFIG_FILE_ERROR)
return config
def setup_config(command, filename, section, vars):
"""Place any commands to setup omnia here"""
conf = appconfig('config:' + filename)
load_environment(conf.global_conf, conf.local_conf)
logging.info("Creating tables")
meta.metadata.create_all(bind=meta.engine)
logging.info("Finished setting up")
def command(self):
config_spec = self.args[0]
section = self.options.section_name
if section is None:
if '#' in config_spec:
config_spec, section = config_spec.split('#', 1)
else:
section = 'main'
if not ':' in section:
plain_section = section
section = 'app:'+section
else:
plain_section = section.split(':', 1)[0]
if not config_spec.startswith('config:'):
config_spec = 'config:' + config_spec
if plain_section != 'main':
config_spec += '#' + plain_section
config_file = config_spec[len('config:'):].split('#', 1)[0]
config_file = os.path.join(os.getcwd(), config_file)
self.logging_file_config(config_file)
conf = appconfig(config_spec, relative_to=os.getcwd())
ep_name = conf.context.entry_point_name
ep_group = conf.context.protocol
dist = conf.context.distribution
if dist is None:
raise BadCommand(
"The section %r is not the application (probably a filter). You should add #section_name, where section_name is the section that configures your application" % plain_section)
installer = self.get_installer(dist, ep_group, ep_name)
installer.setup_config(
self, config_file, section, self.sysconfig_install_vars(installer))
self.call_sysconfig_functions(
'post_setup_hook', installer, config_file)
def get_wsgi_app_settings(self, name=None, defaults=None):
"""
Return an :class:`collections.OrderedDict` representing the
application config for a WSGI application named ``name`` in the
PasteDeploy config file specified by ``self.uri``.
``defaults``, if passed, should be a dictionary used as variable
assignments like ``{'http_port': 8080}``. This is useful if e.g.
``%(http_port)s`` is used in the config file.
If the ``name`` is None, this will attempt to parse the name from
the ``config_uri`` string expecting the format ``inifile#name``.
If no name is found, the name will default to "main".
:param name: The named WSGI app for which to find the settings.
Defaults to ``None`` which becomes ``main``
inside :func:`paste.deploy.loadapp`.
:param defaults: The ``global_conf`` that will be used during settings
generation.
:return: A :class:`plaster_pastedeploy.ConfigDict` of key/value pairs.
"""
name = self._maybe_get_default_name(name)
defaults = self._get_defaults(defaults)
conf = appconfig(
self.pastedeploy_spec,
name=name,
relative_to=self.relative_to,
global_conf=defaults,
)
return ConfigDict(conf.local_conf, conf.global_conf, self)
def load_config(config=None):
from paste.deploy import appconfig
if config:
filename = os.path.abspath(config)
config_source = u'-c parameter'
elif os.environ.get(u'CKAN_INI'):
filename = os.environ.get(u'CKAN_INI')
config_source = u'$CKAN_INI'
else:
default_filename = u'development.ini'
filename = os.path.join(os.getcwd(), default_filename)
if not os.path.exists(filename):
# give really clear error message for this common situation
msg = u'ERROR: You need to specify the CKAN config (.ini) '\
u'file path.'\
u'\nUse the --config parameter or set environment ' \
u'variable CKAN_INI or have {}\nin the current directory.' \
.format(default_filename)
exit(msg)
if not os.path.exists(filename):
msg = u'Config file not found: %s' % filename
msg += u'\n(Given by: %s)' % config_source
exit(msg)
loggingFileConfig(filename)
log.info(u'Using configuration file {}'.format(filename))
return appconfig(u'config:' + filename)
def test_config(self):
config = appconfig('config:test.ini', relative_to=conf_dir)
config.local_conf['ckan.plugins'] = 'googleanalytics'
config.local_conf['googleanalytics.id'] = ''
command = LoadAnalytics("loadanalytics")
command.CONFIG = config.local_conf
self.assertRaises(Exception, command.run, [])
def main(argv):
if len(argv) < 2:
print "call: updatecomponents.py ini_file"
exit(1)
ini_file = argv[1]
dictdata_path = 'quanthistling/dictdata'
conf = appconfig('config:' + ini_file, relative_to='.')
if not pylons.test.pylonsapp:
load_environment(conf.global_conf, conf.local_conf)
# Create the tables if they don't already exist
metadata.create_all(bind=Session.bind)
# update languages
for c in quanthistling.dictdata.components.list:
db_lang = Session.query(model.Component).filter_by(name=c['name']).first()
if db_lang == None:
component = model.Component()
component.name = c['name']
component.description = c['description']
Session.add(component)
Session.commit()
print("Inserted component " + c['name'] + ".")
def setup_config(command, filename, section, vars):
"""Place any commands to setup wohnungswarteliste here"""
conf = appconfig('config:' + filename)
load_environment(conf.global_conf, conf.local_conf)
log.info("Creating database tables")
model.meta.create_all(bind=model.engine)
log.info("Finished setting up")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('config', help='path of configuration file')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(
level=logging.INFO if args.verbose else logging.WARNING,
stream=sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
for package_extra in model.Session.query(model.PackageExtra).filter(
model.PackageExtra.key == 'supplier_id',
model.PackageExtra.package_id.in_(
model.Session.query(model.Package.id).filter(
model.Package.name.like('%-fork-%'))),
):
model.Session.delete(package_extra)
model.repo.commit_and_remove()
return 0
def setup_config(filename, relative_to):
if not relative_to or not os.path.exists(relative_to):
relative_to = "."
print 'Loading config "%s" at path "%s"...' % (filename, relative_to)
conf = appconfig('config:' + filename, relative_to=relative_to)
load_environment(conf.global_conf, conf.local_conf, False)
g._push_object(meta.globj) #zomg teh h4x
def __setup_thelma(self, ini_file):
here_dir = os.getcwd()
config_uri = 'config:%s' % ini_file
self.logging_file_config(ini_file) # pylint: disable=E1101
settings = appconfig(config_uri, 'thelma', relative_to=here_dir)
reg = Registry('thelma')
# Some tools need to resolve URLs, so we need to set up a request
# and a service.
url = 'http://0.0.0.0:6543'
req = DummyRequest(application_url=url,
host_url=url,
path_url=url,
url=url,
registry=reg)
config = create_config(settings, registry=reg)
config.setup_registry(settings=settings)
config.begin(request=req)
config.load_zcml('configure.zcml')
srvc = config.get_registered_utility(IService)
req.root = srvc
# Set up repositories.
repo_mgr = config.get_registered_utility(IRepositoryManager)
repo_mgr.initialize_all()
# Start the everest service.
srvc.start()
# Configure the session maker.
session_maker.configure(extension=ZopeTransactionExtension())
# Set up machinery to enforce a session.flush() just before the
# report is run so we have proper IDs in the output.
# FIXME: This should be encapsulated better, perhaps depending on
# an option that selects the backend.
def on_session_begin(session, trx, conn): # pylint: disable=W0613
self._report_callback = lambda sess = session: sess.flush()
event.listen(Session, 'after_begin', on_session_begin)
return config
def load_paste_config(app_name, options, args, config_dir=None):
"""
Looks for a config file to use for an app and returns the
config file path and a configuration mapping from a paste config file.
We search for the paste config file in the following order:
* If --config-file option is used, use that
* If args[0] is a file, use that
* Search for $app_name.conf in standard directories:
* .
* ~.config_dir/
* ~
* /etc/config_dir
* /etc
:param app_name: Name of the application to load config for, or None.
None signifies to only load the [DEFAULT] section of
the config file.
:param options: Set of typed options returned from parse_options()
:param args: Command line arguments from argv[1:]
:retval Tuple of (conf_file, conf)
:raises RuntimeError when config file cannot be located or there was a
problem loading the configuration file.
"""
conf_file = find_config_file(app_name, options, args, config_dir)
if not conf_file:
raise RuntimeError("Unable to locate any configuration file. " "Cannot load application %s" % app_name)
try:
conf = deploy.appconfig("config:%s" % conf_file, name=app_name)
return conf_file, conf
except Exception, e:
raise RuntimeError("Error trying to load config %s: %s" % (conf_file, e))
def main(argv):
if len(argv) < 2:
print "call: updatelanguages.py ini_file"
exit(1)
ini_file = argv[1]
dictdata_path = 'quanthistling/dictdata'
log = logging.getLogger()
logging.basicConfig(level=logging.INFO)
conf = appconfig('config:' + ini_file, relative_to='.')
if not pylons.test.pylonsapp:
load_environment(conf.global_conf, conf.local_conf)
# Create the tables if they don't already exist
metadata.create_all(bind=Session.bind)
# update languages
for l in quanthistling.dictdata.languages.list:
db_lang = Session.query(model.LanguageIso).filter_by(name=l['name']).first()
if db_lang == None:
lname = l['name']
#lname = importfunctions.normalize_stroke(lname)
#lname = unicodedata.normalize("NFD", lname)
language = model.LanguageIso()
language.name = l['name']
language.langcode = l['langcode']
language.description = l['description']
language.url = l['url']
Session.add(language)
Session.commit()
log.info("Inserted language " + l['name'] + ".")
def get_wsgi_app_settings(self, name=None, defaults=None):
"""
Return an :class:`collections.OrderedDict` representing the
application config for a WSGI application named ``name`` in the
PasteDeploy config file specified by ``self.uri``.
``defaults``, if passed, should be a dictionary used as variable
assignments like ``{'http_port': 8080}``. This is useful if e.g.
``%(http_port)s`` is used in the config file.
If the ``name`` is None, this will attempt to parse the name from
the ``config_uri`` string expecting the format ``inifile#name``.
If no name is found, the name will default to "main".
:param name: The named WSGI app for which to find the settings.
Defaults to ``None`` which becomes ``main``
inside :func:`paste.deploy.loadapp`.
:param defaults: The ``global_conf`` that will be used during settings
generation.
:return: A :class:`plaster_pastedeploy.ConfigDict` of key/value pairs.
"""
name = self._maybe_get_default_name(name)
defaults = self._get_defaults(defaults)
conf = appconfig(self.pastedeploy_spec,
name=name,
relative_to=self.relative_to,
global_conf=defaults)
return ConfigDict(conf.local_conf, conf.global_conf, self)
def main(argv):
if len(argv) < 2:
print "call: updateversion.py ini_file"
exit(1)
ini_file = argv[1]
conf = appconfig('config:' + ini_file, relative_to='.')
if not pylons.test.pylonsapp:
load_environment(conf.global_conf, conf.local_conf)
# Create the tables if they don't already exist
metadata.create_all(bind=Session.bind)
new_version = False
corpusversion = Session.query(model.Corpusversion).filter_by(version=version,revision=revision).first()
if corpusversion == None:
corpusversion = model.Corpusversion()
new_version = True
corpusversion.version = version
corpusversion.revision = revision
corpusversion.updated = datetime.datetime.now()
if new_version:
Session.add(corpusversion)
Session.commit()
def basic_setup(self):
global log, M
if self.args[0]:
# Probably being called from the command line - load the config
# file
self.config = conf = appconfig('config:%s' %
self.args[0], relative_to=os.getcwd())
# ... logging does not understand section#subsection syntax
logging_config = self.args[0].split('#')[0]
logging.config.fileConfig(
logging_config, disable_existing_loggers=False)
log = logging.getLogger('allura.command')
log.info('Initialize command with config %r', self.args[0])
load_environment(conf.global_conf, conf.local_conf)
self.setup_globals()
from allura import model
M = model
ming.configure(**conf)
if asbool(conf.get('activitystream.recording.enabled', False)):
activitystream.configure(**h.convert_bools(conf, prefix='activitystream.'))
pylons.tmpl_context.user = M.User.anonymous()
else:
# Probably being called from another script (websetup, perhaps?)
log = logging.getLogger('allura.command')
conf = pylons.config
self.tools = pylons.app_globals.entry_points['tool'].values()
for ep in h.iter_entry_points('allura.command_init'):
log.info('Running command_init for %s', ep.name)
ep.load()(conf)
log.info('Loaded tools')
def setup_config(command, filename, section, vars):
"""Place any commands to setup masterapp here"""
conf = appconfig('config:' + filename)
load_environment(conf.global_conf, conf.local_conf)
# Upgrade the database
"""
try:
mg.version_control(config['migrate.url'], config['migrate.repo'])
except DatabaseAlreadyControlledError:
log.info("DB exists, not versioning it")
v = mg.version(config['migrate.repo'])
dbv = mg.db_version(config['migrate.url'], config['migrate.repo'])
if v != dbv:
mg.upgrade(config['migrate.url'], config['migrate.repo'])
# Populate with fake data
if config['populate_model'] == 'true':
log.info("Populating data")
import masterapp.lib.populate_model as populate_model
populate_model.populate()
"""
log.info("Successfully setup")
def _setup(self):
"""
Set up logging, import pylons/paste/debexpo modules, parse config file, create config
class and chdir to the incoming directory.
"""
# Look for ini file
if not os.path.isfile(self.ini_file):
self._fail('Cannot find ini file')
self._setup_logging()
# Import debexpo root directory
sys.path.append(os.path.dirname(self.ini_file))
# Initialize Pylons app
conf = appconfig('config:' + self.ini_file)
pylons.config = load_environment(conf.global_conf, conf.local_conf)
# Change into the incoming directory
incoming_dir = pylons.config['debexpo.upload.incoming']
logging.info("Changing dir to %s", incoming_dir)
os.chdir(incoming_dir)
# Look for the changes file
if not os.path.isfile(self.changes_file):
self._fail('Cannot find changes file')
def __init__(self, uploader_conf, plugin_name, regex=None, cutoff=None):
super(LogUploader, self).__init__(uploader_conf)
log_name = "%s-log-uploader" % plugin_name
self.logger = utils.get_logger(uploader_conf, log_name, log_route=plugin_name)
self.log_dir = uploader_conf.get("log_dir", "/var/log/swift/")
self.swift_account = uploader_conf["swift_account"]
self.container_name = uploader_conf["container_name"]
proxy_server_conf_loc = uploader_conf.get("proxy_server_conf", "/etc/swift/proxy-server.conf")
proxy_server_conf = appconfig("config:%s" % proxy_server_conf_loc, name="proxy-server")
memcache = loadfilter("config:%s" % proxy_server_conf_loc, name="cache")(object).memcache
self.internal_proxy = InternalProxy(proxy_server_conf, memcache=memcache)
self.new_log_cutoff = int(cutoff or uploader_conf.get("new_log_cutoff", "7200"))
self.unlink_log = uploader_conf.get("unlink_log", "true").lower() in utils.TRUE_VALUES
self.filename_pattern = regex or uploader_conf.get(
"source_filename_pattern",
"""
^%s-
(?P<year>[0-9]{4})
(?P<month>[0-1][0-9])
(?P<day>[0-3][0-9])
(?P<hour>[0-2][0-9])
.*$"""
% plugin_name,
)
self.content_type = uploader_conf.get("content_type")
def test_for_stale_connection(*args, **kwargs):
try:
return func(*args, **kwargs)
except:
conf = appconfig('config:development.ini',
relative_to=os.path.join(
os.path.dirname(__file__), '../../'))
engine = create_engine(conf.get("mysql_address"))
conn = engine.connect()
metadata = MetaData(bind=engine)
# bind the tables
citations = Table("citations", metadata, autoload=True)
labels = Table("labels", metadata, autoload=True)
reviews = Table("projects", metadata, autoload=True)
users = Table("user", metadata, autoload=True)
labeled_features = Table("labeledfeatures",
metadata,
autoload=True)
encoded_status = Table("encodedstatuses", metadata, autoload=True)
prediction_status = Table("predictionstatuses",
metadata,
autoload=True)
predictions = Table("predictions", metadata, autoload=True)
priorities = Table("priorities", metadata, autoload=True)
# Actually execute.
return func(*args, **kwargs)
def setUp(self):
from paste.deploy import appconfig
os.environ['PYRAMID_SETTINGS'] = 'development.ini#main'
self.settings = appconfig('config:{}'.format(os.environ['PYRAMID_SETTINGS']), relative_to='.')
self.config = testing.setUp(settings=self.settings)
self.created_ids = []
def test_overlay_point_data(self):
conf = appconfig('config:test.ini', relative_to='.')
test_conf = load_environment(conf.global_conf, conf.local_conf)
s= NetCdfService(config=test_conf)
self.assertNotEqual(None, s.overlay_point_data('http://thredds-prod.nerc-lancaster.ac.uk/thredds/dodsC/ECOMAPSDetail/ECOMAPSInputLOI01.nc'))
def setUp(self):
from service import main
from paste.deploy import appconfig
from webtest import TestApp
# set settings
os.environ['PYRAMID_SETTINGS'] = 'development.ini#main'
self.settings = appconfig('config:{}'.format(os.environ['PYRAMID_SETTINGS']), relative_to='.')
app = main({}, **self.settings)
self.testapp = TestApp(app)
self.config = testing.setUp(settings=self.settings)
self.created_ids = []
# init user
self.user = User.create('*****@*****.**', 'hello')
payload = {
'email': '*****@*****.**',
'password': '******',
}
response = self.testapp.post_json('/api/v1/authenticate', payload, status=200)
self.access_token = response.json['access_token']
# init business
address = {
'street1': 'Central Park',
'city': 'New York',
'state': 'NY',
}
self.business = Business.create('Test Location', address)
