def update_from_ini(ini_file, app, commit):
""" Pull in the relavant database and model parameters from pylons .INI
This makes use of the ConfigParser embedded in paste.deploy.loadwsgi.
It looks for a section called [app]
-- the typical value is [app:main]
The database URL is found from ``sqlalchemy.url``
-- eg: ``sqlalchemy.url = sqlite:///%(here)s/development.db``
The metadata for SQLalchemy is taken from a new entry : ``migrate.metadata``
-- eg: ``migrate.metadata = MYPROJ.model.meta:metadata``
Where the file model/meta.py contains the following :
``
from sqlalchemy import schema
metadata = schema.MetaData()
# The declarative DataBase base Object
Base = declarative_base(metadata=metadata)
``
"""
global _debug_messages
ini_file = os.path.abspath(ini_file)
if not os.path.exists(ini_file):
raise OSError("File %s not found" % ini_file)
config_dir = os.path.dirname(ini_file)
cp = NicerConfigParser(ini_file)
cp.read(ini_file)
global_conf = cp.defaults()
cp._defaults.setdefault("here", config_dir)
cp._defaults.setdefault("__file__", ini_file)
app_main = get_config(cp, app, None)
url = app_main['sqlalchemy.url']
# model_egg = app_main['use']
# model = model_egg.replace('egg:', '') + '.model.meta:metadata'
model_metadata = app_main['migrate.metadata']
if _debug_messages:
print "url = '%s'" % url
print "model = '%s'" % model_metadata
update_pylons_db_from_model(url, model_metadata, commit)
def setup_self(setup_globals_func=setup_globals):
"""
Transform this module into a celery config module by reading the
mediagoblin config file. Set the environment variable
MEDIAGOBLIN_CONFIG to specify where this config file is at and
what section it uses.
By default it defaults to 'mediagoblin.ini:app:mediagoblin'.
The first colon ":" is a delimiter between the filename and the
config section, so in this case the filename is 'mediagoblin.ini'
and the section where mediagoblin is defined is 'app:mediagoblin'.
Args:
- 'setup_globals_func': this is for testing purposes only. Don't
set this!
"""
mgoblin_conf_file, mgoblin_section = os.environ.get("MEDIAGOBLIN_CONFIG", "mediagoblin.ini:app:mediagoblin").split(
":", 1
)
if not os.path.exists(mgoblin_conf_file):
raise IOError("MEDIAGOBLIN_CONFIG not set or file does not exist")
parser = NicerConfigParser(mgoblin_conf_file)
parser.read(mgoblin_conf_file)
parser._defaults.setdefault("here", os.path.dirname(os.path.abspath(mgoblin_conf_file)))
parser._defaults.setdefault("__file__", os.path.abspath(mgoblin_conf_file))
mgoblin_section = dict(parser.items(mgoblin_section))
mgoblin_conf = dict([(section_name, dict(parser.items(section_name))) for section_name in parser.sections()])
setup_celery_from_config(mgoblin_section, mgoblin_conf, settings_module=OUR_MODULENAME, set_environ=False)
connection = mongokit.Connection(mgoblin_section.get("db_host"), mgoblin_section.get("db_port"))
db = connection[mgoblin_section.get("db_name", "mediagoblin")]
models.register_models(connection)
# Set up the storage systems.
public_store = storage.storage_system_from_paste_config(mgoblin_section, "publicstore")
queue_store = storage.storage_system_from_paste_config(mgoblin_section, "queuestore")
setup_globals_func(db_connection=connection, database=db, public_store=public_store, queue_store=queue_store)
def _populate_with_variable_assignments(self, section):
if section == 'DEFAULT' or section in self._sections_processed:
return
defaults = self.defaults()
global_vars = dict(defaults)
self._sections_processed[section] = global_vars
get_from_globals = {}
for option in self.options(section):
if option.startswith('set '):
name = option[4:].strip()
global_vars[name] = NicerConfigParser.get(self, section, option)
elif option.startswith('get '):
name = option[4:].strip()
get_from_globals[name] = NicerConfigParser.get(self, section, option)
else:
if option in defaults:
# @@: It's a global option (?), so skip it
continue
get_from_globals.pop(option, None)
for lhs, rhs in get_from_globals.items():
self.set(section, lhs, global_vars[rhs])
def __init__(self, ini_file=None, nologsetup=False):
"""
Set up the configuration ready for main and appmain to use.
:params ini_file: None or ini file used to configure the webapp
and logging. If this is None then the internal version will be
used in the evasion.web.script directory
"""
self.log = logging.getLogger('evasion.web.scripts.runweb.Run')
config_dir = os.path.dirname(__file__)
if not ini_file:
# If nothing was given for the config file use our internal version.
self.iniFile = os.path.join(config_dir, 'development.ini')
self.log = logging.getLogger("init: using internal ini file '%s'." % self.iniFile)
else:
self.iniFile = os.path.abspath(ini_file)
if not nologsetup:
logging.config.fileConfig(self.iniFile)
self.log.debug("init: config dir:'%s'" % config_dir)
self.log.debug("init: self.iniFile:'%s'" % self.iniFile)
self.cp = NicerConfigParser(self.iniFile)
self.cp.read(self.iniFile)
self.globalConf = self.cp.defaults()
self.cp._defaults.setdefault("here", os.path.abspath(os.curdir))
self.cp._defaults.setdefault("__file__", self.iniFile)
self.serverConf = self.getConfig(self.cp, "server:main", "egg:Paste#http")
self.appConf = self.getConfig(self.cp, "app:main", "egg:evasion-web")
m = dict(
state= self.cp.get('Messenger', 'state', 'off'),
host = self.cp.get('Messenger', 'host', '127.0.0.1'),
port = self.cp.get('Messenger', 'port', 61613),
username = self.cp.get('Messenger', 'username', ''),
password = self.cp.get('Messenger', 'password', ''),
channel = self.cp.get('Messenger', 'channel', 'evasion'),
)
self.messengerConf = m
# Only set up if running as part of the director under the webadminctrl
# controller. This runs the evasion.web as a thread instead of a separate
# process.
#
self.directorIntegrationServer = None
self.directorIntegrationIsRunning = False
def setup_self(setup_globals_func=setup_globals):
"""
Transform this module into a celery config module by reading the
mediagoblin config file. Set the environment variable
MEDIAGOBLIN_CONFIG to specify where this config file is at and
what section it uses.
By default it defaults to 'mediagoblin.ini:app:mediagoblin'.
The first colon ":" is a delimiter between the filename and the
config section, so in this case the filename is 'mediagoblin.ini'
and the section where mediagoblin is defined is 'app:mediagoblin'.
Args:
- 'setup_globals_func': this is for testing purposes only. Don't
set this!
"""
mgoblin_conf_file, mgoblin_section = os.environ.get(
'MEDIAGOBLIN_CONFIG', 'mediagoblin.ini:app:mediagoblin').split(':', 1)
if not os.path.exists(mgoblin_conf_file):
raise IOError("MEDIAGOBLIN_CONFIG not set or file does not exist")
parser = NicerConfigParser(mgoblin_conf_file)
parser.read(mgoblin_conf_file)
parser._defaults.setdefault(
'here', os.path.dirname(os.path.abspath(mgoblin_conf_file)))
parser._defaults.setdefault('__file__', os.path.abspath(mgoblin_conf_file))
mgoblin_section = dict(parser.items(mgoblin_section))
mgoblin_conf = dict([(section_name, dict(parser.items(section_name)))
for section_name in parser.sections()])
setup_celery_from_config(mgoblin_section,
mgoblin_conf,
settings_module=OUR_MODULENAME,
set_environ=False)
connection = mongokit.Connection(mgoblin_section.get('db_host'),
mgoblin_section.get('db_port'))
db = connection[mgoblin_section.get('db_name', 'mediagoblin')]
models.register_models(connection)
# Set up the storage systems.
public_store = storage.storage_system_from_paste_config(
mgoblin_section, 'publicstore')
queue_store = storage.storage_system_from_paste_config(
mgoblin_section, 'queuestore')
setup_globals_func(db_connection=connection,
database=db,
public_store=public_store,
queue_store=queue_store)
def from_spec_force_utf8(cls, spec, **kw):
cp = ConfigParser()
cp.readfp(codecs.open(resource_spec(spec), "r", "utf8"))
return cls(cp, **kw)
def read(self, filename):
NicerConfigParser.read(self, filename)
ini_file = Path(filename).absolute()
self._defaults.setdefault("__file__", ini_file)
self._defaults.setdefault("here", ini_file.parent)
def remove_section(self, section):
if section in self._sections_processed:
del self._sections_processed[section]
return NicerConfigParser.remove_section(self, section)
def options(self, section):
self._populate_with_variable_assignments(section)
return NicerConfigParser.options(self, section)
def items(self, section, raw=False, vars=None):
self._populate_with_variable_assignments(section)
return NicerConfigParser.items(self, section, raw=raw, vars=vars)
def get(self, section, option, raw=False, vars=None, fallback=None):
self._populate_with_variable_assignments(section)
if sys.version_info < (3, 0):
return NicerConfigParser.get(self, section, option, raw=raw, vars=vars)
else:
return NicerConfigParser.get(self, section, option, raw=raw, vars=vars, fallback=fallback)
def __init__(self, *args, **kwargs):
NicerConfigParser.__init__(self, *args, **kwargs)
self._sections_processed = {}
class Run(object):
def __init__(self, ini_file=None, nologsetup=False):
"""
Set up the configuration ready for main and appmain to use.
:params ini_file: None or ini file used to configure the webapp
and logging. If this is None then the internal version will be
used in the evasion.web.script directory
"""
self.log = logging.getLogger('evasion.web.scripts.runweb.Run')
config_dir = os.path.dirname(__file__)
if not ini_file:
# If nothing was given for the config file use our internal version.
self.iniFile = os.path.join(config_dir, 'development.ini')
self.log = logging.getLogger("init: using internal ini file '%s'." % self.iniFile)
else:
self.iniFile = os.path.abspath(ini_file)
if not nologsetup:
logging.config.fileConfig(self.iniFile)
self.log.debug("init: config dir:'%s'" % config_dir)
self.log.debug("init: self.iniFile:'%s'" % self.iniFile)
self.cp = NicerConfigParser(self.iniFile)
self.cp.read(self.iniFile)
self.globalConf = self.cp.defaults()
self.cp._defaults.setdefault("here", os.path.abspath(os.curdir))
self.cp._defaults.setdefault("__file__", self.iniFile)
self.serverConf = self.getConfig(self.cp, "server:main", "egg:Paste#http")
self.appConf = self.getConfig(self.cp, "app:main", "egg:evasion-web")
m = dict(
state= self.cp.get('Messenger', 'state', 'off'),
host = self.cp.get('Messenger', 'host', '127.0.0.1'),
port = self.cp.get('Messenger', 'port', 61613),
username = self.cp.get('Messenger', 'username', ''),
password = self.cp.get('Messenger', 'password', ''),
channel = self.cp.get('Messenger', 'channel', 'evasion'),
)
self.messengerConf = m
# Only set up if running as part of the director under the webadminctrl
# controller. This runs the evasion.web as a thread instead of a separate
# process.
#
self.directorIntegrationServer = None
self.directorIntegrationIsRunning = False
def setupapp(self):
"""
Run the equivalent paster setup-app
"""
from evasion.web import websetup
class O:
global_conf = self.globalConf
local_conf = self.appConf
command='?' # not sure what to do here
conf = O()
websetup.setup_app(command, conf, vars)
def getConfig(self, cp, section, expected_use_value):
"""Get a section from an INI-style config file as a dict.
``cp`` -- NicerConfigParser.
``section`` -- the section to read.
``expected_use_value`` -- expected value of ``use`` option in the section.
Aborts if the value of ``use`` doesn't equal the expected value. This
indicates Paster would instantiate a different object than we're expecting.
The ``use`` key is removed from the dict before returning.
"""
defaults = self.cp.defaults()
ret = {}
for option in self.cp.options(section):
if option.startswith("set "): # Override a global option.
option = option[4:]
elif option in defaults: # Don't carry over other global options.
continue
ret[option] = self.cp.get(section, option)
use = ret.pop("use", "")
if use != expected_use_value:
msg = ("unexpected value for 'use=' in section '%s': "
"expected '%s', found '%s'")
msg %= (section, expected_use_value, use)
raise EnvironmentError(msg)
return ret
def appmainSetup(self):
"""
Called to create the wsgi app ready for appmain or
directorIntegration to use.
"""
app = app_factory(self.globalConf, **self.appConf)
return app
def appmain(self, isExit):
"""
Called to run inside its own thread once twisted has taken over
the main loop.
"""
app = self.appmainSetup()
self.log.info("appmain: Serving webapp")
server_runner(app, self.globalConf, **self.serverConf)
def directorIntegrationStart(self):
"""
Create a server which the director webadminctrl controller
will use to run the webapp, when start is called. The
controller will also be able to stop the webapp via shutdown.
"""
# You don't need this in integration mode as it is part
# of the director's messaging system. If this is kept in
# it will cause messages to be resent onto the message
# bus in error.
#
#self.setUpStomp()
self.log.info("directorIntegrationStart: creating wsgi_app")
wsgi_app = self.appmainSetup()
# Use threadpool to also get access to server_close()
self.serverConf['use_threadpool'] = True
# Don't start serving straigh away, return so I can
# store the server handle to close it later.
self.serverConf['start_loop'] = False
self.log.info("directorIntegrationStart: creating server.")
self.directorIntegrationServer = serve(wsgi_app, **self.serverConf)
try:
self.directorIntegrationIsRunning = True
self.log.info("directorIntegrationStart: serving until stopped.")
self.directorIntegrationServer.serve_forever()
except KeyboardInterrupt:
# allow CTRL+C to shutdown
self.log.warn("directorIntegrationStart: KeyboardInterrupt! Stopping... ")
except:
self.log.exception("directorIntegrationStart Error - ")
self.directorIntegrationIsRunning = False
self.log.info("directorIntegrationStart: server stopped.")
def directorIntegrationIsStarted(self):
return self.directorIntegrationIsRunning
def directorIntegrationStop(self):
"""Stop the server handling any more requests."""
if not self.directorIntegrationServer:
self.log.error("directorIntegrationStop: directorIntegrationStart not called to set up server!")
else:
self.log.info("directorIntegrationStop: telling server to close.")
self.directorIntegrationServer.server_close()
self.log.info("directorIntegrationStop: server close called ok.")
def setUpStomp(self):
"""Connect to the broker so we can send/receive messages."""
# Only import if we use it:
from evasion import messenger
stomp_cfg = dict(
host = self.cp.get("Messenger", "host"),
port = int(self.cp.get("Messenger", "port")),
username = self.cp.get("Messenger", "username"),
password = self.cp.get("Messenger", "password"),
channel = self.cp.get("Messenger", "channel"),
)
self.log.info("appmain: setting up stomp connection")
messenger.stompprotocol.setup(stomp_cfg)
def main(self):
"""
Called to run twisted in the mainloop so messaging will work correctly.
The webapp will be run via appmain.
"""
if self.messengerConf['state'] == 'on':
# Only import if we use it:
from evasion import messenger
self.setUpStomp()
self.log.info("main: running mainloop until done.")
messenger.run(self.appmain)
self.log.info("main: Exiting.")
else:
self.log.info("main: running mainloop (no messenger).")
self.appmain(None)
self.log.info("main: Exiting.")
def read(self, filename):
NicerConfigParser.read(self, filename)
ini_file = Path(filename).absolute()
self._defaults.setdefault("__file__", ini_file)
self._defaults.setdefault("here", ini_file.parent)
