def update_from_ini(ini_file, app, commit): """ Pull in the relavant database and model parameters from pylons .INI This makes use of the ConfigParser embedded in paste.deploy.loadwsgi. It looks for a section called [app] -- the typical value is [app:main] The database URL is found from ``sqlalchemy.url`` -- eg: ``sqlalchemy.url = sqlite:///%(here)s/development.db`` The metadata for SQLalchemy is taken from a new entry : ``migrate.metadata`` -- eg: ``migrate.metadata = MYPROJ.model.meta:metadata`` Where the file model/meta.py contains the following : `` from sqlalchemy import schema metadata = schema.MetaData() # The declarative DataBase base Object Base = declarative_base(metadata=metadata) `` """ global _debug_messages ini_file = os.path.abspath(ini_file) if not os.path.exists(ini_file): raise OSError("File %s not found" % ini_file) config_dir = os.path.dirname(ini_file) cp = NicerConfigParser(ini_file) cp.read(ini_file) global_conf = cp.defaults() cp._defaults.setdefault("here", config_dir) cp._defaults.setdefault("__file__", ini_file) app_main = get_config(cp, app, None) url = app_main['sqlalchemy.url'] # model_egg = app_main['use'] # model = model_egg.replace('egg:', '') + '.model.meta:metadata' model_metadata = app_main['migrate.metadata'] if _debug_messages: print "url = '%s'" % url print "model = '%s'" % model_metadata update_pylons_db_from_model(url, model_metadata, commit)
def setup_self(setup_globals_func=setup_globals): """ Transform this module into a celery config module by reading the mediagoblin config file. Set the environment variable MEDIAGOBLIN_CONFIG to specify where this config file is at and what section it uses. By default it defaults to 'mediagoblin.ini:app:mediagoblin'. The first colon ":" is a delimiter between the filename and the config section, so in this case the filename is 'mediagoblin.ini' and the section where mediagoblin is defined is 'app:mediagoblin'. Args: - 'setup_globals_func': this is for testing purposes only. Don't set this! """ mgoblin_conf_file, mgoblin_section = os.environ.get("MEDIAGOBLIN_CONFIG", "mediagoblin.ini:app:mediagoblin").split( ":", 1 ) if not os.path.exists(mgoblin_conf_file): raise IOError("MEDIAGOBLIN_CONFIG not set or file does not exist") parser = NicerConfigParser(mgoblin_conf_file) parser.read(mgoblin_conf_file) parser._defaults.setdefault("here", os.path.dirname(os.path.abspath(mgoblin_conf_file))) parser._defaults.setdefault("__file__", os.path.abspath(mgoblin_conf_file)) mgoblin_section = dict(parser.items(mgoblin_section)) mgoblin_conf = dict([(section_name, dict(parser.items(section_name))) for section_name in parser.sections()]) setup_celery_from_config(mgoblin_section, mgoblin_conf, settings_module=OUR_MODULENAME, set_environ=False) connection = mongokit.Connection(mgoblin_section.get("db_host"), mgoblin_section.get("db_port")) db = connection[mgoblin_section.get("db_name", "mediagoblin")] models.register_models(connection) # Set up the storage systems. public_store = storage.storage_system_from_paste_config(mgoblin_section, "publicstore") queue_store = storage.storage_system_from_paste_config(mgoblin_section, "queuestore") setup_globals_func(db_connection=connection, database=db, public_store=public_store, queue_store=queue_store)
def _populate_with_variable_assignments(self, section): if section == 'DEFAULT' or section in self._sections_processed: return defaults = self.defaults() global_vars = dict(defaults) self._sections_processed[section] = global_vars get_from_globals = {} for option in self.options(section): if option.startswith('set '): name = option[4:].strip() global_vars[name] = NicerConfigParser.get(self, section, option) elif option.startswith('get '): name = option[4:].strip() get_from_globals[name] = NicerConfigParser.get(self, section, option) else: if option in defaults: # @@: It's a global option (?), so skip it continue get_from_globals.pop(option, None) for lhs, rhs in get_from_globals.items(): self.set(section, lhs, global_vars[rhs])
def __init__(self, ini_file=None, nologsetup=False): """ Set up the configuration ready for main and appmain to use. :params ini_file: None or ini file used to configure the webapp and logging. If this is None then the internal version will be used in the evasion.web.script directory """ self.log = logging.getLogger('evasion.web.scripts.runweb.Run') config_dir = os.path.dirname(__file__) if not ini_file: # If nothing was given for the config file use our internal version. self.iniFile = os.path.join(config_dir, 'development.ini') self.log = logging.getLogger("init: using internal ini file '%s'." % self.iniFile) else: self.iniFile = os.path.abspath(ini_file) if not nologsetup: logging.config.fileConfig(self.iniFile) self.log.debug("init: config dir:'%s'" % config_dir) self.log.debug("init: self.iniFile:'%s'" % self.iniFile) self.cp = NicerConfigParser(self.iniFile) self.cp.read(self.iniFile) self.globalConf = self.cp.defaults() self.cp._defaults.setdefault("here", os.path.abspath(os.curdir)) self.cp._defaults.setdefault("__file__", self.iniFile) self.serverConf = self.getConfig(self.cp, "server:main", "egg:Paste#http") self.appConf = self.getConfig(self.cp, "app:main", "egg:evasion-web") m = dict( state= self.cp.get('Messenger', 'state', 'off'), host = self.cp.get('Messenger', 'host', '127.0.0.1'), port = self.cp.get('Messenger', 'port', 61613), username = self.cp.get('Messenger', 'username', ''), password = self.cp.get('Messenger', 'password', ''), channel = self.cp.get('Messenger', 'channel', 'evasion'), ) self.messengerConf = m # Only set up if running as part of the director under the webadminctrl # controller. This runs the evasion.web as a thread instead of a separate # process. # self.directorIntegrationServer = None self.directorIntegrationIsRunning = False
def setup_self(setup_globals_func=setup_globals): """ Transform this module into a celery config module by reading the mediagoblin config file. Set the environment variable MEDIAGOBLIN_CONFIG to specify where this config file is at and what section it uses. By default it defaults to 'mediagoblin.ini:app:mediagoblin'. The first colon ":" is a delimiter between the filename and the config section, so in this case the filename is 'mediagoblin.ini' and the section where mediagoblin is defined is 'app:mediagoblin'. Args: - 'setup_globals_func': this is for testing purposes only. Don't set this! """ mgoblin_conf_file, mgoblin_section = os.environ.get( 'MEDIAGOBLIN_CONFIG', 'mediagoblin.ini:app:mediagoblin').split(':', 1) if not os.path.exists(mgoblin_conf_file): raise IOError("MEDIAGOBLIN_CONFIG not set or file does not exist") parser = NicerConfigParser(mgoblin_conf_file) parser.read(mgoblin_conf_file) parser._defaults.setdefault( 'here', os.path.dirname(os.path.abspath(mgoblin_conf_file))) parser._defaults.setdefault('__file__', os.path.abspath(mgoblin_conf_file)) mgoblin_section = dict(parser.items(mgoblin_section)) mgoblin_conf = dict([(section_name, dict(parser.items(section_name))) for section_name in parser.sections()]) setup_celery_from_config(mgoblin_section, mgoblin_conf, settings_module=OUR_MODULENAME, set_environ=False) connection = mongokit.Connection(mgoblin_section.get('db_host'), mgoblin_section.get('db_port')) db = connection[mgoblin_section.get('db_name', 'mediagoblin')] models.register_models(connection) # Set up the storage systems. public_store = storage.storage_system_from_paste_config( mgoblin_section, 'publicstore') queue_store = storage.storage_system_from_paste_config( mgoblin_section, 'queuestore') setup_globals_func(db_connection=connection, database=db, public_store=public_store, queue_store=queue_store)
def from_spec_force_utf8(cls, spec, **kw): cp = ConfigParser() cp.readfp(codecs.open(resource_spec(spec), "r", "utf8")) return cls(cp, **kw)
def read(self, filename): NicerConfigParser.read(self, filename) ini_file = Path(filename).absolute() self._defaults.setdefault("__file__", ini_file) self._defaults.setdefault("here", ini_file.parent)
def remove_section(self, section): if section in self._sections_processed: del self._sections_processed[section] return NicerConfigParser.remove_section(self, section)
def options(self, section): self._populate_with_variable_assignments(section) return NicerConfigParser.options(self, section)
def items(self, section, raw=False, vars=None): self._populate_with_variable_assignments(section) return NicerConfigParser.items(self, section, raw=raw, vars=vars)
def get(self, section, option, raw=False, vars=None, fallback=None): self._populate_with_variable_assignments(section) if sys.version_info < (3, 0): return NicerConfigParser.get(self, section, option, raw=raw, vars=vars) else: return NicerConfigParser.get(self, section, option, raw=raw, vars=vars, fallback=fallback)
def __init__(self, *args, **kwargs): NicerConfigParser.__init__(self, *args, **kwargs) self._sections_processed = {}
class Run(object): def __init__(self, ini_file=None, nologsetup=False): """ Set up the configuration ready for main and appmain to use. :params ini_file: None or ini file used to configure the webapp and logging. If this is None then the internal version will be used in the evasion.web.script directory """ self.log = logging.getLogger('evasion.web.scripts.runweb.Run') config_dir = os.path.dirname(__file__) if not ini_file: # If nothing was given for the config file use our internal version. self.iniFile = os.path.join(config_dir, 'development.ini') self.log = logging.getLogger("init: using internal ini file '%s'." % self.iniFile) else: self.iniFile = os.path.abspath(ini_file) if not nologsetup: logging.config.fileConfig(self.iniFile) self.log.debug("init: config dir:'%s'" % config_dir) self.log.debug("init: self.iniFile:'%s'" % self.iniFile) self.cp = NicerConfigParser(self.iniFile) self.cp.read(self.iniFile) self.globalConf = self.cp.defaults() self.cp._defaults.setdefault("here", os.path.abspath(os.curdir)) self.cp._defaults.setdefault("__file__", self.iniFile) self.serverConf = self.getConfig(self.cp, "server:main", "egg:Paste#http") self.appConf = self.getConfig(self.cp, "app:main", "egg:evasion-web") m = dict( state= self.cp.get('Messenger', 'state', 'off'), host = self.cp.get('Messenger', 'host', '127.0.0.1'), port = self.cp.get('Messenger', 'port', 61613), username = self.cp.get('Messenger', 'username', ''), password = self.cp.get('Messenger', 'password', ''), channel = self.cp.get('Messenger', 'channel', 'evasion'), ) self.messengerConf = m # Only set up if running as part of the director under the webadminctrl # controller. This runs the evasion.web as a thread instead of a separate # process. # self.directorIntegrationServer = None self.directorIntegrationIsRunning = False def setupapp(self): """ Run the equivalent paster setup-app """ from evasion.web import websetup class O: global_conf = self.globalConf local_conf = self.appConf command='?' # not sure what to do here conf = O() websetup.setup_app(command, conf, vars) def getConfig(self, cp, section, expected_use_value): """Get a section from an INI-style config file as a dict. ``cp`` -- NicerConfigParser. ``section`` -- the section to read. ``expected_use_value`` -- expected value of ``use`` option in the section. Aborts if the value of ``use`` doesn't equal the expected value. This indicates Paster would instantiate a different object than we're expecting. The ``use`` key is removed from the dict before returning. """ defaults = self.cp.defaults() ret = {} for option in self.cp.options(section): if option.startswith("set "): # Override a global option. option = option[4:] elif option in defaults: # Don't carry over other global options. continue ret[option] = self.cp.get(section, option) use = ret.pop("use", "") if use != expected_use_value: msg = ("unexpected value for 'use=' in section '%s': " "expected '%s', found '%s'") msg %= (section, expected_use_value, use) raise EnvironmentError(msg) return ret def appmainSetup(self): """ Called to create the wsgi app ready for appmain or directorIntegration to use. """ app = app_factory(self.globalConf, **self.appConf) return app def appmain(self, isExit): """ Called to run inside its own thread once twisted has taken over the main loop. """ app = self.appmainSetup() self.log.info("appmain: Serving webapp") server_runner(app, self.globalConf, **self.serverConf) def directorIntegrationStart(self): """ Create a server which the director webadminctrl controller will use to run the webapp, when start is called. The controller will also be able to stop the webapp via shutdown. """ # You don't need this in integration mode as it is part # of the director's messaging system. If this is kept in # it will cause messages to be resent onto the message # bus in error. # #self.setUpStomp() self.log.info("directorIntegrationStart: creating wsgi_app") wsgi_app = self.appmainSetup() # Use threadpool to also get access to server_close() self.serverConf['use_threadpool'] = True # Don't start serving straigh away, return so I can # store the server handle to close it later. self.serverConf['start_loop'] = False self.log.info("directorIntegrationStart: creating server.") self.directorIntegrationServer = serve(wsgi_app, **self.serverConf) try: self.directorIntegrationIsRunning = True self.log.info("directorIntegrationStart: serving until stopped.") self.directorIntegrationServer.serve_forever() except KeyboardInterrupt: # allow CTRL+C to shutdown self.log.warn("directorIntegrationStart: KeyboardInterrupt! Stopping... ") except: self.log.exception("directorIntegrationStart Error - ") self.directorIntegrationIsRunning = False self.log.info("directorIntegrationStart: server stopped.") def directorIntegrationIsStarted(self): return self.directorIntegrationIsRunning def directorIntegrationStop(self): """Stop the server handling any more requests.""" if not self.directorIntegrationServer: self.log.error("directorIntegrationStop: directorIntegrationStart not called to set up server!") else: self.log.info("directorIntegrationStop: telling server to close.") self.directorIntegrationServer.server_close() self.log.info("directorIntegrationStop: server close called ok.") def setUpStomp(self): """Connect to the broker so we can send/receive messages.""" # Only import if we use it: from evasion import messenger stomp_cfg = dict( host = self.cp.get("Messenger", "host"), port = int(self.cp.get("Messenger", "port")), username = self.cp.get("Messenger", "username"), password = self.cp.get("Messenger", "password"), channel = self.cp.get("Messenger", "channel"), ) self.log.info("appmain: setting up stomp connection") messenger.stompprotocol.setup(stomp_cfg) def main(self): """ Called to run twisted in the mainloop so messaging will work correctly. The webapp will be run via appmain. """ if self.messengerConf['state'] == 'on': # Only import if we use it: from evasion import messenger self.setUpStomp() self.log.info("main: running mainloop until done.") messenger.run(self.appmain) self.log.info("main: Exiting.") else: self.log.info("main: running mainloop (no messenger).") self.appmain(None) self.log.info("main: Exiting.")