def main(): parser = argparse.ArgumentParser(description="Winchester pipeline worker") parser.add_argument('--config', '-c', default='winchester.yaml', help='The name of the winchester config file') parser.add_argument('--daemon', '-d', help='Run in daemon mode.') args = parser.parse_args() conf = ConfigManager.load_config_file(args.config) if 'logging_config' in conf: fileConfig(conf['logging_config']) else: logging.basicConfig() if 'log_level' in conf: level = conf['log_level'] level = getattr(logging, level.upper()) logging.getLogger('winchester').setLevel(level) pipe = PipelineManager(conf) if args.daemon: print "Backgrounding for daemon mode." with daemon.DaemonContext(): pipe.run() else: pipe.run()
def __init__(self, conf): super(EventProcessor, self).__init__(conf) self._winchester_config = conf.winchester.winchester_config self._config_mgr = ConfigManager.load_config_file(self._winchester_config) self._trigger_manager = TriggerManager(self._config_mgr) self._group = conf.kafka.stream_def_group self._tm_lock = threading.Lock()
def __init__(self, conf): super(PipelineProcessor, self).__init__(conf) self._winchester_config = conf.winchester.winchester_config self._config_mgr = ConfigManager.load_config_file( self._winchester_config) self._group = conf.kafka.stream_def_pipe_group self._pm_lock = threading.Lock() self._pipe = PipelineManager(self._config_mgr)
def main(): parser = argparse.ArgumentParser(description="Winchester pipeline worker") parser.add_argument('--config', '-c', default='winchester.yaml', help='The name of the winchester config file') parser.add_argument('--name', '-n', default='pipeline_worker', help='The name of this process for logging purposes') parser.add_argument('--daemon', '-d', help='Run in daemon mode.') args = parser.parse_args() conf = ConfigManager.load_config_file(args.config) proc_name = args.name if 'log_level' in conf: level = conf['log_level'] level = getattr(logging, level.upper()) else: level = logging.INFO if 'log_file' in conf: log_file = conf['log_file'] % dict(proc_name=proc_name) else: log_file = '%(proc_name)s.log' % dict(proc_name=proc_name) # This is a hack, but it's needed to pass the logfile name & default # loglevel into log handlers configured with a config file. (mdragon) logging.LOCAL_LOG_FILE = log_file logging.LOCAL_DEFAULT_LEVEL = level if 'logging_config' in conf: fileConfig(conf['logging_config']) else: logging.basicConfig() logging.getLogger('winchester').setLevel(level) timesync = time_sync.TimeSync(conf) pipe = PipelineManager(conf, time_sync=timesync, proc_name=proc_name) if args.daemon: print("Backgrounding for daemon mode.") with daemon.DaemonContext(): pipe.run() else: pipe.run()
def __init__(self, config, scratchpad): """config is a ConfigParser object. Use the scratchpad to ensure we don't create multiple connections to the db. """ if 'quincy_config' not in scratchpad: target = config.get('quince', 'winchester_config') logger.debug("Quince is using Winchester config from %s" % target) quincy_config = ConfigManager.load_config_file(target) quincy_config = ConfigManager.wrap(quincy_config, self.config_description()) scratchpad['quincy_config'] = quincy_config scratchpad['quincy_driver'] = DBInterface( quincy_config['database']) self.winchester_config = scratchpad['quincy_config'] self.driver = scratchpad['quincy_driver']
def __init__(self, app=None, queue_name=None): super(WinchesterHandler, self).__init__(app=app, queue_name=queue_name) conf_file = self.config_get("config_file") config = ConfigManager.load_config_file(conf_file) self.time_sync = time_sync.TimeSync(config, publishes=True) self.trigger_manager = TriggerManager(config, time_sync=self.time_sync)
def __init__(self, winchester_config): self.winchester_config = winchester_config self.config = ConfigManager.load_config_file(winchester_config) self.trigger_manager = TriggerManager(self.config) self.pipe = PipelineManager(self.config)
def __init__(self, kafka_config, winchester_config): self.winchester_config = winchester_config self.kafka_config = kafka_config self.config_mgr = ConfigManager.load_config_file( self.winchester_config)
config = context.config # Interpret the config file for Python logging. # This line sets up loggers basically. # fileConfig(config.config_file_name) # add your model's MetaData object here # for 'autogenerate' support # from myapp import mymodel # target_metadata = mymodel.Base.metadata from winchester.config import ConfigManager from winchester.models import Base target_metadata = Base.metadata winchester_config = ConfigManager.load_config_file( config.get_main_option("winchester_config")) # other values from the config, defined by the needs of env.py, # can be acquired: # my_important_option = config.get_main_option("my_important_option") # ... etc. def run_migrations_offline(): """Run migrations in 'offline' mode. This configures the context with just a URL and not an Engine, though an Engine is acceptable here as well. By skipping the Engine creation we don't even need a DBAPI to be available. Calls to context.execute() here emit the given string to the
def __init__(self, app=None, queue_name=None): super(WinchesterHandler, self).__init__(app=app, queue_name=queue_name) conf_file = self.config_get("config_file") config = ConfigManager.load_config_file(conf_file) self.trigger_manager = TriggerManager(config)