def main():
parser = argparse.ArgumentParser(description="Winchester pipeline worker")
parser.add_argument('--config',
'-c',
default='winchester.yaml',
help='The name of the winchester config file')
parser.add_argument('--daemon', '-d', help='Run in daemon mode.')
args = parser.parse_args()
conf = ConfigManager.load_config_file(args.config)
if 'logging_config' in conf:
fileConfig(conf['logging_config'])
else:
logging.basicConfig()
if 'log_level' in conf:
level = conf['log_level']
level = getattr(logging, level.upper())
logging.getLogger('winchester').setLevel(level)
pipe = PipelineManager(conf)
if args.daemon:
print "Backgrounding for daemon mode."
with daemon.DaemonContext():
pipe.run()
else:
pipe.run()
def __init__(self, conf):
super(EventProcessor, self).__init__(conf)
self._winchester_config = conf.winchester.winchester_config
self._config_mgr = ConfigManager.load_config_file(self._winchester_config)
self._trigger_manager = TriggerManager(self._config_mgr)
self._group = conf.kafka.stream_def_group
self._tm_lock = threading.Lock()
def __init__(self, conf):
super(PipelineProcessor, self).__init__(conf)
self._winchester_config = conf.winchester.winchester_config
self._config_mgr = ConfigManager.load_config_file(
self._winchester_config)
self._group = conf.kafka.stream_def_pipe_group
self._pm_lock = threading.Lock()
self._pipe = PipelineManager(self._config_mgr)
def main():
parser = argparse.ArgumentParser(description="Winchester pipeline worker")
parser.add_argument('--config',
'-c',
default='winchester.yaml',
help='The name of the winchester config file')
parser.add_argument('--name',
'-n',
default='pipeline_worker',
help='The name of this process for logging purposes')
parser.add_argument('--daemon', '-d', help='Run in daemon mode.')
args = parser.parse_args()
conf = ConfigManager.load_config_file(args.config)
proc_name = args.name
if 'log_level' in conf:
level = conf['log_level']
level = getattr(logging, level.upper())
else:
level = logging.INFO
if 'log_file' in conf:
log_file = conf['log_file'] % dict(proc_name=proc_name)
else:
log_file = '%(proc_name)s.log' % dict(proc_name=proc_name)
# This is a hack, but it's needed to pass the logfile name & default
# loglevel into log handlers configured with a config file. (mdragon)
logging.LOCAL_LOG_FILE = log_file
logging.LOCAL_DEFAULT_LEVEL = level
if 'logging_config' in conf:
fileConfig(conf['logging_config'])
else:
logging.basicConfig()
logging.getLogger('winchester').setLevel(level)
timesync = time_sync.TimeSync(conf)
pipe = PipelineManager(conf, time_sync=timesync, proc_name=proc_name)
if args.daemon:
print("Backgrounding for daemon mode.")
with daemon.DaemonContext():
pipe.run()
else:
pipe.run()
def __init__(self, config, scratchpad):
"""config is a ConfigParser object.
Use the scratchpad to ensure we don't create multiple
connections to the db.
"""
if 'quincy_config' not in scratchpad:
target = config.get('quince', 'winchester_config')
logger.debug("Quince is using Winchester config from %s" % target)
quincy_config = ConfigManager.load_config_file(target)
quincy_config = ConfigManager.wrap(quincy_config,
self.config_description())
scratchpad['quincy_config'] = quincy_config
scratchpad['quincy_driver'] = DBInterface(
quincy_config['database'])
self.winchester_config = scratchpad['quincy_config']
self.driver = scratchpad['quincy_driver']
def main():
parser = argparse.ArgumentParser(description="Winchester pipeline worker")
parser.add_argument('--config', '-c', default='winchester.yaml',
help='The name of the winchester config file')
parser.add_argument('--name', '-n', default='pipeline_worker',
help='The name of this process for logging purposes')
parser.add_argument('--daemon', '-d', help='Run in daemon mode.')
args = parser.parse_args()
conf = ConfigManager.load_config_file(args.config)
proc_name = args.name
if 'log_level' in conf:
level = conf['log_level']
level = getattr(logging, level.upper())
else:
level = logging.INFO
if 'log_file' in conf:
log_file = conf['log_file'] % dict(proc_name=proc_name)
else:
log_file = '%(proc_name)s.log' % dict(proc_name=proc_name)
# This is a hack, but it's needed to pass the logfile name & default
# loglevel into log handlers configured with a config file. (mdragon)
logging.LOCAL_LOG_FILE = log_file
logging.LOCAL_DEFAULT_LEVEL = level
if 'logging_config' in conf:
fileConfig(conf['logging_config'])
else:
logging.basicConfig()
logging.getLogger('winchester').setLevel(level)
timesync = time_sync.TimeSync(conf)
pipe = PipelineManager(conf, time_sync=timesync, proc_name=proc_name)
if args.daemon:
print("Backgrounding for daemon mode.")
with daemon.DaemonContext():
pipe.run()
else:
pipe.run()
def main():
parser = argparse.ArgumentParser(description="Winchester pipeline worker")
parser.add_argument('--config', '-c', default='winchester.yaml',
help='The name of the winchester config file')
parser.add_argument('--daemon', '-d', help='Run in daemon mode.')
args = parser.parse_args()
conf = ConfigManager.load_config_file(args.config)
if 'logging_config' in conf:
fileConfig(conf['logging_config'])
else:
logging.basicConfig()
if 'log_level' in conf:
level = conf['log_level']
level = getattr(logging, level.upper())
logging.getLogger('winchester').setLevel(level)
pipe = PipelineManager(conf)
if args.daemon:
print "Backgrounding for daemon mode."
with daemon.DaemonContext():
pipe.run()
else:
pipe.run()
def __init__(self, app=None, queue_name=None):
super(WinchesterHandler, self).__init__(app=app, queue_name=queue_name)
conf_file = self.config_get("config_file")
config = ConfigManager.load_config_file(conf_file)
self.time_sync = time_sync.TimeSync(config, publishes=True)
self.trigger_manager = TriggerManager(config, time_sync=self.time_sync)
def __init__(self, app=None, queue_name=None):
super(WinchesterHandler, self).__init__(app=app, queue_name=queue_name)
conf_file = self.config_get("config_file")
config = ConfigManager.load_config_file(conf_file)
self.time_sync = time_sync.TimeSync(config, publishes=True)
self.trigger_manager = TriggerManager(config, time_sync=self.time_sync)
def __init__(self, winchester_config):
self.winchester_config = winchester_config
self.config = ConfigManager.load_config_file(winchester_config)
self.trigger_manager = TriggerManager(self.config)
self.pipe = PipelineManager(self.config)
def __init__(self, kafka_config, winchester_config):
self.winchester_config = winchester_config
self.kafka_config = kafka_config
self.config_mgr = ConfigManager.load_config_file(
self.winchester_config)
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
# fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
from winchester.config import ConfigManager
from winchester.models import Base
target_metadata = Base.metadata
winchester_config = ConfigManager.load_config_file(
config.get_main_option("winchester_config"))
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
# fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
from winchester.config import ConfigManager
from winchester.models import Base
target_metadata = Base.metadata
winchester_config = ConfigManager.load_config_file(
config.get_main_option("winchester_config"))
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
def __init__(self, app=None, queue_name=None):
super(WinchesterHandler, self).__init__(app=app, queue_name=queue_name)
conf_file = self.config_get("config_file")
config = ConfigManager.load_config_file(conf_file)
self.trigger_manager = TriggerManager(config)
def __init__(self, winchester_config):
self.winchester_config = winchester_config
self.config = ConfigManager.load_config_file(winchester_config)
self.trigger_manager = TriggerManager(self.config)
self.pipe = PipelineManager(self.config)
