def main():
parser = argparse.ArgumentParser(description="Winchester pipeline worker")
parser.add_argument('--config',
'-c',
default='winchester.yaml',
help='The name of the winchester config file')
parser.add_argument('--daemon', '-d', help='Run in daemon mode.')
args = parser.parse_args()
conf = ConfigManager.load_config_file(args.config)
if 'logging_config' in conf:
fileConfig(conf['logging_config'])
else:
logging.basicConfig()
if 'log_level' in conf:
level = conf['log_level']
level = getattr(logging, level.upper())
logging.getLogger('winchester').setLevel(level)
pipe = PipelineManager(conf)
if args.daemon:
print "Backgrounding for daemon mode."
with daemon.DaemonContext():
pipe.run()
else:
pipe.run()
def run(self):
if 'logging_config' in self.config_mgr:
fileConfig(self.config_mgr['logging_config'])
else:
logging.basicConfig()
if 'log_level' in self.config_mgr:
level = self.config_mgr['log_level']
level = getattr(logging, level.upper())
logging.getLogger('winchester').setLevel(level)
self.pm_lock = threading.Lock()
self.pipe = PipelineManager(self.config_mgr)
# TODO add trigger defs from the DB at startup
# start threads
self.stream_def_thread = threading.Thread(name='stream_defs_pipe',
target=pipe_stream_definition_consumer,
args=(self.kafka_config, self.pm_lock, self.pipe,))
self.pipeline_ready_thread = threading.Thread(name='pipeline',
target=self.pipeline_ready_processor,
args=(self.pm_lock, self.pipe,))
self.stream_def_thread.start()
self.pipeline_ready_thread.start()
self.stream_def_thread.join()
self.pipeline_ready_thread.join()
log.debug('Exiting')
def main():
parser = argparse.ArgumentParser(description="Winchester pipeline worker")
parser.add_argument('--config',
'-c',
default='winchester.yaml',
help='The name of the winchester config file')
parser.add_argument('--name',
'-n',
default='pipeline_worker',
help='The name of this process for logging purposes')
parser.add_argument('--daemon', '-d', help='Run in daemon mode.')
args = parser.parse_args()
conf = ConfigManager.load_config_file(args.config)
proc_name = args.name
if 'log_level' in conf:
level = conf['log_level']
level = getattr(logging, level.upper())
else:
level = logging.INFO
if 'log_file' in conf:
log_file = conf['log_file'] % dict(proc_name=proc_name)
else:
log_file = '%(proc_name)s.log' % dict(proc_name=proc_name)
# This is a hack, but it's needed to pass the logfile name & default
# loglevel into log handlers configured with a config file. (mdragon)
logging.LOCAL_LOG_FILE = log_file
logging.LOCAL_DEFAULT_LEVEL = level
if 'logging_config' in conf:
fileConfig(conf['logging_config'])
else:
logging.basicConfig()
logging.getLogger('winchester').setLevel(level)
timesync = time_sync.TimeSync(conf)
pipe = PipelineManager(conf, time_sync=timesync, proc_name=proc_name)
if args.daemon:
print("Backgrounding for daemon mode.")
with daemon.DaemonContext():
pipe.run()
else:
pipe.run()
def __init__(self, winchester_config):
self.winchester_config = winchester_config
self.config = ConfigManager.load_config_file(winchester_config)
self.trigger_manager = TriggerManager(self.config)
self.pipe = PipelineManager(self.config)
