def __init__(self, config, db=None, pipeline_handlers=None, pipeline_config=None, trigger_defs=None): logger.debug("PipelineManager: Using config: %s" % str(config)) config = ConfigManager.wrap(config, self.config_description()) self.config = config config.check_config() config.add_config_path(*config['config_path']) if db is not None: self.db = db else: self.db = DBInterface(config['database']) if pipeline_handlers is not None: self.pipeline_handlers = pipeline_handlers else: self.pipeline_handlers = self._load_plugins( config['pipeline_handlers']) logger.debug("Pipeline handlers: %s" % str(self.pipeline_handlers)) if pipeline_config is not None: self.pipeline_config = pipeline_config else: self.pipeline_config = config.load_file(config['pipeline_config']) logger.debug("Pipeline config: %s" % str(self.pipeline_config)) for pipeline, handler_configs in self.pipeline_config.items(): self.pipeline_config[pipeline] = [ Pipeline.check_handler_config(conf, self.pipeline_handlers) for conf in handler_configs ] if trigger_defs is not None: self.trigger_definitions = trigger_defs else: defs = config.load_file(config['trigger_definitions']) logger.debug("Loaded trigger definitions %s" % str(defs)) self.trigger_definitions = [ TriggerDefinition(conf, None) for conf in defs ] self.trigger_map = dict( (tdef.name, tdef) for tdef in self.trigger_definitions) self.trigger_manager = TriggerManager( self.config, db=self.db, trigger_defs=self.trigger_definitions) self.pipeline_worker_batch_size = config['pipeline_worker_batch_size'] self.pipeline_worker_delay = config['pipeline_worker_delay'] self.statistics_period = config['statistics_period'] self.purge_completed_streams = config['purge_completed_streams'] self.streams_fired = 0 self.streams_expired = 0 self.streams_loaded = 0 self.last_status = self.current_time()
def __init__(self, config, db=None, stackdistiller=None, trigger_defs=None, time_sync=None): config = ConfigManager.wrap(config, self.config_description()) self.config = config self.debug_manager = debugging.DebugManager() self.trigger_definitions = [] config.check_config() config.add_config_path(*config['config_path']) if time_sync is None: time_sync = ts.TimeSync() self.time_sync = time_sync if db is not None: self.db = db else: self.db = DBInterface(config['database']) if stackdistiller is not None: self.distiller = stackdistiller else: # distiller_config is optional if config.contains('distiller_config'): dist_config = config.load_file(config['distiller_config']) plugmap = self._load_plugins(config['distiller_trait_plugins'], distiller.DEFAULT_PLUGINMAP) self.distiller = distiller.Distiller( dist_config, trait_plugin_map=plugmap, catchall=config['catch_all_notifications']) if trigger_defs is not None: self.trigger_definitions = trigger_defs for t in self.trigger_definitions: t.set_debugger(self.debug_manager) else: # trigger_definition config file is optional if config.contains('trigger_definitions'): defs = config.load_file(config['trigger_definitions']) self.trigger_definitions = [ TriggerDefinition(conf, self.debug_manager) for conf in defs ] # trigger_map is used to quickly access existing trigger_defs self.trigger_map = dict( (tdef.name, tdef) for tdef in self.trigger_definitions) self.saved_events = 0 self.received = 0 self.last_status = self.current_time()
def __init__(self, config, db=None, stackdistiller=None, trigger_defs=None): config = ConfigManager.wrap(config, self.config_description()) self.config = config self.debug_manager = debugging.DebugManager() config.check_config() config.add_config_path(*config['config_path']) if db is not None: self.db = db else: self.db = DBInterface(config['database']) if stackdistiller is not None: self.distiller = stackdistiller else: dist_config = config.load_file(config['distiller_config']) plugmap = self._load_plugins(config['distiller_trait_plugins'], distiller.DEFAULT_PLUGINMAP) self.distiller = distiller.Distiller( dist_config, trait_plugin_map=plugmap, catchall=config['catch_all_notifications']) if trigger_defs is not None: self.trigger_definitions = trigger_defs for t in self.trigger_definitions: t.set_debugger(self.debug_manager) else: defs = config.load_file(config['trigger_definitions']) self.trigger_definitions = [ TriggerDefinition(conf, self.debug_manager) for conf in defs ] self.saved_events = 0 self.received = 0 self.last_status = self.current_time()
def __init__(self, config, db=None, pipeline_handlers=None, pipeline_config=None, trigger_defs=None, time_sync=None, proc_name='pipeline_worker'): # name used to distinguish worker processes in logs self.proc_name = proc_name logger.debug("PipelineManager(%s): Using config: %s" % (self.proc_name, str(config))) config = ConfigManager.wrap(config, self.config_description()) self.config = config self.trigger_definitions = [] config.check_config() config.add_config_path(*config['config_path']) if time_sync is None: time_sync = ts.TimeSync() self.time_sync = time_sync if db is not None: self.db = db else: self.db = DBInterface(config['database']) if pipeline_handlers is not None: self.pipeline_handlers = pipeline_handlers else: self.pipeline_handlers = self._load_plugins( config['pipeline_handlers']) logger.debug("Pipeline handlers: %s" % str(self.pipeline_handlers)) if pipeline_config is not None: self.pipeline_config = pipeline_config else: self.pipeline_config = config.load_file(config['pipeline_config']) logger.debug("Pipeline config: %s" % str(self.pipeline_config)) for pipeline, handler_configs in self.pipeline_config.items(): self.pipeline_config[pipeline] = [ Pipeline.check_handler_config(conf, self.pipeline_handlers) for conf in handler_configs ] if trigger_defs is not None: self.trigger_definitions = trigger_defs else: # trigger_definition config file is optional if config.contains('trigger_definitions'): defs = config.load_file(config['trigger_definitions']) logger.debug("Loaded trigger definitions %s" % str(defs)) self.trigger_definitions = [ TriggerDefinition(conf, None) for conf in defs ] self.trigger_manager = TriggerManager( self.config, db=self.db, trigger_defs=self.trigger_definitions, time_sync=time_sync) self.pipeline_worker_batch_size = config['pipeline_worker_batch_size'] self.pipeline_worker_delay = config['pipeline_worker_delay'] self.statistics_period = config['statistics_period'] self.purge_completed_streams = config['purge_completed_streams'] self.trim_events = config['trim_events'] self.trim_events_batch_size = config['trim_events_batch_size'] try: self.trim_events_age = timex.parse(str(config['trim_events_age'])) except timex.TimexError: logger.error("Invalid trim event expression: %s Event trimming " "disabled." % config['trim_events_age']) self.trim_events_age = None self.trim_events = False self.streams_fired = 0 self.streams_expired = 0 self.streams_loaded = 0 self.last_status = self.current_time()