def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.batch_size = self.getConfigurationValue('batch_size') self.backlog_size = self.getConfigurationValue('backlog_size') self.file_name = self.getConfigurationValue('file_name') self.format = self.getConfigurationValue('format') self.compress = self.getConfigurationValue('compress') self.file_handles = {} if self.compress == 'gzip': try: # Import module into namespace of object. Otherwise it will not be accessible when process was forked. self.gzip_module = __import__('gzip') except ImportError: self.logger.error( 'Gzip compression selected but gzip module could not be loaded.' ) self.lumbermill.shutDown() if self.compress == 'snappy': try: self.snappy_module = __import__('snappy') except ImportError: self.logger.error( 'Snappy compression selected but snappy module could not be loaded.' ) self.lumbermill.shutDown() self.buffer = Buffer( self.batch_size, self.storeData, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.backlog_size) TimedFunctionManager.startTimedFunction(self.closeStaleFileHandles)
def initAfterFork(self): self.evaluate_facet_data_func = setInterval(self.getConfigurationValue('interval'))(self.evaluateFacets) #self.getEvaluateFunc() self.timed_func_handler_a = TimedFunctionManager.startTimedFunction(self.evaluate_facet_data_func) if self.cache: self.store_facets_in_cache_func = setInterval(1)(self.storeFacetsInCache) self.timed_func_handler_b = TimedFunctionManager.startTimedFunction(self.store_facets_in_cache_func) BaseThreadedModule.initAfterFork(self)
def initAfterFork(self): # Get all configured queues for waiting event stats. self.module_queues = self.lumbermill.getAllQueues() self.psutil_processes.append(psutil.Process(self.lumbermill.getMainProcessId())) for worker in self.lumbermill.child_processes: self.psutil_processes.append(psutil.Process(worker.pid)) TimedFunctionManager.startTimedFunction(self.getRunTimedFunctionsFunc()) BaseThreadedModule.initAfterFork(self)
def initAfterFork(self): # Get all configured queues for waiting event stats. for module_name, module_info in self.lumbermill.modules.items(): instance = module_info['instances'][0] if not hasattr(instance, 'getInputQueue') or not instance.getInputQueue(): continue self.module_queues[module_name] = instance.getInputQueue() TimedFunctionManager.startTimedFunction(self.getRunTimedFunctionsFunc()) BaseThreadedModule.initAfterFork()
def initAfterFork(self): # Get all configured queues for waiting event stats. for module_name, module_info in self.lumbermill.modules.items(): instance = module_info['instances'][0] if not hasattr(instance, 'getInputQueue') or not instance.getInputQueue(): continue self.module_queues[module_name] = instance.getInputQueue() TimedFunctionManager.startTimedFunction( self.getRunTimedFunctionsFunc()) BaseThreadedModule.initAfterFork()
def initAfterFork(self): self.evaluate_facet_data_func = setInterval( self.getConfigurationValue('interval'))( self.evaluateFacets) #self.getEvaluateFunc() self.timed_func_handler_a = TimedFunctionManager.startTimedFunction( self.evaluate_facet_data_func) if self.persistence_backend: self.store_facets_in_backend_func = setInterval(1)( self.storeFacetsInRedis) self.timed_func_handler_b = TimedFunctionManager.startTimedFunction( self.store_facets_in_backend_func) BaseThreadedModule.initAfterFork(self)
def run(self): if self.is_leader: TimedFunctionManager.startTimedFunction(self.sendAliveRequests) TimedFunctionManager.startTimedFunction(self.sendDiscoverBroadcast) TimedFunctionManager.startTimedFunction(self.dropDeadPackFollowers) else: TimedFunctionManager.startTimedFunction(self.dropDeadPackLeader) self.logger.info("Running as pack %s." % self.getConfigurationValue('pack')) while self.alive: try: message, host = self.socket.recvfrom(64536) except socket.timeout: continue try: # Decrypt and msgpack decode message. message = msgpack.unpackb(self.decrypt(message)) except: etype, evalue, etb = sys.exc_info() self.logger.warning("Could not parse cluster message %s. Maybe your secrets differ? Exception: %s, Error: %s." % (message, etype, evalue)) continue # Ignore messages for other clusters and from self. if message['cluster'] != self.cluster_name or message['sender'] == self.hostname: continue self.logger.info("Received message %s from %s." % (message, host[0])) if message['action'] not in self.handlers.keys(): self.logger.warning('Got request for unknown handler %s.' % message['action']) # Excecute callbacks for callback in self.handlers["%s" % message['action']]: self.logger.debug('Calling callback %s for %s.' % (callback, message['action'])) callback(host, message)
def shutDown(self, signum=False, frame=False): if self.is_master(): self.logger.info("Shutting down LumberMill.") # Send SIGINT to workers for good measure. for worker in list(self.child_processes): os.kill(worker.pid, signal.SIGINT) if not self.alive: sys.exit(0) self.alive = False self.shutDownModules() TimedFunctionManager.stopTimedFunctions() tornado.ioloop.IOLoop.instance().stop() if self.is_master(): self.logger.info("Shutdown complete.")
def __init__(self, flush_size=None, callback=None, interval=1, maxsize=5000): self.logger = logging.getLogger(self.__class__.__name__) self.flush_size = flush_size self.buffer = [] self.maxsize = maxsize self.append = self.put self.flush_interval = interval self.flush_callback = callback self.flush_timed_func = self.getTimedFlushMethod() self.timed_func_handle = TimedFunctionManager.startTimedFunction( self.flush_timed_func) self.is_flushing = False
def run(self): if self.is_leader: TimedFunctionManager.startTimedFunction(self.sendAliveRequests) TimedFunctionManager.startTimedFunction(self.sendDiscoverBroadcast) TimedFunctionManager.startTimedFunction(self.dropDeadPackFollowers) else: TimedFunctionManager.startTimedFunction(self.dropDeadPackLeader) self.logger.info("Running as pack %s." % self.getConfigurationValue('pack')) while self.alive: try: message, host = self.socket.recvfrom(64536) except socket.timeout: continue try: # Decrypt and msgpack decode message. message = msgpack.unpackb(self.decrypt(message)) except: etype, evalue, etb = sys.exc_info() self.logger.warning( "Could not parse cluster message %s. Maybe your secrets differ? Exception: %s, Error: %s." % (message, etype, evalue)) continue # Ignore messages for other clusters and from self. if message['cluster'] != self.cluster_name or message[ 'sender'] == self.hostname: continue self.logger.info("Received message %s from %s." % (message, host[0])) if message['action'] not in self.handlers.keys(): self.logger.warning('Got request for unknown handler %s.' % message['action']) # Excecute callbacks for callback in self.handlers["%s" % message['action']]: self.logger.debug('Calling callback %s for %s.' % (callback, message['action'])) callback(host, message)
def start(self): # Get currently running configuration. self.filtered_startup_config = self.filterIgnoredModules(self.lumbermill.getConfiguration()) if self.pack.is_leader: TimedFunctionManager.startTimedFunction(self.update_config_func)
def startInterval(self): if self.timed_func_handle: self.stopInterval() self.timed_func_handle = TimedFunctionManager.startTimedFunction( self.flush_timed_func)
def stopInterval(self): TimedFunctionManager.stopTimedFunctions(self.timed_func_handle) self.timed_func_handle = False
def initAfterFork(self): # Get all configured queues for waiting event stats. self.module_queues = self.lumbermill.getAllQueues() TimedFunctionManager.startTimedFunction( self.getRunTimedFunctionsFunc()) BaseThreadedModule.initAfterFork(self)
def initAfterFork(self): timed_func = self.getRunTimedFunctionsFunc() TimedFunctionManager.startTimedFunction(timed_func) BaseThreadedModule.initAfterFork(self)
def initAfterFork(self): self.hdfs = self.getHdfsClient() TimedFunctionManager.startTimedFunction(self.timed_store_func) # Call parent run method BaseThreadedModule.initAfterFork(self)
def initAfterFork(self): if self.interval: TimedFunctionManager.startTimedFunction( self.getRunTimedFunctionsFunc()) BaseThreadedModule.initAfterFork(self)
def initAfterFork(self): timed_func = self.getRunTimedFunctionsFunc() TimedFunctionManager.startTimedFunction(timed_func) BaseThreadedModule.initAfterFork(self)
def initAfterFork(self): if self.hot_rules_first: resort_fieldextraction_regexpressions_func = self.getResortFieldextractionRegexpressionsFunc() self.timed_func_handler = TimedFunctionManager.startTimedFunction(resort_fieldextraction_regexpressions_func) BaseThreadedModule.initAfterFork(self)
def start(self): self.scanner = nmap.PortScanner() timed_func = self.getScannerFunc() self.timed_func_handler = TimedFunctionManager.startTimedFunction(timed_func)
def initAfterFork(self): if self.interval: TimedFunctionManager.startTimedFunction(self.getRunTimedFunctionsFunc()) BaseThreadedModule.initAfterFork(self)
def start(self): self.timedFuncHandle = TimedFunctionManager.startTimedFunction(self.getTimedGarbageCollectFunc())
def initAfterFork(self): self.hdfs = self.getHdfsClient() TimedFunctionManager.startTimedFunction(self.timed_store_func) # Call parent run method BaseThreadedModule.initAfterFork(self)
def initAfterFork(self): if self.hot_rules_first: resort_fieldextraction_regexpressions_func = self.getResortFieldextractionRegexpressionsFunc() self.timed_func_handler = TimedFunctionManager.startTimedFunction(resort_fieldextraction_regexpressions_func) BaseThreadedModule.initAfterFork(self)
def start(self): self.scanner = nmap.PortScanner() timed_func = self.getScannerFunc() self.timed_func_handler = TimedFunctionManager.startTimedFunction( timed_func)
def initAfterFork(self): if self.interval: self.evaluate_facet_data_func = self.getEvaluateFunc() self.timed_func_handler = TimedFunctionManager.startTimedFunction(self.evaluate_facet_data_func) BaseThreadedModule.initAfterFork(self)
def initAfterFork(self): self.gc_throttled_events_info = self.getGcThrottledEventsInfoFunc() self.timed_func_handler = TimedFunctionManager.startTimedFunction(self.gc_throttled_events_info) BaseThreadedModule.initAfterFork(self)
def start(self): # Get currently running configuration. self.filtered_startup_config = self.filterIgnoredModules( self.lumbermill.getConfiguration()) if self.pack.is_leader: TimedFunctionManager.startTimedFunction(self.update_config_func)
def start(self): self.timedFuncHandle = TimedFunctionManager.startTimedFunction( self.getTimedGarbageCollectFunc())