def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.batch_size = self.getConfigurationValue('batch_size')
self.backlog_size = self.getConfigurationValue('backlog_size')
self.file_name = self.getConfigurationValue('file_name')
self.format = self.getConfigurationValue('format')
self.compress = self.getConfigurationValue('compress')
self.file_handles = {}
if self.compress == 'gzip':
try:
# Import module into namespace of object. Otherwise it will not be accessible when process was forked.
self.gzip_module = __import__('gzip')
except ImportError:
self.logger.error(
'Gzip compression selected but gzip module could not be loaded.'
)
self.lumbermill.shutDown()
if self.compress == 'snappy':
try:
self.snappy_module = __import__('snappy')
except ImportError:
self.logger.error(
'Snappy compression selected but snappy module could not be loaded.'
)
self.lumbermill.shutDown()
self.buffer = Buffer(
self.batch_size,
self.storeData,
self.getConfigurationValue('store_interval_in_secs'),
maxsize=self.backlog_size)
TimedFunctionManager.startTimedFunction(self.closeStaleFileHandles)
def initAfterFork(self):
self.evaluate_facet_data_func = setInterval(self.getConfigurationValue('interval'))(self.evaluateFacets) #self.getEvaluateFunc()
self.timed_func_handler_a = TimedFunctionManager.startTimedFunction(self.evaluate_facet_data_func)
if self.cache:
self.store_facets_in_cache_func = setInterval(1)(self.storeFacetsInCache)
self.timed_func_handler_b = TimedFunctionManager.startTimedFunction(self.store_facets_in_cache_func)
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
# Get all configured queues for waiting event stats.
self.module_queues = self.lumbermill.getAllQueues()
self.psutil_processes.append(psutil.Process(self.lumbermill.getMainProcessId()))
for worker in self.lumbermill.child_processes:
self.psutil_processes.append(psutil.Process(worker.pid))
TimedFunctionManager.startTimedFunction(self.getRunTimedFunctionsFunc())
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
# Get all configured queues for waiting event stats.
for module_name, module_info in self.lumbermill.modules.items():
instance = module_info['instances'][0]
if not hasattr(instance, 'getInputQueue') or not instance.getInputQueue():
continue
self.module_queues[module_name] = instance.getInputQueue()
TimedFunctionManager.startTimedFunction(self.getRunTimedFunctionsFunc())
BaseThreadedModule.initAfterFork()
def initAfterFork(self):
# Get all configured queues for waiting event stats.
for module_name, module_info in self.lumbermill.modules.items():
instance = module_info['instances'][0]
if not hasattr(instance,
'getInputQueue') or not instance.getInputQueue():
continue
self.module_queues[module_name] = instance.getInputQueue()
TimedFunctionManager.startTimedFunction(
self.getRunTimedFunctionsFunc())
BaseThreadedModule.initAfterFork()
def initAfterFork(self):
self.evaluate_facet_data_func = setInterval(
self.getConfigurationValue('interval'))(
self.evaluateFacets) #self.getEvaluateFunc()
self.timed_func_handler_a = TimedFunctionManager.startTimedFunction(
self.evaluate_facet_data_func)
if self.persistence_backend:
self.store_facets_in_backend_func = setInterval(1)(
self.storeFacetsInRedis)
self.timed_func_handler_b = TimedFunctionManager.startTimedFunction(
self.store_facets_in_backend_func)
BaseThreadedModule.initAfterFork(self)
def run(self):
if self.is_leader:
TimedFunctionManager.startTimedFunction(self.sendAliveRequests)
TimedFunctionManager.startTimedFunction(self.sendDiscoverBroadcast)
TimedFunctionManager.startTimedFunction(self.dropDeadPackFollowers)
else:
TimedFunctionManager.startTimedFunction(self.dropDeadPackLeader)
self.logger.info("Running as pack %s." % self.getConfigurationValue('pack'))
while self.alive:
try:
message, host = self.socket.recvfrom(64536)
except socket.timeout:
continue
try:
# Decrypt and msgpack decode message.
message = msgpack.unpackb(self.decrypt(message))
except:
etype, evalue, etb = sys.exc_info()
self.logger.warning("Could not parse cluster message %s. Maybe your secrets differ? Exception: %s, Error: %s." % (message, etype, evalue))
continue
# Ignore messages for other clusters and from self.
if message['cluster'] != self.cluster_name or message['sender'] == self.hostname:
continue
self.logger.info("Received message %s from %s." % (message, host[0]))
if message['action'] not in self.handlers.keys():
self.logger.warning('Got request for unknown handler %s.' % message['action'])
# Excecute callbacks
for callback in self.handlers["%s" % message['action']]:
self.logger.debug('Calling callback %s for %s.' % (callback, message['action']))
callback(host, message)
def shutDown(self, signum=False, frame=False):
if self.is_master():
self.logger.info("Shutting down LumberMill.")
# Send SIGINT to workers for good measure.
for worker in list(self.child_processes):
os.kill(worker.pid, signal.SIGINT)
if not self.alive:
sys.exit(0)
self.alive = False
self.shutDownModules()
TimedFunctionManager.stopTimedFunctions()
tornado.ioloop.IOLoop.instance().stop()
if self.is_master():
self.logger.info("Shutdown complete.")
def __init__(self,
flush_size=None,
callback=None,
interval=1,
maxsize=5000):
self.logger = logging.getLogger(self.__class__.__name__)
self.flush_size = flush_size
self.buffer = []
self.maxsize = maxsize
self.append = self.put
self.flush_interval = interval
self.flush_callback = callback
self.flush_timed_func = self.getTimedFlushMethod()
self.timed_func_handle = TimedFunctionManager.startTimedFunction(
self.flush_timed_func)
self.is_flushing = False
def run(self):
if self.is_leader:
TimedFunctionManager.startTimedFunction(self.sendAliveRequests)
TimedFunctionManager.startTimedFunction(self.sendDiscoverBroadcast)
TimedFunctionManager.startTimedFunction(self.dropDeadPackFollowers)
else:
TimedFunctionManager.startTimedFunction(self.dropDeadPackLeader)
self.logger.info("Running as pack %s." %
self.getConfigurationValue('pack'))
while self.alive:
try:
message, host = self.socket.recvfrom(64536)
except socket.timeout:
continue
try:
# Decrypt and msgpack decode message.
message = msgpack.unpackb(self.decrypt(message))
except:
etype, evalue, etb = sys.exc_info()
self.logger.warning(
"Could not parse cluster message %s. Maybe your secrets differ? Exception: %s, Error: %s."
% (message, etype, evalue))
continue
# Ignore messages for other clusters and from self.
if message['cluster'] != self.cluster_name or message[
'sender'] == self.hostname:
continue
self.logger.info("Received message %s from %s." %
(message, host[0]))
if message['action'] not in self.handlers.keys():
self.logger.warning('Got request for unknown handler %s.' %
message['action'])
# Excecute callbacks
for callback in self.handlers["%s" % message['action']]:
self.logger.debug('Calling callback %s for %s.' %
(callback, message['action']))
callback(host, message)
def start(self):
# Get currently running configuration.
self.filtered_startup_config = self.filterIgnoredModules(self.lumbermill.getConfiguration())
if self.pack.is_leader:
TimedFunctionManager.startTimedFunction(self.update_config_func)
def startInterval(self):
if self.timed_func_handle:
self.stopInterval()
self.timed_func_handle = TimedFunctionManager.startTimedFunction(
self.flush_timed_func)
def stopInterval(self):
TimedFunctionManager.stopTimedFunctions(self.timed_func_handle)
self.timed_func_handle = False
def initAfterFork(self):
# Get all configured queues for waiting event stats.
self.module_queues = self.lumbermill.getAllQueues()
TimedFunctionManager.startTimedFunction(
self.getRunTimedFunctionsFunc())
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
timed_func = self.getRunTimedFunctionsFunc()
TimedFunctionManager.startTimedFunction(timed_func)
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
self.hdfs = self.getHdfsClient()
TimedFunctionManager.startTimedFunction(self.timed_store_func)
# Call parent run method
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
if self.interval:
TimedFunctionManager.startTimedFunction(
self.getRunTimedFunctionsFunc())
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
timed_func = self.getRunTimedFunctionsFunc()
TimedFunctionManager.startTimedFunction(timed_func)
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
if self.hot_rules_first:
resort_fieldextraction_regexpressions_func = self.getResortFieldextractionRegexpressionsFunc()
self.timed_func_handler = TimedFunctionManager.startTimedFunction(resort_fieldextraction_regexpressions_func)
BaseThreadedModule.initAfterFork(self)
def start(self):
self.scanner = nmap.PortScanner()
timed_func = self.getScannerFunc()
self.timed_func_handler = TimedFunctionManager.startTimedFunction(timed_func)
def initAfterFork(self):
if self.interval:
TimedFunctionManager.startTimedFunction(self.getRunTimedFunctionsFunc())
BaseThreadedModule.initAfterFork(self)
def start(self):
self.timedFuncHandle = TimedFunctionManager.startTimedFunction(self.getTimedGarbageCollectFunc())
def initAfterFork(self):
self.hdfs = self.getHdfsClient()
TimedFunctionManager.startTimedFunction(self.timed_store_func)
# Call parent run method
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
if self.hot_rules_first:
resort_fieldextraction_regexpressions_func = self.getResortFieldextractionRegexpressionsFunc()
self.timed_func_handler = TimedFunctionManager.startTimedFunction(resort_fieldextraction_regexpressions_func)
BaseThreadedModule.initAfterFork(self)
def start(self):
self.scanner = nmap.PortScanner()
timed_func = self.getScannerFunc()
self.timed_func_handler = TimedFunctionManager.startTimedFunction(
timed_func)
def initAfterFork(self):
if self.interval:
self.evaluate_facet_data_func = self.getEvaluateFunc()
self.timed_func_handler = TimedFunctionManager.startTimedFunction(self.evaluate_facet_data_func)
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
self.gc_throttled_events_info = self.getGcThrottledEventsInfoFunc()
self.timed_func_handler = TimedFunctionManager.startTimedFunction(self.gc_throttled_events_info)
BaseThreadedModule.initAfterFork(self)
def start(self):
# Get currently running configuration.
self.filtered_startup_config = self.filterIgnoredModules(
self.lumbermill.getConfiguration())
if self.pack.is_leader:
TimedFunctionManager.startTimedFunction(self.update_config_func)
def start(self):
self.timedFuncHandle = TimedFunctionManager.startTimedFunction(
self.getTimedGarbageCollectFunc())