def initAfterFork(self):
BaseThreadedModule.initAfterFork(self)
self.buffer = Buffer(
self.getConfigurationValue('batch_size'),
self.storeData,
self.getConfigurationValue('store_interval_in_secs'),
maxsize=self.getConfigurationValue('backlog_size'))
def initAfterFork(self):
# As the buffer uses a threaded timed function to flush its buffer and thread will not survive a fork, init buffer here.
self.buffers = collections.defaultdict(lambda: Buffer(flush_size=self.buffer_size,
callback=self.sendMergedEvent,
interval=self.flush_interval_in_secs,
maxsize=self.buffer_size))
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
self.evaluate_facet_data_func = setInterval(self.getConfigurationValue('interval'))(self.evaluateFacets) #self.getEvaluateFunc()
self.timed_func_handler_a = TimedFunctionManager.startTimedFunction(self.evaluate_facet_data_func)
if self.cache:
self.store_facets_in_cache_func = setInterval(1)(self.storeFacetsInCache)
self.timed_func_handler_b = TimedFunctionManager.startTimedFunction(self.store_facets_in_cache_func)
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
BaseThreadedModule.initAfterFork(self)
self.buffer = Buffer(
self.getConfigurationValue('batch_size'),
self.storeData,
self.getConfigurationValue('store_interval_in_secs'),
maxsize=self.getConfigurationValue('backlog_size'))
try:
self.sqs_resource = boto3.resource(
'sqs',
region_name=self.getConfigurationValue('region'),
api_version=None,
use_ssl=True,
verify=None,
endpoint_url=None,
aws_access_key_id=self.getConfigurationValue(
'aws_access_key_id'),
aws_secret_access_key=self.getConfigurationValue(
'aws_secret_access_key'),
aws_session_token=None,
config=None)
self.sqs_queue = self.sqs_resource.get_queue_by_name(
QueueName=self.getConfigurationValue('queue'))
except:
etype, evalue, etb = sys.exc_info()
self.logger.error(
"Could not connect to sqs service. Exception: %s, Error: %s." %
(etype, evalue))
self.lumbermill.shutDown()
def initAfterFork(self):
BaseThreadedModule.initAfterFork(self)
self.buffer = Buffer(self.getConfigurationValue('batch_size'), self.storeData, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
self.connection = self.connect()
if not self.connection:
self.lumbermill.shutDown()
return
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
# Get all configured queues for waiting event stats.
self.module_queues = self.lumbermill.getAllQueues()
self.psutil_processes.append(psutil.Process(self.lumbermill.getMainProcessId()))
for worker in self.lumbermill.child_processes:
self.psutil_processes.append(psutil.Process(worker.pid))
TimedFunctionManager.startTimedFunction(self.getRunTimedFunctionsFunc())
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
# As the buffer uses a threaded timed function to flush its buffer and thread will not survive a fork, init buffer here.
self.buffers = collections.defaultdict(
lambda: Buffer(flush_size=self.buffer_size,
callback=self.sendMergedEvent,
interval=self.flush_interval_in_secs,
maxsize=self.buffer_size))
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
# Get all configured queues for waiting event stats.
for module_name, module_info in self.lumbermill.modules.items():
instance = module_info['instances'][0]
if not hasattr(instance, 'getInputQueue') or not instance.getInputQueue():
continue
self.module_queues[module_name] = instance.getInputQueue()
TimedFunctionManager.startTimedFunction(self.getRunTimedFunctionsFunc())
BaseThreadedModule.initAfterFork()
def initAfterFork(self):
BaseThreadedModule.initAfterFork(self)
# Init monogdb client after fork.
self.mongodb = self.connect()
if not self.mongodb:
self.lumbermill.shutDown()
return
# As the buffer uses a threaded timed function to flush its buffer and thread will not survive a fork, init buffer here.
self.buffer = Buffer(self.getConfigurationValue('batch_size'), self.storeData, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
def initAfterFork(self):
BaseThreadedModule.initAfterFork(self)
# Init es client after fork as mentioned in https://elasticsearch-py.readthedocs.org/en/master/
self.es = self.connect()
if not self.es:
self.lumbermill.shutDown()
return
# As the buffer uses a threaded timed function to flush its buffer and thread will not survive a fork, init buffer here.
self.buffer = Buffer(self.getConfigurationValue('batch_size'), self.storeData, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
def initAfterFork(self):
self.resolver = resolver.Resolver()
self.resolver.timeout = self.timeout
self.resolver.lifetime = self.timeout
if self.nameservers:
self.resolver.nameservers = self.nameservers
self.queue = Queue.Queue(20)
self.lookup_threads = [LookupThread(self.queue, self.lookup_type, self) for _ in range(0, self.lookup_threads_pool_size)]
for thread in self.lookup_threads:
thread.start()
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
# Get all configured queues for waiting event stats.
for module_name, module_info in self.lumbermill.modules.items():
instance = module_info['instances'][0]
if not hasattr(instance,
'getInputQueue') or not instance.getInputQueue():
continue
self.module_queues[module_name] = instance.getInputQueue()
TimedFunctionManager.startTimedFunction(
self.getRunTimedFunctionsFunc())
BaseThreadedModule.initAfterFork()
def initAfterFork(self):
BaseThreadedModule.initAfterFork(self)
# Calculate event count when running in multiple processes.
if self.max_events_count == 0:
return
self.max_events_count = int(self.getConfigurationValue("events_count")/self.lumbermill.getWorkerCount())
if self.lumbermill.is_master():
remainder = self.getConfigurationValue("events_count") % self.lumbermill.getWorkerCount()
self.max_events_count += remainder
if self.max_events_count == 0:
self.shutDown()
def initAfterFork(self):
BaseThreadedModule.initAfterFork(self)
self.buffer = Buffer(
self.getConfigurationValue('batch_size'),
self.storeData,
self.getConfigurationValue('store_interval_in_secs'),
maxsize=self.getConfigurationValue('backlog_size'))
self.connection = self.connect()
if not self.connection:
self.lumbermill.shutDown()
return
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
self.evaluate_facet_data_func = setInterval(
self.getConfigurationValue('interval'))(
self.evaluateFacets) #self.getEvaluateFunc()
self.timed_func_handler_a = TimedFunctionManager.startTimedFunction(
self.evaluate_facet_data_func)
if self.persistence_backend:
self.store_facets_in_backend_func = setInterval(1)(
self.storeFacetsInRedis)
self.timed_func_handler_b = TimedFunctionManager.startTimedFunction(
self.store_facets_in_backend_func)
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
BaseThreadedModule.initAfterFork(self)
# Init monogdb client after fork.
self.mongodb = self.connect()
if not self.mongodb:
self.lumbermill.shutDown()
return
# As the buffer uses a threaded timed function to flush its buffer and thread will not survive a fork, init buffer here.
self.buffer = Buffer(
self.getConfigurationValue('batch_size'),
self.storeData,
self.getConfigurationValue('store_interval_in_secs'),
maxsize=self.getConfigurationValue('backlog_size'))
def initAfterFork(self):
BaseThreadedModule.initAfterFork(self)
# Init es client after fork as mentioned in https://elasticsearch-py.readthedocs.org/en/master/
self.es = self.connect()
if not self.es:
self.lumbermill.shutDown()
return
# As the buffer uses a threaded timed function to flush its buffer and thread will not survive a fork, init buffer here.
self.buffer = Buffer(
self.getConfigurationValue('batch_size'),
self.storeData,
self.getConfigurationValue('store_interval_in_secs'),
maxsize=self.getConfigurationValue('backlog_size'))
def initAfterFork(self):
self.resolver = resolver.Resolver()
self.resolver.timeout = self.timeout
self.resolver.lifetime = self.timeout
if self.nameservers:
self.resolver.nameservers = self.nameservers
self.queue = Queue.Queue(20)
self.lookup_threads = [
LookupThread(self.queue, self.lookup_type, self)
for _ in range(0, self.lookup_threads_pool_size)
]
for thread in self.lookup_threads:
thread.start()
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
BaseThreadedModule.initAfterFork(self)
# Calculate event count when running in multiple processes.
if self.max_events_count == 0:
return
self.max_events_count = int(
self.getConfigurationValue("events_count") /
self.lumbermill.getWorkerCount())
if self.lumbermill.is_master():
remainder = self.getConfigurationValue(
"events_count") % self.lumbermill.getWorkerCount()
self.max_events_count += remainder
if self.max_events_count == 0:
self.shutDown()
def initAfterFork(self):
"""
When running with multiple processes, calculate the the number of files each processes should work on.
If dividing the number of files by the number of workers yields a remainder, master process will take care of it.
"""
BaseThreadedModule.initAfterFork(self)
datastore = self.lumbermill.getInternalDataStore()
datastore.acquireLock()
from_file_list_idx = datastore.getDataDict()[self.datastore_key]
to_file_list_idx = from_file_list_idx + int(len(self.files) / self.lumbermill.getWorkerCount())
if self.lumbermill.is_master():
to_file_list_idx += len(self.files) % self.lumbermill.getWorkerCount()
self.files = self.files[from_file_list_idx:to_file_list_idx]
datastore.getDataDict()[self.datastore_key] = to_file_list_idx
datastore.releaseLock()
def initAfterFork(self):
"""
When running with multiple processes, calculate the the number of files each processes should work on.
If dividing the number of files by the number of workers yields a remainder, master process will take care of it.
"""
BaseThreadedModule.initAfterFork(self)
datastore = self.lumbermill.getInternalDataStore()
datastore.acquireLock()
from_file_list_idx = datastore.getDataDict()[self.datastore_key]
to_file_list_idx = from_file_list_idx + int(
len(self.files) / self.lumbermill.getWorkerCount())
if self.lumbermill.is_master():
to_file_list_idx += len(
self.files) % self.lumbermill.getWorkerCount()
self.files = self.files[from_file_list_idx:to_file_list_idx]
datastore.getDataDict()[self.datastore_key] = to_file_list_idx
datastore.releaseLock()
def initAfterFork(self):
#self.buffer = Utils.Buffer(self.getConfigurationValue('batch_size'), self.storeData, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
timed_func = self.getRunTimedFunctionsFunc()
TimedFunctionManager.startTimedFunction(timed_func)
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
# Get all configured queues for waiting event stats.
self.module_queues = self.lumbermill.getAllQueues()
TimedFunctionManager.startTimedFunction(
self.getRunTimedFunctionsFunc())
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
timed_func = self.getRunTimedFunctionsFunc()
TimedFunctionManager.startTimedFunction(timed_func)
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
BaseThreadedModule.initAfterFork(self)
self.simple_es_client = requests.Session()
def initAfterFork(self):
if self.hot_rules_first:
resort_fieldextraction_regexpressions_func = self.getResortFieldextractionRegexpressionsFunc()
self.timed_func_handler = TimedFunctionManager.startTimedFunction(resort_fieldextraction_regexpressions_func)
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
if self.interval:
self.evaluate_facet_data_func = self.getEvaluateFunc()
self.timed_func_handler = TimedFunctionManager.startTimedFunction(self.evaluate_facet_data_func)
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
if self.hot_rules_first:
resort_fieldextraction_regexpressions_func = self.getResortFieldextractionRegexpressionsFunc()
self.timed_func_handler = TimedFunctionManager.startTimedFunction(resort_fieldextraction_regexpressions_func)
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
self.hdfs = self.getHdfsClient()
TimedFunctionManager.startTimedFunction(self.timed_store_func)
# Call parent run method
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
BaseThreadedModule.initAfterFork(self)
self.simple_es_client = requests.Session()
def initAfterFork(self):
if self.interval:
TimedFunctionManager.startTimedFunction(self.getRunTimedFunctionsFunc())
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
self.hdfs = self.getHdfsClient()
TimedFunctionManager.startTimedFunction(self.timed_store_func)
# Call parent run method
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
self.gc_throttled_events_info = self.getGcThrottledEventsInfoFunc()
self.timed_func_handler = TimedFunctionManager.startTimedFunction(self.gc_throttled_events_info)
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
if self.interval:
TimedFunctionManager.startTimedFunction(
self.getRunTimedFunctionsFunc())
BaseThreadedModule.initAfterFork(self)