def configure(self, configuration):
# Call parent configure method.
BaseThreadedModule.configure(self, configuration)
for module_name in ['elasticsearch', 'urllib3', 'requests']:
if self.getConfigurationValue('log_level') == 'info':
logging.getLogger(module_name).setLevel(logging.WARN)
else:
# Set log level for elasticsarch library if configured to other than default.
logging.getLogger(module_name).setLevel(self.logger.level)
self.action = self.getConfigurationValue('action')
self.fields = self.getConfigurationValue('fields')
self.ttl = self.getConfigurationValue("ttl")
self.index_name = self.getConfigurationValue("index_name")
self.routing_pattern = self.getConfigurationValue("routing")
self.doc_id_pattern = self.getConfigurationValue("doc_id")
self.doc_type_pattern = self.getConfigurationValue("doc_type")
self.doc_type_is_dynamic = self.isDynamicConfigurationValue("doc_type")
self.es_nodes = self.getConfigurationValue("nodes")
self.read_timeout = self.getConfigurationValue("read_timeout")
if not isinstance(self.es_nodes, list):
self.es_nodes = [self.es_nodes]
if self.getConfigurationValue("connection_type") == 'urllib3':
self.connection_class = elasticsearch.connection.Urllib3HttpConnection
elif self.getConfigurationValue("connection_type") == 'requests':
self.connection_class = elasticsearch.connection.RequestsHttpConnection
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
if len(self.getConfigurationValue('cluster')) == 0:
redis_store = self.getConfigurationValue('server')
self.client = self.getRedisClient()
else:
redis_store = self.getConfigurationValue('cluster')
self.client = self.getClusterRedisClient()
try:
self.client.ping()
except:
etype, evalue, etb = sys.exc_info()
self.logger.error(
"Could not connect to redis store at %s. Exception: %s, Error: %s."
% (redis_store, etype, evalue))
self.lumbermill.shutDown()
self.set_buffer = None
if self.getConfigurationValue(
'store_interval_in_secs') or self.getConfigurationValue(
'batch_size'):
self.set_buffer = Buffer(
self.getConfigurationValue('batch_size'),
self.setBufferedCallback,
self.getConfigurationValue('store_interval_in_secs'),
maxsize=self.getConfigurationValue('backlog_size'))
self._set = self.set
self.set = self.setBuffered
self._get = self.get
self.get = self.getBuffered
self._delete = self.delete
self.delete = self.deleteBuffered
self._pop = self.pop
self.pop = self.popBuffered
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.source_fields = self.getConfigurationValue('source_fields')
# Allow single string as well.
if isinstance(self.source_fields, types.StringTypes):
self.source_fields = [self.source_fields]
self.target_fields = self.getConfigurationValue('target_fields')
# Allow single string as well.
if isinstance(self.target_fields, types.StringTypes):
self.target_fields = [self.target_fields]
# If target_fields were provided they need to be of same length as source_fields.
if self.target_fields and len(self.source_fields) != len(
self.target_fields):
self.logger.error(
"Count of configured source and target fields need to match. Please check your configuration."
)
self.lumbermill.shutDown()
self.compression = self.getConfigurationValue('compression')
# Get compression specific handler.
try:
self.deflater = getattr(self, "deflate_with_%s" % self.compression)
except AttributeError:
etype, evalue, etb = sys.exc_info()
self.logger.error(
"Unknown decompression lib: %s. Exception: %s, Error: %s" %
(self.compression, etype, evalue))
self.lumbermill.shutDown()
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
# Set boto log level.
logging.getLogger('boto3').setLevel(logging.CRITICAL)
logging.getLogger('botocore').setLevel(logging.CRITICAL)
self.sqs_queue_name = self.getConfigurationValue('queue')
self.attribute_names = self.getConfigurationValue('attribute_names')
self.message_attribute_names = self.getConfigurationValue('message_attribute_names')
self.poll_interval = self.getConfigurationValue('poll_interval_in_secs')
self.batch_size = self.getConfigurationValue('batch_size')
try:
self.sqs_client = boto3.client('sqs', region_name=self.getConfigurationValue('region'),
api_version=None,
use_ssl=True,
verify=None,
endpoint_url=None,
aws_access_key_id=self.getConfigurationValue('aws_access_key_id'),
aws_secret_access_key=self.getConfigurationValue('aws_secret_access_key'),
aws_session_token=None,
config=None)
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("Could not connect to sqs service. Exception: %s, Error: %s." % (etype, evalue))
self.lumbermill.shutDown()
try:
self.sqs_queue_url = self.sqs_client.get_queue_url(QueueName=self.sqs_queue_name)['QueueUrl']
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("Could not get queue url for sqs queue %s. Exception: %s, Error: %s." % (self.sqs_queue_name, etype, evalue))
self.lumbermill.shutDown()
def shutDown(self):
if self.server:
self.server.stop()
# Give os time to free the socket. Otherwise a reload will fail with 'address already in use'
time.sleep(.2)
# Call parent shutDown method.
BaseThreadedModule.shutDown(self)
def initAfterFork(self):
BaseThreadedModule.initAfterFork(self)
self.buffer = Buffer(
self.getConfigurationValue('batch_size'),
self.storeData,
self.getConfigurationValue('store_interval_in_secs'),
maxsize=self.getConfigurationValue('backlog_size'))
def initAfterFork(self):
# As the buffer uses a threaded timed function to flush its buffer and thread will not survive a fork, init buffer here.
self.buffers = collections.defaultdict(lambda: Buffer(flush_size=self.buffer_size,
callback=self.sendMergedEvent,
interval=self.flush_interval_in_secs,
maxsize=self.buffer_size))
BaseThreadedModule.initAfterFork(self)
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.lists = self.getConfigurationValue('lists')
if not isinstance(self.lists, list):
self.lists = [self.lists]
self.timeout = self.getConfigurationValue('timeout')
self.batch_size = self.getConfigurationValue('batch_size')
self.client = redis.StrictRedis(
host=self.getConfigurationValue('server'),
port=self.getConfigurationValue('port'),
password=self.getConfigurationValue('password'),
db=self.getConfigurationValue('db'))
try:
self.client.ping()
except:
etype, evalue, etb = sys.exc_info()
self.logger.error(
"Could not connect to redis store at %s. Exception: %s, Error: %s."
% (self.getConfigurationValue('server'), etype, evalue))
self.lumbermill.shutDown()
# Monkeypatch run method to use the correct handle event method.
if self.batch_size == 1:
self.run = self.handleSingleEvent
else:
self.run = self.handleBatchEvents
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.source_field = self.getConfigurationValue('source_field')
self.seperator = self.getConfigurationValue('seperator')
self.target_field = self.getConfigurationValue('target_field')
self.drop_original = not self.getConfigurationValue('keep_original')
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.source_field = self.getConfigurationValue('source_field')
self.add_event_fields = self.getConfigurationValue('add_event_fields')
self.backend_lock_name = "Lumbermill:Facet:FacetLock-%s" % self.lumbermill.getMainProcessId(
)
self.backend_key_name = "Lumbermill:Facet:FacetKeys-%s" % self.lumbermill.getMainProcessId(
)
self.backend_ttl = self.getConfigurationValue('backend_ttl')
if (self.backend_ttl < self.getConfigurationValue('interval')):
self.logger.error(
'backend_ttl setting is smaller then interval setting. Please check.'
)
self.lumbermill.shutDown()
return
backend_info = self.lumbermill.getModuleInfoById(
self.getConfigurationValue('backend'))
if not backend_info:
self.logger.error(
"Could not find %s backend for persistant storage." %
(self.getConfigurationValue('backend')))
self.lumbermill.shutDown()
return
self.persistence_backend = backend_info['instances'][0]
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.interval = self.getConfigurationValue('interval')
self.fields = self.getConfigurationValue('fields')
self.stats_collector = StatisticCollector()
self.module_queues = {}
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.batch_size = self.getConfigurationValue('batch_size')
self.backlog_size = self.getConfigurationValue('backlog_size')
self.file_name = self.getConfigurationValue('file_name')
self.format = self.getConfigurationValue('format')
self.compress = self.getConfigurationValue('compress')
self.file_handles = {}
if self.compress == 'gzip':
try:
# Import module into namespace of object. Otherwise it will not be accessible when process was forked.
self.gzip_module = __import__('gzip')
except ImportError:
self.logger.error(
'Gzip compression selected but gzip module could not be loaded.'
)
self.lumbermill.shutDown()
if self.compress == 'snappy':
try:
self.snappy_module = __import__('snappy')
except ImportError:
self.logger.error(
'Snappy compression selected but snappy module could not be loaded.'
)
self.lumbermill.shutDown()
self.buffer = Buffer(
self.batch_size,
self.storeData,
self.getConfigurationValue('store_interval_in_secs'),
maxsize=self.backlog_size)
TimedFunctionManager.startTimedFunction(self.closeStaleFileHandles)
def configure(self, configuration):
# self.logger.setLevel(logging.DEBUG)
# Call parent configure method.
BaseThreadedModule.configure(self, configuration)
self.is_leader = True if self.getConfigurationValue(
'pack') == 'leader' else False
self.pack_followers = {}
self.cluster_name = self.getConfigurationValue('name')
self.discovered_leader = None
self.secret = hashlib.sha256(
self.getConfigurationValue('secret')).digest()
self.handlers = collections.defaultdict(list)
self.lock = threading.Lock()
# Setup socket.
self.interface_addr = self.getConfigurationValue('interface')
self.interface_port = self.getConfigurationValue('port')
self.broadcast_addr = self.getConfigurationValue('broadcast')
self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
self.socket.settimeout(1)
self.hostname = socket.gethostname()
try:
self.socket.bind((self.interface_addr, self.interface_port))
except:
etype, evalue, etb = sys.exc_info()
self.logger.error(
"Could not listen on %s:%s. Exception: %s, Error: %s." %
(self.getConfigurationValue("interface"),
self.getConfigurationValue("port"), etype, evalue))
self.alive = False
self.lumbermill.shutDown()
return
self.addHandlers()
def shutDown(self):
self.accumulateReceiveRateStats()
self.accumulateEventTypeStats()
if self.lumbermill.is_master():
self.printIntervalStatistics()
self.mp_stats_collector.shutDown()
BaseThreadedModule.shutDown(self)
def configure(self, configuration):
# Call parent configure method.
BaseThreadedModule.configure(self, configuration)
self.format = self.getConfigurationValue('format')
self.collection = self.getConfigurationValue('collection')
self.database = self.getConfigurationValue('database')
self.doc_id_pattern = self.getConfigurationValue("doc_id")
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.syslogger = logging.getLogger(self.__class__.__name__)
self.syslogger.propagate = False
self.format = self.getConfigurationValue('format')
if os.path.exists(self.getConfigurationValue('address')):
address = self.getConfigurationValue('address')
else:
server, port = self.getConfigurationValue('address').split(':')
address = (server, int(port))
if self.getConfigurationValue('proto') == 'tcp':
socket_type = socket.SOCK_STREAM
else:
socket_type = socket.SOCK_DGRAM
try:
facility = logging.handlers.SysLogHandler.facility_names[
self.getConfigurationValue('facility')]
except KeyError:
self.logger.error("The configured facility %s is unknown." %
(self.getConfigurationValue('facility')))
self.syslog_handler = logging.handlers.SysLogHandler(
address, facility=facility, socktype=socket_type)
self.syslogger.addHandler(self.syslog_handler)
self.format = self.getConfigurationValue('format')
def configure(self, configuration):
# Call parent configure method.
BaseThreadedModule.configure(self, configuration)
self.format = self.getConfigurationValue('format')
self.collection = self.getConfigurationValue('collection')
self.database = self.getConfigurationValue('database')
self.doc_id_pattern = self.getConfigurationValue("doc_id")
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
backend = self.getConfigurationValue('backend')
self.backend_client = None
if backend == 'DictStore':
import simplekv.memory
self.backend_client = None
self.kv_store = simplekv.memory.DictStore()
elif backend == 'RedisStore':
import simplekv.memory.redisstore
self.backend_client = self.getRedisClient()
self.kv_store = simplekv.memory.redisstore.RedisStore(self.backend_client)
elif backend == 'MemcacheStore':
import simplekv.memory.memcachestore
self.backend_client = self.getMemcacheClient()
self.kv_store = simplekv.memory.memcachestore.MemcacheStore(self.backend_client)
self.set_buffer = None
if self.getConfigurationValue('store_interval_in_secs') or self.getConfigurationValue('batch_size'):
if backend == 'RedisStore':
self.set_buffer = Buffer(self.getConfigurationValue('batch_size'), self.setRedisBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
else:
self.set_buffer = Buffer(self.getConfigurationValue('batch_size'), self.setBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
self._set = self.set
self.set = self.setBuffered
self._get = self.get
self.get = self.getBuffered
self._delete = self.delete
self.delete = self.deleteBuffered
self._pop = self.pop
self.pop = self.popBuffered
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.interval = self.getConfigurationValue('interval')
self.fields = self.getConfigurationValue('fields')
self.stats_collector = StatisticCollector()
self.module_queues = {}
def configure(self, configuration):
BaseThreadedModule.configure(self, configuration)
self.hostname = self.getConfigurationValue("hostname")
self.fields = self.getConfigurationValue("fields")
self.field_prefix = self.getConfigurationValue("field_prefix")
self.timestamp_field = self.getConfigurationValue("timestamp_field")
self.batch_size = self.getConfigurationValue('batch_size')
self.backlog_size = self.getConfigurationValue('backlog_size')
self.agent_conf = self.getConfigurationValue("agent_conf")
if self.agent_conf:
if self.agent_conf is True:
self.agent_conf = "/etc/zabbix/zabbix_agentd.conf"
if not os.path.isfile(self.agent_conf):
self.logger.error("%s does not point to an existing file." % self.agent_conf)
self.lumbermill.shutDown()
self.zabbix_sender = ZabbixSender(use_config=self.agent_conf)
else:
self.logger.error("asdads")
server = self.getConfigurationValue("server")
port = 10051
if ":" in self.server:
server, port = self.server.split(":")
self.zabbix_sender = ZabbixSender(zabbix_server=server, port=port)
self.buffer = Buffer(self.getConfigurationValue('batch_size'), self.storeData,
self.getConfigurationValue('store_interval_in_secs'),
maxsize=self.getConfigurationValue('backlog_size'))
def configure(self, configuration):
# self.logger.setLevel(logging.DEBUG)
# Call parent configure method.
BaseThreadedModule.configure(self, configuration)
self.is_leader = True if self.getConfigurationValue('pack') == 'leader' else False
self.pack_followers = {}
self.cluster_name = self.getConfigurationValue('name')
self.discovered_leader = None
self.secret = hashlib.sha256(self.getConfigurationValue('secret')).digest()
self.handlers = collections.defaultdict(list)
self.lock = threading.Lock()
# Setup socket.
self.interface_addr = self.getConfigurationValue('interface')
self.interface_port = self.getConfigurationValue('port')
self.broadcast_addr = self.getConfigurationValue('broadcast')
self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
self.socket.settimeout(1)
self.hostname = socket.gethostname()
try:
self.socket.bind((self.interface_addr, self.interface_port))
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("Could not listen on %s:%s. Exception: %s, Error: %s." % (self.getConfigurationValue("interface"),
self.getConfigurationValue("port"), etype, evalue))
self.alive = False
self.lumbermill.shutDown()
return
self.addHandlers()
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.pretty_print = self.getConfigurationValue('pretty_print')
self.fields = self.getConfigurationValue('fields')
self.format = self.getConfigurationValue('format')
self.printing = False
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.backend = self.getConfigurationValue('backend')
self.backend_client = None
self.kv_store = None
self.set_buffer = None
if self.backend == 'DictStore':
import simplekv.memory
self.kv_store = simplekv.memory.DictStore()
elif self.backend == 'RedisStore':
import simplekv.memory.redisstore
self.backend_client = self._getRedisClient()
self.kv_store = simplekv.memory.redisstore.RedisStore(self.backend_client)
elif self.backend == 'MemcacheStore':
import simplekv.memory.memcachestore
self.backend_client = self._getMemcacheClient()
self.kv_store = simplekv.memory.memcachestore.MemcacheStore(self.backend_client)
else:
self.logger("Unknown backend type %s. Please check." % backend)
self.lumbermill.shutDown();
if self.getConfigurationValue('store_interval_in_secs') or self.getConfigurationValue('batch_size'):
if self.backend == 'RedisStore':
self.set_buffer = Buffer(self.getConfigurationValue('batch_size'), self._setRedisBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
else:
self.set_buffer = Buffer(self.getConfigurationValue('batch_size'), self._setBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
self._set = self.set
self.set = self._setBuffered
self._get = self.get
self.get = self._getBuffered
self._delete = self.delete
self.delete = self._deleteBuffered
self._pop = self.pop
self.pop = self._popBuffered
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.source_field = self.getConfigurationValue('source_field')
self.seperator = self.getConfigurationValue('seperator')
self.target_field = self.getConfigurationValue('target_field')
self.drop_original = not self.getConfigurationValue('keep_original')
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
# Make urllib3s logging less verbose.
urllib3_logger = logging.getLogger('requests.packages.urllib3.connectionpool')
urllib3_logger.setLevel(logging.CRITICAL)
self.server, self.port = self.getConfigurationValue('server').split(':')
self.user = self.getConfigurationValue('user')
self.events_container = []
self.batch_size = self.getConfigurationValue('batch_size')
self.backlog_size = self.getConfigurationValue('backlog_size')
self.path = self.getConfigurationValue('path')
self.name_pattern = self.getConfigurationValue('name_pattern')
self.format = self.getConfigurationValue('format')
self.compress = self.getConfigurationValue('compress')
if self.compress == 'gzip':
try:
import gzip
except ImportError:
self.logger.error('Gzip compression selected but gzip module could not be loaded.')
self.lumbermill.shutDown()
if self.compress == 'snappy':
try:
import snappy
except ImportError:
self.logger.error('Snappy compression selected but snappy module could not be loaded.')
self.lumbermill.shutDown()
self.is_storing = False
self.lock = multiprocessing.Lock()
self.timed_store_func = self.getTimedStoreFunc()
def initAfterFork(self):
BaseThreadedModule.initAfterFork(self)
self.buffer = Buffer(
self.getConfigurationValue('batch_size'),
self.storeData,
self.getConfigurationValue('store_interval_in_secs'),
maxsize=self.getConfigurationValue('backlog_size'))
try:
self.sqs_resource = boto3.resource(
'sqs',
region_name=self.getConfigurationValue('region'),
api_version=None,
use_ssl=True,
verify=None,
endpoint_url=None,
aws_access_key_id=self.getConfigurationValue(
'aws_access_key_id'),
aws_secret_access_key=self.getConfigurationValue(
'aws_secret_access_key'),
aws_session_token=None,
config=None)
self.sqs_queue = self.sqs_resource.get_queue_by_name(
QueueName=self.getConfigurationValue('queue'))
except:
etype, evalue, etb = sys.exc_info()
self.logger.error(
"Could not connect to sqs service. Exception: %s, Error: %s." %
(etype, evalue))
self.lumbermill.shutDown()
def shutDown(self):
if self.server:
self.server.stop()
# Give os time to free the socket. Otherwise a reload will fail with 'address already in use'
time.sleep(.2)
# Call parent shutDown method.
BaseThreadedModule.shutDown(self)
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
if len(self.getConfigurationValue('cluster')) == 0:
redis_store = self.getConfigurationValue('server')
self.client = self.getRedisClient()
else:
redis_store = self.getConfigurationValue('cluster')
self.client = self.getClusterRedisClient()
try:
self.client.ping()
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("Could not connect to redis store at %s. Exception: %s, Error: %s." % (redis_store,etype, evalue))
self.lumbermill.shutDown()
self.set_buffer = None
if self.getConfigurationValue('store_interval_in_secs') or self.getConfigurationValue('batch_size'):
self.set_buffer = Buffer(self.getConfigurationValue('batch_size'), self.setBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
self._set = self.set
self.set = self.setBuffered
self._get = self.get
self.get = self.getBuffered
self._delete = self.delete
self.delete = self.deleteBuffered
self._pop = self.pop
self.pop = self.popBuffered
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.formats = self.getConfigurationValue('formats')
self.connection_data = (self.getConfigurationValue('server'),
self.getConfigurationValue('port'))
self.connection = None
def initAfterFork(self):
self.evaluate_facet_data_func = setInterval(self.getConfigurationValue('interval'))(self.evaluateFacets) #self.getEvaluateFunc()
self.timed_func_handler_a = TimedFunctionManager.startTimedFunction(self.evaluate_facet_data_func)
if self.cache:
self.store_facets_in_cache_func = setInterval(1)(self.storeFacetsInCache)
self.timed_func_handler_b = TimedFunctionManager.startTimedFunction(self.store_facets_in_cache_func)
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
BaseThreadedModule.initAfterFork(self)
self.buffer = Buffer(self.getConfigurationValue('batch_size'), self.storeData, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
self.connection = self.connect()
if not self.connection:
self.lumbermill.shutDown()
return
BaseThreadedModule.initAfterFork(self)
def initAfterFork(self):
# Get all configured queues for waiting event stats.
self.module_queues = self.lumbermill.getAllQueues()
self.psutil_processes.append(psutil.Process(self.lumbermill.getMainProcessId()))
for worker in self.lumbermill.child_processes:
self.psutil_processes.append(psutil.Process(worker.pid))
TimedFunctionManager.startTimedFunction(self.getRunTimedFunctionsFunc())
BaseThreadedModule.initAfterFork(self)
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.events = self.getConfigurationValue("event")
if not isinstance(self.events, list):
self.events = [self.events]
self.sleep = self.getConfigurationValue("sleep")
self.max_events_count = self.getConfigurationValue("events_count")
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
# Set boto log level.
logging.getLogger('boto3').setLevel(logging.CRITICAL)
logging.getLogger('botocore').setLevel(logging.CRITICAL)
self.batch_size = self.getConfigurationValue('batch_size')
self.format = self.getConfigurationValue('format')
def initAfterFork(self):
# As the buffer uses a threaded timed function to flush its buffer and thread will not survive a fork, init buffer here.
self.buffers = collections.defaultdict(
lambda: Buffer(flush_size=self.buffer_size,
callback=self.sendMergedEvent,
interval=self.flush_interval_in_secs,
maxsize=self.buffer_size))
BaseThreadedModule.initAfterFork(self)
def shutDown(self):
# Call parent shutDown method.
BaseThreadedModule.shutDown(self)
try:
self.socket.close()
self.context.term()
except AttributeError:
pass
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.events = self.getConfigurationValue("event")
if not isinstance(self.events, list):
self.events = [self.events]
self.sleep = self.getConfigurationValue("sleep")
self.max_events_count = self.getConfigurationValue("events_count")
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.server = None
self.topic = self.getConfigurationValue('topic')
self.format = self.getConfigurationValue('format')
self.mode = self.getConfigurationValue('mode')
if self.mode == "bind":
self.can_run_forked = False
def initAfterFork(self):
BaseThreadedModule.initAfterFork(self)
# Init es client after fork as mentioned in https://elasticsearch-py.readthedocs.org/en/master/
self.es = self.connect()
if not self.es:
self.lumbermill.shutDown()
return
# As the buffer uses a threaded timed function to flush its buffer and thread will not survive a fork, init buffer here.
self.buffer = Buffer(self.getConfigurationValue('batch_size'), self.storeData, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.emit_as_event = self.getConfigurationValue('emit_as_event')
self.interval = self.getConfigurationValue('interval')
self.stats_collector = StatisticCollector()
for counter_name in ['events_received', 'event_type_Unknown', 'event_type_httpd_access_log']:
MultiProcessStatisticCollector().initCounter(counter_name)
self.module_queues = {}
def initAfterFork(self):
# Get all configured queues for waiting event stats.
for module_name, module_info in self.lumbermill.modules.items():
instance = module_info['instances'][0]
if not hasattr(instance, 'getInputQueue') or not instance.getInputQueue():
continue
self.module_queues[module_name] = instance.getInputQueue()
TimedFunctionManager.startTimedFunction(self.getRunTimedFunctionsFunc())
BaseThreadedModule.initAfterFork()
def initAfterFork(self):
BaseThreadedModule.initAfterFork(self)
# Init monogdb client after fork.
self.mongodb = self.connect()
if not self.mongodb:
self.lumbermill.shutDown()
return
# As the buffer uses a threaded timed function to flush its buffer and thread will not survive a fork, init buffer here.
self.buffer = Buffer(self.getConfigurationValue('batch_size'), self.storeData, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.source_fields = self.getConfigurationValue('source_fields')
# Allow single string as well.
if isinstance(self.source_fields, types.StringTypes):
self.source_fields = [self.source_fields]
self.target_field = self.getConfigurationValue('target_field')
self.in_mem_cache = MemoryCache(size=1000)
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.source_fields = self.getConfigurationValue('source_fields')
# Allow single string as well.
if isinstance(self.source_fields, types.StringTypes):
self.source_fields = [self.source_fields]
self.target_field = self.getConfigurationValue('target_field')
self.in_mem_cache = MemoryCache(size=1000)
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.emit_as_event = self.getConfigurationValue('emit_as_event')
self.interval = self.getConfigurationValue('interval')
self.event_type_statistics = self.getConfigurationValue(
'event_type_statistics')
self.stats_collector = StatisticCollector()
self.mp_stats_collector = MultiProcessStatisticCollector()
self.module_queues = {}
def configure(self, configuration):
# Call parent configure method.
BaseThreadedModule.configure(self, configuration)
self.file_tailer = None
self.files = self.scanPaths()
self.line_by_line = self.getConfigurationValue('line_by_line')
self.mode = self.getConfigurationValue('mode')
self.datastore_key = "%032x%s" % (random.getrandbits(128), self.process_id)
self.total_file_count = len(self.files)
self.lumbermill.setInInternalDataStore(self.datastore_key, 0)
def configure(self, configuration):
BaseThreadedModule.configure(self, configuration)
self.redis_ttl = self.getConfigurationValue('redis_ttl')
# Get redis client module.
if self.getConfigurationValue('redis_store'):
mod_info = self.lumbermill.getModuleInfoById(
self.getConfigurationValue('redis_store'))
self.redis_store = mod_info['instances'][0]
else:
self.redis_store = None
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.source_fields = self.getConfigurationValue('source_fields')
# Allow single string as well.
if isinstance(self.source_fields, types.StringTypes):
self.source_fields = [self.source_fields]
self.seperator = self.getConfigurationValue('seperator')
self.target_field = self.getConfigurationValue('target_field')
self.drop_original = not self.getConfigurationValue('keep_original')
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.source_field = self.getConfigurationValue('source_field')
self.target_field = self.getConfigurationValue('target_field')
self.drop_original = not self.getConfigurationValue('keep_original')
if self.getConfigurationValue('action') == 'decode':
self.handleEvent = self.decodeBase64
else:
self.handleEvent = self.encodeBase64
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.source_field = self.getConfigurationValue('source_field')
self.target_field = self.getConfigurationValue('target_field')
self.drop_original = not self.getConfigurationValue('keep_original')
if self.getConfigurationValue('action') == 'decode':
self.handleEvent = self.decodeBase64
else:
self.handleEvent = self.encodeBase64
def initAfterFork(self):
# Get all configured queues for waiting event stats.
for module_name, module_info in self.lumbermill.modules.items():
instance = module_info['instances'][0]
if not hasattr(instance,
'getInputQueue') or not instance.getInputQueue():
continue
self.module_queues[module_name] = instance.getInputQueue()
TimedFunctionManager.startTimedFunction(
self.getRunTimedFunctionsFunc())
BaseThreadedModule.initAfterFork()
def initAfterFork(self):
BaseThreadedModule.initAfterFork(self)
# Calculate event count when running in multiple processes.
if self.max_events_count == 0:
return
self.max_events_count = int(self.getConfigurationValue("events_count")/self.lumbermill.getWorkerCount())
if self.lumbermill.is_master():
remainder = self.getConfigurationValue("events_count") % self.lumbermill.getWorkerCount()
self.max_events_count += remainder
if self.max_events_count == 0:
self.shutDown()
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.results = []
function_str = "lambda event: " + re.sub('%\((.*?)\)s', r"event.get('\1', False)", self.getConfigurationValue('function'))
self.function = self.compileFunction(function_str)
if self.getConfigurationValue('results_function'):
function_str = "lambda results: "+ re.sub('%\((.*?)\)s', r"results", self.getConfigurationValue('results_function'))
self.results_function = self.compileFunction(function_str)
self.target_field = self.getConfigurationValue('target_field')
self.interval = self.getConfigurationValue('interval')
def initAfterFork(self):
self.resolver = resolver.Resolver()
self.resolver.timeout = self.timeout
self.resolver.lifetime = self.timeout
if self.nameservers:
self.resolver.nameservers = self.nameservers
self.queue = Queue.Queue(20)
self.lookup_threads = [LookupThread(self.queue, self.lookup_type, self) for _ in range(0, self.lookup_threads_pool_size)]
for thread in self.lookup_threads:
thread.start()
BaseThreadedModule.initAfterFork(self)
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.format = self.getConfigurationValue('format')
try:
self.client = redis.Redis(host=self.getConfigurationValue('server'),
port=self.getConfigurationValue('port'),
password=self.getConfigurationValue('password'),
db=self.getConfigurationValue('db'))
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("Could not connect to redis store at %s. Exception: %s, Error: %s." % (self.getConfigurationValue('server'),etype, evalue))
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.emit_as_event = self.getConfigurationValue('emit_as_event')
self.interval = self.getConfigurationValue('interval')
self.stats_collector = StatisticCollector()
for counter_name in [
'events_received', 'event_type_Unknown',
'event_type_httpd_access_log'
]:
MultiProcessStatisticCollector().initCounter(counter_name)
self.module_queues = {}
def initAfterFork(self):
BaseThreadedModule.initAfterFork(self)
self.buffer = Buffer(
self.getConfigurationValue('batch_size'),
self.storeData,
self.getConfigurationValue('store_interval_in_secs'),
maxsize=self.getConfigurationValue('backlog_size'))
self.connection = self.connect()
if not self.connection:
self.lumbermill.shutDown()
return
BaseThreadedModule.initAfterFork(self)
def configure(self, configuration):
BaseThreadedModule.configure(self, configuration)
socket.setdefaulttimeout(self.getConfigurationValue('socket_timeout'))
self.get_metadata = self.getConfigurationValue('get_metadata')
self.interval = self.getConfigurationValue('interval')
# Get redis client module.
if self.getConfigurationValue('redis_store'):
mod_info = self.lumbermill.getModuleInfoById(
self.getConfigurationValue('redis_store'))
self.redis_store = mod_info['instances'][0]
else:
self.redis_store = None
def shutDown(self):
try:
self.buffer.flush()
except:
pass
try:
self.client.close()
self.zmq_context.term()
except AttributeError:
pass
# Call parent shutDown method.
BaseThreadedModule.shutDown(self)
