def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.source_field = self.getConfigurationValue('source_field')
self.seperator = self.getConfigurationValue('seperator')
self.target_field = self.getConfigurationValue('target_field')
self.drop_original = not self.getConfigurationValue('keep_original')
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
# Make urllib3s logging less verbose.
urllib3_logger = logging.getLogger('requests.packages.urllib3.connectionpool')
urllib3_logger.setLevel(logging.CRITICAL)
self.server, self.port = self.getConfigurationValue('server').split(':')
self.user = self.getConfigurationValue('user')
self.events_container = []
self.batch_size = self.getConfigurationValue('batch_size')
self.backlog_size = self.getConfigurationValue('backlog_size')
self.path = self.getConfigurationValue('path')
self.name_pattern = self.getConfigurationValue('name_pattern')
self.format = self.getConfigurationValue('format')
self.compress = self.getConfigurationValue('compress')
if self.compress == 'gzip':
try:
import gzip
except ImportError:
self.logger.error('Gzip compression selected but gzip module could not be loaded.')
self.lumbermill.shutDown()
if self.compress == 'snappy':
try:
import snappy
except ImportError:
self.logger.error('Snappy compression selected but snappy module could not be loaded.')
self.lumbermill.shutDown()
self.is_storing = False
self.lock = multiprocessing.Lock()
self.timed_store_func = self.getTimedStoreFunc()
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.backend = self.getConfigurationValue('backend')
self.backend_client = None
self.kv_store = None
self.set_buffer = None
if self.backend == 'DictStore':
import simplekv.memory
self.kv_store = simplekv.memory.DictStore()
elif self.backend == 'RedisStore':
import simplekv.memory.redisstore
self.backend_client = self._getRedisClient()
self.kv_store = simplekv.memory.redisstore.RedisStore(self.backend_client)
elif self.backend == 'MemcacheStore':
import simplekv.memory.memcachestore
self.backend_client = self._getMemcacheClient()
self.kv_store = simplekv.memory.memcachestore.MemcacheStore(self.backend_client)
else:
self.logger("Unknown backend type %s. Please check." % backend)
self.lumbermill.shutDown();
if self.getConfigurationValue('store_interval_in_secs') or self.getConfigurationValue('batch_size'):
if self.backend == 'RedisStore':
self.set_buffer = Buffer(self.getConfigurationValue('batch_size'), self._setRedisBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
else:
self.set_buffer = Buffer(self.getConfigurationValue('batch_size'), self._setBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
self._set = self.set
self.set = self._setBuffered
self._get = self.get
self.get = self._getBuffered
self._delete = self.delete
self.delete = self._deleteBuffered
self._pop = self.pop
self.pop = self._popBuffered
def configure(self, configuration):
# Call parent configure method.
BaseThreadedModule.configure(self, configuration)
for module_name in ['elasticsearch', 'urllib3', 'requests']:
if self.getConfigurationValue('log_level') == 'info':
logging.getLogger(module_name).setLevel(logging.WARN)
else:
# Set log level for elasticsarch library if configured to other than default.
logging.getLogger(module_name).setLevel(self.logger.level)
self.action = self.getConfigurationValue('action')
self.fields = self.getConfigurationValue('fields')
self.ttl = self.getConfigurationValue("ttl")
self.index_name = self.getConfigurationValue("index_name")
self.routing_pattern = self.getConfigurationValue("routing")
self.doc_id_pattern = self.getConfigurationValue("doc_id")
self.doc_type_pattern = self.getConfigurationValue("doc_type")
self.doc_type_is_dynamic = self.isDynamicConfigurationValue("doc_type")
self.es_nodes = self.getConfigurationValue("nodes")
self.read_timeout = self.getConfigurationValue("read_timeout")
if not isinstance(self.es_nodes, list):
self.es_nodes = [self.es_nodes]
if self.getConfigurationValue("connection_type") == 'urllib3':
self.connection_class = elasticsearch.connection.Urllib3HttpConnection
elif self.getConfigurationValue("connection_type") == 'requests':
self.connection_class = elasticsearch.connection.RequestsHttpConnection
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.interval = self.getConfigurationValue('interval')
self.fields = self.getConfigurationValue('fields')
self.stats_collector = StatisticCollector()
self.module_queues = {}
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.source_field = self.getConfigurationValue('source_field')
self.seperator = self.getConfigurationValue('seperator')
self.target_field = self.getConfigurationValue('target_field')
self.drop_original = not self.getConfigurationValue('keep_original')
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.source_field = self.getConfigurationValue('source_field')
self.add_event_fields = self.getConfigurationValue('add_event_fields')
self.backend_lock_name = "Lumbermill:Facet:FacetLock-%s" % self.lumbermill.getMainProcessId(
)
self.backend_key_name = "Lumbermill:Facet:FacetKeys-%s" % self.lumbermill.getMainProcessId(
)
self.backend_ttl = self.getConfigurationValue('backend_ttl')
if (self.backend_ttl < self.getConfigurationValue('interval')):
self.logger.error(
'backend_ttl setting is smaller then interval setting. Please check.'
)
self.lumbermill.shutDown()
return
backend_info = self.lumbermill.getModuleInfoById(
self.getConfigurationValue('backend'))
if not backend_info:
self.logger.error(
"Could not find %s backend for persistant storage." %
(self.getConfigurationValue('backend')))
self.lumbermill.shutDown()
return
self.persistence_backend = backend_info['instances'][0]
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
backend = self.getConfigurationValue('backend')
self.backend_client = None
if backend == 'DictStore':
import simplekv.memory
self.backend_client = None
self.kv_store = simplekv.memory.DictStore()
elif backend == 'RedisStore':
import simplekv.memory.redisstore
self.backend_client = self.getRedisClient()
self.kv_store = simplekv.memory.redisstore.RedisStore(self.backend_client)
elif backend == 'MemcacheStore':
import simplekv.memory.memcachestore
self.backend_client = self.getMemcacheClient()
self.kv_store = simplekv.memory.memcachestore.MemcacheStore(self.backend_client)
self.set_buffer = None
if self.getConfigurationValue('store_interval_in_secs') or self.getConfigurationValue('batch_size'):
if backend == 'RedisStore':
self.set_buffer = Buffer(self.getConfigurationValue('batch_size'), self.setRedisBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
else:
self.set_buffer = Buffer(self.getConfigurationValue('batch_size'), self.setBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
self._set = self.set
self.set = self.setBuffered
self._get = self.get
self.get = self.getBuffered
self._delete = self.delete
self.delete = self.deleteBuffered
self._pop = self.pop
self.pop = self.popBuffered
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
if len(self.getConfigurationValue('cluster')) == 0:
redis_store = self.getConfigurationValue('server')
self.client = self.getRedisClient()
else:
redis_store = self.getConfigurationValue('cluster')
self.client = self.getClusterRedisClient()
try:
self.client.ping()
except:
etype, evalue, etb = sys.exc_info()
self.logger.error(
"Could not connect to redis store at %s. Exception: %s, Error: %s."
% (redis_store, etype, evalue))
self.lumbermill.shutDown()
self.set_buffer = None
if self.getConfigurationValue(
'store_interval_in_secs') or self.getConfigurationValue(
'batch_size'):
self.set_buffer = Buffer(
self.getConfigurationValue('batch_size'),
self.setBufferedCallback,
self.getConfigurationValue('store_interval_in_secs'),
maxsize=self.getConfigurationValue('backlog_size'))
self._set = self.set
self.set = self.setBuffered
self._get = self.get
self.get = self.getBuffered
self._delete = self.delete
self.delete = self.deleteBuffered
self._pop = self.pop
self.pop = self.popBuffered
def configure(self, configuration):
# self.logger.setLevel(logging.DEBUG)
# Call parent configure method.
BaseThreadedModule.configure(self, configuration)
self.is_leader = True if self.getConfigurationValue(
'pack') == 'leader' else False
self.pack_followers = {}
self.cluster_name = self.getConfigurationValue('name')
self.discovered_leader = None
self.secret = hashlib.sha256(
self.getConfigurationValue('secret')).digest()
self.handlers = collections.defaultdict(list)
self.lock = threading.Lock()
# Setup socket.
self.interface_addr = self.getConfigurationValue('interface')
self.interface_port = self.getConfigurationValue('port')
self.broadcast_addr = self.getConfigurationValue('broadcast')
self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
self.socket.settimeout(1)
self.hostname = socket.gethostname()
try:
self.socket.bind((self.interface_addr, self.interface_port))
except:
etype, evalue, etb = sys.exc_info()
self.logger.error(
"Could not listen on %s:%s. Exception: %s, Error: %s." %
(self.getConfigurationValue("interface"),
self.getConfigurationValue("port"), etype, evalue))
self.alive = False
self.lumbermill.shutDown()
return
self.addHandlers()
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.interval = self.getConfigurationValue('interval')
self.fields = self.getConfigurationValue('fields')
self.stats_collector = StatisticCollector()
self.module_queues = {}
def configure(self, configuration):
# self.logger.setLevel(logging.DEBUG)
# Call parent configure method.
BaseThreadedModule.configure(self, configuration)
self.is_leader = True if self.getConfigurationValue('pack') == 'leader' else False
self.pack_followers = {}
self.cluster_name = self.getConfigurationValue('name')
self.discovered_leader = None
self.secret = hashlib.sha256(self.getConfigurationValue('secret')).digest()
self.handlers = collections.defaultdict(list)
self.lock = threading.Lock()
# Setup socket.
self.interface_addr = self.getConfigurationValue('interface')
self.interface_port = self.getConfigurationValue('port')
self.broadcast_addr = self.getConfigurationValue('broadcast')
self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
self.socket.settimeout(1)
self.hostname = socket.gethostname()
try:
self.socket.bind((self.interface_addr, self.interface_port))
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("Could not listen on %s:%s. Exception: %s, Error: %s." % (self.getConfigurationValue("interface"),
self.getConfigurationValue("port"), etype, evalue))
self.alive = False
self.lumbermill.shutDown()
return
self.addHandlers()
def configure(self, configuration):
BaseThreadedModule.configure(self, configuration)
self.hostname = self.getConfigurationValue("hostname")
self.fields = self.getConfigurationValue("fields")
self.field_prefix = self.getConfigurationValue("field_prefix")
self.timestamp_field = self.getConfigurationValue("timestamp_field")
self.batch_size = self.getConfigurationValue('batch_size')
self.backlog_size = self.getConfigurationValue('backlog_size')
self.agent_conf = self.getConfigurationValue("agent_conf")
if self.agent_conf:
if self.agent_conf is True:
self.agent_conf = "/etc/zabbix/zabbix_agentd.conf"
if not os.path.isfile(self.agent_conf):
self.logger.error("%s does not point to an existing file." % self.agent_conf)
self.lumbermill.shutDown()
self.zabbix_sender = ZabbixSender(use_config=self.agent_conf)
else:
self.logger.error("asdads")
server = self.getConfigurationValue("server")
port = 10051
if ":" in self.server:
server, port = self.server.split(":")
self.zabbix_sender = ZabbixSender(zabbix_server=server, port=port)
self.buffer = Buffer(self.getConfigurationValue('batch_size'), self.storeData,
self.getConfigurationValue('store_interval_in_secs'),
maxsize=self.getConfigurationValue('backlog_size'))
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.formats = self.getConfigurationValue('formats')
self.connection_data = (self.getConfigurationValue('server'),
self.getConfigurationValue('port'))
self.connection = None
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.lists = self.getConfigurationValue('lists')
if not isinstance(self.lists, list):
self.lists = [self.lists]
self.timeout = self.getConfigurationValue('timeout')
self.batch_size = self.getConfigurationValue('batch_size')
self.client = redis.StrictRedis(
host=self.getConfigurationValue('server'),
port=self.getConfigurationValue('port'),
password=self.getConfigurationValue('password'),
db=self.getConfigurationValue('db'))
try:
self.client.ping()
except:
etype, evalue, etb = sys.exc_info()
self.logger.error(
"Could not connect to redis store at %s. Exception: %s, Error: %s."
% (self.getConfigurationValue('server'), etype, evalue))
self.lumbermill.shutDown()
# Monkeypatch run method to use the correct handle event method.
if self.batch_size == 1:
self.run = self.handleSingleEvent
else:
self.run = self.handleBatchEvents
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
if len(self.getConfigurationValue('cluster')) == 0:
redis_store = self.getConfigurationValue('server')
self.client = self.getRedisClient()
else:
redis_store = self.getConfigurationValue('cluster')
self.client = self.getClusterRedisClient()
try:
self.client.ping()
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("Could not connect to redis store at %s. Exception: %s, Error: %s." % (redis_store,etype, evalue))
self.lumbermill.shutDown()
self.set_buffer = None
if self.getConfigurationValue('store_interval_in_secs') or self.getConfigurationValue('batch_size'):
self.set_buffer = Buffer(self.getConfigurationValue('batch_size'), self.setBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
self._set = self.set
self.set = self.setBuffered
self._get = self.get
self.get = self.getBuffered
self._delete = self.delete
self.delete = self.deleteBuffered
self._pop = self.pop
self.pop = self.popBuffered
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.source_fields = self.getConfigurationValue('source_fields')
# Allow single string as well.
if isinstance(self.source_fields, types.StringTypes):
self.source_fields = [self.source_fields]
self.target_fields = self.getConfigurationValue('target_fields')
# Allow single string as well.
if isinstance(self.target_fields, types.StringTypes):
self.target_fields = [self.target_fields]
# If target_fields were provided they need to be of same length as source_fields.
if self.target_fields and len(self.source_fields) != len(
self.target_fields):
self.logger.error(
"Count of configured source and target fields need to match. Please check your configuration."
)
self.lumbermill.shutDown()
self.compression = self.getConfigurationValue('compression')
# Get compression specific handler.
try:
self.deflater = getattr(self, "deflate_with_%s" % self.compression)
except AttributeError:
etype, evalue, etb = sys.exc_info()
self.logger.error(
"Unknown decompression lib: %s. Exception: %s, Error: %s" %
(self.compression, etype, evalue))
self.lumbermill.shutDown()
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
# Set boto log level.
logging.getLogger('boto3').setLevel(logging.CRITICAL)
logging.getLogger('botocore').setLevel(logging.CRITICAL)
self.sqs_queue_name = self.getConfigurationValue('queue')
self.attribute_names = self.getConfigurationValue('attribute_names')
self.message_attribute_names = self.getConfigurationValue('message_attribute_names')
self.poll_interval = self.getConfigurationValue('poll_interval_in_secs')
self.batch_size = self.getConfigurationValue('batch_size')
try:
self.sqs_client = boto3.client('sqs', region_name=self.getConfigurationValue('region'),
api_version=None,
use_ssl=True,
verify=None,
endpoint_url=None,
aws_access_key_id=self.getConfigurationValue('aws_access_key_id'),
aws_secret_access_key=self.getConfigurationValue('aws_secret_access_key'),
aws_session_token=None,
config=None)
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("Could not connect to sqs service. Exception: %s, Error: %s." % (etype, evalue))
self.lumbermill.shutDown()
try:
self.sqs_queue_url = self.sqs_client.get_queue_url(QueueName=self.sqs_queue_name)['QueueUrl']
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("Could not get queue url for sqs queue %s. Exception: %s, Error: %s." % (self.sqs_queue_name, etype, evalue))
self.lumbermill.shutDown()
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.batch_size = self.getConfigurationValue('batch_size')
self.backlog_size = self.getConfigurationValue('backlog_size')
self.file_name = self.getConfigurationValue('file_name')
self.format = self.getConfigurationValue('format')
self.compress = self.getConfigurationValue('compress')
self.file_handles = {}
if self.compress == 'gzip':
try:
# Import module into namespace of object. Otherwise it will not be accessible when process was forked.
self.gzip_module = __import__('gzip')
except ImportError:
self.logger.error(
'Gzip compression selected but gzip module could not be loaded.'
)
self.lumbermill.shutDown()
if self.compress == 'snappy':
try:
self.snappy_module = __import__('snappy')
except ImportError:
self.logger.error(
'Snappy compression selected but snappy module could not be loaded.'
)
self.lumbermill.shutDown()
self.buffer = Buffer(
self.batch_size,
self.storeData,
self.getConfigurationValue('store_interval_in_secs'),
maxsize=self.backlog_size)
TimedFunctionManager.startTimedFunction(self.closeStaleFileHandles)
def configure(self, configuration):
# Call parent configure method.
BaseThreadedModule.configure(self, configuration)
self.format = self.getConfigurationValue('format')
self.collection = self.getConfigurationValue('collection')
self.database = self.getConfigurationValue('database')
self.doc_id_pattern = self.getConfigurationValue("doc_id")
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.syslogger = logging.getLogger(self.__class__.__name__)
self.syslogger.propagate = False
self.format = self.getConfigurationValue('format')
if os.path.exists(self.getConfigurationValue('address')):
address = self.getConfigurationValue('address')
else:
server, port = self.getConfigurationValue('address').split(':')
address = (server, int(port))
if self.getConfigurationValue('proto') == 'tcp':
socket_type = socket.SOCK_STREAM
else:
socket_type = socket.SOCK_DGRAM
try:
facility = logging.handlers.SysLogHandler.facility_names[
self.getConfigurationValue('facility')]
except KeyError:
self.logger.error("The configured facility %s is unknown." %
(self.getConfigurationValue('facility')))
self.syslog_handler = logging.handlers.SysLogHandler(
address, facility=facility, socktype=socket_type)
self.syslogger.addHandler(self.syslog_handler)
self.format = self.getConfigurationValue('format')
def configure(self, configuration):
# Call parent configure method.
BaseThreadedModule.configure(self, configuration)
self.format = self.getConfigurationValue('format')
self.collection = self.getConfigurationValue('collection')
self.database = self.getConfigurationValue('database')
self.doc_id_pattern = self.getConfigurationValue("doc_id")
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.pretty_print = self.getConfigurationValue('pretty_print')
self.fields = self.getConfigurationValue('fields')
self.format = self.getConfigurationValue('format')
self.printing = False
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
# Set boto log level.
logging.getLogger('boto3').setLevel(logging.CRITICAL)
logging.getLogger('botocore').setLevel(logging.CRITICAL)
self.batch_size = self.getConfigurationValue('batch_size')
self.format = self.getConfigurationValue('format')
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.events = self.getConfigurationValue("event")
if not isinstance(self.events, list):
self.events = [self.events]
self.sleep = self.getConfigurationValue("sleep")
self.max_events_count = self.getConfigurationValue("events_count")
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.events = self.getConfigurationValue("event")
if not isinstance(self.events, list):
self.events = [self.events]
self.sleep = self.getConfigurationValue("sleep")
self.max_events_count = self.getConfigurationValue("events_count")
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.server = None
self.topic = self.getConfigurationValue('topic')
self.format = self.getConfigurationValue('format')
self.mode = self.getConfigurationValue('mode')
if self.mode == "bind":
self.can_run_forked = False
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.source_fields = self.getConfigurationValue('source_fields')
# Allow single string as well.
if isinstance(self.source_fields, types.StringTypes):
self.source_fields = [self.source_fields]
self.target_field = self.getConfigurationValue('target_field')
self.in_mem_cache = MemoryCache(size=1000)
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.emit_as_event = self.getConfigurationValue('emit_as_event')
self.interval = self.getConfigurationValue('interval')
self.stats_collector = StatisticCollector()
for counter_name in ['events_received', 'event_type_Unknown', 'event_type_httpd_access_log']:
MultiProcessStatisticCollector().initCounter(counter_name)
self.module_queues = {}
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.source_fields = self.getConfigurationValue('source_fields')
# Allow single string as well.
if isinstance(self.source_fields, types.StringTypes):
self.source_fields = [self.source_fields]
self.target_field = self.getConfigurationValue('target_field')
self.in_mem_cache = MemoryCache(size=1000)
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.source_field = self.getConfigurationValue('source_field')
self.target_field = self.getConfigurationValue('target_field')
self.drop_original = not self.getConfigurationValue('keep_original')
if self.getConfigurationValue('action') == 'decode':
self.handleEvent = self.decodeBase64
else:
self.handleEvent = self.encodeBase64
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.source_fields = self.getConfigurationValue('source_fields')
# Allow single string as well.
if isinstance(self.source_fields, types.StringTypes):
self.source_fields = [self.source_fields]
self.seperator = self.getConfigurationValue('seperator')
self.target_field = self.getConfigurationValue('target_field')
self.drop_original = not self.getConfigurationValue('keep_original')
def configure(self, configuration):
# Call parent configure method.
BaseThreadedModule.configure(self, configuration)
self.file_tailer = None
self.files = self.scanPaths()
self.line_by_line = self.getConfigurationValue('line_by_line')
self.mode = self.getConfigurationValue('mode')
self.datastore_key = "%032x%s" % (random.getrandbits(128), self.process_id)
self.total_file_count = len(self.files)
self.lumbermill.setInInternalDataStore(self.datastore_key, 0)
def configure(self, configuration):
BaseThreadedModule.configure(self, configuration)
self.redis_ttl = self.getConfigurationValue('redis_ttl')
# Get redis client module.
if self.getConfigurationValue('redis_store'):
mod_info = self.lumbermill.getModuleInfoById(
self.getConfigurationValue('redis_store'))
self.redis_store = mod_info['instances'][0]
else:
self.redis_store = None
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.source_field = self.getConfigurationValue('source_field')
self.target_field = self.getConfigurationValue('target_field')
self.drop_original = not self.getConfigurationValue('keep_original')
if self.getConfigurationValue('action') == 'decode':
self.handleEvent = self.decodeBase64
else:
self.handleEvent = self.encodeBase64
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.emit_as_event = self.getConfigurationValue('emit_as_event')
self.interval = self.getConfigurationValue('interval')
self.event_type_statistics = self.getConfigurationValue(
'event_type_statistics')
self.stats_collector = StatisticCollector()
self.mp_stats_collector = MultiProcessStatisticCollector()
self.module_queues = {}
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.results = []
function_str = "lambda event: " + re.sub('%\((.*?)\)s', r"event.get('\1', False)", self.getConfigurationValue('function'))
self.function = self.compileFunction(function_str)
if self.getConfigurationValue('results_function'):
function_str = "lambda results: "+ re.sub('%\((.*?)\)s', r"results", self.getConfigurationValue('results_function'))
self.results_function = self.compileFunction(function_str)
self.target_field = self.getConfigurationValue('target_field')
self.interval = self.getConfigurationValue('interval')
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.emit_as_event = self.getConfigurationValue('emit_as_event')
self.interval = self.getConfigurationValue('interval')
self.stats_collector = StatisticCollector()
for counter_name in [
'events_received', 'event_type_Unknown',
'event_type_httpd_access_log'
]:
MultiProcessStatisticCollector().initCounter(counter_name)
self.module_queues = {}
def configure(self, configuration):
BaseThreadedModule.configure(self, configuration)
socket.setdefaulttimeout(self.getConfigurationValue('socket_timeout'))
self.get_metadata = self.getConfigurationValue('get_metadata')
self.interval = self.getConfigurationValue('interval')
# Get redis client module.
if self.getConfigurationValue('redis_store'):
mod_info = self.lumbermill.getModuleInfoById(
self.getConfigurationValue('redis_store'))
self.redis_store = mod_info['instances'][0]
else:
self.redis_store = None
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.format = self.getConfigurationValue('format')
try:
self.client = redis.Redis(host=self.getConfigurationValue('server'),
port=self.getConfigurationValue('port'),
password=self.getConfigurationValue('password'),
db=self.getConfigurationValue('db'))
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("Could not connect to redis store at %s. Exception: %s, Error: %s." % (self.getConfigurationValue('server'),etype, evalue))
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.in_mem_cache = MemoryCache(size=5000)
self.lookup_type = self.getConfigurationValue('action')
self.source_field = self.getConfigurationValue('source_field')
self.target_field = self.getConfigurationValue('target_field')
self.nameservers = self.getConfigurationValue('nameservers')
self.timeout = self.getConfigurationValue('timeout')
# Allow single string as well.
if isinstance(self.nameservers, types.StringTypes):
self.nameservers = [self.nameservers]
self.lookup_threads_pool_size = 3
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.emit_as_event = self.getConfigurationValue('emit_as_event')
self.interval = self.getConfigurationValue('interval')
self.event_type_statistics = self.getConfigurationValue('event_type_statistics')
self.process_statistics = self.getConfigurationValue('process_statistics')
self.stats_collector = StatisticCollector()
self.mp_stats_collector = MultiProcessStatisticCollector()
self.module_queues = {}
self.psutil_processes = []
self.last_values = {'events_received': 0}
self.methods = dir(self)
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.in_mem_cache = MemoryCache(size=5000)
self.lookup_type = self.getConfigurationValue('action')
self.source_field = self.getConfigurationValue('source_field')
self.target_field = self.getConfigurationValue('target_field')
self.nameservers = self.getConfigurationValue('nameservers')
self.timeout = self.getConfigurationValue('timeout')
# Allow single string as well.
if isinstance(self.nameservers, types.StringTypes):
self.nameservers = [self.nameservers]
self.lookup_threads_pool_size = 3
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
# Set defaults
supported_regex_match_types = ['search', 'findall']
self.timed_func_handler = None
self.source_field = self.getConfigurationValue('source_field')
self.mark_unmatched_as = self.getConfigurationValue('mark_unmatched_as')
self.break_on_match = self.getConfigurationValue('break_on_match')
self.hot_rules_first = self.getConfigurationValue('hot_rules_first')
self.event_types = []
self.fieldextraction_regexpressions = []
self.logstash_patterns = {}
self.readLogstashPatterns()
for regex_config in configuration['field_extraction_patterns']:
event_type = regex_config.keys()[0]
regex_pattern = regex_config[event_type]
regex_options = 0
regex_match_type = 'search'
if isinstance(regex_pattern, list):
i = iter(regex_pattern)
# Pattern is the first entry
regex_pattern = i.next()
# Regex options the second
try:
regex_options = eval(i.next())
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("RegEx error for options %s. Exception: %s, Error: %s." % (regex_options, etype, evalue))
self.lumbermill.shutDown()
return
# Regex match type the third (optional)
try:
regex_match_type = i.next()
except:
pass
# Make sure regex_match_type is valid
# At the moment only search and findall are supported
if regex_match_type not in supported_regex_match_types:
self.logger.error("RegEx error for match type %s. Only %s are supported." % (regex_options, supported_regex_match_types))
self.lumbermill.shutDown()
return
try:
regex_pattern = self.replaceLogstashPatterns(regex_pattern)
regex = re.compile(regex_pattern, regex_options)
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("RegEx error for %s pattern %s. Exception: %s, Error: %s." % (event_type, regex_pattern, etype, evalue))
self.lumbermill.shutDown()
self.fieldextraction_regexpressions.append({'event_type': event_type, 'pattern': regex, 'match_type': regex_match_type, 'hitcounter': 0})
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
# Set defaults
supported_regex_match_types = ['search', 'findall']
self.timed_func_handler = None
self.source_field = self.getConfigurationValue('source_field')
self.mark_unmatched_as = self.getConfigurationValue('mark_unmatched_as')
self.break_on_match = self.getConfigurationValue('break_on_match')
self.hot_rules_first = self.getConfigurationValue('hot_rules_first')
self.event_types = []
self.fieldextraction_regexpressions = []
self.logstash_patterns = {}
self.readLogstashPatterns()
for regex_config in configuration['field_extraction_patterns']:
event_type = regex_config.keys()[0]
regex_pattern = regex_config[event_type]
regex_options = 0
regex_match_type = 'search'
if isinstance(regex_pattern, list):
i = iter(regex_pattern)
# Pattern is the first entry
regex_pattern = i.next()
# Regex options the second
try:
regex_options = eval(i.next())
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("RegEx error for options %s. Exception: %s, Error: %s." % (regex_options, etype, evalue))
self.lumbermill.shutDown()
return
# Regex match type the third (optional)
try:
regex_match_type = i.next()
except StopIteration:
pass
# Make sure regex_match_type is valid
# At the moment only search and findall are supported
if regex_match_type not in supported_regex_match_types:
self.logger.error("RegEx error for match type %s. Only %s are supported." % (regex_options, supported_regex_match_types))
self.lumbermill.shutDown()
return
try:
regex_pattern = self.replaceLogstashPatterns(regex_pattern)
regex = re.compile(regex_pattern, regex_options)
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("RegEx error for %s pattern %s. Exception: %s, Error: %s." % (event_type, regex_pattern, etype, evalue))
self.lumbermill.shutDown()
self.fieldextraction_regexpressions.append({'event_type': event_type, 'pattern': regex, 'match_type': regex_match_type, 'hitcounter': 0})
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.format = self.getConfigurationValue('format')
try:
self.client = redis.Redis(
host=self.getConfigurationValue('server'),
port=self.getConfigurationValue('port'),
password=self.getConfigurationValue('password'),
db=self.getConfigurationValue('db'))
except:
etype, evalue, etb = sys.exc_info()
self.logger.error(
"Could not connect to redis store at %s. Exception: %s, Error: %s."
% (self.getConfigurationValue('server'), etype, evalue))
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.source_fields = self.getConfigurationValue('source_fields')
# Allow single string as well.
if isinstance(self.source_fields, types.StringTypes):
self.source_fields = [self.source_fields]
self.target_field = self.getConfigurationValue('target_field')
self.drop_original = not self.getConfigurationValue('keep_original')
if self.getConfigurationValue('action') == 'decode':
self.default_list_attributes = [attrib for attrib in dir(list) if not attrib.startswith('__')]
self.parser = Parser()
self.handleEvent = self.decodeEvent
else:
self.handleEvent = self.encodeEvent
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
# Set defaultså
self.typecast_switch = { 'int': self.cast_to_int,
'integer': self.cast_to_int,
'float': self.cast_to_float,
'str': self.cast_to_str,
'string': self.cast_to_str,
'bool': self.cast_to_bool,
'boolean': self.cast_to_bool,
}
self.action = configuration['action']
# Precompile regex for replacement if defined
if 'regex' in configuration:
regex_pattern = configuration['regex']
regex_options = 0
if isinstance(regex_pattern, list):
i = iter(regex_pattern)
# Pattern is the first entry
regex_pattern = i.next()
# Regex options the second
try:
regex_options = eval(i.next())
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("RegEx error for options %s. Exception: %s, Error: %s" % (regex_options, etype, evalue))
self.lumbermill.shutDown()
try:
self.regex = re.compile(regex_pattern, regex_options)
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("RegEx error for pattern %s. Exception: %s, Error: %s" % (regex_pattern, etype, evalue))
self.lumbermill.shutDown()
self.source_field = self.getConfigurationValue('source_field') if "source_field" in self.configuration_data else []
self.source_fields = self.getConfigurationValue('source_fields') if "source_fields" in self.configuration_data else []
self.target_field = self.getConfigurationValue('target_field') if "target_field" in self.configuration_data else []
self.target_fields = self.getConfigurationValue('target_fields') if "target_fields" in self.configuration_data else []
# Call action specific configure method.
if "configure_%s_action" % self.action in dir(self):
getattr(self, "configure_%s_action" % self.action)()
# Get action specific method
try:
self.event_handler = getattr(self, "%s" % self.action)
except AttributeError:
etype, evalue, etb = sys.exc_info()
self.logger.error("ModifyFields action called that does not exist: %s. Exception: %s, Error: %s" % (self.action, etype, evalue))
self.lumbermill.shutDown()
def configure(self, configuration):
# Call parent configure method.
BaseThreadedModule.configure(self, configuration)
paths = self.getConfigurationValue('paths')
if isinstance(paths, types.StringType):
paths = [paths]
self.line_by_line = self.getConfigurationValue('line_by_line')
self.files = []
for path in paths:
self.files += self.getFilesInPath(
path, self.getConfigurationValue('pattern'),
self.getConfigurationValue('recursive'))
self.datastore_key = "%032x%s" % (random.getrandbits(128),
self.process_id)
self.total_file_count = len(self.files)
self.lumbermill.setInInternalDataStore(self.datastore_key, 0)
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.server = False
self.settings = self.getSettings()
self.application = tornado.web.Application([], **self.settings)
# Default handlers.
handlers = [ # REST ActionHandler
(r"/rest/server/restart", handler.ActionHandler.RestartHandler),
(r"/rest/server/info", handler.ActionHandler.GetServerInformation),
(r"/rest/server/statistics", handler.ActionHandler.GetServerStatistics),
(r"/rest/server/configuration", handler.ActionHandler.GetServerConfiguration),
# WebsocketHandler
(r"/websockets/statistics", handler.WebsocketHandler.StatisticsWebSocketHandler),
(r"/websockets/get_logs", handler.WebsocketHandler.LogToWebSocketHandler) ]
self.addHandlers(handlers)
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.source_fields = self.getConfigurationValue('source_fields')
# Allow single string as well.
if isinstance(self.source_fields, types.StringTypes):
self.source_fields = [self.source_fields]
self.target_field = self.getConfigurationValue('target_field')
self.drop_original = not self.getConfigurationValue('keep_original')
if self.getConfigurationValue('action') == 'decode':
if self.getConfigurationValue('mode') == 'line':
self.handleEvent = self.decodeEventLine
else:
self.handleEvent = self.decodeEventStream
else:
self.handleEvent = self.encodeEvent
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
for module in self.getConfigurationValue("imports"):
exec("import %s" % module)
source = "def UserFunc(event):\n%s" % self.getConfigurationValue("source")
if self.getConfigurationValue("debug"):
print source
exec(source)
try:
pass
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("ExecPython failed while creating user function. Exception: %s, Error: %s" % (etype, evalue))
self.lumbermill.shutDown()
self._func = locals()["UserFunc"]
def configure(self, configuration):
# Call parent configure method.
BaseThreadedModule.configure(self, configuration)
# Set log level for elasticsarch library if configured to other than default.
if self.getConfigurationValue('log_level') != 'info':
logging.getLogger('elasticsearch').setLevel(self.logger.level)
logging.getLogger('requests').setLevel(self.logger.level)
else:
logging.getLogger('elasticsearch').setLevel(logging.WARN)
logging.getLogger('requests').setLevel(logging.WARN)
self.query = self.getConfigurationValue('query')
# Test if query is valid json.
try:
json.loads(self.query)
except:
etype, evalue, etb = sys.exc_info()
self.logger.error(
"Parsing json query %s failed. Exception: %s, Error: %s." %
(self.query, etype, evalue))
self.lumbermill.shutDown()
self.search_type = self.getConfigurationValue('search_type')
self.batch_size = self.getConfigurationValue('batch_size')
self.field_mappings = self.getConfigurationValue('field_mappings')
self.es_nodes = self.getConfigurationValue('nodes')
self.read_timeout = self.getConfigurationValue("read_timeout")
if not isinstance(self.es_nodes, list):
self.es_nodes = [self.es_nodes]
self.index_name_pattern = self.getConfigurationValue('index_name')
self.index_name = mapDynamicValue(self.index_name_pattern,
use_strftime=True).lower()
if self.getConfigurationValue("connection_type") == 'urllib3':
self.connection_class = connection.Urllib3HttpConnection
elif self.getConfigurationValue('connection_type') == 'requests':
self.connection_class = connection.RequestsHttpConnection
self.lock = Lock()
self.manager = Manager()
if self.search_type == 'scan':
self.can_run_forked = True
scroll_id = self.getInitalialScrollId()
if not scroll_id:
self.lumbermill.shutDown()
self.shared_scroll_id = self.manager.Value(c_char_p, scroll_id)
elif self.search_type == 'normal':
self.query_from = 0
self.query = json.loads(self.query)
self.query['size'] = self.batch_size
self.es = self.connect()
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.source_fields = self.getConfigurationValue('source_fields')
# Allow single string as well.
if isinstance(self.source_fields, types.StringTypes):
self.source_fields = [self.source_fields]
self.target_field = self.getConfigurationValue('target_field')
self.drop_original = not self.getConfigurationValue('keep_original')
if self.getConfigurationValue('action') == 'decode':
if self.getConfigurationValue('mode') == 'line':
self.handleEvent = self.decodeEventLine
else:
self.unpacker = msgpack.Unpacker()
self.handleEvent = self.decodeEventStream
else:
self.handleEvent = self.encodeEvent
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.gi = False
try:
self.gi = pygeoip.GeoIP(configuration['geoip_dat_path'],
pygeoip.MEMORY_CACHE)
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("Could not init %s. Exception: %s, Error: %s. " %
(self.__class__.__name__, etype, evalue))
if not self.gi:
self.lumbermill.shutDown()
return False
self.geo_info_fields = self.getConfigurationValue('geo_info_fields')
self.source_fields = self.getConfigurationValue('source_fields')
self.target_field = self.getConfigurationValue('target_field')
def configure(self, configuration):
BaseThreadedModule.configure(self, configuration)
self.interface = self.getConfigurationValue('interface')
self.protocols = self.getConfigurationValue('protocols')
self.promiscous_mode = 1 if self.getConfigurationValue('promiscous') else 0
self.target_field = self.getConfigurationValue('target_field')
self.kv_store = self.getConfigurationValue('key_value_store') if self.getConfigurationValue('key_value_store') else {}
self.packet_decoders = {}
try:
self.sniffer = pcapy.open_live(self.interface, 65536, self.promiscous_mode, 100)
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("Sniffer could not be created. Exception: %s, Error: %s." % (etype, evalue))
self.lumbermill.shutDown()
if self.getConfigurationValue('packetfilter'):
self.sniffer.setfilter(self.getConfigurationValue('packetfilter'))
self.link_layer = self.sniffer.datalink()
def configure(self, configuration):
# Call parent configure method.
BaseThreadedModule.configure(self, configuration)
# Set log level for elasticsarch library if configured to other than default.
if self.getConfigurationValue('log_level') != 'info':
logging.getLogger('elasticsearch').setLevel(self.logger.level)
logging.getLogger('requests').setLevel(self.logger.level)
else:
logging.getLogger('elasticsearch').setLevel(logging.WARN)
logging.getLogger('requests').setLevel(logging.WARN)
self.query = self.getConfigurationValue('query')
# Test if query is valid json.
try:
json.loads(self.query)
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("Parsing json query %s failed. Exception: %s, Error: %s." % (self.query, etype, evalue))
self.lumbermill.shutDown()
self.search_type = self.getConfigurationValue('search_type')
self.batch_size = self.getConfigurationValue('batch_size')
self.field_mappings = self.getConfigurationValue('field_mappings')
self.es_nodes = self.getConfigurationValue('nodes')
self.read_timeout = self.getConfigurationValue("read_timeout")
if not isinstance(self.es_nodes, list):
self.es_nodes = [self.es_nodes]
self.index_name_pattern = self.getConfigurationValue('index_name')
self.index_name = mapDynamicValue(self.index_name_pattern, use_strftime=True).lower()
if self.getConfigurationValue("connection_type") == 'urllib3':
self.connection_class = connection.Urllib3HttpConnection
elif self.getConfigurationValue('connection_type') == 'requests':
self.connection_class = connection.RequestsHttpConnection
self.lock = Lock()
self.manager = Manager()
if self.search_type == 'scroll':
self.can_run_forked = True
scroll_id = self.getInitalialScrollId()
if not scroll_id:
self.lumbermill.shutDown()
self.shared_scroll_id = self.manager.Value(c_char_p, scroll_id)
elif self.search_type == 'normal':
self.query_from = 0
self.query = json.loads(self.query)
self.query['size'] = self.batch_size
self.es = self.connect()
def configure(self, configuration):
# Call parent configure method
BaseThreadedModule.configure(self, configuration)
self.source_field = self.getConfigurationValue('source_field')
self.source_date_pattern = self.getConfigurationValue('source_date_pattern')
try:
self.source_timezone = pytz.timezone(self.getConfigurationValue('source_timezone'))
except pytz.UnknownTimeZoneError:
etype, evalue, etb = sys.exc_info()
self.logger.error("Unknown source timezone %s. Exception: %s, Error: %s." % (self.getConfigurationValue('source_timezone'), etype, evalue))
self.lumbermill.shutDown()
self.target_field = self.getConfigurationValue('target_field') if self.getConfigurationValue('target_field') else self.source_field
self.target_date_pattern = self.getConfigurationValue('target_date_pattern')
try:
self.target_timezone = pytz.timezone(self.getConfigurationValue('target_timezone'))
except pytz.UnknownTimeZoneError:
etype, evalue, etb = sys.exc_info()
self.logger.error("Unknown target timezone %s. Exception: %s, Error: %s." % (self.getConfigurationValue('target_timezone'), etype, evalue))
self.lumbermill.shutDown()