def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.source_field = self.getConfigurationValue('source_field') self.seperator = self.getConfigurationValue('seperator') self.target_field = self.getConfigurationValue('target_field') self.drop_original = not self.getConfigurationValue('keep_original')
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) # Make urllib3s logging less verbose. urllib3_logger = logging.getLogger('requests.packages.urllib3.connectionpool') urllib3_logger.setLevel(logging.CRITICAL) self.server, self.port = self.getConfigurationValue('server').split(':') self.user = self.getConfigurationValue('user') self.events_container = [] self.batch_size = self.getConfigurationValue('batch_size') self.backlog_size = self.getConfigurationValue('backlog_size') self.path = self.getConfigurationValue('path') self.name_pattern = self.getConfigurationValue('name_pattern') self.format = self.getConfigurationValue('format') self.compress = self.getConfigurationValue('compress') if self.compress == 'gzip': try: import gzip except ImportError: self.logger.error('Gzip compression selected but gzip module could not be loaded.') self.lumbermill.shutDown() if self.compress == 'snappy': try: import snappy except ImportError: self.logger.error('Snappy compression selected but snappy module could not be loaded.') self.lumbermill.shutDown() self.is_storing = False self.lock = multiprocessing.Lock() self.timed_store_func = self.getTimedStoreFunc()
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.backend = self.getConfigurationValue('backend') self.backend_client = None self.kv_store = None self.set_buffer = None if self.backend == 'DictStore': import simplekv.memory self.kv_store = simplekv.memory.DictStore() elif self.backend == 'RedisStore': import simplekv.memory.redisstore self.backend_client = self._getRedisClient() self.kv_store = simplekv.memory.redisstore.RedisStore(self.backend_client) elif self.backend == 'MemcacheStore': import simplekv.memory.memcachestore self.backend_client = self._getMemcacheClient() self.kv_store = simplekv.memory.memcachestore.MemcacheStore(self.backend_client) else: self.logger("Unknown backend type %s. Please check." % backend) self.lumbermill.shutDown(); if self.getConfigurationValue('store_interval_in_secs') or self.getConfigurationValue('batch_size'): if self.backend == 'RedisStore': self.set_buffer = Buffer(self.getConfigurationValue('batch_size'), self._setRedisBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size')) else: self.set_buffer = Buffer(self.getConfigurationValue('batch_size'), self._setBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size')) self._set = self.set self.set = self._setBuffered self._get = self.get self.get = self._getBuffered self._delete = self.delete self.delete = self._deleteBuffered self._pop = self.pop self.pop = self._popBuffered
def configure(self, configuration): # Call parent configure method. BaseThreadedModule.configure(self, configuration) for module_name in ['elasticsearch', 'urllib3', 'requests']: if self.getConfigurationValue('log_level') == 'info': logging.getLogger(module_name).setLevel(logging.WARN) else: # Set log level for elasticsarch library if configured to other than default. logging.getLogger(module_name).setLevel(self.logger.level) self.action = self.getConfigurationValue('action') self.fields = self.getConfigurationValue('fields') self.ttl = self.getConfigurationValue("ttl") self.index_name = self.getConfigurationValue("index_name") self.routing_pattern = self.getConfigurationValue("routing") self.doc_id_pattern = self.getConfigurationValue("doc_id") self.doc_type_pattern = self.getConfigurationValue("doc_type") self.doc_type_is_dynamic = self.isDynamicConfigurationValue("doc_type") self.es_nodes = self.getConfigurationValue("nodes") self.read_timeout = self.getConfigurationValue("read_timeout") if not isinstance(self.es_nodes, list): self.es_nodes = [self.es_nodes] if self.getConfigurationValue("connection_type") == 'urllib3': self.connection_class = elasticsearch.connection.Urllib3HttpConnection elif self.getConfigurationValue("connection_type") == 'requests': self.connection_class = elasticsearch.connection.RequestsHttpConnection
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.interval = self.getConfigurationValue('interval') self.fields = self.getConfigurationValue('fields') self.stats_collector = StatisticCollector() self.module_queues = {}
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.source_field = self.getConfigurationValue('source_field') self.add_event_fields = self.getConfigurationValue('add_event_fields') self.backend_lock_name = "Lumbermill:Facet:FacetLock-%s" % self.lumbermill.getMainProcessId( ) self.backend_key_name = "Lumbermill:Facet:FacetKeys-%s" % self.lumbermill.getMainProcessId( ) self.backend_ttl = self.getConfigurationValue('backend_ttl') if (self.backend_ttl < self.getConfigurationValue('interval')): self.logger.error( 'backend_ttl setting is smaller then interval setting. Please check.' ) self.lumbermill.shutDown() return backend_info = self.lumbermill.getModuleInfoById( self.getConfigurationValue('backend')) if not backend_info: self.logger.error( "Could not find %s backend for persistant storage." % (self.getConfigurationValue('backend'))) self.lumbermill.shutDown() return self.persistence_backend = backend_info['instances'][0]
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) backend = self.getConfigurationValue('backend') self.backend_client = None if backend == 'DictStore': import simplekv.memory self.backend_client = None self.kv_store = simplekv.memory.DictStore() elif backend == 'RedisStore': import simplekv.memory.redisstore self.backend_client = self.getRedisClient() self.kv_store = simplekv.memory.redisstore.RedisStore(self.backend_client) elif backend == 'MemcacheStore': import simplekv.memory.memcachestore self.backend_client = self.getMemcacheClient() self.kv_store = simplekv.memory.memcachestore.MemcacheStore(self.backend_client) self.set_buffer = None if self.getConfigurationValue('store_interval_in_secs') or self.getConfigurationValue('batch_size'): if backend == 'RedisStore': self.set_buffer = Buffer(self.getConfigurationValue('batch_size'), self.setRedisBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size')) else: self.set_buffer = Buffer(self.getConfigurationValue('batch_size'), self.setBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size')) self._set = self.set self.set = self.setBuffered self._get = self.get self.get = self.getBuffered self._delete = self.delete self.delete = self.deleteBuffered self._pop = self.pop self.pop = self.popBuffered
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) if len(self.getConfigurationValue('cluster')) == 0: redis_store = self.getConfigurationValue('server') self.client = self.getRedisClient() else: redis_store = self.getConfigurationValue('cluster') self.client = self.getClusterRedisClient() try: self.client.ping() except: etype, evalue, etb = sys.exc_info() self.logger.error( "Could not connect to redis store at %s. Exception: %s, Error: %s." % (redis_store, etype, evalue)) self.lumbermill.shutDown() self.set_buffer = None if self.getConfigurationValue( 'store_interval_in_secs') or self.getConfigurationValue( 'batch_size'): self.set_buffer = Buffer( self.getConfigurationValue('batch_size'), self.setBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size')) self._set = self.set self.set = self.setBuffered self._get = self.get self.get = self.getBuffered self._delete = self.delete self.delete = self.deleteBuffered self._pop = self.pop self.pop = self.popBuffered
def configure(self, configuration): # self.logger.setLevel(logging.DEBUG) # Call parent configure method. BaseThreadedModule.configure(self, configuration) self.is_leader = True if self.getConfigurationValue( 'pack') == 'leader' else False self.pack_followers = {} self.cluster_name = self.getConfigurationValue('name') self.discovered_leader = None self.secret = hashlib.sha256( self.getConfigurationValue('secret')).digest() self.handlers = collections.defaultdict(list) self.lock = threading.Lock() # Setup socket. self.interface_addr = self.getConfigurationValue('interface') self.interface_port = self.getConfigurationValue('port') self.broadcast_addr = self.getConfigurationValue('broadcast') self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) self.socket.settimeout(1) self.hostname = socket.gethostname() try: self.socket.bind((self.interface_addr, self.interface_port)) except: etype, evalue, etb = sys.exc_info() self.logger.error( "Could not listen on %s:%s. Exception: %s, Error: %s." % (self.getConfigurationValue("interface"), self.getConfigurationValue("port"), etype, evalue)) self.alive = False self.lumbermill.shutDown() return self.addHandlers()
def configure(self, configuration): # self.logger.setLevel(logging.DEBUG) # Call parent configure method. BaseThreadedModule.configure(self, configuration) self.is_leader = True if self.getConfigurationValue('pack') == 'leader' else False self.pack_followers = {} self.cluster_name = self.getConfigurationValue('name') self.discovered_leader = None self.secret = hashlib.sha256(self.getConfigurationValue('secret')).digest() self.handlers = collections.defaultdict(list) self.lock = threading.Lock() # Setup socket. self.interface_addr = self.getConfigurationValue('interface') self.interface_port = self.getConfigurationValue('port') self.broadcast_addr = self.getConfigurationValue('broadcast') self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) self.socket.settimeout(1) self.hostname = socket.gethostname() try: self.socket.bind((self.interface_addr, self.interface_port)) except: etype, evalue, etb = sys.exc_info() self.logger.error("Could not listen on %s:%s. Exception: %s, Error: %s." % (self.getConfigurationValue("interface"), self.getConfigurationValue("port"), etype, evalue)) self.alive = False self.lumbermill.shutDown() return self.addHandlers()
def configure(self, configuration): BaseThreadedModule.configure(self, configuration) self.hostname = self.getConfigurationValue("hostname") self.fields = self.getConfigurationValue("fields") self.field_prefix = self.getConfigurationValue("field_prefix") self.timestamp_field = self.getConfigurationValue("timestamp_field") self.batch_size = self.getConfigurationValue('batch_size') self.backlog_size = self.getConfigurationValue('backlog_size') self.agent_conf = self.getConfigurationValue("agent_conf") if self.agent_conf: if self.agent_conf is True: self.agent_conf = "/etc/zabbix/zabbix_agentd.conf" if not os.path.isfile(self.agent_conf): self.logger.error("%s does not point to an existing file." % self.agent_conf) self.lumbermill.shutDown() self.zabbix_sender = ZabbixSender(use_config=self.agent_conf) else: self.logger.error("asdads") server = self.getConfigurationValue("server") port = 10051 if ":" in self.server: server, port = self.server.split(":") self.zabbix_sender = ZabbixSender(zabbix_server=server, port=port) self.buffer = Buffer(self.getConfigurationValue('batch_size'), self.storeData, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.formats = self.getConfigurationValue('formats') self.connection_data = (self.getConfigurationValue('server'), self.getConfigurationValue('port')) self.connection = None
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.lists = self.getConfigurationValue('lists') if not isinstance(self.lists, list): self.lists = [self.lists] self.timeout = self.getConfigurationValue('timeout') self.batch_size = self.getConfigurationValue('batch_size') self.client = redis.StrictRedis( host=self.getConfigurationValue('server'), port=self.getConfigurationValue('port'), password=self.getConfigurationValue('password'), db=self.getConfigurationValue('db')) try: self.client.ping() except: etype, evalue, etb = sys.exc_info() self.logger.error( "Could not connect to redis store at %s. Exception: %s, Error: %s." % (self.getConfigurationValue('server'), etype, evalue)) self.lumbermill.shutDown() # Monkeypatch run method to use the correct handle event method. if self.batch_size == 1: self.run = self.handleSingleEvent else: self.run = self.handleBatchEvents
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) if len(self.getConfigurationValue('cluster')) == 0: redis_store = self.getConfigurationValue('server') self.client = self.getRedisClient() else: redis_store = self.getConfigurationValue('cluster') self.client = self.getClusterRedisClient() try: self.client.ping() except: etype, evalue, etb = sys.exc_info() self.logger.error("Could not connect to redis store at %s. Exception: %s, Error: %s." % (redis_store,etype, evalue)) self.lumbermill.shutDown() self.set_buffer = None if self.getConfigurationValue('store_interval_in_secs') or self.getConfigurationValue('batch_size'): self.set_buffer = Buffer(self.getConfigurationValue('batch_size'), self.setBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size')) self._set = self.set self.set = self.setBuffered self._get = self.get self.get = self.getBuffered self._delete = self.delete self.delete = self.deleteBuffered self._pop = self.pop self.pop = self.popBuffered
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.source_fields = self.getConfigurationValue('source_fields') # Allow single string as well. if isinstance(self.source_fields, types.StringTypes): self.source_fields = [self.source_fields] self.target_fields = self.getConfigurationValue('target_fields') # Allow single string as well. if isinstance(self.target_fields, types.StringTypes): self.target_fields = [self.target_fields] # If target_fields were provided they need to be of same length as source_fields. if self.target_fields and len(self.source_fields) != len( self.target_fields): self.logger.error( "Count of configured source and target fields need to match. Please check your configuration." ) self.lumbermill.shutDown() self.compression = self.getConfigurationValue('compression') # Get compression specific handler. try: self.deflater = getattr(self, "deflate_with_%s" % self.compression) except AttributeError: etype, evalue, etb = sys.exc_info() self.logger.error( "Unknown decompression lib: %s. Exception: %s, Error: %s" % (self.compression, etype, evalue)) self.lumbermill.shutDown()
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) # Set boto log level. logging.getLogger('boto3').setLevel(logging.CRITICAL) logging.getLogger('botocore').setLevel(logging.CRITICAL) self.sqs_queue_name = self.getConfigurationValue('queue') self.attribute_names = self.getConfigurationValue('attribute_names') self.message_attribute_names = self.getConfigurationValue('message_attribute_names') self.poll_interval = self.getConfigurationValue('poll_interval_in_secs') self.batch_size = self.getConfigurationValue('batch_size') try: self.sqs_client = boto3.client('sqs', region_name=self.getConfigurationValue('region'), api_version=None, use_ssl=True, verify=None, endpoint_url=None, aws_access_key_id=self.getConfigurationValue('aws_access_key_id'), aws_secret_access_key=self.getConfigurationValue('aws_secret_access_key'), aws_session_token=None, config=None) except: etype, evalue, etb = sys.exc_info() self.logger.error("Could not connect to sqs service. Exception: %s, Error: %s." % (etype, evalue)) self.lumbermill.shutDown() try: self.sqs_queue_url = self.sqs_client.get_queue_url(QueueName=self.sqs_queue_name)['QueueUrl'] except: etype, evalue, etb = sys.exc_info() self.logger.error("Could not get queue url for sqs queue %s. Exception: %s, Error: %s." % (self.sqs_queue_name, etype, evalue)) self.lumbermill.shutDown()
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.batch_size = self.getConfigurationValue('batch_size') self.backlog_size = self.getConfigurationValue('backlog_size') self.file_name = self.getConfigurationValue('file_name') self.format = self.getConfigurationValue('format') self.compress = self.getConfigurationValue('compress') self.file_handles = {} if self.compress == 'gzip': try: # Import module into namespace of object. Otherwise it will not be accessible when process was forked. self.gzip_module = __import__('gzip') except ImportError: self.logger.error( 'Gzip compression selected but gzip module could not be loaded.' ) self.lumbermill.shutDown() if self.compress == 'snappy': try: self.snappy_module = __import__('snappy') except ImportError: self.logger.error( 'Snappy compression selected but snappy module could not be loaded.' ) self.lumbermill.shutDown() self.buffer = Buffer( self.batch_size, self.storeData, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.backlog_size) TimedFunctionManager.startTimedFunction(self.closeStaleFileHandles)
def configure(self, configuration): # Call parent configure method. BaseThreadedModule.configure(self, configuration) self.format = self.getConfigurationValue('format') self.collection = self.getConfigurationValue('collection') self.database = self.getConfigurationValue('database') self.doc_id_pattern = self.getConfigurationValue("doc_id")
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.syslogger = logging.getLogger(self.__class__.__name__) self.syslogger.propagate = False self.format = self.getConfigurationValue('format') if os.path.exists(self.getConfigurationValue('address')): address = self.getConfigurationValue('address') else: server, port = self.getConfigurationValue('address').split(':') address = (server, int(port)) if self.getConfigurationValue('proto') == 'tcp': socket_type = socket.SOCK_STREAM else: socket_type = socket.SOCK_DGRAM try: facility = logging.handlers.SysLogHandler.facility_names[ self.getConfigurationValue('facility')] except KeyError: self.logger.error("The configured facility %s is unknown." % (self.getConfigurationValue('facility'))) self.syslog_handler = logging.handlers.SysLogHandler( address, facility=facility, socktype=socket_type) self.syslogger.addHandler(self.syslog_handler) self.format = self.getConfigurationValue('format')
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.pretty_print = self.getConfigurationValue('pretty_print') self.fields = self.getConfigurationValue('fields') self.format = self.getConfigurationValue('format') self.printing = False
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) # Set boto log level. logging.getLogger('boto3').setLevel(logging.CRITICAL) logging.getLogger('botocore').setLevel(logging.CRITICAL) self.batch_size = self.getConfigurationValue('batch_size') self.format = self.getConfigurationValue('format')
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.events = self.getConfigurationValue("event") if not isinstance(self.events, list): self.events = [self.events] self.sleep = self.getConfigurationValue("sleep") self.max_events_count = self.getConfigurationValue("events_count")
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.server = None self.topic = self.getConfigurationValue('topic') self.format = self.getConfigurationValue('format') self.mode = self.getConfigurationValue('mode') if self.mode == "bind": self.can_run_forked = False
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.source_fields = self.getConfigurationValue('source_fields') # Allow single string as well. if isinstance(self.source_fields, types.StringTypes): self.source_fields = [self.source_fields] self.target_field = self.getConfigurationValue('target_field') self.in_mem_cache = MemoryCache(size=1000)
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.emit_as_event = self.getConfigurationValue('emit_as_event') self.interval = self.getConfigurationValue('interval') self.stats_collector = StatisticCollector() for counter_name in ['events_received', 'event_type_Unknown', 'event_type_httpd_access_log']: MultiProcessStatisticCollector().initCounter(counter_name) self.module_queues = {}
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.source_field = self.getConfigurationValue('source_field') self.target_field = self.getConfigurationValue('target_field') self.drop_original = not self.getConfigurationValue('keep_original') if self.getConfigurationValue('action') == 'decode': self.handleEvent = self.decodeBase64 else: self.handleEvent = self.encodeBase64
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.source_fields = self.getConfigurationValue('source_fields') # Allow single string as well. if isinstance(self.source_fields, types.StringTypes): self.source_fields = [self.source_fields] self.seperator = self.getConfigurationValue('seperator') self.target_field = self.getConfigurationValue('target_field') self.drop_original = not self.getConfigurationValue('keep_original')
def configure(self, configuration): # Call parent configure method. BaseThreadedModule.configure(self, configuration) self.file_tailer = None self.files = self.scanPaths() self.line_by_line = self.getConfigurationValue('line_by_line') self.mode = self.getConfigurationValue('mode') self.datastore_key = "%032x%s" % (random.getrandbits(128), self.process_id) self.total_file_count = len(self.files) self.lumbermill.setInInternalDataStore(self.datastore_key, 0)
def configure(self, configuration): BaseThreadedModule.configure(self, configuration) self.redis_ttl = self.getConfigurationValue('redis_ttl') # Get redis client module. if self.getConfigurationValue('redis_store'): mod_info = self.lumbermill.getModuleInfoById( self.getConfigurationValue('redis_store')) self.redis_store = mod_info['instances'][0] else: self.redis_store = None
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.emit_as_event = self.getConfigurationValue('emit_as_event') self.interval = self.getConfigurationValue('interval') self.event_type_statistics = self.getConfigurationValue( 'event_type_statistics') self.stats_collector = StatisticCollector() self.mp_stats_collector = MultiProcessStatisticCollector() self.module_queues = {}
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.results = [] function_str = "lambda event: " + re.sub('%\((.*?)\)s', r"event.get('\1', False)", self.getConfigurationValue('function')) self.function = self.compileFunction(function_str) if self.getConfigurationValue('results_function'): function_str = "lambda results: "+ re.sub('%\((.*?)\)s', r"results", self.getConfigurationValue('results_function')) self.results_function = self.compileFunction(function_str) self.target_field = self.getConfigurationValue('target_field') self.interval = self.getConfigurationValue('interval')
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.emit_as_event = self.getConfigurationValue('emit_as_event') self.interval = self.getConfigurationValue('interval') self.stats_collector = StatisticCollector() for counter_name in [ 'events_received', 'event_type_Unknown', 'event_type_httpd_access_log' ]: MultiProcessStatisticCollector().initCounter(counter_name) self.module_queues = {}
def configure(self, configuration): BaseThreadedModule.configure(self, configuration) socket.setdefaulttimeout(self.getConfigurationValue('socket_timeout')) self.get_metadata = self.getConfigurationValue('get_metadata') self.interval = self.getConfigurationValue('interval') # Get redis client module. if self.getConfigurationValue('redis_store'): mod_info = self.lumbermill.getModuleInfoById( self.getConfigurationValue('redis_store')) self.redis_store = mod_info['instances'][0] else: self.redis_store = None
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.format = self.getConfigurationValue('format') try: self.client = redis.Redis(host=self.getConfigurationValue('server'), port=self.getConfigurationValue('port'), password=self.getConfigurationValue('password'), db=self.getConfigurationValue('db')) except: etype, evalue, etb = sys.exc_info() self.logger.error("Could not connect to redis store at %s. Exception: %s, Error: %s." % (self.getConfigurationValue('server'),etype, evalue))
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.in_mem_cache = MemoryCache(size=5000) self.lookup_type = self.getConfigurationValue('action') self.source_field = self.getConfigurationValue('source_field') self.target_field = self.getConfigurationValue('target_field') self.nameservers = self.getConfigurationValue('nameservers') self.timeout = self.getConfigurationValue('timeout') # Allow single string as well. if isinstance(self.nameservers, types.StringTypes): self.nameservers = [self.nameservers] self.lookup_threads_pool_size = 3
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.emit_as_event = self.getConfigurationValue('emit_as_event') self.interval = self.getConfigurationValue('interval') self.event_type_statistics = self.getConfigurationValue('event_type_statistics') self.process_statistics = self.getConfigurationValue('process_statistics') self.stats_collector = StatisticCollector() self.mp_stats_collector = MultiProcessStatisticCollector() self.module_queues = {} self.psutil_processes = [] self.last_values = {'events_received': 0} self.methods = dir(self)
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) # Set defaults supported_regex_match_types = ['search', 'findall'] self.timed_func_handler = None self.source_field = self.getConfigurationValue('source_field') self.mark_unmatched_as = self.getConfigurationValue('mark_unmatched_as') self.break_on_match = self.getConfigurationValue('break_on_match') self.hot_rules_first = self.getConfigurationValue('hot_rules_first') self.event_types = [] self.fieldextraction_regexpressions = [] self.logstash_patterns = {} self.readLogstashPatterns() for regex_config in configuration['field_extraction_patterns']: event_type = regex_config.keys()[0] regex_pattern = regex_config[event_type] regex_options = 0 regex_match_type = 'search' if isinstance(regex_pattern, list): i = iter(regex_pattern) # Pattern is the first entry regex_pattern = i.next() # Regex options the second try: regex_options = eval(i.next()) except: etype, evalue, etb = sys.exc_info() self.logger.error("RegEx error for options %s. Exception: %s, Error: %s." % (regex_options, etype, evalue)) self.lumbermill.shutDown() return # Regex match type the third (optional) try: regex_match_type = i.next() except: pass # Make sure regex_match_type is valid # At the moment only search and findall are supported if regex_match_type not in supported_regex_match_types: self.logger.error("RegEx error for match type %s. Only %s are supported." % (regex_options, supported_regex_match_types)) self.lumbermill.shutDown() return try: regex_pattern = self.replaceLogstashPatterns(regex_pattern) regex = re.compile(regex_pattern, regex_options) except: etype, evalue, etb = sys.exc_info() self.logger.error("RegEx error for %s pattern %s. Exception: %s, Error: %s." % (event_type, regex_pattern, etype, evalue)) self.lumbermill.shutDown() self.fieldextraction_regexpressions.append({'event_type': event_type, 'pattern': regex, 'match_type': regex_match_type, 'hitcounter': 0})
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) # Set defaults supported_regex_match_types = ['search', 'findall'] self.timed_func_handler = None self.source_field = self.getConfigurationValue('source_field') self.mark_unmatched_as = self.getConfigurationValue('mark_unmatched_as') self.break_on_match = self.getConfigurationValue('break_on_match') self.hot_rules_first = self.getConfigurationValue('hot_rules_first') self.event_types = [] self.fieldextraction_regexpressions = [] self.logstash_patterns = {} self.readLogstashPatterns() for regex_config in configuration['field_extraction_patterns']: event_type = regex_config.keys()[0] regex_pattern = regex_config[event_type] regex_options = 0 regex_match_type = 'search' if isinstance(regex_pattern, list): i = iter(regex_pattern) # Pattern is the first entry regex_pattern = i.next() # Regex options the second try: regex_options = eval(i.next()) except: etype, evalue, etb = sys.exc_info() self.logger.error("RegEx error for options %s. Exception: %s, Error: %s." % (regex_options, etype, evalue)) self.lumbermill.shutDown() return # Regex match type the third (optional) try: regex_match_type = i.next() except StopIteration: pass # Make sure regex_match_type is valid # At the moment only search and findall are supported if regex_match_type not in supported_regex_match_types: self.logger.error("RegEx error for match type %s. Only %s are supported." % (regex_options, supported_regex_match_types)) self.lumbermill.shutDown() return try: regex_pattern = self.replaceLogstashPatterns(regex_pattern) regex = re.compile(regex_pattern, regex_options) except: etype, evalue, etb = sys.exc_info() self.logger.error("RegEx error for %s pattern %s. Exception: %s, Error: %s." % (event_type, regex_pattern, etype, evalue)) self.lumbermill.shutDown() self.fieldextraction_regexpressions.append({'event_type': event_type, 'pattern': regex, 'match_type': regex_match_type, 'hitcounter': 0})
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.format = self.getConfigurationValue('format') try: self.client = redis.Redis( host=self.getConfigurationValue('server'), port=self.getConfigurationValue('port'), password=self.getConfigurationValue('password'), db=self.getConfigurationValue('db')) except: etype, evalue, etb = sys.exc_info() self.logger.error( "Could not connect to redis store at %s. Exception: %s, Error: %s." % (self.getConfigurationValue('server'), etype, evalue))
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.source_fields = self.getConfigurationValue('source_fields') # Allow single string as well. if isinstance(self.source_fields, types.StringTypes): self.source_fields = [self.source_fields] self.target_field = self.getConfigurationValue('target_field') self.drop_original = not self.getConfigurationValue('keep_original') if self.getConfigurationValue('action') == 'decode': self.default_list_attributes = [attrib for attrib in dir(list) if not attrib.startswith('__')] self.parser = Parser() self.handleEvent = self.decodeEvent else: self.handleEvent = self.encodeEvent
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) # Set defaultså self.typecast_switch = { 'int': self.cast_to_int, 'integer': self.cast_to_int, 'float': self.cast_to_float, 'str': self.cast_to_str, 'string': self.cast_to_str, 'bool': self.cast_to_bool, 'boolean': self.cast_to_bool, } self.action = configuration['action'] # Precompile regex for replacement if defined if 'regex' in configuration: regex_pattern = configuration['regex'] regex_options = 0 if isinstance(regex_pattern, list): i = iter(regex_pattern) # Pattern is the first entry regex_pattern = i.next() # Regex options the second try: regex_options = eval(i.next()) except: etype, evalue, etb = sys.exc_info() self.logger.error("RegEx error for options %s. Exception: %s, Error: %s" % (regex_options, etype, evalue)) self.lumbermill.shutDown() try: self.regex = re.compile(regex_pattern, regex_options) except: etype, evalue, etb = sys.exc_info() self.logger.error("RegEx error for pattern %s. Exception: %s, Error: %s" % (regex_pattern, etype, evalue)) self.lumbermill.shutDown() self.source_field = self.getConfigurationValue('source_field') if "source_field" in self.configuration_data else [] self.source_fields = self.getConfigurationValue('source_fields') if "source_fields" in self.configuration_data else [] self.target_field = self.getConfigurationValue('target_field') if "target_field" in self.configuration_data else [] self.target_fields = self.getConfigurationValue('target_fields') if "target_fields" in self.configuration_data else [] # Call action specific configure method. if "configure_%s_action" % self.action in dir(self): getattr(self, "configure_%s_action" % self.action)() # Get action specific method try: self.event_handler = getattr(self, "%s" % self.action) except AttributeError: etype, evalue, etb = sys.exc_info() self.logger.error("ModifyFields action called that does not exist: %s. Exception: %s, Error: %s" % (self.action, etype, evalue)) self.lumbermill.shutDown()
def configure(self, configuration): # Call parent configure method. BaseThreadedModule.configure(self, configuration) paths = self.getConfigurationValue('paths') if isinstance(paths, types.StringType): paths = [paths] self.line_by_line = self.getConfigurationValue('line_by_line') self.files = [] for path in paths: self.files += self.getFilesInPath( path, self.getConfigurationValue('pattern'), self.getConfigurationValue('recursive')) self.datastore_key = "%032x%s" % (random.getrandbits(128), self.process_id) self.total_file_count = len(self.files) self.lumbermill.setInInternalDataStore(self.datastore_key, 0)
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.server = False self.settings = self.getSettings() self.application = tornado.web.Application([], **self.settings) # Default handlers. handlers = [ # REST ActionHandler (r"/rest/server/restart", handler.ActionHandler.RestartHandler), (r"/rest/server/info", handler.ActionHandler.GetServerInformation), (r"/rest/server/statistics", handler.ActionHandler.GetServerStatistics), (r"/rest/server/configuration", handler.ActionHandler.GetServerConfiguration), # WebsocketHandler (r"/websockets/statistics", handler.WebsocketHandler.StatisticsWebSocketHandler), (r"/websockets/get_logs", handler.WebsocketHandler.LogToWebSocketHandler) ] self.addHandlers(handlers)
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.source_fields = self.getConfigurationValue('source_fields') # Allow single string as well. if isinstance(self.source_fields, types.StringTypes): self.source_fields = [self.source_fields] self.target_field = self.getConfigurationValue('target_field') self.drop_original = not self.getConfigurationValue('keep_original') if self.getConfigurationValue('action') == 'decode': if self.getConfigurationValue('mode') == 'line': self.handleEvent = self.decodeEventLine else: self.handleEvent = self.decodeEventStream else: self.handleEvent = self.encodeEvent
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) for module in self.getConfigurationValue("imports"): exec("import %s" % module) source = "def UserFunc(event):\n%s" % self.getConfigurationValue("source") if self.getConfigurationValue("debug"): print source exec(source) try: pass except: etype, evalue, etb = sys.exc_info() self.logger.error("ExecPython failed while creating user function. Exception: %s, Error: %s" % (etype, evalue)) self.lumbermill.shutDown() self._func = locals()["UserFunc"]
def configure(self, configuration): # Call parent configure method. BaseThreadedModule.configure(self, configuration) # Set log level for elasticsarch library if configured to other than default. if self.getConfigurationValue('log_level') != 'info': logging.getLogger('elasticsearch').setLevel(self.logger.level) logging.getLogger('requests').setLevel(self.logger.level) else: logging.getLogger('elasticsearch').setLevel(logging.WARN) logging.getLogger('requests').setLevel(logging.WARN) self.query = self.getConfigurationValue('query') # Test if query is valid json. try: json.loads(self.query) except: etype, evalue, etb = sys.exc_info() self.logger.error( "Parsing json query %s failed. Exception: %s, Error: %s." % (self.query, etype, evalue)) self.lumbermill.shutDown() self.search_type = self.getConfigurationValue('search_type') self.batch_size = self.getConfigurationValue('batch_size') self.field_mappings = self.getConfigurationValue('field_mappings') self.es_nodes = self.getConfigurationValue('nodes') self.read_timeout = self.getConfigurationValue("read_timeout") if not isinstance(self.es_nodes, list): self.es_nodes = [self.es_nodes] self.index_name_pattern = self.getConfigurationValue('index_name') self.index_name = mapDynamicValue(self.index_name_pattern, use_strftime=True).lower() if self.getConfigurationValue("connection_type") == 'urllib3': self.connection_class = connection.Urllib3HttpConnection elif self.getConfigurationValue('connection_type') == 'requests': self.connection_class = connection.RequestsHttpConnection self.lock = Lock() self.manager = Manager() if self.search_type == 'scan': self.can_run_forked = True scroll_id = self.getInitalialScrollId() if not scroll_id: self.lumbermill.shutDown() self.shared_scroll_id = self.manager.Value(c_char_p, scroll_id) elif self.search_type == 'normal': self.query_from = 0 self.query = json.loads(self.query) self.query['size'] = self.batch_size self.es = self.connect()
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.source_fields = self.getConfigurationValue('source_fields') # Allow single string as well. if isinstance(self.source_fields, types.StringTypes): self.source_fields = [self.source_fields] self.target_field = self.getConfigurationValue('target_field') self.drop_original = not self.getConfigurationValue('keep_original') if self.getConfigurationValue('action') == 'decode': if self.getConfigurationValue('mode') == 'line': self.handleEvent = self.decodeEventLine else: self.unpacker = msgpack.Unpacker() self.handleEvent = self.decodeEventStream else: self.handleEvent = self.encodeEvent
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.gi = False try: self.gi = pygeoip.GeoIP(configuration['geoip_dat_path'], pygeoip.MEMORY_CACHE) except: etype, evalue, etb = sys.exc_info() self.logger.error("Could not init %s. Exception: %s, Error: %s. " % (self.__class__.__name__, etype, evalue)) if not self.gi: self.lumbermill.shutDown() return False self.geo_info_fields = self.getConfigurationValue('geo_info_fields') self.source_fields = self.getConfigurationValue('source_fields') self.target_field = self.getConfigurationValue('target_field')
def configure(self, configuration): BaseThreadedModule.configure(self, configuration) self.interface = self.getConfigurationValue('interface') self.protocols = self.getConfigurationValue('protocols') self.promiscous_mode = 1 if self.getConfigurationValue('promiscous') else 0 self.target_field = self.getConfigurationValue('target_field') self.kv_store = self.getConfigurationValue('key_value_store') if self.getConfigurationValue('key_value_store') else {} self.packet_decoders = {} try: self.sniffer = pcapy.open_live(self.interface, 65536, self.promiscous_mode, 100) except: etype, evalue, etb = sys.exc_info() self.logger.error("Sniffer could not be created. Exception: %s, Error: %s." % (etype, evalue)) self.lumbermill.shutDown() if self.getConfigurationValue('packetfilter'): self.sniffer.setfilter(self.getConfigurationValue('packetfilter')) self.link_layer = self.sniffer.datalink()
def configure(self, configuration): # Call parent configure method. BaseThreadedModule.configure(self, configuration) # Set log level for elasticsarch library if configured to other than default. if self.getConfigurationValue('log_level') != 'info': logging.getLogger('elasticsearch').setLevel(self.logger.level) logging.getLogger('requests').setLevel(self.logger.level) else: logging.getLogger('elasticsearch').setLevel(logging.WARN) logging.getLogger('requests').setLevel(logging.WARN) self.query = self.getConfigurationValue('query') # Test if query is valid json. try: json.loads(self.query) except: etype, evalue, etb = sys.exc_info() self.logger.error("Parsing json query %s failed. Exception: %s, Error: %s." % (self.query, etype, evalue)) self.lumbermill.shutDown() self.search_type = self.getConfigurationValue('search_type') self.batch_size = self.getConfigurationValue('batch_size') self.field_mappings = self.getConfigurationValue('field_mappings') self.es_nodes = self.getConfigurationValue('nodes') self.read_timeout = self.getConfigurationValue("read_timeout") if not isinstance(self.es_nodes, list): self.es_nodes = [self.es_nodes] self.index_name_pattern = self.getConfigurationValue('index_name') self.index_name = mapDynamicValue(self.index_name_pattern, use_strftime=True).lower() if self.getConfigurationValue("connection_type") == 'urllib3': self.connection_class = connection.Urllib3HttpConnection elif self.getConfigurationValue('connection_type') == 'requests': self.connection_class = connection.RequestsHttpConnection self.lock = Lock() self.manager = Manager() if self.search_type == 'scroll': self.can_run_forked = True scroll_id = self.getInitalialScrollId() if not scroll_id: self.lumbermill.shutDown() self.shared_scroll_id = self.manager.Value(c_char_p, scroll_id) elif self.search_type == 'normal': self.query_from = 0 self.query = json.loads(self.query) self.query['size'] = self.batch_size self.es = self.connect()
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.source_field = self.getConfigurationValue('source_field') self.source_date_pattern = self.getConfigurationValue('source_date_pattern') try: self.source_timezone = pytz.timezone(self.getConfigurationValue('source_timezone')) except pytz.UnknownTimeZoneError: etype, evalue, etb = sys.exc_info() self.logger.error("Unknown source timezone %s. Exception: %s, Error: %s." % (self.getConfigurationValue('source_timezone'), etype, evalue)) self.lumbermill.shutDown() self.target_field = self.getConfigurationValue('target_field') if self.getConfigurationValue('target_field') else self.source_field self.target_date_pattern = self.getConfigurationValue('target_date_pattern') try: self.target_timezone = pytz.timezone(self.getConfigurationValue('target_timezone')) except pytz.UnknownTimeZoneError: etype, evalue, etb = sys.exc_info() self.logger.error("Unknown target timezone %s. Exception: %s, Error: %s." % (self.getConfigurationValue('target_timezone'), etype, evalue)) self.lumbermill.shutDown()