def configure(self, configuration): # Call parent configure method. BaseThreadedModule.configure(self, configuration) for module_name in ['elasticsearch', 'urllib3', 'requests']: if self.getConfigurationValue('log_level') == 'info': logging.getLogger(module_name).setLevel(logging.WARN) else: # Set log level for elasticsarch library if configured to other than default. logging.getLogger(module_name).setLevel(self.logger.level) self.action = self.getConfigurationValue('action') self.fields = self.getConfigurationValue('fields') self.ttl = self.getConfigurationValue("ttl") self.index_name = self.getConfigurationValue("index_name") self.routing_pattern = self.getConfigurationValue("routing") self.doc_id_pattern = self.getConfigurationValue("doc_id") self.doc_type_pattern = self.getConfigurationValue("doc_type") self.doc_type_is_dynamic = self.isDynamicConfigurationValue("doc_type") self.es_nodes = self.getConfigurationValue("nodes") self.read_timeout = self.getConfigurationValue("read_timeout") if not isinstance(self.es_nodes, list): self.es_nodes = [self.es_nodes] if self.getConfigurationValue("connection_type") == 'urllib3': self.connection_class = elasticsearch.connection.Urllib3HttpConnection elif self.getConfigurationValue("connection_type") == 'requests': self.connection_class = elasticsearch.connection.RequestsHttpConnection
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) if len(self.getConfigurationValue('cluster')) == 0: redis_store = self.getConfigurationValue('server') self.client = self.getRedisClient() else: redis_store = self.getConfigurationValue('cluster') self.client = self.getClusterRedisClient() try: self.client.ping() except: etype, evalue, etb = sys.exc_info() self.logger.error( "Could not connect to redis store at %s. Exception: %s, Error: %s." % (redis_store, etype, evalue)) self.lumbermill.shutDown() self.set_buffer = None if self.getConfigurationValue( 'store_interval_in_secs') or self.getConfigurationValue( 'batch_size'): self.set_buffer = Buffer( self.getConfigurationValue('batch_size'), self.setBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size')) self._set = self.set self.set = self.setBuffered self._get = self.get self.get = self.getBuffered self._delete = self.delete self.delete = self.deleteBuffered self._pop = self.pop self.pop = self.popBuffered
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.source_fields = self.getConfigurationValue('source_fields') # Allow single string as well. if isinstance(self.source_fields, types.StringTypes): self.source_fields = [self.source_fields] self.target_fields = self.getConfigurationValue('target_fields') # Allow single string as well. if isinstance(self.target_fields, types.StringTypes): self.target_fields = [self.target_fields] # If target_fields were provided they need to be of same length as source_fields. if self.target_fields and len(self.source_fields) != len( self.target_fields): self.logger.error( "Count of configured source and target fields need to match. Please check your configuration." ) self.lumbermill.shutDown() self.compression = self.getConfigurationValue('compression') # Get compression specific handler. try: self.deflater = getattr(self, "deflate_with_%s" % self.compression) except AttributeError: etype, evalue, etb = sys.exc_info() self.logger.error( "Unknown decompression lib: %s. Exception: %s, Error: %s" % (self.compression, etype, evalue)) self.lumbermill.shutDown()
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) # Set boto log level. logging.getLogger('boto3').setLevel(logging.CRITICAL) logging.getLogger('botocore').setLevel(logging.CRITICAL) self.sqs_queue_name = self.getConfigurationValue('queue') self.attribute_names = self.getConfigurationValue('attribute_names') self.message_attribute_names = self.getConfigurationValue('message_attribute_names') self.poll_interval = self.getConfigurationValue('poll_interval_in_secs') self.batch_size = self.getConfigurationValue('batch_size') try: self.sqs_client = boto3.client('sqs', region_name=self.getConfigurationValue('region'), api_version=None, use_ssl=True, verify=None, endpoint_url=None, aws_access_key_id=self.getConfigurationValue('aws_access_key_id'), aws_secret_access_key=self.getConfigurationValue('aws_secret_access_key'), aws_session_token=None, config=None) except: etype, evalue, etb = sys.exc_info() self.logger.error("Could not connect to sqs service. Exception: %s, Error: %s." % (etype, evalue)) self.lumbermill.shutDown() try: self.sqs_queue_url = self.sqs_client.get_queue_url(QueueName=self.sqs_queue_name)['QueueUrl'] except: etype, evalue, etb = sys.exc_info() self.logger.error("Could not get queue url for sqs queue %s. Exception: %s, Error: %s." % (self.sqs_queue_name, etype, evalue)) self.lumbermill.shutDown()
def shutDown(self): if self.server: self.server.stop() # Give os time to free the socket. Otherwise a reload will fail with 'address already in use' time.sleep(.2) # Call parent shutDown method. BaseThreadedModule.shutDown(self)
def initAfterFork(self): BaseThreadedModule.initAfterFork(self) self.buffer = Buffer( self.getConfigurationValue('batch_size'), self.storeData, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
def initAfterFork(self): # As the buffer uses a threaded timed function to flush its buffer and thread will not survive a fork, init buffer here. self.buffers = collections.defaultdict(lambda: Buffer(flush_size=self.buffer_size, callback=self.sendMergedEvent, interval=self.flush_interval_in_secs, maxsize=self.buffer_size)) BaseThreadedModule.initAfterFork(self)
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.lists = self.getConfigurationValue('lists') if not isinstance(self.lists, list): self.lists = [self.lists] self.timeout = self.getConfigurationValue('timeout') self.batch_size = self.getConfigurationValue('batch_size') self.client = redis.StrictRedis( host=self.getConfigurationValue('server'), port=self.getConfigurationValue('port'), password=self.getConfigurationValue('password'), db=self.getConfigurationValue('db')) try: self.client.ping() except: etype, evalue, etb = sys.exc_info() self.logger.error( "Could not connect to redis store at %s. Exception: %s, Error: %s." % (self.getConfigurationValue('server'), etype, evalue)) self.lumbermill.shutDown() # Monkeypatch run method to use the correct handle event method. if self.batch_size == 1: self.run = self.handleSingleEvent else: self.run = self.handleBatchEvents
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.source_field = self.getConfigurationValue('source_field') self.seperator = self.getConfigurationValue('seperator') self.target_field = self.getConfigurationValue('target_field') self.drop_original = not self.getConfigurationValue('keep_original')
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.source_field = self.getConfigurationValue('source_field') self.add_event_fields = self.getConfigurationValue('add_event_fields') self.backend_lock_name = "Lumbermill:Facet:FacetLock-%s" % self.lumbermill.getMainProcessId( ) self.backend_key_name = "Lumbermill:Facet:FacetKeys-%s" % self.lumbermill.getMainProcessId( ) self.backend_ttl = self.getConfigurationValue('backend_ttl') if (self.backend_ttl < self.getConfigurationValue('interval')): self.logger.error( 'backend_ttl setting is smaller then interval setting. Please check.' ) self.lumbermill.shutDown() return backend_info = self.lumbermill.getModuleInfoById( self.getConfigurationValue('backend')) if not backend_info: self.logger.error( "Could not find %s backend for persistant storage." % (self.getConfigurationValue('backend'))) self.lumbermill.shutDown() return self.persistence_backend = backend_info['instances'][0]
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.interval = self.getConfigurationValue('interval') self.fields = self.getConfigurationValue('fields') self.stats_collector = StatisticCollector() self.module_queues = {}
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.batch_size = self.getConfigurationValue('batch_size') self.backlog_size = self.getConfigurationValue('backlog_size') self.file_name = self.getConfigurationValue('file_name') self.format = self.getConfigurationValue('format') self.compress = self.getConfigurationValue('compress') self.file_handles = {} if self.compress == 'gzip': try: # Import module into namespace of object. Otherwise it will not be accessible when process was forked. self.gzip_module = __import__('gzip') except ImportError: self.logger.error( 'Gzip compression selected but gzip module could not be loaded.' ) self.lumbermill.shutDown() if self.compress == 'snappy': try: self.snappy_module = __import__('snappy') except ImportError: self.logger.error( 'Snappy compression selected but snappy module could not be loaded.' ) self.lumbermill.shutDown() self.buffer = Buffer( self.batch_size, self.storeData, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.backlog_size) TimedFunctionManager.startTimedFunction(self.closeStaleFileHandles)
def configure(self, configuration): # self.logger.setLevel(logging.DEBUG) # Call parent configure method. BaseThreadedModule.configure(self, configuration) self.is_leader = True if self.getConfigurationValue( 'pack') == 'leader' else False self.pack_followers = {} self.cluster_name = self.getConfigurationValue('name') self.discovered_leader = None self.secret = hashlib.sha256( self.getConfigurationValue('secret')).digest() self.handlers = collections.defaultdict(list) self.lock = threading.Lock() # Setup socket. self.interface_addr = self.getConfigurationValue('interface') self.interface_port = self.getConfigurationValue('port') self.broadcast_addr = self.getConfigurationValue('broadcast') self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) self.socket.settimeout(1) self.hostname = socket.gethostname() try: self.socket.bind((self.interface_addr, self.interface_port)) except: etype, evalue, etb = sys.exc_info() self.logger.error( "Could not listen on %s:%s. Exception: %s, Error: %s." % (self.getConfigurationValue("interface"), self.getConfigurationValue("port"), etype, evalue)) self.alive = False self.lumbermill.shutDown() return self.addHandlers()
def shutDown(self): self.accumulateReceiveRateStats() self.accumulateEventTypeStats() if self.lumbermill.is_master(): self.printIntervalStatistics() self.mp_stats_collector.shutDown() BaseThreadedModule.shutDown(self)
def configure(self, configuration): # Call parent configure method. BaseThreadedModule.configure(self, configuration) self.format = self.getConfigurationValue('format') self.collection = self.getConfigurationValue('collection') self.database = self.getConfigurationValue('database') self.doc_id_pattern = self.getConfigurationValue("doc_id")
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.syslogger = logging.getLogger(self.__class__.__name__) self.syslogger.propagate = False self.format = self.getConfigurationValue('format') if os.path.exists(self.getConfigurationValue('address')): address = self.getConfigurationValue('address') else: server, port = self.getConfigurationValue('address').split(':') address = (server, int(port)) if self.getConfigurationValue('proto') == 'tcp': socket_type = socket.SOCK_STREAM else: socket_type = socket.SOCK_DGRAM try: facility = logging.handlers.SysLogHandler.facility_names[ self.getConfigurationValue('facility')] except KeyError: self.logger.error("The configured facility %s is unknown." % (self.getConfigurationValue('facility'))) self.syslog_handler = logging.handlers.SysLogHandler( address, facility=facility, socktype=socket_type) self.syslogger.addHandler(self.syslog_handler) self.format = self.getConfigurationValue('format')
def configure(self, configuration): # Call parent configure method. BaseThreadedModule.configure(self, configuration) self.format = self.getConfigurationValue('format') self.collection = self.getConfigurationValue('collection') self.database = self.getConfigurationValue('database') self.doc_id_pattern = self.getConfigurationValue("doc_id")
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) backend = self.getConfigurationValue('backend') self.backend_client = None if backend == 'DictStore': import simplekv.memory self.backend_client = None self.kv_store = simplekv.memory.DictStore() elif backend == 'RedisStore': import simplekv.memory.redisstore self.backend_client = self.getRedisClient() self.kv_store = simplekv.memory.redisstore.RedisStore(self.backend_client) elif backend == 'MemcacheStore': import simplekv.memory.memcachestore self.backend_client = self.getMemcacheClient() self.kv_store = simplekv.memory.memcachestore.MemcacheStore(self.backend_client) self.set_buffer = None if self.getConfigurationValue('store_interval_in_secs') or self.getConfigurationValue('batch_size'): if backend == 'RedisStore': self.set_buffer = Buffer(self.getConfigurationValue('batch_size'), self.setRedisBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size')) else: self.set_buffer = Buffer(self.getConfigurationValue('batch_size'), self.setBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size')) self._set = self.set self.set = self.setBuffered self._get = self.get self.get = self.getBuffered self._delete = self.delete self.delete = self.deleteBuffered self._pop = self.pop self.pop = self.popBuffered
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.interval = self.getConfigurationValue('interval') self.fields = self.getConfigurationValue('fields') self.stats_collector = StatisticCollector() self.module_queues = {}
def configure(self, configuration): BaseThreadedModule.configure(self, configuration) self.hostname = self.getConfigurationValue("hostname") self.fields = self.getConfigurationValue("fields") self.field_prefix = self.getConfigurationValue("field_prefix") self.timestamp_field = self.getConfigurationValue("timestamp_field") self.batch_size = self.getConfigurationValue('batch_size') self.backlog_size = self.getConfigurationValue('backlog_size') self.agent_conf = self.getConfigurationValue("agent_conf") if self.agent_conf: if self.agent_conf is True: self.agent_conf = "/etc/zabbix/zabbix_agentd.conf" if not os.path.isfile(self.agent_conf): self.logger.error("%s does not point to an existing file." % self.agent_conf) self.lumbermill.shutDown() self.zabbix_sender = ZabbixSender(use_config=self.agent_conf) else: self.logger.error("asdads") server = self.getConfigurationValue("server") port = 10051 if ":" in self.server: server, port = self.server.split(":") self.zabbix_sender = ZabbixSender(zabbix_server=server, port=port) self.buffer = Buffer(self.getConfigurationValue('batch_size'), self.storeData, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
def configure(self, configuration): # self.logger.setLevel(logging.DEBUG) # Call parent configure method. BaseThreadedModule.configure(self, configuration) self.is_leader = True if self.getConfigurationValue('pack') == 'leader' else False self.pack_followers = {} self.cluster_name = self.getConfigurationValue('name') self.discovered_leader = None self.secret = hashlib.sha256(self.getConfigurationValue('secret')).digest() self.handlers = collections.defaultdict(list) self.lock = threading.Lock() # Setup socket. self.interface_addr = self.getConfigurationValue('interface') self.interface_port = self.getConfigurationValue('port') self.broadcast_addr = self.getConfigurationValue('broadcast') self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) self.socket.settimeout(1) self.hostname = socket.gethostname() try: self.socket.bind((self.interface_addr, self.interface_port)) except: etype, evalue, etb = sys.exc_info() self.logger.error("Could not listen on %s:%s. Exception: %s, Error: %s." % (self.getConfigurationValue("interface"), self.getConfigurationValue("port"), etype, evalue)) self.alive = False self.lumbermill.shutDown() return self.addHandlers()
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.pretty_print = self.getConfigurationValue('pretty_print') self.fields = self.getConfigurationValue('fields') self.format = self.getConfigurationValue('format') self.printing = False
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.backend = self.getConfigurationValue('backend') self.backend_client = None self.kv_store = None self.set_buffer = None if self.backend == 'DictStore': import simplekv.memory self.kv_store = simplekv.memory.DictStore() elif self.backend == 'RedisStore': import simplekv.memory.redisstore self.backend_client = self._getRedisClient() self.kv_store = simplekv.memory.redisstore.RedisStore(self.backend_client) elif self.backend == 'MemcacheStore': import simplekv.memory.memcachestore self.backend_client = self._getMemcacheClient() self.kv_store = simplekv.memory.memcachestore.MemcacheStore(self.backend_client) else: self.logger("Unknown backend type %s. Please check." % backend) self.lumbermill.shutDown(); if self.getConfigurationValue('store_interval_in_secs') or self.getConfigurationValue('batch_size'): if self.backend == 'RedisStore': self.set_buffer = Buffer(self.getConfigurationValue('batch_size'), self._setRedisBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size')) else: self.set_buffer = Buffer(self.getConfigurationValue('batch_size'), self._setBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size')) self._set = self.set self.set = self._setBuffered self._get = self.get self.get = self._getBuffered self._delete = self.delete self.delete = self._deleteBuffered self._pop = self.pop self.pop = self._popBuffered
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.source_field = self.getConfigurationValue('source_field') self.seperator = self.getConfigurationValue('seperator') self.target_field = self.getConfigurationValue('target_field') self.drop_original = not self.getConfigurationValue('keep_original')
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) # Make urllib3s logging less verbose. urllib3_logger = logging.getLogger('requests.packages.urllib3.connectionpool') urllib3_logger.setLevel(logging.CRITICAL) self.server, self.port = self.getConfigurationValue('server').split(':') self.user = self.getConfigurationValue('user') self.events_container = [] self.batch_size = self.getConfigurationValue('batch_size') self.backlog_size = self.getConfigurationValue('backlog_size') self.path = self.getConfigurationValue('path') self.name_pattern = self.getConfigurationValue('name_pattern') self.format = self.getConfigurationValue('format') self.compress = self.getConfigurationValue('compress') if self.compress == 'gzip': try: import gzip except ImportError: self.logger.error('Gzip compression selected but gzip module could not be loaded.') self.lumbermill.shutDown() if self.compress == 'snappy': try: import snappy except ImportError: self.logger.error('Snappy compression selected but snappy module could not be loaded.') self.lumbermill.shutDown() self.is_storing = False self.lock = multiprocessing.Lock() self.timed_store_func = self.getTimedStoreFunc()
def initAfterFork(self): BaseThreadedModule.initAfterFork(self) self.buffer = Buffer( self.getConfigurationValue('batch_size'), self.storeData, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size')) try: self.sqs_resource = boto3.resource( 'sqs', region_name=self.getConfigurationValue('region'), api_version=None, use_ssl=True, verify=None, endpoint_url=None, aws_access_key_id=self.getConfigurationValue( 'aws_access_key_id'), aws_secret_access_key=self.getConfigurationValue( 'aws_secret_access_key'), aws_session_token=None, config=None) self.sqs_queue = self.sqs_resource.get_queue_by_name( QueueName=self.getConfigurationValue('queue')) except: etype, evalue, etb = sys.exc_info() self.logger.error( "Could not connect to sqs service. Exception: %s, Error: %s." % (etype, evalue)) self.lumbermill.shutDown()
def shutDown(self): if self.server: self.server.stop() # Give os time to free the socket. Otherwise a reload will fail with 'address already in use' time.sleep(.2) # Call parent shutDown method. BaseThreadedModule.shutDown(self)
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) if len(self.getConfigurationValue('cluster')) == 0: redis_store = self.getConfigurationValue('server') self.client = self.getRedisClient() else: redis_store = self.getConfigurationValue('cluster') self.client = self.getClusterRedisClient() try: self.client.ping() except: etype, evalue, etb = sys.exc_info() self.logger.error("Could not connect to redis store at %s. Exception: %s, Error: %s." % (redis_store,etype, evalue)) self.lumbermill.shutDown() self.set_buffer = None if self.getConfigurationValue('store_interval_in_secs') or self.getConfigurationValue('batch_size'): self.set_buffer = Buffer(self.getConfigurationValue('batch_size'), self.setBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size')) self._set = self.set self.set = self.setBuffered self._get = self.get self.get = self.getBuffered self._delete = self.delete self.delete = self.deleteBuffered self._pop = self.pop self.pop = self.popBuffered
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.formats = self.getConfigurationValue('formats') self.connection_data = (self.getConfigurationValue('server'), self.getConfigurationValue('port')) self.connection = None
def initAfterFork(self): self.evaluate_facet_data_func = setInterval(self.getConfigurationValue('interval'))(self.evaluateFacets) #self.getEvaluateFunc() self.timed_func_handler_a = TimedFunctionManager.startTimedFunction(self.evaluate_facet_data_func) if self.cache: self.store_facets_in_cache_func = setInterval(1)(self.storeFacetsInCache) self.timed_func_handler_b = TimedFunctionManager.startTimedFunction(self.store_facets_in_cache_func) BaseThreadedModule.initAfterFork(self)
def initAfterFork(self): BaseThreadedModule.initAfterFork(self) self.buffer = Buffer(self.getConfigurationValue('batch_size'), self.storeData, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size')) self.connection = self.connect() if not self.connection: self.lumbermill.shutDown() return BaseThreadedModule.initAfterFork(self)
def initAfterFork(self): # Get all configured queues for waiting event stats. self.module_queues = self.lumbermill.getAllQueues() self.psutil_processes.append(psutil.Process(self.lumbermill.getMainProcessId())) for worker in self.lumbermill.child_processes: self.psutil_processes.append(psutil.Process(worker.pid)) TimedFunctionManager.startTimedFunction(self.getRunTimedFunctionsFunc()) BaseThreadedModule.initAfterFork(self)
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.events = self.getConfigurationValue("event") if not isinstance(self.events, list): self.events = [self.events] self.sleep = self.getConfigurationValue("sleep") self.max_events_count = self.getConfigurationValue("events_count")
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) # Set boto log level. logging.getLogger('boto3').setLevel(logging.CRITICAL) logging.getLogger('botocore').setLevel(logging.CRITICAL) self.batch_size = self.getConfigurationValue('batch_size') self.format = self.getConfigurationValue('format')
def initAfterFork(self): # As the buffer uses a threaded timed function to flush its buffer and thread will not survive a fork, init buffer here. self.buffers = collections.defaultdict( lambda: Buffer(flush_size=self.buffer_size, callback=self.sendMergedEvent, interval=self.flush_interval_in_secs, maxsize=self.buffer_size)) BaseThreadedModule.initAfterFork(self)
def shutDown(self): # Call parent shutDown method. BaseThreadedModule.shutDown(self) try: self.socket.close() self.context.term() except AttributeError: pass
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.events = self.getConfigurationValue("event") if not isinstance(self.events, list): self.events = [self.events] self.sleep = self.getConfigurationValue("sleep") self.max_events_count = self.getConfigurationValue("events_count")
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.server = None self.topic = self.getConfigurationValue('topic') self.format = self.getConfigurationValue('format') self.mode = self.getConfigurationValue('mode') if self.mode == "bind": self.can_run_forked = False
def initAfterFork(self): BaseThreadedModule.initAfterFork(self) # Init es client after fork as mentioned in https://elasticsearch-py.readthedocs.org/en/master/ self.es = self.connect() if not self.es: self.lumbermill.shutDown() return # As the buffer uses a threaded timed function to flush its buffer and thread will not survive a fork, init buffer here. self.buffer = Buffer(self.getConfigurationValue('batch_size'), self.storeData, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.emit_as_event = self.getConfigurationValue('emit_as_event') self.interval = self.getConfigurationValue('interval') self.stats_collector = StatisticCollector() for counter_name in ['events_received', 'event_type_Unknown', 'event_type_httpd_access_log']: MultiProcessStatisticCollector().initCounter(counter_name) self.module_queues = {}
def initAfterFork(self): # Get all configured queues for waiting event stats. for module_name, module_info in self.lumbermill.modules.items(): instance = module_info['instances'][0] if not hasattr(instance, 'getInputQueue') or not instance.getInputQueue(): continue self.module_queues[module_name] = instance.getInputQueue() TimedFunctionManager.startTimedFunction(self.getRunTimedFunctionsFunc()) BaseThreadedModule.initAfterFork()
def initAfterFork(self): BaseThreadedModule.initAfterFork(self) # Init monogdb client after fork. self.mongodb = self.connect() if not self.mongodb: self.lumbermill.shutDown() return # As the buffer uses a threaded timed function to flush its buffer and thread will not survive a fork, init buffer here. self.buffer = Buffer(self.getConfigurationValue('batch_size'), self.storeData, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.source_fields = self.getConfigurationValue('source_fields') # Allow single string as well. if isinstance(self.source_fields, types.StringTypes): self.source_fields = [self.source_fields] self.target_field = self.getConfigurationValue('target_field') self.in_mem_cache = MemoryCache(size=1000)
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.source_fields = self.getConfigurationValue('source_fields') # Allow single string as well. if isinstance(self.source_fields, types.StringTypes): self.source_fields = [self.source_fields] self.target_field = self.getConfigurationValue('target_field') self.in_mem_cache = MemoryCache(size=1000)
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.emit_as_event = self.getConfigurationValue('emit_as_event') self.interval = self.getConfigurationValue('interval') self.event_type_statistics = self.getConfigurationValue( 'event_type_statistics') self.stats_collector = StatisticCollector() self.mp_stats_collector = MultiProcessStatisticCollector() self.module_queues = {}
def configure(self, configuration): # Call parent configure method. BaseThreadedModule.configure(self, configuration) self.file_tailer = None self.files = self.scanPaths() self.line_by_line = self.getConfigurationValue('line_by_line') self.mode = self.getConfigurationValue('mode') self.datastore_key = "%032x%s" % (random.getrandbits(128), self.process_id) self.total_file_count = len(self.files) self.lumbermill.setInInternalDataStore(self.datastore_key, 0)
def configure(self, configuration): BaseThreadedModule.configure(self, configuration) self.redis_ttl = self.getConfigurationValue('redis_ttl') # Get redis client module. if self.getConfigurationValue('redis_store'): mod_info = self.lumbermill.getModuleInfoById( self.getConfigurationValue('redis_store')) self.redis_store = mod_info['instances'][0] else: self.redis_store = None
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.source_fields = self.getConfigurationValue('source_fields') # Allow single string as well. if isinstance(self.source_fields, types.StringTypes): self.source_fields = [self.source_fields] self.seperator = self.getConfigurationValue('seperator') self.target_field = self.getConfigurationValue('target_field') self.drop_original = not self.getConfigurationValue('keep_original')
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.source_field = self.getConfigurationValue('source_field') self.target_field = self.getConfigurationValue('target_field') self.drop_original = not self.getConfigurationValue('keep_original') if self.getConfigurationValue('action') == 'decode': self.handleEvent = self.decodeBase64 else: self.handleEvent = self.encodeBase64
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.source_field = self.getConfigurationValue('source_field') self.target_field = self.getConfigurationValue('target_field') self.drop_original = not self.getConfigurationValue('keep_original') if self.getConfigurationValue('action') == 'decode': self.handleEvent = self.decodeBase64 else: self.handleEvent = self.encodeBase64
def initAfterFork(self): # Get all configured queues for waiting event stats. for module_name, module_info in self.lumbermill.modules.items(): instance = module_info['instances'][0] if not hasattr(instance, 'getInputQueue') or not instance.getInputQueue(): continue self.module_queues[module_name] = instance.getInputQueue() TimedFunctionManager.startTimedFunction( self.getRunTimedFunctionsFunc()) BaseThreadedModule.initAfterFork()
def initAfterFork(self): BaseThreadedModule.initAfterFork(self) # Calculate event count when running in multiple processes. if self.max_events_count == 0: return self.max_events_count = int(self.getConfigurationValue("events_count")/self.lumbermill.getWorkerCount()) if self.lumbermill.is_master(): remainder = self.getConfigurationValue("events_count") % self.lumbermill.getWorkerCount() self.max_events_count += remainder if self.max_events_count == 0: self.shutDown()
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.results = [] function_str = "lambda event: " + re.sub('%\((.*?)\)s', r"event.get('\1', False)", self.getConfigurationValue('function')) self.function = self.compileFunction(function_str) if self.getConfigurationValue('results_function'): function_str = "lambda results: "+ re.sub('%\((.*?)\)s', r"results", self.getConfigurationValue('results_function')) self.results_function = self.compileFunction(function_str) self.target_field = self.getConfigurationValue('target_field') self.interval = self.getConfigurationValue('interval')
def initAfterFork(self): self.resolver = resolver.Resolver() self.resolver.timeout = self.timeout self.resolver.lifetime = self.timeout if self.nameservers: self.resolver.nameservers = self.nameservers self.queue = Queue.Queue(20) self.lookup_threads = [LookupThread(self.queue, self.lookup_type, self) for _ in range(0, self.lookup_threads_pool_size)] for thread in self.lookup_threads: thread.start() BaseThreadedModule.initAfterFork(self)
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.format = self.getConfigurationValue('format') try: self.client = redis.Redis(host=self.getConfigurationValue('server'), port=self.getConfigurationValue('port'), password=self.getConfigurationValue('password'), db=self.getConfigurationValue('db')) except: etype, evalue, etb = sys.exc_info() self.logger.error("Could not connect to redis store at %s. Exception: %s, Error: %s." % (self.getConfigurationValue('server'),etype, evalue))
def configure(self, configuration): # Call parent configure method BaseThreadedModule.configure(self, configuration) self.emit_as_event = self.getConfigurationValue('emit_as_event') self.interval = self.getConfigurationValue('interval') self.stats_collector = StatisticCollector() for counter_name in [ 'events_received', 'event_type_Unknown', 'event_type_httpd_access_log' ]: MultiProcessStatisticCollector().initCounter(counter_name) self.module_queues = {}
def initAfterFork(self): BaseThreadedModule.initAfterFork(self) self.buffer = Buffer( self.getConfigurationValue('batch_size'), self.storeData, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size')) self.connection = self.connect() if not self.connection: self.lumbermill.shutDown() return BaseThreadedModule.initAfterFork(self)
def configure(self, configuration): BaseThreadedModule.configure(self, configuration) socket.setdefaulttimeout(self.getConfigurationValue('socket_timeout')) self.get_metadata = self.getConfigurationValue('get_metadata') self.interval = self.getConfigurationValue('interval') # Get redis client module. if self.getConfigurationValue('redis_store'): mod_info = self.lumbermill.getModuleInfoById( self.getConfigurationValue('redis_store')) self.redis_store = mod_info['instances'][0] else: self.redis_store = None
def shutDown(self): try: self.buffer.flush() except: pass try: self.client.close() self.zmq_context.term() except AttributeError: pass # Call parent shutDown method. BaseThreadedModule.shutDown(self)