class DeduplicatorExpertBot(Bot): def init(self): self.cache = Cache( self.parameters.redis_cache_host, self.parameters.redis_cache_port, self.parameters.redis_cache_db, self.parameters.redis_cache_ttl, ) def process(self): message = self.receive_message() ignore_keys = set(k.strip() for k in self.parameters.ignore_keys.split(',')) message_hash = message.hash(ignore_keys) old_hash = hash(int(message_hash, 16)) if not (self.cache.exists(message_hash) or self.cache.exists(old_hash)): self.cache.set(message_hash, 'hash') self.send_message(message) else: self.logger.debug('Dropped message.') self.acknowledge_message()
class DeduplicatorExpertBot(Bot): def init(self): self.cache = Cache(self.parameters.redis_cache_host, self.parameters.redis_cache_port, self.parameters.redis_cache_db, self.parameters.redis_cache_ttl, ) def process(self): message = self.receive_message() ignore_keys = set(k.strip() for k in self.parameters.ignore_keys.split(',')) message_hash = message.hash(ignore_keys) old_hash = hash(int(message_hash, 16)) if not (self.cache.exists(message_hash) or self.cache.exists(old_hash)): self.cache.set(message_hash, 'hash') self.send_message(message) else: self.logger.debug('Dropped message.') self.acknowledge_message()
class DeduplicatorExpertBot(Bot): def init(self): self.cache = Cache(self.parameters.redis_cache_host, self.parameters.redis_cache_port, self.parameters.redis_cache_db, self.parameters.redis_cache_ttl, ) def process(self): message = self.receive_message() if message is None: self.acknowledge_message() return auxiliar_message = copy.copy(message) ignore_keys = self.parameters.ignore_keys.split(',') for ignore_key in ignore_keys: ignore_key = ignore_key.strip() if ignore_key in auxiliar_message: del auxiliar_message[ignore_key] message_hash = hash(auxiliar_message) if not self.cache.exists(message_hash): self.cache.set(message_hash, 'hash') self.send_message(message) self.acknowledge_message()
class DeduplicatorExpertBot(Bot): def init(self): self.cache = Cache( self.parameters.redis_cache_host, self.parameters.redis_cache_port, self.parameters.redis_cache_db, self.parameters.redis_cache_ttl, ) def process(self): message = self.receive_message() if message is None: self.acknowledge_message() return auxiliar_message = copy.copy(message) ignore_keys = self.parameters.ignore_keys.split(',') for ignore_key in ignore_keys: ignore_key = ignore_key.strip() if ignore_key in auxiliar_message: auxiliar_message.clear(ignore_key) message_hash = hash(auxiliar_message) if not self.cache.exists(message_hash): self.cache.set(message_hash, 'hash') self.send_message(message) self.acknowledge_message()
class DeduplicatorExpertBot(Bot): def init(self): self.cache = Cache(self.parameters.redis_cache_host, self.parameters.redis_cache_port, self.parameters.redis_cache_db, self.parameters.redis_cache_ttl, getattr(self.parameters, "redis_cache_password", None) ) self.filter_keys = set(k.strip() for k in self.parameters.filter_keys.split(',')) def process(self): message = self.receive_message() message_hash = message.hash(filter_keys=self.filter_keys, filter_type=self.parameters.filter_type) if not self.cache.exists(message_hash): self.cache.set(message_hash, 'hash') self.send_message(message) else: self.logger.debug('Dropped message.') self.acknowledge_message()
class DeduplicatorBot(Bot): def init(self): self.cache = Cache( self.parameters.redis_cache_host, self.parameters.redis_cache_port, self.parameters.redis_cache_db, self.parameters.redis_cache_ttl ) def process(self): message = self.receive_message() if message: # Event deduplication if isinstance(message, Event): event = deepcopy(message) event.clear("observation_time") if event.value("type")=="vulnerable service" or event.value("type")=="compromised": event.clear("source_time") event.clear("source_reverse_dns") message_hash = hash(event) # Generic message deduplication else: message_hash = hash(message) if not self.cache.exists(message_hash): self.cache.set(message_hash, 'hash') self.send_message(message) self.acknowledge_message()
class DeduplicatorExpertBot(Bot): _message_processed_verb = 'Forwarded' def init(self): self.cache = Cache(self.parameters.redis_cache_host, self.parameters.redis_cache_port, self.parameters.redis_cache_db, self.parameters.redis_cache_ttl, getattr(self.parameters, "redis_cache_password", None) ) self.filter_keys = {k.strip() for k in self.parameters.filter_keys.split(',')} self.bypass = getattr(self.parameters, "bypass", False) def process(self): message = self.receive_message() if self.bypass: self.send_message(message) else: message_hash = message.hash(filter_keys=self.filter_keys, filter_type=self.parameters.filter_type) if not self.cache.exists(message_hash): self.cache.set(message_hash, 'hash') self.send_message(message) else: self.logger.debug('Dropped message.') self.acknowledge_message()
class DeduplicatorBot(Bot): def init(self): self.cache = Cache( self.parameters.redis_cache_host, self.parameters.redis_cache_port, self.parameters.redis_cache_db, self.parameters.redis_cache_ttl ) def process(self): message = self.receive_message() if message: # Event deduplication if isinstance(message, Event): event = copy(message) event.clear("observation_time") message_hash = hash(event) # Generic message deduplication else: message_hash = hash(message) if not self.cache.exists(message_hash): self.send_message(message) self.cache.set(message_hash, 'hash') self.acknowledge_message()
class DeduplicatorBot(Bot): def init(self): self.cache = Cache( self.parameters.redis_cache_host, self.parameters.redis_cache_port, self.parameters.redis_cache_db, self.parameters.redis_cache_ttl ) def process(self): message = self.receive_message() message_hash = hash(message) if not self.cache.exists(message_hash): self.cache.set(message_hash, 'hash') self.send_message(message) self.acknowledge_message()