class DeduplicatorExpertBot(Bot):
def init(self):
self.cache = Cache(
self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
)
def process(self):
message = self.receive_message()
ignore_keys = set(k.strip()
for k in self.parameters.ignore_keys.split(','))
message_hash = message.hash(ignore_keys)
old_hash = hash(int(message_hash, 16))
if not (self.cache.exists(message_hash)
or self.cache.exists(old_hash)):
self.cache.set(message_hash, 'hash')
self.send_message(message)
else:
self.logger.debug('Dropped message.')
self.acknowledge_message()
class DeduplicatorExpertBot(Bot):
def init(self):
self.cache = Cache(self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
)
def process(self):
message = self.receive_message()
ignore_keys = set(k.strip()
for k in self.parameters.ignore_keys.split(','))
message_hash = message.hash(ignore_keys)
old_hash = hash(int(message_hash, 16))
if not (self.cache.exists(message_hash) or self.cache.exists(old_hash)):
self.cache.set(message_hash, 'hash')
self.send_message(message)
else:
self.logger.debug('Dropped message.')
self.acknowledge_message()
class DeduplicatorExpertBot(Bot):
def init(self):
self.cache = Cache(self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
)
def process(self):
message = self.receive_message()
if message is None:
self.acknowledge_message()
return
auxiliar_message = copy.copy(message)
ignore_keys = self.parameters.ignore_keys.split(',')
for ignore_key in ignore_keys:
ignore_key = ignore_key.strip()
if ignore_key in auxiliar_message:
del auxiliar_message[ignore_key]
message_hash = hash(auxiliar_message)
if not self.cache.exists(message_hash):
self.cache.set(message_hash, 'hash')
self.send_message(message)
self.acknowledge_message()
class DeduplicatorExpertBot(Bot):
def init(self):
self.cache = Cache(
self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
)
def process(self):
message = self.receive_message()
if message is None:
self.acknowledge_message()
return
auxiliar_message = copy.copy(message)
ignore_keys = self.parameters.ignore_keys.split(',')
for ignore_key in ignore_keys:
ignore_key = ignore_key.strip()
if ignore_key in auxiliar_message:
auxiliar_message.clear(ignore_key)
message_hash = hash(auxiliar_message)
if not self.cache.exists(message_hash):
self.cache.set(message_hash, 'hash')
self.send_message(message)
self.acknowledge_message()
class DeduplicatorExpertBot(Bot):
def init(self):
self.cache = Cache(self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
getattr(self.parameters, "redis_cache_password",
None)
)
self.filter_keys = set(k.strip() for k in
self.parameters.filter_keys.split(','))
def process(self):
message = self.receive_message()
message_hash = message.hash(filter_keys=self.filter_keys,
filter_type=self.parameters.filter_type)
if not self.cache.exists(message_hash):
self.cache.set(message_hash, 'hash')
self.send_message(message)
else:
self.logger.debug('Dropped message.')
self.acknowledge_message()
class DeduplicatorBot(Bot):
def init(self):
self.cache = Cache(
self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl
)
def process(self):
message = self.receive_message()
if message:
# Event deduplication
if isinstance(message, Event):
event = deepcopy(message)
event.clear("observation_time")
if event.value("type")=="vulnerable service" or event.value("type")=="compromised":
event.clear("source_time")
event.clear("source_reverse_dns")
message_hash = hash(event)
# Generic message deduplication
else:
message_hash = hash(message)
if not self.cache.exists(message_hash):
self.cache.set(message_hash, 'hash')
self.send_message(message)
self.acknowledge_message()
class DeduplicatorExpertBot(Bot):
_message_processed_verb = 'Forwarded'
def init(self):
self.cache = Cache(self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
getattr(self.parameters, "redis_cache_password",
None)
)
self.filter_keys = {k.strip() for k in
self.parameters.filter_keys.split(',')}
self.bypass = getattr(self.parameters, "bypass", False)
def process(self):
message = self.receive_message()
if self.bypass:
self.send_message(message)
else:
message_hash = message.hash(filter_keys=self.filter_keys,
filter_type=self.parameters.filter_type)
if not self.cache.exists(message_hash):
self.cache.set(message_hash, 'hash')
self.send_message(message)
else:
self.logger.debug('Dropped message.')
self.acknowledge_message()
class DeduplicatorBot(Bot):
def init(self):
self.cache = Cache(
self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl
)
def process(self):
message = self.receive_message()
if message:
# Event deduplication
if isinstance(message, Event):
event = copy(message)
event.clear("observation_time")
message_hash = hash(event)
# Generic message deduplication
else:
message_hash = hash(message)
if not self.cache.exists(message_hash):
self.send_message(message)
self.cache.set(message_hash, 'hash')
self.acknowledge_message()
class DeduplicatorBot(Bot):
def init(self):
self.cache = Cache(
self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl
)
def process(self):
message = self.receive_message()
message_hash = hash(message)
if not self.cache.exists(message_hash):
self.cache.set(message_hash, 'hash')
self.send_message(message)
self.acknowledge_message()