Exemplo n.º 1
0
class DeduplicatorExpertBot(Bot):
    def init(self):
        self.cache = Cache(
            self.parameters.redis_cache_host,
            self.parameters.redis_cache_port,
            self.parameters.redis_cache_db,
            self.parameters.redis_cache_ttl,
        )

    def process(self):
        message = self.receive_message()

        ignore_keys = set(k.strip()
                          for k in self.parameters.ignore_keys.split(','))
        message_hash = message.hash(ignore_keys)

        old_hash = hash(int(message_hash, 16))

        if not (self.cache.exists(message_hash)
                or self.cache.exists(old_hash)):
            self.cache.set(message_hash, 'hash')
            self.send_message(message)
        else:
            self.logger.debug('Dropped message.')

        self.acknowledge_message()
Exemplo n.º 2
0
class DeduplicatorExpertBot(Bot):

    def init(self):
        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           )

    def process(self):
        message = self.receive_message()

        ignore_keys = set(k.strip()
                          for k in self.parameters.ignore_keys.split(','))
        message_hash = message.hash(ignore_keys)

        old_hash = hash(int(message_hash, 16))

        if not (self.cache.exists(message_hash) or self.cache.exists(old_hash)):
            self.cache.set(message_hash, 'hash')
            self.send_message(message)
        else:
            self.logger.debug('Dropped message.')

        self.acknowledge_message()
Exemplo n.º 3
0
class DeduplicatorExpertBot(Bot):

    def init(self):
        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           )

    def process(self):
        message = self.receive_message()

        if message is None:
            self.acknowledge_message()
            return

        auxiliar_message = copy.copy(message)

        ignore_keys = self.parameters.ignore_keys.split(',')

        for ignore_key in ignore_keys:
            ignore_key = ignore_key.strip()
            if ignore_key in auxiliar_message:
                del auxiliar_message[ignore_key]

        message_hash = hash(auxiliar_message)

        if not self.cache.exists(message_hash):
            self.cache.set(message_hash, 'hash')
            self.send_message(message)

        self.acknowledge_message()
Exemplo n.º 4
0
class DeduplicatorExpertBot(Bot):
    def init(self):
        self.cache = Cache(
            self.parameters.redis_cache_host,
            self.parameters.redis_cache_port,
            self.parameters.redis_cache_db,
            self.parameters.redis_cache_ttl,
        )

    def process(self):
        message = self.receive_message()

        if message is None:
            self.acknowledge_message()
            return

        auxiliar_message = copy.copy(message)

        ignore_keys = self.parameters.ignore_keys.split(',')

        for ignore_key in ignore_keys:
            ignore_key = ignore_key.strip()
            if ignore_key in auxiliar_message:
                auxiliar_message.clear(ignore_key)

        message_hash = hash(auxiliar_message)

        if not self.cache.exists(message_hash):
            self.cache.set(message_hash, 'hash')
            self.send_message(message)

        self.acknowledge_message()
Exemplo n.º 5
0
class DeduplicatorExpertBot(Bot):

    def init(self):
        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           getattr(self.parameters, "redis_cache_password",
                                   None)
                           )
        self.filter_keys = set(k.strip() for k in
                               self.parameters.filter_keys.split(','))

    def process(self):
        message = self.receive_message()

        message_hash = message.hash(filter_keys=self.filter_keys,
                                    filter_type=self.parameters.filter_type)

        if not self.cache.exists(message_hash):
            self.cache.set(message_hash, 'hash')
            self.send_message(message)
        else:
            self.logger.debug('Dropped message.')

        self.acknowledge_message()
class DeduplicatorBot(Bot):
    
    def init(self):
        self.cache = Cache(
                            self.parameters.redis_cache_host,
                            self.parameters.redis_cache_port,
                            self.parameters.redis_cache_db,
                            self.parameters.redis_cache_ttl
                          )


    def process(self):
        message = self.receive_message()

        if message:
            
            # Event deduplication
            if isinstance(message, Event):
                event = deepcopy(message)
                event.clear("observation_time")
		if event.value("type")=="vulnerable service" or event.value("type")=="compromised":
		   event.clear("source_time")
		   event.clear("source_reverse_dns")
                message_hash = hash(event)                            

            # Generic message deduplication
            else:
                message_hash = hash(message)

            if not self.cache.exists(message_hash):
                self.cache.set(message_hash, 'hash')
                self.send_message(message)

        self.acknowledge_message()
Exemplo n.º 7
0
class DeduplicatorExpertBot(Bot):

    _message_processed_verb = 'Forwarded'

    def init(self):
        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           getattr(self.parameters, "redis_cache_password",
                                   None)
                           )
        self.filter_keys = {k.strip() for k in
                            self.parameters.filter_keys.split(',')}
        self.bypass = getattr(self.parameters, "bypass", False)

    def process(self):
        message = self.receive_message()

        if self.bypass:
            self.send_message(message)
        else:
            message_hash = message.hash(filter_keys=self.filter_keys,
                                        filter_type=self.parameters.filter_type)

            if not self.cache.exists(message_hash):
                self.cache.set(message_hash, 'hash')
                self.send_message(message)
            else:
                self.logger.debug('Dropped message.')

        self.acknowledge_message()
Exemplo n.º 8
0
class DeduplicatorBot(Bot):
    
    def init(self):
        self.cache = Cache(
                            self.parameters.redis_cache_host,
                            self.parameters.redis_cache_port,
                            self.parameters.redis_cache_db,
                            self.parameters.redis_cache_ttl
                          )


    def process(self):
        message = self.receive_message()

        if message:
            
            # Event deduplication
            if isinstance(message, Event):
                event = copy(message)
                event.clear("observation_time")
                message_hash = hash(event)                            

            # Generic message deduplication
            else:
                message_hash = hash(message)

            if not self.cache.exists(message_hash):
                self.send_message(message)
                self.cache.set(message_hash, 'hash')

        self.acknowledge_message()
Exemplo n.º 9
0
class DeduplicatorBot(Bot):
    
    def init(self):
        self.cache = Cache(
                            self.parameters.redis_cache_host,
                            self.parameters.redis_cache_port,
                            self.parameters.redis_cache_db,
                            self.parameters.redis_cache_ttl
                          )

    def process(self):
        message = self.receive_message()
        message_hash = hash(message)

        if not self.cache.exists(message_hash):
            self.cache.set(message_hash, 'hash')
            self.send_message(message)

        self.acknowledge_message()