Ejemplo n.º 1
0
class ReverseDnsExpertBot(Bot):

    def init(self):
        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           )

    def process(self):
        event = self.receive_message()

        keys = ["source.%s", "destination.%s"]

        for key in keys:
            ip_key = key % "ip"

            if not event.contains(ip_key):
                continue

            ip = event.get(ip_key)
            ip_version = IPAddress.version(ip)
            ip_integer = IPAddress.to_int(ip)

            if ip_version == 4:
                minimum = MINIMUM_BGP_PREFIX_IPV4

            elif ip_version == 6:
                minimum = MINIMUM_BGP_PREFIX_IPV6

            else:
                self.logger.warning("Invalid IP version {}".format(ip_version))
                self.send_message(event)
                self.acknowledge_message()

            cache_key = bin(ip_integer)[2: minimum + 2]
            cachevalue = self.cache.get(cache_key)

            result = None
            if cachevalue:
                result = cachevalue
            else:
                rev_name = reversename.from_address(ip)
                try:
                    result = resolver.query(rev_name, "PTR")
                    expiration = result.expiration
                    result = result[0]
                except dns.exception.DNSException as e:
                    if isinstance(e, dns.resolver.NXDOMAIN):
                        continue
                else:
                    ttl = datetime.fromtimestamp(expiration) - datetime.now()
                    self.cache.set(cache_key, str(result),
                                   ttl=int(ttl.total_seconds()))

            if result is not None:
                event.add(key % 'reverse_dns', str(result), force=True)

        self.send_message(event)
        self.acknowledge_message()
Ejemplo n.º 2
0
 def init(self):
     self.cache = Cache(
         self.parameters.redis_cache_host,
         self.parameters.redis_cache_port,
         self.parameters.redis_cache_db,
         self.parameters.redis_cache_ttl,
     )
Ejemplo n.º 3
0
    def init(self):
        if requests is None:
            raise ValueError('Could not import requests. Please install it.')

        if hasattr(self.parameters, 'query_ripe_stat'):
            self.logger.warning(
                "The parameter 'query_ripe_stat' is deprecated and will be "
                "removed in 2.0. Use 'query_ripe_stat_asn' and "
                "'query_ripe_stat_ip' instead'.")
        self.query_db_asn = getattr(self.parameters, 'query_ripe_db_asn', True)
        self.query_db_ip = getattr(self.parameters, 'query_ripe_db_ip', True)
        self.query_stat_asn = getattr(
            self.parameters, 'query_ripe_stat_asn',
            getattr(self.parameters, 'query_ripe_stat', True))
        self.query_stat_ip = getattr(
            self.parameters, 'query_ripe_stat_ip',
            getattr(self.parameters, 'query_ripe_stat', True))
        self.mode = getattr(self.parameters, 'mode', 'append')

        self.set_request_parameters()

        cache_host = getattr(self.parameters, 'redis_cache_host')
        cache_port = getattr(self.parameters, 'redis_cache_port')
        cache_db = getattr(self.parameters, 'redis_cache_db')
        cache_ttl = getattr(self.parameters, 'redis_cache_ttl')
        if cache_host and cache_port and cache_db and cache_ttl:
            self.cache = Cache(
                cache_host, cache_port, cache_db, cache_ttl,
                getattr(self.parameters, "redis_cache_password", None))
Ejemplo n.º 4
0
class DeduplicatorBot(Bot):
    
    def init(self):
        self.cache = Cache(
                            self.parameters.redis_cache_host,
                            self.parameters.redis_cache_port,
                            self.parameters.redis_cache_db,
                            self.parameters.redis_cache_ttl
                          )


    def process(self):
        message = self.receive_message()

        if message:
            
            # Event deduplication
            if isinstance(message, Event):
                event = copy(message)
                event.clear("observation_time")
                message_hash = hash(event)                            

            # Generic message deduplication
            else:
                message_hash = hash(message)

            if not self.cache.exists(message_hash):
                self.send_message(message)
                self.cache.set(message_hash, 'hash')

        self.acknowledge_message()
Ejemplo n.º 5
0
    def init(self):
        self.apikey = getattr(self.parameters, "api_key", None)
        if self.apikey is None:
            raise ValueError('No api_key provided.')
        self.secret = getattr(self.parameters, "secret", None)
        if self.secret is None:
            raise ValueError('No secret provided.')
        self.country = getattr(self.parameters, "country", None)
        if self.country is None:
            raise ValueError('No country provided.')

        self.types = getattr(self.parameters, 'types', None)
        if isinstance(self.types, str):
            self.types = self.types.split(',')

        self.preamble = '{{ "apikey": "{}" '.format(self.apikey)

        self.set_request_parameters()
        self.session = create_request_session(self)

        self.cache = Cache(
            self.parameters.redis_cache_host,
            self.parameters.redis_cache_port,
            self.parameters.redis_cache_db,
            getattr(self.parameters, 'redis_cache_ttl', 864000),  # 10 days
            getattr(self.parameters, "redis_cache_password", None))
Ejemplo n.º 6
0
 def tearDownClass(cls):
     cache = Cache(test.BOT_CONFIG['redis_cache_host'],
                   test.BOT_CONFIG['redis_cache_port'],
                   test.BOT_CONFIG['redis_cache_db'],
                   test.BOT_CONFIG['redis_cache_ttl'],
                   )
     cache.flush()
Ejemplo n.º 7
0
class DeduplicatorExpertBot(Bot):

    def init(self):
        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           )

    def process(self):
        message = self.receive_message()

        if message is None:
            self.acknowledge_message()
            return

        auxiliar_message = copy.copy(message)

        ignore_keys = self.parameters.ignore_keys.split(',')

        for ignore_key in ignore_keys:
            ignore_key = ignore_key.strip()
            if ignore_key in auxiliar_message:
                del auxiliar_message[ignore_key]

        message_hash = hash(auxiliar_message)

        if not self.cache.exists(message_hash):
            self.cache.set(message_hash, 'hash')
            self.send_message(message)

        self.acknowledge_message()
Ejemplo n.º 8
0
 def init(self):
     self.cache = Cache(
         self.parameters.redis_cache_host, self.parameters.redis_cache_port,
         self.parameters.redis_cache_db, self.parameters.redis_cache_ttl,
         getattr(self.parameters, "redis_cache_password", None))
     self.filter_keys = set(k.strip()
                            for k in self.parameters.filter_keys.split(','))
Ejemplo n.º 9
0
class DeduplicatorExpertBot(Bot):
    def init(self):
        self.cache = Cache(
            self.parameters.redis_cache_host,
            self.parameters.redis_cache_port,
            self.parameters.redis_cache_db,
            self.parameters.redis_cache_ttl,
        )

    def process(self):
        message = self.receive_message()

        if message is None:
            self.acknowledge_message()
            return

        auxiliar_message = copy.copy(message)

        ignore_keys = self.parameters.ignore_keys.split(',')

        for ignore_key in ignore_keys:
            ignore_key = ignore_key.strip()
            if ignore_key in auxiliar_message:
                auxiliar_message.clear(ignore_key)

        message_hash = hash(auxiliar_message)

        if not self.cache.exists(message_hash):
            self.cache.set(message_hash, 'hash')
            self.send_message(message)

        self.acknowledge_message()
Ejemplo n.º 10
0
class DeduplicatorExpertBot(Bot):

    def init(self):
        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           )

    def process(self):
        message = self.receive_message()

        ignore_keys = set(k.strip()
                          for k in self.parameters.ignore_keys.split(','))
        message_hash = message.hash(ignore_keys)

        old_hash = hash(int(message_hash, 16))

        if not (self.cache.exists(message_hash) or self.cache.exists(old_hash)):
            self.cache.set(message_hash, 'hash')
            self.send_message(message)
        else:
            self.logger.debug('Dropped message.')

        self.acknowledge_message()
Ejemplo n.º 11
0
 def tearDownClass(cls):
     cache = Cache(test.BOT_CONFIG['redis_cache_host'],
                   test.BOT_CONFIG['redis_cache_port'],
                   test.BOT_CONFIG['redis_cache_db'],
                   test.BOT_CONFIG['redis_cache_ttl'],
                   )
     cache.flush()
Ejemplo n.º 12
0
class ReverseDnsExpertBot(Bot):
    def init(self):
        self.cache = Cache(
            self.parameters.redis_cache_host,
            self.parameters.redis_cache_port,
            self.parameters.redis_cache_db,
            self.parameters.redis_cache_ttl,
        )

    def process(self):
        event = self.receive_message()

        keys = ["source.%s", "destination.%s"]

        for key in keys:
            ip_key = key % "ip"

            if not event.contains(ip_key):
                continue

            ip = event.get(ip_key)
            ip_version = IPAddress.version(ip)
            ip_integer = IPAddress.to_int(ip)

            if ip_version == 4:
                minimum = MINIMUM_BGP_PREFIX_IPV4

            elif ip_version == 6:
                minimum = MINIMUM_BGP_PREFIX_IPV6

            cache_key = bin(ip_integer)[2:minimum + 2]
            cachevalue = self.cache.get(cache_key)

            result = None
            if cachevalue:
                result = cachevalue
            else:
                rev_name = reversename.from_address(ip)
                try:
                    result = resolver.query(rev_name, "PTR")
                    expiration = result.expiration
                    result = result[0]

                    if str(result) == '.':
                        result = None
                        raise ValueError
                except (dns.exception.DNSException, ValueError) as e:
                    if isinstance(e, dns.resolver.NXDOMAIN):
                        continue
                else:
                    ttl = datetime.fromtimestamp(expiration) - datetime.now()
                    self.cache.set(cache_key,
                                   str(result),
                                   ttl=int(ttl.total_seconds()))

            if result is not None:
                event.add(key % 'reverse_dns', str(result), force=True)

        self.send_message(event)
        self.acknowledge_message()
Ejemplo n.º 13
0
    def init(self):
        self.set_request_parameters()
        self.http_header['Ocp-Apim-Subscription-Key'] = self.parameters.api_key
        if self.parameters.file_match:
            self.file_match = re.compile(self.parameters.file_match)
        else:
            self.file_match = None

        if self.parameters.not_older_than:
            try:
                self.time_match = timedelta(
                    minutes=parse_relative(self.parameters.not_older_than))
            except ValueError:
                self.time_match = parser.parse(self.parameters.not_older_than)
                self.logger.info("Filtering files absolute %r.",
                                 self.time_match)
            else:
                self.logger.info("Filtering files relative %r.",
                                 self.time_match)
        else:
            self.time_match = None

        self.cache = Cache(
            self.parameters.redis_cache_host, self.parameters.redis_cache_port,
            self.parameters.redis_cache_db, self.parameters.redis_cache_ttl,
            getattr(self.parameters, "redis_cache_password", None))
Ejemplo n.º 14
0
class DeduplicatorExpertBot(Bot):

    def init(self):
        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           getattr(self.parameters, "redis_cache_password",
                                   None)
                           )
        self.filter_keys = set(k.strip() for k in
                               self.parameters.filter_keys.split(','))

    def process(self):
        message = self.receive_message()

        message_hash = message.hash(filter_keys=self.filter_keys,
                                    filter_type=self.parameters.filter_type)

        if not self.cache.exists(message_hash):
            self.cache.set(message_hash, 'hash')
            self.send_message(message)
        else:
            self.logger.debug('Dropped message.')

        self.acknowledge_message()
class DeduplicatorBot(Bot):
    
    def init(self):
        self.cache = Cache(
                            self.parameters.redis_cache_host,
                            self.parameters.redis_cache_port,
                            self.parameters.redis_cache_db,
                            self.parameters.redis_cache_ttl
                          )


    def process(self):
        message = self.receive_message()

        if message:
            
            # Event deduplication
            if isinstance(message, Event):
                event = deepcopy(message)
                event.clear("observation_time")
		if event.value("type")=="vulnerable service" or event.value("type")=="compromised":
		   event.clear("source_time")
		   event.clear("source_reverse_dns")
                message_hash = hash(event)                            

            # Generic message deduplication
            else:
                message_hash = hash(message)

            if not self.cache.exists(message_hash):
                self.cache.set(message_hash, 'hash')
                self.send_message(message)

        self.acknowledge_message()
Ejemplo n.º 16
0
class CymruExpertBot(Bot):
    def init(self):
        self.cache = Cache(
            self.parameters.redis_cache_host, self.parameters.redis_cache_port,
            self.parameters.redis_cache_db, self.parameters.redis_cache_ttl,
            getattr(self.parameters, "redis_cache_password", None))

        if not hasattr(self.parameters, 'overwrite'):
            self.logger.warning(
                "Parameter 'overwrite' is not given, assuming 'True'. "
                "Please set it explicitly, default will change to "
                "'False' in version 3.0.0'.")
        self.overwrite = getattr(self.parameters, 'overwrite', True)

    def process(self):
        event = self.receive_message()

        keys = ["source.%s", "destination.%s"]

        for key in keys:
            ip_key = key % "ip"

            if ip_key not in event:
                continue

            ip = event.get(ip_key)
            ip_version = IPAddress.version(ip)
            ip_integer = IPAddress.to_int(ip)

            if ip_version == 4:
                minimum = MINIMUM_BGP_PREFIX_IPV4

            elif ip_version == 6:
                minimum = MINIMUM_BGP_PREFIX_IPV6

            else:
                raise ValueError('Unexpected IP version '
                                 '{!r}.'.format(ip_version))

            cache_key = bin(ip_integer)[2:minimum + 2]
            result_json = self.cache.get(cache_key)

            if result_json:
                result = json.loads(result_json)
            else:
                result = Cymru.query(ip)
                if not result:
                    continue
                result_json = json.dumps(result)
                self.cache.set(cache_key, result_json)

            for result_key, result_value in result.items():
                if result_key == 'registry' and result_value == 'other':
                    continue
                event.add(key % result_key,
                          result_value,
                          overwrite=self.overwrite)

        self.send_message(event)
        self.acknowledge_message()
Ejemplo n.º 17
0
class DeduplicatorExpertBot(Bot):
    def init(self):
        self.cache = Cache(
            self.parameters.redis_cache_host,
            self.parameters.redis_cache_port,
            self.parameters.redis_cache_db,
            self.parameters.redis_cache_ttl,
        )

    def process(self):
        message = self.receive_message()

        ignore_keys = set(k.strip()
                          for k in self.parameters.ignore_keys.split(','))
        message_hash = message.hash(ignore_keys)

        old_hash = hash(int(message_hash, 16))

        if not (self.cache.exists(message_hash)
                or self.cache.exists(old_hash)):
            self.cache.set(message_hash, 'hash')
            self.send_message(message)
        else:
            self.logger.debug('Dropped message.')

        self.acknowledge_message()
Ejemplo n.º 18
0
class DeduplicatorExpertBot(Bot):

    _message_processed_verb = 'Forwarded'

    def init(self):
        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           getattr(self.parameters, "redis_cache_password",
                                   None)
                           )
        self.filter_keys = {k.strip() for k in
                            self.parameters.filter_keys.split(',')}
        self.bypass = getattr(self.parameters, "bypass", False)

    def process(self):
        message = self.receive_message()

        if self.bypass:
            self.send_message(message)
        else:
            message_hash = message.hash(filter_keys=self.filter_keys,
                                        filter_type=self.parameters.filter_type)

            if not self.cache.exists(message_hash):
                self.cache.set(message_hash, 'hash')
                self.send_message(message)
            else:
                self.logger.debug('Dropped message.')

        self.acknowledge_message()
Ejemplo n.º 19
0
 def __initialize_cache(self):
     cache_host = getattr(self.parameters, 'redis_cache_host')
     cache_port = getattr(self.parameters, 'redis_cache_port')
     cache_db = getattr(self.parameters, 'redis_cache_db')
     cache_ttl = getattr(self.parameters, 'redis_cache_ttl')
     if cache_host and cache_port and cache_db and cache_ttl:
         self.__cache = Cache(cache_host, cache_port, cache_db, cache_ttl,
                              getattr(self.parameters, "redis_cache_password", None))
Ejemplo n.º 20
0
 def init(self):
     print("joooo")
     self.cache = Cache(
         self.parameters.redis_cache_host,
         self.parameters.redis_cache_port,
         self.parameters.redis_cache_db,
         self.parameters.redis_cache_ttl,
     )
Ejemplo n.º 21
0
 def tearDownClass(cls):
     cache = Cache(
         test.BOT_CONFIG["redis_cache_host"],
         test.BOT_CONFIG["redis_cache_port"],
         test.BOT_CONFIG["redis_cache_db"],
         test.BOT_CONFIG["redis_cache_ttl"],
     )
     cache.flush()
Ejemplo n.º 22
0
 def init(self):
     self.cache = Cache(self.parameters.redis_cache_host,
                        self.parameters.redis_cache_port,
                        self.parameters.redis_cache_db,
                        self.parameters.redis_cache_ttl,
                        getattr(self.parameters, "redis_cache_password",
                                None)
                        )
Ejemplo n.º 23
0
class CymruExpertBot(Bot):
    def init(self):
        self.cache = Cache(
            self.parameters.redis_cache_host,
            self.parameters.redis_cache_port,
            self.parameters.redis_cache_db,
            self.parameters.redis_cache_ttl,
        )

    def process(self):
        event = self.receive_message()

        if event is None:
            self.acknowledge_message()
            return

        keys = ["source.%s", "destination.%s"]

        for key in keys:
            ip_key = key % "ip"

            if not event.contains(ip_key):
                continue

            ip = event.value(ip_key)
            ip_version = IPAddress.version(ip)
            ip_integer = IPAddress.to_int(ip)

            if ip_version == 4:
                minimum = MINIMUM_BGP_PREFIX_IPV4

            elif ip_version == 6:
                minimum = MINIMUM_BGP_PREFIX_IPV6

            else:
                raise ValueError('Unexpected IP version '
                                 '{!r}.'.format(ip_version))

            cache_key = bin(ip_integer)[2:minimum + 2]
            result_json = self.cache.get(cache_key)

            if result_json:
                result = json.loads(result_json)
            else:
                result = Cymru.query(ip)
                result_json = json.dumps(result)
                self.cache.set(cache_key, result_json)

            for result_key, result_value in result.items():
                event.add(key % result_key,
                          result_value,
                          sanitize=True,
                          force=True)

        self.send_message(event)
        self.acknowledge_message()
Ejemplo n.º 24
0
class CymruExpertBot(Bot):

    def init(self):
        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           getattr(self.parameters, "redis_cache_password",
                                   None)
                           )

    def process(self):
        event = self.receive_message()

        keys = ["source.%s", "destination.%s"]

        for key in keys:
            ip_key = key % "ip"

            if ip_key not in event:
                continue

            ip = event.get(ip_key)
            ip_version = IPAddress.version(ip)
            ip_integer = IPAddress.to_int(ip)

            if ip_version == 4:
                minimum = MINIMUM_BGP_PREFIX_IPV4

            elif ip_version == 6:
                minimum = MINIMUM_BGP_PREFIX_IPV6

            else:
                raise ValueError('Unexpected IP version '
                                 '{!r}.'.format(ip_version))

            cache_key = bin(ip_integer)[2: minimum + 2]
            result_json = self.cache.get(cache_key)

            if result_json:
                result = json.loads(result_json)
            else:
                result = Cymru.query(ip)
                if not result:
                    continue
                result_json = json.dumps(result)
                self.cache.set(cache_key, result_json)

            for result_key, result_value in result.items():
                if result_key == 'registry' and result_value == 'other':
                    continue
                event.add(key % result_key, result_value, overwrite=True)

        self.send_message(event)
        self.acknowledge_message()
Ejemplo n.º 25
0
class CymruExpertBot(Bot):

    def init(self):
        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           )

    def process(self):
        event = self.receive_message()

        if event is None:
            self.acknowledge_message()
            return

        keys = ["source.%s", "destination.%s"]

        for key in keys:
            ip_key = key % "ip"

            if not event.contains(ip_key):
                continue

            ip = event.value(ip_key)
            ip_version = IPAddress.version(ip)
            ip_integer = IPAddress.to_int(ip)

            if ip_version == 4:
                minimum = MINIMUM_BGP_PREFIX_IPV4

            elif ip_version == 6:
                minimum = MINIMUM_BGP_PREFIX_IPV6

            else:
                raise ValueError('Unexpected IP version '
                                 '{!r}.'.format(ip_version))

            cache_key = bin(ip_integer)[2: minimum + 2]
            result_json = self.cache.get(cache_key)

            if result_json:
                result = json.loads(result_json)
            else:
                result = Cymru.query(ip)
                result_json = json.dumps(result)
                self.cache.set(cache_key, result_json)

            for result_key, result_value in result.items():
                event.add(key % result_key, result_value, sanitize=True,
                          force=True)

        self.send_message(event)
        self.acknowledge_message()
Ejemplo n.º 26
0
 def init(self):
     self.query_db_asn = getattr(self.parameters, 'query_ripe_db_asn', True)
     self.query_db_ip = getattr(self.parameters, 'query_ripe_db_ip', True)
     self.query_stat_asn = getattr(self.parameters, 'query_ripe_stat', True)
     self.query_stat_ip = getattr(self.parameters, 'query_ripe_stat', True)
     self.cache = Cache(
         self.parameters.redis_cache_host,
         self.parameters.redis_cache_port,
         self.parameters.redis_cache_db,
         self.parameters.redis_cache_ttl,
     )
Ejemplo n.º 27
0
    def init(self):
        self.cache = Cache(
            self.parameters.redis_cache_host, self.parameters.redis_cache_port,
            self.parameters.redis_cache_db, self.parameters.redis_cache_ttl,
            getattr(self.parameters, "redis_cache_password", None))

        if not hasattr(self.parameters, 'overwrite'):
            self.logger.warning(
                "Parameter 'overwrite' is not given, assuming 'True'. "
                "Please set it explicitly, default will change to "
                "'False' in version 3.0.0'.")
        self.overwrite = getattr(self.parameters, 'overwrite', True)
Ejemplo n.º 28
0
class CymruExpertBot(Bot):
    def init(self):
        self.cache = Cache(
            self.parameters.redis_cache_host, self.parameters.redis_cache_port,
            self.parameters.redis_cache_db, self.parameters.redis_cache_ttl,
            getattr(self.parameters, "redis_cache_password", None))

        if not hasattr(self.parameters, 'overwrite'):
            self.logger.warning(
                "Parameter 'overwrite' is not given, assuming 'True'. "
                "Please set it explicitly, default will change to "
                "'False' in version 3.0.0'.")
        self.overwrite = getattr(self.parameters, 'overwrite', True)

    def process(self):
        event = self.receive_message()

        keys = ["source.%s", "destination.%s"]

        for key in keys:
            ip_key = key % "ip"

            if ip_key not in event:
                continue

            address = event.get(ip_key)
            cache_key = CACHE_KEY % (IPAddress.version(address), address)
            result_json = self.cache.get(cache_key)

            if result_json:
                result = json.loads(result_json)
            else:
                result = Cymru.query(address)
                if not result:
                    self.logger.info(
                        'Got no result from Cymru for IP address %r.', address)
                result_json = json.dumps(result)
                self.cache.set(cache_key, result_json)

            if not result:
                continue

            for result_key, result_value in result.items():
                if result_key == 'registry' and result_value == 'other':
                    continue
                event.add(key % result_key,
                          result_value,
                          overwrite=self.overwrite)

        self.send_message(event)
        self.acknowledge_message()
Ejemplo n.º 29
0
class ForwardDnsExpertBot(Bot):
    def init(self):
        print("joooo")
        self.cache = Cache(
            self.parameters.redis_cache_host,
            self.parameters.redis_cache_port,
            self.parameters.redis_cache_db,
            self.parameters.redis_cache_ttl,
        )

    def process(self):
        event = self.receive_message()
        if event is None:
            self.acknowledge_message()
            return

        messageSent = False
        for key in ["source.%s", "destination.%s"]:
            ip_key = key % "ip"
            fqdn_key = key % "fqdn"

            if event.contains(ip_key) or not event.contains(fqdn_key):
                continue

            fqdn = event[fqdn_key]
            cache_key = hash(fqdn)
            cachevalue = self.cache.get(cache_key)

            result = None
            if cachevalue:
                result = cachevalue
            else:
                soc = socket.getaddrinfo(fqdn, 0)
                try:
                    result = set([address[4][0] for address in soc])
                except socket.error as msg:
                    print(msg)
                    continue
                else:
                    self.cache.set(cache_key, result)

            print("VEEOF", result)
            for ip in result:
                print("ZDEE!!", ip, ip_key, fqdn_key)
                event.add(ip_key, ip, sanitize=True, force=True)
                self.send_message(event)
                messageSent = True

        if not messageSent:
            self.send_message(event)
        self.acknowledge_message()
Ejemplo n.º 30
0
class ForwardDnsExpertBot(Bot):

    def init(self):
        print("joooo")
        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           )

    def process(self):
        event = self.receive_message()
        if event is None:
            self.acknowledge_message()
            return

        messageSent = False
        for key in ["source.%s", "destination.%s"]:
            ip_key = key % "ip"
            fqdn_key = key % "fqdn"

            if event.contains(ip_key) or not event.contains(fqdn_key):
                continue

            fqdn = event[fqdn_key]
            cache_key = hash(fqdn)
            cachevalue = self.cache.get(cache_key)

            result = None
            if cachevalue:
                result = cachevalue
            else:
                soc = socket.getaddrinfo(fqdn, 0)
                try:
                    result = set([address[4][0] for address in soc])
                except socket.error as msg:
                    print(msg)
                    continue
                else:
                    self.cache.set(cache_key, result)

            print("VEEOF",result)
            for ip in result:
                print("ZDEE!!",ip,ip_key,fqdn_key)
                event.add(ip_key, ip, sanitize=True, force=True)
                self.send_message(event)
                messageSent = True

        if not messageSent:
            self.send_message(event)
        self.acknowledge_message()
Ejemplo n.º 31
0
class ThresholdExpertBot(Bot):

    _message_processed_verb = 'Forwarded'

    is_multithreadable = False

    def init(self):
        self.cache = Cache(
            self.parameters.redis_cache_host, self.parameters.redis_cache_port,
            self.parameters.redis_cache_db, self.parameters.timeout,
            getattr(self.parameters, "redis_cache_password", None))
        self.filter_keys = getattr(self.parameters, "filter_keys", [])
        self.filter_type = getattr(self.parameters, "filter_type", "whitelist")
        self.bypass = getattr(self.parameters, "bypass", False)
        self.timeout = getattr(self.parameters, "timeout", -1)
        if self.timeout <= 0:
            raise ConfigurationError(
                'Timeout',
                'Invalid timeout specified, use positive integer seconds.')
        self.threshold = getattr(self.parameters, "threshold", -1)
        if self.threshold <= 0:
            raise ConfigurationError(
                'Threshold',
                'Invalid threshold specified, use positive integer count.')
        self.add_keys = getattr(self.parameters, "add_keys", {})

    def process(self):
        message = self.receive_message()

        if self.bypass:
            self.send_message(message)
        else:
            message_hash = message.hash(filter_keys=self.filter_keys,
                                        filter_type=self.filter_type)
            old_count = int(self.cache.get(message_hash) or 0)
            self.logger.debug('Message %s has been seen %i times before.',
                              message_hash, old_count)
            # Use Redis "set" instead of "incr" to reset the timeout
            # every time
            self.cache.set(message_hash, str(old_count + 1))
            if old_count + 1 == self.threshold:
                self.logger.debug('Threshold reached, forwarding message.')
                message.update(self.add_keys)
                message.add('extra.count', old_count + 1, overwrite=True)
                self.send_message(message)
            else:
                self.logger.debug('Dropped message.')

        self.acknowledge_message()
Ejemplo n.º 32
0
 def init(self):
     print("joooo")
     self.cache = Cache(self.parameters.redis_cache_host,
                        self.parameters.redis_cache_port,
                        self.parameters.redis_cache_db,
                        self.parameters.redis_cache_ttl,
                        )
 def init(self):
     self.cache = Cache(
                         self.parameters.redis_cache_host,
                         self.parameters.redis_cache_port,
                         self.parameters.redis_cache_db,
                         self.parameters.redis_cache_ttl
                       )
Ejemplo n.º 34
0
    def init(self):
        self.set_request_parameters()
        self.http_header['Ocp-Apim-Subscription-Key'] = self.parameters.api_key
        if self.parameters.file_match:
            self.file_match = re.compile(self.parameters.file_match)
        else:
            self.file_match = None

        if self.parameters.not_older_than:
            try:
                self.time_match = timedelta(minutes=parse_relative(self.parameters.not_older_than))
            except ValueError:
                self.time_match = parser.parse(self.parameters.not_older_than)
                self.logger.info("Filtering files absolute %r.", self.time_match)
            else:
                self.logger.info("Filtering files relative %r.", self.time_match)
        else:
            self.time_match = None

        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           getattr(self.parameters, "redis_cache_password",
                                   None)
                           )
Ejemplo n.º 35
0
 def init(self):
     self.cache = Cache(
                         self.parameters.redis_cache_host,
                         self.parameters.redis_cache_port,
                         self.parameters.redis_cache_db,
                         self.parameters.redis_cache_ttl
                       )
     self.logger.warning("Bot init succesful.")
Ejemplo n.º 36
0
 def __initialize_cache(self):
     cache_host = getattr(self.parameters, 'redis_cache_host')
     cache_port = getattr(self.parameters, 'redis_cache_port')
     cache_db = getattr(self.parameters, 'redis_cache_db')
     cache_ttl = getattr(self.parameters, 'redis_cache_ttl')
     if cache_host and cache_port and cache_db and cache_ttl:
         self.__cache = Cache(cache_host, cache_port, cache_db, cache_ttl,
                              getattr(self.parameters, "redis_cache_password", None))
Ejemplo n.º 37
0
 def init(self):
     self.cache = Cache(self.parameters.redis_cache_host,
                        self.parameters.redis_cache_port,
                        self.parameters.redis_cache_db,
                        self.parameters.redis_cache_ttl,
                        getattr(self.parameters, "redis_cache_password",
                                None)
                        )
Ejemplo n.º 38
0
    def init(self):
        if requests is None:
            raise ValueError('Could not import requests. Please install it.')

        self.set_request_parameters()

        self.http_header['Ocp-Apim-Subscription-Key'] = self.parameters.api_key
        if self.parameters.file_match:
            self.file_match = re.compile(self.parameters.file_match)
        else:
            self.file_match = None

        if self.parameters.not_older_than:
            try:
                self.time_match = timedelta(
                    minutes=parse_relative(self.parameters.not_older_than))
            except ValueError:
                if sys.version_info >= (3, 6):
                    self.time_match = parser.parse(
                        self.parameters.not_older_than).astimezone(pytz.utc)
                else:  # "astimezone() cannot be applied to a naive datetime" otherwise
                    if '+' not in self.parameters.not_older_than:
                        self.parameters.not_older_than += '+00:00'
                    self.time_match = parser.parse(
                        self.parameters.not_older_than)
                self.logger.info("Filtering files absolute %r.",
                                 self.time_match)
                self.check_ttl_time()
            else:
                self.logger.info("Filtering files relative %r.",
                                 self.time_match)
                if timedelta(seconds=self.parameters.redis_cache_ttl
                             ) < self.time_match:
                    raise ValueError(
                        "The cache's TTL must be higher than 'not_older_than', "
                        "otherwise the bot is processing the same data over and over again."
                    )
        else:
            self.time_match = None
        self.session = create_request_session_from_bot(self)

        self.cache = Cache(
            self.parameters.redis_cache_host, self.parameters.redis_cache_port,
            self.parameters.redis_cache_db, self.parameters.redis_cache_ttl,
            getattr(self.parameters, "redis_cache_password", None))
Ejemplo n.º 39
0
    def init(self):
        if ContainerClient is None or create_configuration is None:
            raise MissingDependencyError("azure.storage")

        self.config = create_configuration(storage_sdk='blob')
        if hasattr(self.parameters, 'https_proxy'):
            # Create a storage configuration object and update the proxy policy
            self.config.proxy_policy.proxies = {
                'http': self.parameters.http_proxy,
                'https': self.parameters.https_proxy,
            }

        self.cache = Cache(
            self.parameters.redis_cache_host,
            self.parameters.redis_cache_port,
            self.parameters.redis_cache_db,
            getattr(self.parameters, 'redis_cache_ttl', 864000),  # 10 days
            getattr(self.parameters, "redis_cache_password", None))
Ejemplo n.º 40
0
 def init(self):
     self.cache = Cache(self.parameters.redis_cache_host,
                        self.parameters.redis_cache_port,
                        self.parameters.redis_cache_db,
                        self.parameters.redis_cache_ttl,
                        getattr(self.parameters, "redis_cache_password",
                                None)
                        )
     self.filter_keys = set(k.strip() for k in
                            self.parameters.filter_keys.split(','))
Ejemplo n.º 41
0
class DeduplicatorBot(Bot):
    
    def init(self):
        self.cache = Cache(
                            self.parameters.redis_cache_host,
                            self.parameters.redis_cache_port,
                            self.parameters.redis_cache_db,
                            self.parameters.redis_cache_ttl
                          )

    def process(self):
        message = self.receive_message()
        message_hash = hash(message)

        if not self.cache.exists(message_hash):
            self.cache.set(message_hash, 'hash')
            self.send_message(message)

        self.acknowledge_message()
Ejemplo n.º 42
0
 def init(self):
     self.query_db_asn = getattr(self.parameters, 'query_ripe_db_asn', True)
     self.query_db_ip = getattr(self.parameters, 'query_ripe_db_ip', True)
     self.query_stat_asn = getattr(self.parameters, 'query_ripe_stat', True)
     self.query_stat_ip = getattr(self.parameters, 'query_ripe_stat', True)
     self.cache = Cache(self.parameters.redis_cache_host,
                        self.parameters.redis_cache_port,
                        self.parameters.redis_cache_db,
                        self.parameters.redis_cache_ttl,
                        )
Ejemplo n.º 43
0
 def init(self):
     self.cache = Cache(
         self.parameters.redis_cache_host, self.parameters.redis_cache_port,
         self.parameters.redis_cache_db, self.parameters.timeout,
         getattr(self.parameters, "redis_cache_password", None))
     self.filter_keys = getattr(self.parameters, "filter_keys", [])
     self.filter_type = getattr(self.parameters, "filter_type", "whitelist")
     self.bypass = getattr(self.parameters, "bypass", False)
     self.timeout = getattr(self.parameters, "timeout", -1)
     if self.timeout <= 0:
         raise ConfigurationError(
             'Timeout',
             'Invalid timeout specified, use positive integer seconds.')
     self.threshold = getattr(self.parameters, "threshold", -1)
     if self.threshold <= 0:
         raise ConfigurationError(
             'Threshold',
             'Invalid threshold specified, use positive integer count.')
     self.add_keys = getattr(self.parameters, "add_keys", {})
Ejemplo n.º 44
0
    def init(self):
        if requests is None:
            raise MissingDependencyError("requests")

        self.set_request_parameters()

        self.http_header['Ocp-Apim-Subscription-Key'] = self.parameters.api_key
        if self.parameters.file_match:
            self.file_match = re.compile(self.parameters.file_match)
        else:
            self.file_match = None

        if self.parameters.not_older_than:
            try:
                self.time_match = timedelta(
                    minutes=parse_relative(self.parameters.not_older_than))
            except ValueError:
                self.time_match = parser.parse(
                    self.parameters.not_older_than).astimezone(pytz.utc)
                self.logger.info("Filtering files absolute %r.",
                                 self.time_match)
                self.check_ttl_time()
            else:
                self.logger.info("Filtering files relative %r.",
                                 self.time_match)
                if timedelta(seconds=self.parameters.redis_cache_ttl
                             ) < self.time_match:
                    raise ValueError(
                        "The cache's TTL must be higher than 'not_older_than', "
                        "otherwise the bot is processing the same data over and over again."
                    )
        else:
            self.time_match = None
        self.session = create_request_session(self)

        self.cache = Cache(
            self.parameters.redis_cache_host, self.parameters.redis_cache_port,
            self.parameters.redis_cache_db, self.parameters.redis_cache_ttl,
            getattr(self.parameters, "redis_cache_password", None))
Ejemplo n.º 45
0
class MicrosoftAzureCollectorBot(CollectorBot):
    def init(self):
        if ContainerClient is None or create_configuration is None:
            raise MissingDependencyError("azure.storage")

        self.config = create_configuration(storage_sdk='blob')
        if hasattr(self.parameters, 'https_proxy'):
            # Create a storage configuration object and update the proxy policy
            self.config.proxy_policy.proxies = {
                'http': self.parameters.http_proxy,
                'https': self.parameters.https_proxy,
            }

        self.cache = Cache(
            self.parameters.redis_cache_host,
            self.parameters.redis_cache_port,
            self.parameters.redis_cache_db,
            getattr(self.parameters, 'redis_cache_ttl', 864000),  # 10 days
            getattr(self.parameters, "redis_cache_password", None))

    def process(self):
        container_client = ContainerClient.from_connection_string(
            conn_str=self.parameters.connection_string,
            container_name=self.parameters.container_name,
            _configuration=self.config)
        for blob in container_client.list_blobs():
            if self.cache.get(blob.name):
                self.logger.debug('Processed file %r already.', blob.name)
                continue
            self.logger.debug('Processing blob %r.', blob.name)
            blob_obj = io.BytesIO()
            container_client.download_blob(blob).readinto(blob_obj)
            blob_obj.seek(0)
            report = self.new_report()
            report.add('raw', gzip.GzipFile(fileobj=blob_obj).read().decode())
            self.send_message(report)
            self.cache.set(blob.name,
                           1)  # Redis-py >= 3.0.0 does not allow True
Ejemplo n.º 46
0
    def init(self):
        if requests is None:
            raise ValueError('Could not import requests. Please install it.')

        self.set_request_parameters()
        self.http_header['Ocp-Apim-Subscription-Key'] = self.parameters.api_key
        if self.parameters.file_match:
            self.file_match = re.compile(self.parameters.file_match)
        else:
            self.file_match = None

        if self.parameters.not_older_than:
            try:
                self.time_match = timedelta(minutes=parse_relative(self.parameters.not_older_than))
            except ValueError:
                if sys.version_info >= (3, 6):
                    self.time_match = parser.parse(self.parameters.not_older_than).astimezone(pytz.utc)
                else:  # "astimezone() cannot be applied to a naive datetime" otherwise
                    if '+' not in self.parameters.not_older_than:
                        self.parameters.not_older_than += '+00:00'
                    self.time_match = parser.parse(self.parameters.not_older_than)
                self.logger.info("Filtering files absolute %r.", self.time_match)
                self.check_ttl_time()
            else:
                self.logger.info("Filtering files relative %r.", self.time_match)
                if timedelta(seconds=self.parameters.redis_cache_ttl) < self.time_match:
                    raise ValueError("The cache's TTL must be higher than 'not_older_than', "
                                     "otherwise the bot is processing the same data over and over again.")
        else:
            self.time_match = None

        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           getattr(self.parameters, "redis_cache_password",
                                   None)
                           )
Ejemplo n.º 47
0
class RIPENCCExpertBot(Bot):

    def init(self):
        self.query_db_asn = getattr(self.parameters, 'query_ripe_db_asn', True)
        self.query_db_ip = getattr(self.parameters, 'query_ripe_db_ip', True)
        self.query_stat_asn = getattr(self.parameters, 'query_ripe_stat', True)
        self.query_stat_ip = getattr(self.parameters, 'query_ripe_stat', True)
        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           )

    def process(self):
        event = self.receive_message()

        if event is None:
            self.acknowledge_message()
            return

        for key in ['source.', 'destination.']:
            ip_key = key + "ip"
            abuse_key = key + "abuse_contact"
            asn_key = key + "asn"

            ip = event.get(ip_key, None)
            if not ip:
                continue
            ip_version = IPAddress.version(ip)
            ip_integer = IPAddress.to_int(ip)

            if ip_version == 4:
                minimum = MINIMUM_BGP_PREFIX_IPV4

            elif ip_version == 6:
                minimum = MINIMUM_BGP_PREFIX_IPV6

            else:
                raise ValueError('Unexpected IP version '
                                 '{!r}.'.format(ip_version))

            cache_key = bin(ip_integer)[2: minimum + 2]
            cache_result = self.cache.get(cache_key)

            abuse = (event.get(abuse_key).split(',') if abuse_key in event
                     else [])

            if cache_result:
                cache_result = ast.literal_eval(cache_result)
                cache_result = [n.strip() for n in cache_result]
                abuse.extend(cache_result)

            else:
                asn = event.get(asn_key, None)
                if self.query_db_asn and asn:
                    abuse.extend(lib.query_asn(asn))
                if self.query_db_ip and ip:
                    abuse.extend(lib.query_ripedb(ip))
                if self.query_stat_asn and asn:
                    abuse.extend(lib.query_ripestat(asn))
                if self.query_stat_ip and ip:
                    abuse.extend(lib.query_ripestat(ip))
                self.cache.set(cache_key,abuse)

            event.add(abuse_key, ','.join(filter(None, set(abuse))), force=True)

        self.send_message(event)
        self.acknowledge_message()
Ejemplo n.º 48
0
class RIPEExpertBot(Bot):
    QUERY = {
        'db_ip': 'https://rest.db.ripe.net/abuse-contact/{}.json',
        'db_asn': 'https://rest.db.ripe.net/abuse-contact/as{}.json',
        'stat': 'https://stat.ripe.net/data/abuse-contact-finder/data.json?resource={}',
        'stat_geolocation': 'https://stat.ripe.net/data/maxmind-geo-lite/data.json?resource={}',
    }

    REPLY_TO_DATA = {
        'db_ip': lambda x: clean_string(x['abuse-contacts']['email']),
        'db_asn': lambda x: clean_string(x['abuse-contacts']['email']),
        'stat': lambda x: clean_string(x['data']['anti_abuse_contacts']['abuse_c'][0]['email']),
        'stat_geolocation': lambda x: clean_geo(x['data']['located_resources'][0]['locations'][0]),
    }

    GEOLOCATION_REPLY_TO_INTERNAL = {
        ('cc', 'country'),
        ('latitude', 'latitude'),
        ('longitude', 'longitude'),
        ('city', 'city')
    }

    def init(self):
        if requests is None:
            raise ValueError("Could not import 'requests'. Please install the package.")

        self.__mode = getattr(self.parameters, 'mode', 'append')
        self.__query = {
            "db_asn": getattr(self.parameters, 'query_ripe_db_asn', True),
            "db_ip": getattr(self.parameters, 'query_ripe_db_ip', True),
            "stat_asn": getattr(self.parameters, 'query_ripe_stat_asn', True),
            "stat_ip": getattr(self.parameters, 'query_ripe_stat_ip', True),
            "stat_geo": getattr(self.parameters, 'query_ripe_stat_geolocation', True)
        }

        self.__initialize_http_session()
        self.__initialize_cache()

    def __initialize_http_session(self):
        self.http_session = requests.Session()
        self.set_request_parameters()
        self.http_session.proxies.update(self.proxy)
        self.http_session.headers.update(self.http_header)
        self.http_session.verify = self.http_verify_cert
        self.http_session.cert = self.ssl_client_cert

    def __initialize_cache(self):
        cache_host = getattr(self.parameters, 'redis_cache_host')
        cache_port = getattr(self.parameters, 'redis_cache_port')
        cache_db = getattr(self.parameters, 'redis_cache_db')
        cache_ttl = getattr(self.parameters, 'redis_cache_ttl')
        if cache_host and cache_port and cache_db and cache_ttl:
            self.__cache = Cache(cache_host, cache_port, cache_db, cache_ttl,
                                 getattr(self.parameters, "redis_cache_password", None))

    def process(self):
        with self.event_context() as event:
            for target in {'source.', 'destination.'}:
                abuse_key = target + "abuse_contact"
                abuse = set(event.get(abuse_key).split(',')) if self.__mode == 'append' and abuse_key in event else set()

                asn = event.get(target + "asn", None)
                if asn:
                    if self.__query['stat_asn']:
                        abuse.update(self.__perform_cached_query('stat', asn))
                    if self.__query['db_asn']:
                        abuse.update(self.__perform_cached_query('db_asn', asn))

                ip = event.get(target + "ip", None)
                if ip:
                    if self.__query['stat_ip']:
                        abuse.update(self.__perform_cached_query('stat', ip))
                    if self.__query['db_ip']:
                        abuse.update(self.__perform_cached_query('db_ip', ip))
                    if self.__query['stat_geo']:
                        info = self.__perform_cached_query('stat_geolocation', ip)

                        should_overwrite = self.__mode == 'replace'

                        for local_key, ripe_key in self.GEOLOCATION_REPLY_TO_INTERNAL:
                            if ripe_key in info:
                                event.add(target + "geolocation." + local_key, info[ripe_key], overwrite=should_overwrite)

                event.add(abuse_key, ','.join(abuse), overwrite=True)

    @contextmanager
    def event_context(self):
        event = self.receive_message()
        try:
            yield event
        finally:
            self.send_message(event)
            self.acknowledge_message()

    def __perform_cached_query(self, type, resource):
        cached_value = self.__cache.get('{}:{}'.format(type, resource))
        if cached_value:
            if cached_value == CACHE_NO_VALUE:
                return {}
            else:
                return json.loads(cached_value)
        else:
            response = self.http_session.get(self.QUERY[type].format(resource), data="", timeout=self.http_timeout_sec)
            if response.status_code != 200:
                if type == 'db_asn' and response.status_code == 404:
                    """ If no abuse contact could be found, a 404 is given. """
                    try:
                        if response.json()['message'].startswith('No abuse contact found for '):
                            self.__cache.set('{}:{}'.format(type, resource), CACHE_NO_VALUE)
                            return {}
                    except ValueError:
                        pass
                raise ValueError(STATUS_CODE_ERROR.format(response.status_code))
            try:
                data = self.REPLY_TO_DATA[type](response.json())
                self.__cache.set('{}:{}'.format(type, resource),
                                 (json.dumps(list(data) if isinstance(data, set) else data) if data else CACHE_NO_VALUE))
                return data
            except (KeyError, IndexError):
                self.__cache.set('{}:{}'.format(type, resource), CACHE_NO_VALUE)

            return {}
Ejemplo n.º 49
0
class CymruExpertBot(Bot):
    
    def init(self):
        self.cache = Cache(
                            self.parameters.redis_cache_host,
                            self.parameters.redis_cache_port,
                            self.parameters.redis_cache_db,
                            self.parameters.redis_cache_ttl
                          )

    
    def process(self):
        event = self.receive_message()
        
        if not event:
            self.acknowledge_message()
            return

        keys = ["source_%s", "destination_%s"]
        
        for key in keys:
            ip = event.value(key % "ip")
            
            if not ip:
                self.send_message(event)
                self.acknowledge_message()
                return

            elif utils.is_ipv4(ip):
                ip_version = 4
                ip_integer = utils.ip_to_int(ip)
                cache_key = bin(ip_integer)[2 : MINIMUM_BGP_PREFIX_IPV4 + 2]

            elif utils.is_ipv6(ip):
                ip_version = 6
                ip_integer = utils.ip_to_int(ip)
                cache_key = bin(ip_integer)[2 : MINIMUM_BGP_PREFIX_IPV6 + 2]

            else:
                self.send_message(event)
                self.acknowledge_message()
                return


            result_json = self.cache.get(cache_key)

            if result_json:
                result = json.loads(result_json)
            else:
                result = Cymru.query(ip, ip_version)
                result_json = json.dumps(result)
                self.cache.set(cache_key, result_json)
            
            if "asn" in result:
                event.clear(key % 'asn')
                event.add(key % 'asn',        result['asn'])
                
            if "bgp_prefix" in result:
                event.clear(key % 'bgp_prefix')
                event.add(key % 'bgp_prefix', result['bgp_prefix'])
                
            if "registry" in result:
                event.clear(key % 'registry')
                event.add(key % 'registry',   result['registry'])
                
            if "allocated" in result:
                event.clear(key % 'allocated')
                event.add(key % 'allocated',  result['allocated'])
                
            if "as_name" in result:
                event.clear(key % 'as_name')
                event.add(key % 'as_name',    result['as_name'])
                
            if "cc" in result:
                event.clear(key % 'cymru_cc')
                event.add(key % 'cymru_cc',   result['cc'])

        self.send_message(event)
        self.acknowledge_message()
Ejemplo n.º 50
0
class MicrosoftInterflowCollectorBot(CollectorBot):
    def init(self):
        self.set_request_parameters()
        self.http_header['Ocp-Apim-Subscription-Key'] = self.parameters.api_key
        if self.parameters.file_match:
            self.file_match = re.compile(self.parameters.file_match)
        else:
            self.file_match = None

        if self.parameters.not_older_than:
            try:
                self.time_match = timedelta(minutes=parse_relative(self.parameters.not_older_than))
            except ValueError:
                self.time_match = parser.parse(self.parameters.not_older_than)
                self.logger.info("Filtering files absolute %r.", self.time_match)
            else:
                self.logger.info("Filtering files relative %r.", self.time_match)
        else:
            self.time_match = None

        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           getattr(self.parameters, "redis_cache_password",
                                   None)
                           )

    def process(self):
        self.logger.debug('Downloading file list.')
        files = requests.get(URL_LIST,
                             auth=self.auth,
                             proxies=self.proxy,
                             headers=self.http_header,
                             verify=self.http_verify_cert,
                             cert=self.ssl_client_cert,
                             timeout=self.http_timeout_sec)
        files.raise_for_status()
        self.logger.debug('Downloaded file list, %s entries.', len(files.json()))
        for file in files.json():
            if self.cache.get(file['Name']):
                self.logger.debug('Processed file %s already.', file['Name'])
                continue
            if self.file_match and not self.file_match.match(file['Name']):
                self.logger.debug('File %r does not match filename filter.', file['Name'])
                continue
            filetime = parser.parse(file['LastModified'])
            if isinstance(self.time_match, datetime) and filetime < self.time_match:
                self.logger.debug('File %r does not match absolute time filter.', file['Name'])
                continue
            else:
                now = datetime.now(tz=pytz.timezone('UTC'))
                if isinstance(self.time_match, timedelta) and filetime < (now - self.time_match):
                    self.logger.debug('File %r does not match relative time filter.', file['Name'])
                    continue

            self.logger.debug('Processing file %r.', file['Name'])
            download_url = URL_DOWNLOAD % file['Name']
            download = requests.get(download_url,
                                    auth=self.auth,
                                    proxies=self.proxy,
                                    headers=self.http_header,
                                    verify=self.http_verify_cert,
                                    cert=self.ssl_client_cert,
                                    timeout=self.http_timeout_sec)
            download.raise_for_status()
            if download_url.endswith('.gz'):
                raw = gzip.open(io.BytesIO(download.content)).read().decode()
            else:
                raw = download.text
            report = self.new_report()
            report.add('feed.url', download_url)
            report.add('raw', raw)
            self.send_message(report)
            self.cache.set(file['Name'], True)
Ejemplo n.º 51
0
class ShadowServerAPICollectorBot(CollectorBot):
    """
    Shadowserver Reports API Collector Bot

    Parameters
    ----------
    api_key: str
        Your Shadowserver API key
    secret: str
        Your Shadowserver API secret
    country: str
        The country you want to download reports for (i.e. 'austria')
    types: list
        A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined').
    """
    def init(self):
        self.apikey = getattr(self.parameters, "api_key", None)
        if self.apikey is None:
            raise ValueError('No api_key provided.')
        self.secret = getattr(self.parameters, "secret", None)
        if self.secret is None:
            raise ValueError('No secret provided.')
        self.country = getattr(self.parameters, "country", None)
        if self.country is None:
            raise ValueError('No country provided.')

        self.types = getattr(self.parameters, 'types', None)
        if isinstance(self.types, str):
            self.types = self.types.split(',')

        self.preamble = '{{ "apikey": "{}" '.format(self.apikey)

        self.set_request_parameters()
        self.session = create_request_session(self)

        self.cache = Cache(
            self.parameters.redis_cache_host,
            self.parameters.redis_cache_port,
            self.parameters.redis_cache_db,
            getattr(self.parameters, 'redis_cache_ttl', 864000),  # 10 days
            getattr(self.parameters, "redis_cache_password", None))

    def _headers(self, data):
        return {
            'HMAC2':
            hmac.new(self.secret.encode(),
                     data.encode('utf-8'),
                     digestmod=hashlib.sha256).hexdigest()
        }

    def _reports_list(self, date=None):
        """
        Get a list of all the reports shadowserver has for a specific country
        via the reports/list endpoint. If a list of types is set in the
        parameters, we only process reports with those types.
        To be on the safe side regarding different calculations of timestamps,
        we request reports over a timespan of four days: two days in the past
        until one day in the future.
        The names of processed reports are cached and therefore not processed
        again.
        """
        if date is None:
            date = datetime.today().date()
        daybefore = date - timedelta(2)
        dayafter = date + timedelta(1)

        data = self.preamble
        data += ',"report": ["{}"] '.format(self.country)
        data += ',"date": "{}:{}" '.format(daybefore.isoformat(),
                                           dayafter.isoformat())
        data += '}'
        self.logger.debug('Downloading report list with data: %s.', data)

        response = self.session.post(APIROOT + 'reports/list',
                                     data=data,
                                     headers=self._headers(data))
        response.raise_for_status()

        reports = response.json()
        self.logger.debug('Downloaded report list, %s entries.', len(reports))

        if 'error' in reports:
            self.logger.debug('There was an error downloading the reports: %s',
                              reports['error'])
            return None

        if self.types is not None:
            reports = [
                report for report in reports
                if any(rtype in report['file'] for rtype in self.types)
            ]

        return reports

    def _report_download(self, reportid: str):
        """
        Download one report from the shadowserver API via the reports/download endpoint
        """
        data = self.preamble
        data += ',"id": "{}"}}'.format(reportid)
        self.logger.debug('Downloading report with data: %s.', data)

        response = self.session.post(APIROOT + 'reports/download',
                                     data=data,
                                     headers=self._headers(data))
        response.raise_for_status()

        return response.text

    def process(self):
        """
        Download reports and send them.
        Cache the filename of the report to not download the same report again.
        """
        reportslist = self._reports_list()
        self.logger.debug('Reports list contains %s entries after filtering.',
                          len(reportslist))

        for item in reportslist:
            filename = item['file']
            if self.cache.get(filename):
                self.logger.debug('Processed file %r already.', filename)
                continue
            self.logger.debug('Processing file %s.', filename)
            reportdata = self._report_download(item['id'])
            report = self.new_report()
            report.add('extra.file_name', filename)
            report.add('raw', str(reportdata))
            self.send_message(report)
            self.cache.set(filename, 1)
            self.logger.debug('Sent report: %s.', filename)
Ejemplo n.º 52
0
class ReverseDnsExpertBot(Bot):
    def init(self):
        self.cache = Cache(
            self.parameters.redis_cache_host, self.parameters.redis_cache_port,
            self.parameters.redis_cache_db, self.parameters.redis_cache_ttl,
            getattr(self.parameters, "redis_cache_password", None))

    def process(self):
        event = self.receive_message()

        keys = ["source.%s", "destination.%s"]

        for key in keys:
            ip_key = key % "ip"

            if ip_key not in event:
                continue

            ip = event.get(ip_key)
            ip_version = IPAddress.version(ip)
            ip_integer = IPAddress.to_int(ip)

            if ip_version == 4:
                minimum = MINIMUM_BGP_PREFIX_IPV4

            elif ip_version == 6:
                minimum = MINIMUM_BGP_PREFIX_IPV6

            cache_key = bin(ip_integer)[2:minimum + 2]
            cachevalue = self.cache.get(cache_key)

            result = None
            if cachevalue == DNS_EXCEPTION_VALUE:
                continue
            elif cachevalue:
                result = cachevalue
            else:
                rev_name = reversename.from_address(ip)
                try:
                    results = resolver.query(rev_name, "PTR")
                    expiration = results.expiration
                    for result in results:
                        # use first valid result
                        if event.is_valid('source.reverse_dns', str(result)):
                            break
                    else:
                        raise InvalidPTRResult
                except (dns.exception.DNSException, InvalidPTRResult) as e:
                    # Set default TTL for 'DNS query name does not exist' error
                    ttl = None if isinstance(e, dns.resolver.NXDOMAIN) else \
                        getattr(self.parameters, "cache_ttl_invalid_response",
                                60)
                    self.cache.set(cache_key, DNS_EXCEPTION_VALUE, ttl)
                    result = None

                else:
                    ttl = datetime.fromtimestamp(expiration) - datetime.now()
                    self.cache.set(cache_key,
                                   str(result),
                                   ttl=int(ttl.total_seconds()))

            if result is not None:
                event.add(key % 'reverse_dns', str(result), overwrite=True)

        self.send_message(event)
        self.acknowledge_message()
Ejemplo n.º 53
0
class MicrosoftInterflowCollectorBot(CollectorBot):
    def check_ttl_time(self):
        """
        Checks if the cache's TTL is big enough compared to the chosen
        time frame so that the bot does not process the same data over and
        over.
        """
        if isinstance(self.time_match, datetime):  # absolute
            now = datetime.now(tz=pytz.timezone('UTC'))
            if now - timedelta(
                    seconds=self.parameters.redis_cache_ttl) > self.time_match:
                raise ValueError(
                    "The cache's TTL must be higher than 'not_older_than', "
                    "otherwise the bot is processing the same data over and over again."
                )

    def init(self):
        if requests is None:
            raise MissingDependencyError("requests")

        self.set_request_parameters()

        self.http_header['Ocp-Apim-Subscription-Key'] = self.parameters.api_key
        if self.parameters.file_match:
            self.file_match = re.compile(self.parameters.file_match)
        else:
            self.file_match = None

        if self.parameters.not_older_than:
            try:
                self.time_match = timedelta(
                    minutes=parse_relative(self.parameters.not_older_than))
            except ValueError:
                self.time_match = parser.parse(
                    self.parameters.not_older_than).astimezone(pytz.utc)
                self.logger.info("Filtering files absolute %r.",
                                 self.time_match)
                self.check_ttl_time()
            else:
                self.logger.info("Filtering files relative %r.",
                                 self.time_match)
                if timedelta(seconds=self.parameters.redis_cache_ttl
                             ) < self.time_match:
                    raise ValueError(
                        "The cache's TTL must be higher than 'not_older_than', "
                        "otherwise the bot is processing the same data over and over again."
                    )
        else:
            self.time_match = None
        self.session = create_request_session(self)

        self.cache = Cache(
            self.parameters.redis_cache_host, self.parameters.redis_cache_port,
            self.parameters.redis_cache_db, self.parameters.redis_cache_ttl,
            getattr(self.parameters, "redis_cache_password", None))

    def process(self):
        self.check_ttl_time()
        self.logger.debug('Downloading file list.')
        files = self.session.get(URL_LIST)
        files.raise_for_status()
        self.logger.debug('Downloaded file list, %s entries.',
                          len(files.json()))
        for file in files.json():
            if self.cache.get(file['Name']):
                self.logger.debug('Processed file %s already.', file['Name'])
                continue
            if self.file_match and not self.file_match.match(file['Name']):
                self.logger.debug('File %r does not match filename filter.',
                                  file['Name'])
                continue
            filetime = parser.parse(file['LastModified'])
            if isinstance(self.time_match,
                          datetime) and filetime < self.time_match:
                self.logger.debug(
                    'File %r does not match absolute time filter.',
                    file['Name'])
                continue
            else:
                now = datetime.now(tz=pytz.timezone('UTC'))
                if isinstance(
                        self.time_match,
                        timedelta) and filetime < (now - self.time_match):
                    self.logger.debug(
                        'File %r does not match relative time filter.',
                        file['Name'])
                    continue

            self.logger.debug('Processing file %r.', file['Name'])
            download_url = URL_DOWNLOAD % file['Name']
            download = self.session.get(download_url)
            download.raise_for_status()
            if download_url.endswith('.gz'):
                raw = gzip.open(io.BytesIO(download.content)).read().decode()
            else:
                raw = download.text
            report = self.new_report()
            report.add('feed.url', download_url)
            report.add('raw', raw)
            self.send_message(report)
            # redis-py >= 3.0.0 does no longer support boolean values, cast to string explicitly, also for backwards compatibility
            self.cache.set(file['Name'], "True")

    def print_filelist(self):
        """ Can be called from the debugger for example. """
        self.logger.debug('Downloading file list.')
        files = self.session.get(URL_LIST)
        files.raise_for_status()
        self.logger.debug('Downloaded file list, %s entries.',
                          len(files.json()))
        print(files.text)
Ejemplo n.º 54
0
class ReverseDnsExpertBot(Bot):

    def init(self):
        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           getattr(self.parameters, "redis_cache_password",
                                   None)
                           )

    def process(self):
        event = self.receive_message()

        keys = ["source.%s", "destination.%s"]

        for key in keys:
            ip_key = key % "ip"

            if ip_key not in event:
                continue

            ip = event.get(ip_key)
            ip_version = IPAddress.version(ip)
            ip_integer = IPAddress.to_int(ip)

            if ip_version == 4:
                minimum = MINIMUM_BGP_PREFIX_IPV4

            elif ip_version == 6:
                minimum = MINIMUM_BGP_PREFIX_IPV6

            cache_key = bin(ip_integer)[2: minimum + 2]
            cachevalue = self.cache.get(cache_key)
            
            result = None
            if cachevalue == DNS_EXCEPTION_VALUE:
                continue
            elif cachevalue:
                result = cachevalue
            else:
                rev_name = reversename.from_address(ip)
                try:
                    results = resolver.query(rev_name, "PTR")
                    expiration = results.expiration
                    for result in results:
                        # use first valid result
                        if event.is_valid('source.reverse_dns', str(result)):
                            break
                    else:
                        raise InvalidPTRResult
                except (dns.exception.DNSException, InvalidPTRResult) as e:
                    # Set default TTL for 'DNS query name does not exist' error
                    ttl = None if isinstance(e, dns.resolver.NXDOMAIN) else \
                        getattr(self.parameters, "cache_ttl_invalid_response",
                                60)
                    self.cache.set(cache_key, DNS_EXCEPTION_VALUE, ttl)
                    result = None

                else:
                    ttl = datetime.fromtimestamp(expiration) - datetime.now()
                    self.cache.set(cache_key, str(result),
                                   ttl=int(ttl.total_seconds()))

            if result is not None:
                event.add(key % 'reverse_dns', str(result), overwrite=True)

        self.send_message(event)
        self.acknowledge_message()
Ejemplo n.º 55
0
class MicrosoftInterflowCollectorBot(CollectorBot):

    def check_ttl_time(self):
        """
        Checks if the cache's TTL is big enough compared to the chosen
        time frame so that the bot does not process the same data over and
        over.
        """
        if isinstance(self.time_match, datetime):  # absolute
            now = datetime.now(tz=pytz.timezone('UTC'))
            if now - timedelta(seconds=self.parameters.redis_cache_ttl) > self.time_match:
                raise ValueError("The cache's TTL must be higher than 'not_older_than', "
                                 "otherwise the bot is processing the same data over and over again.")

    def init(self):
        if requests is None:
            raise ValueError('Could not import requests. Please install it.')

        self.set_request_parameters()
        self.http_header['Ocp-Apim-Subscription-Key'] = self.parameters.api_key
        if self.parameters.file_match:
            self.file_match = re.compile(self.parameters.file_match)
        else:
            self.file_match = None

        if self.parameters.not_older_than:
            try:
                self.time_match = timedelta(minutes=parse_relative(self.parameters.not_older_than))
            except ValueError:
                if sys.version_info >= (3, 6):
                    self.time_match = parser.parse(self.parameters.not_older_than).astimezone(pytz.utc)
                else:  # "astimezone() cannot be applied to a naive datetime" otherwise
                    if '+' not in self.parameters.not_older_than:
                        self.parameters.not_older_than += '+00:00'
                    self.time_match = parser.parse(self.parameters.not_older_than)
                self.logger.info("Filtering files absolute %r.", self.time_match)
                self.check_ttl_time()
            else:
                self.logger.info("Filtering files relative %r.", self.time_match)
                if timedelta(seconds=self.parameters.redis_cache_ttl) < self.time_match:
                    raise ValueError("The cache's TTL must be higher than 'not_older_than', "
                                     "otherwise the bot is processing the same data over and over again.")
        else:
            self.time_match = None

        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           getattr(self.parameters, "redis_cache_password",
                                   None)
                           )

    def process(self):
        self.check_ttl_time()
        self.logger.debug('Downloading file list.')
        files = requests.get(URL_LIST,
                             auth=self.auth,
                             proxies=self.proxy,
                             headers=self.http_header,
                             verify=self.http_verify_cert,
                             cert=self.ssl_client_cert,
                             timeout=self.http_timeout_sec)
        files.raise_for_status()
        self.logger.debug('Downloaded file list, %s entries.', len(files.json()))
        for file in files.json():
            if self.cache.get(file['Name']):
                self.logger.debug('Processed file %s already.', file['Name'])
                continue
            if self.file_match and not self.file_match.match(file['Name']):
                self.logger.debug('File %r does not match filename filter.', file['Name'])
                continue
            filetime = parser.parse(file['LastModified'])
            if isinstance(self.time_match, datetime) and filetime < self.time_match:
                self.logger.debug('File %r does not match absolute time filter.', file['Name'])
                continue
            else:
                now = datetime.now(tz=pytz.timezone('UTC'))
                if isinstance(self.time_match, timedelta) and filetime < (now - self.time_match):
                    self.logger.debug('File %r does not match relative time filter.', file['Name'])
                    continue

            self.logger.debug('Processing file %r.', file['Name'])
            download_url = URL_DOWNLOAD % file['Name']
            download = requests.get(download_url,
                                    auth=self.auth,
                                    proxies=self.proxy,
                                    headers=self.http_header,
                                    verify=self.http_verify_cert,
                                    cert=self.ssl_client_cert,
                                    timeout=self.http_timeout_sec)
            download.raise_for_status()
            if download_url.endswith('.gz'):
                raw = gzip.open(io.BytesIO(download.content)).read().decode()
            else:
                raw = download.text
            report = self.new_report()
            report.add('feed.url', download_url)
            report.add('raw', raw)
            self.send_message(report)
            self.cache.set(file['Name'], True)
Ejemplo n.º 56
0
class RIPEExpertBot(Bot):
    QUERY = {
        'db_ip':
        'https://rest.db.ripe.net/abuse-contact/{}.json',
        'db_asn':
        'https://rest.db.ripe.net/abuse-contact/as{}.json',
        'stat':
        'https://stat.ripe.net/data/abuse-contact-finder/data.json?resource={}',
        'stat_geolocation':
        'https://stat.ripe.net/data/maxmind-geo-lite/data.json?resource={}',
    }

    REPLY_TO_DATA = {
        'db_ip':
        lambda x: clean_string(x['abuse-contacts']['email']),
        'db_asn':
        lambda x: clean_string(x['abuse-contacts']['email']),
        'stat':
        lambda x: clean_string(x['data']['anti_abuse_contacts']['abuse_c'][0][
            'email']),
        'stat_geolocation':
        lambda x: clean_geo(x['data']['located_resources'][0]['locations'][0]),
    }

    GEOLOCATION_REPLY_TO_INTERNAL = {('cc', 'country'),
                                     ('latitude', 'latitude'),
                                     ('longitude', 'longitude'),
                                     ('city', 'city')}

    def init(self):
        if requests is None:
            raise ValueError(
                "Could not import 'requests'. Please install the package.")

        self.__mode = getattr(self.parameters, 'mode', 'append')
        self.__query = {
            "db_asn":
            getattr(self.parameters, 'query_ripe_db_asn', True),
            "db_ip":
            getattr(self.parameters, 'query_ripe_db_ip', True),
            "stat_asn":
            getattr(self.parameters, 'query_ripe_stat_asn', True),
            "stat_ip":
            getattr(self.parameters, 'query_ripe_stat_ip', True),
            "stat_geo":
            getattr(self.parameters, 'query_ripe_stat_geolocation', True)
        }

        self.__initialize_http_session()
        self.__initialize_cache()

    def __initialize_http_session(self):
        self.set_request_parameters()
        self.http_session = utils.create_request_session_from_bot(self)

    def __initialize_cache(self):
        cache_host = getattr(self.parameters, 'redis_cache_host')
        cache_port = getattr(self.parameters, 'redis_cache_port')
        cache_db = getattr(self.parameters, 'redis_cache_db')
        cache_ttl = getattr(self.parameters, 'redis_cache_ttl')
        if cache_host and cache_port and cache_db and cache_ttl:
            self.__cache = Cache(
                cache_host, cache_port, cache_db, cache_ttl,
                getattr(self.parameters, "redis_cache_password", None))

    def process(self):
        event = self.receive_message()
        for target in {'source.', 'destination.'}:
            abuse_key = target + "abuse_contact"
            abuse = set(
                event.get(abuse_key).split(',')
            ) if self.__mode == 'append' and abuse_key in event else set()

            asn = event.get(target + "asn", None)
            if asn:
                if self.__query['stat_asn']:
                    abuse.update(self.__perform_cached_query('stat', asn))
                if self.__query['db_asn']:
                    abuse.update(self.__perform_cached_query('db_asn', asn))

            ip = event.get(target + "ip", None)
            if ip:
                if self.__query['stat_ip']:
                    abuse.update(self.__perform_cached_query('stat', ip))
                if self.__query['db_ip']:
                    abuse.update(self.__perform_cached_query('db_ip', ip))
                if self.__query['stat_geo']:
                    info = self.__perform_cached_query('stat_geolocation', ip)

                    should_overwrite = self.__mode == 'replace'

                    for local_key, ripe_key in self.GEOLOCATION_REPLY_TO_INTERNAL:
                        if ripe_key in info:
                            event.add(target + "geolocation." + local_key,
                                      info[ripe_key],
                                      overwrite=should_overwrite)

            event.add(abuse_key, ','.join(abuse), overwrite=True)
        self.send_message(event)
        self.acknowledge_message()

    def __perform_cached_query(self, type, resource):
        cached_value = self.__cache.get('{}:{}'.format(type, resource))
        if cached_value:
            if cached_value == CACHE_NO_VALUE:
                return {}
            else:
                return json.loads(cached_value)
        else:
            response = self.http_session.get(self.QUERY[type].format(resource),
                                             data="",
                                             timeout=self.http_timeout_sec)

            if response.status_code != 200:
                if type == 'db_asn' and response.status_code == 404:
                    """ If no abuse contact could be found, a 404 is given. """
                    try:
                        if response.json()['message'].startswith(
                                'No abuse contact found for '):
                            self.__cache.set('{}:{}'.format(type, resource),
                                             CACHE_NO_VALUE)
                            return {}
                    except ValueError:
                        pass
                raise ValueError(STATUS_CODE_ERROR.format(
                    response.status_code))
            try:
                response_data = response.json()

                # geolocation was marked as under maintenance by this, see
                # https://lists.cert.at/pipermail/intelmq-users/2020-March/000140.html
                status = response_data.get('data_call_status', '')
                if status.startswith('maintenance'):
                    warnings.warn(
                        'The API call %s is currently under maintenance. '
                        'Response: %r. This warning is only given once per bot run.'
                        '' % (type, status))

                data = self.REPLY_TO_DATA[type](response_data)
                self.__cache.set(
                    '{}:{}'.format(type, resource),
                    (json.dumps(list(data) if isinstance(data, set) else data)
                     if data else CACHE_NO_VALUE))
                return data
            except (KeyError, IndexError):
                self.__cache.set('{}:{}'.format(type, resource),
                                 CACHE_NO_VALUE)

            return {}