class ReverseDnsExpertBot(Bot):
def init(self):
self.cache = Cache(self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
)
def process(self):
event = self.receive_message()
keys = ["source.%s", "destination.%s"]
for key in keys:
ip_key = key % "ip"
if not event.contains(ip_key):
continue
ip = event.get(ip_key)
ip_version = IPAddress.version(ip)
ip_integer = IPAddress.to_int(ip)
if ip_version == 4:
minimum = MINIMUM_BGP_PREFIX_IPV4
elif ip_version == 6:
minimum = MINIMUM_BGP_PREFIX_IPV6
else:
self.logger.warning("Invalid IP version {}".format(ip_version))
self.send_message(event)
self.acknowledge_message()
cache_key = bin(ip_integer)[2: minimum + 2]
cachevalue = self.cache.get(cache_key)
result = None
if cachevalue:
result = cachevalue
else:
rev_name = reversename.from_address(ip)
try:
result = resolver.query(rev_name, "PTR")
expiration = result.expiration
result = result[0]
except dns.exception.DNSException as e:
if isinstance(e, dns.resolver.NXDOMAIN):
continue
else:
ttl = datetime.fromtimestamp(expiration) - datetime.now()
self.cache.set(cache_key, str(result),
ttl=int(ttl.total_seconds()))
if result is not None:
event.add(key % 'reverse_dns', str(result), force=True)
self.send_message(event)
self.acknowledge_message()
def init(self):
self.cache = Cache(
self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
)
def init(self):
if requests is None:
raise ValueError('Could not import requests. Please install it.')
if hasattr(self.parameters, 'query_ripe_stat'):
self.logger.warning(
"The parameter 'query_ripe_stat' is deprecated and will be "
"removed in 2.0. Use 'query_ripe_stat_asn' and "
"'query_ripe_stat_ip' instead'.")
self.query_db_asn = getattr(self.parameters, 'query_ripe_db_asn', True)
self.query_db_ip = getattr(self.parameters, 'query_ripe_db_ip', True)
self.query_stat_asn = getattr(
self.parameters, 'query_ripe_stat_asn',
getattr(self.parameters, 'query_ripe_stat', True))
self.query_stat_ip = getattr(
self.parameters, 'query_ripe_stat_ip',
getattr(self.parameters, 'query_ripe_stat', True))
self.mode = getattr(self.parameters, 'mode', 'append')
self.set_request_parameters()
cache_host = getattr(self.parameters, 'redis_cache_host')
cache_port = getattr(self.parameters, 'redis_cache_port')
cache_db = getattr(self.parameters, 'redis_cache_db')
cache_ttl = getattr(self.parameters, 'redis_cache_ttl')
if cache_host and cache_port and cache_db and cache_ttl:
self.cache = Cache(
cache_host, cache_port, cache_db, cache_ttl,
getattr(self.parameters, "redis_cache_password", None))
class DeduplicatorBot(Bot):
def init(self):
self.cache = Cache(
self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl
)
def process(self):
message = self.receive_message()
if message:
# Event deduplication
if isinstance(message, Event):
event = copy(message)
event.clear("observation_time")
message_hash = hash(event)
# Generic message deduplication
else:
message_hash = hash(message)
if not self.cache.exists(message_hash):
self.send_message(message)
self.cache.set(message_hash, 'hash')
self.acknowledge_message()
def init(self):
self.apikey = getattr(self.parameters, "api_key", None)
if self.apikey is None:
raise ValueError('No api_key provided.')
self.secret = getattr(self.parameters, "secret", None)
if self.secret is None:
raise ValueError('No secret provided.')
self.country = getattr(self.parameters, "country", None)
if self.country is None:
raise ValueError('No country provided.')
self.types = getattr(self.parameters, 'types', None)
if isinstance(self.types, str):
self.types = self.types.split(',')
self.preamble = '{{ "apikey": "{}" '.format(self.apikey)
self.set_request_parameters()
self.session = create_request_session(self)
self.cache = Cache(
self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
getattr(self.parameters, 'redis_cache_ttl', 864000), # 10 days
getattr(self.parameters, "redis_cache_password", None))
def tearDownClass(cls):
cache = Cache(test.BOT_CONFIG['redis_cache_host'],
test.BOT_CONFIG['redis_cache_port'],
test.BOT_CONFIG['redis_cache_db'],
test.BOT_CONFIG['redis_cache_ttl'],
)
cache.flush()
class DeduplicatorExpertBot(Bot):
def init(self):
self.cache = Cache(self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
)
def process(self):
message = self.receive_message()
if message is None:
self.acknowledge_message()
return
auxiliar_message = copy.copy(message)
ignore_keys = self.parameters.ignore_keys.split(',')
for ignore_key in ignore_keys:
ignore_key = ignore_key.strip()
if ignore_key in auxiliar_message:
del auxiliar_message[ignore_key]
message_hash = hash(auxiliar_message)
if not self.cache.exists(message_hash):
self.cache.set(message_hash, 'hash')
self.send_message(message)
self.acknowledge_message()
def init(self):
self.cache = Cache(
self.parameters.redis_cache_host, self.parameters.redis_cache_port,
self.parameters.redis_cache_db, self.parameters.redis_cache_ttl,
getattr(self.parameters, "redis_cache_password", None))
self.filter_keys = set(k.strip()
for k in self.parameters.filter_keys.split(','))
class DeduplicatorExpertBot(Bot):
def init(self):
self.cache = Cache(
self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
)
def process(self):
message = self.receive_message()
if message is None:
self.acknowledge_message()
return
auxiliar_message = copy.copy(message)
ignore_keys = self.parameters.ignore_keys.split(',')
for ignore_key in ignore_keys:
ignore_key = ignore_key.strip()
if ignore_key in auxiliar_message:
auxiliar_message.clear(ignore_key)
message_hash = hash(auxiliar_message)
if not self.cache.exists(message_hash):
self.cache.set(message_hash, 'hash')
self.send_message(message)
self.acknowledge_message()
class DeduplicatorExpertBot(Bot):
def init(self):
self.cache = Cache(self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
)
def process(self):
message = self.receive_message()
ignore_keys = set(k.strip()
for k in self.parameters.ignore_keys.split(','))
message_hash = message.hash(ignore_keys)
old_hash = hash(int(message_hash, 16))
if not (self.cache.exists(message_hash) or self.cache.exists(old_hash)):
self.cache.set(message_hash, 'hash')
self.send_message(message)
else:
self.logger.debug('Dropped message.')
self.acknowledge_message()
def tearDownClass(cls):
cache = Cache(test.BOT_CONFIG['redis_cache_host'],
test.BOT_CONFIG['redis_cache_port'],
test.BOT_CONFIG['redis_cache_db'],
test.BOT_CONFIG['redis_cache_ttl'],
)
cache.flush()
class ReverseDnsExpertBot(Bot):
def init(self):
self.cache = Cache(
self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
)
def process(self):
event = self.receive_message()
keys = ["source.%s", "destination.%s"]
for key in keys:
ip_key = key % "ip"
if not event.contains(ip_key):
continue
ip = event.get(ip_key)
ip_version = IPAddress.version(ip)
ip_integer = IPAddress.to_int(ip)
if ip_version == 4:
minimum = MINIMUM_BGP_PREFIX_IPV4
elif ip_version == 6:
minimum = MINIMUM_BGP_PREFIX_IPV6
cache_key = bin(ip_integer)[2:minimum + 2]
cachevalue = self.cache.get(cache_key)
result = None
if cachevalue:
result = cachevalue
else:
rev_name = reversename.from_address(ip)
try:
result = resolver.query(rev_name, "PTR")
expiration = result.expiration
result = result[0]
if str(result) == '.':
result = None
raise ValueError
except (dns.exception.DNSException, ValueError) as e:
if isinstance(e, dns.resolver.NXDOMAIN):
continue
else:
ttl = datetime.fromtimestamp(expiration) - datetime.now()
self.cache.set(cache_key,
str(result),
ttl=int(ttl.total_seconds()))
if result is not None:
event.add(key % 'reverse_dns', str(result), force=True)
self.send_message(event)
self.acknowledge_message()
def init(self):
self.set_request_parameters()
self.http_header['Ocp-Apim-Subscription-Key'] = self.parameters.api_key
if self.parameters.file_match:
self.file_match = re.compile(self.parameters.file_match)
else:
self.file_match = None
if self.parameters.not_older_than:
try:
self.time_match = timedelta(
minutes=parse_relative(self.parameters.not_older_than))
except ValueError:
self.time_match = parser.parse(self.parameters.not_older_than)
self.logger.info("Filtering files absolute %r.",
self.time_match)
else:
self.logger.info("Filtering files relative %r.",
self.time_match)
else:
self.time_match = None
self.cache = Cache(
self.parameters.redis_cache_host, self.parameters.redis_cache_port,
self.parameters.redis_cache_db, self.parameters.redis_cache_ttl,
getattr(self.parameters, "redis_cache_password", None))
class DeduplicatorExpertBot(Bot):
def init(self):
self.cache = Cache(self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
getattr(self.parameters, "redis_cache_password",
None)
)
self.filter_keys = set(k.strip() for k in
self.parameters.filter_keys.split(','))
def process(self):
message = self.receive_message()
message_hash = message.hash(filter_keys=self.filter_keys,
filter_type=self.parameters.filter_type)
if not self.cache.exists(message_hash):
self.cache.set(message_hash, 'hash')
self.send_message(message)
else:
self.logger.debug('Dropped message.')
self.acknowledge_message()
class DeduplicatorBot(Bot):
def init(self):
self.cache = Cache(
self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl
)
def process(self):
message = self.receive_message()
if message:
# Event deduplication
if isinstance(message, Event):
event = deepcopy(message)
event.clear("observation_time")
if event.value("type")=="vulnerable service" or event.value("type")=="compromised":
event.clear("source_time")
event.clear("source_reverse_dns")
message_hash = hash(event)
# Generic message deduplication
else:
message_hash = hash(message)
if not self.cache.exists(message_hash):
self.cache.set(message_hash, 'hash')
self.send_message(message)
self.acknowledge_message()
class CymruExpertBot(Bot):
def init(self):
self.cache = Cache(
self.parameters.redis_cache_host, self.parameters.redis_cache_port,
self.parameters.redis_cache_db, self.parameters.redis_cache_ttl,
getattr(self.parameters, "redis_cache_password", None))
if not hasattr(self.parameters, 'overwrite'):
self.logger.warning(
"Parameter 'overwrite' is not given, assuming 'True'. "
"Please set it explicitly, default will change to "
"'False' in version 3.0.0'.")
self.overwrite = getattr(self.parameters, 'overwrite', True)
def process(self):
event = self.receive_message()
keys = ["source.%s", "destination.%s"]
for key in keys:
ip_key = key % "ip"
if ip_key not in event:
continue
ip = event.get(ip_key)
ip_version = IPAddress.version(ip)
ip_integer = IPAddress.to_int(ip)
if ip_version == 4:
minimum = MINIMUM_BGP_PREFIX_IPV4
elif ip_version == 6:
minimum = MINIMUM_BGP_PREFIX_IPV6
else:
raise ValueError('Unexpected IP version '
'{!r}.'.format(ip_version))
cache_key = bin(ip_integer)[2:minimum + 2]
result_json = self.cache.get(cache_key)
if result_json:
result = json.loads(result_json)
else:
result = Cymru.query(ip)
if not result:
continue
result_json = json.dumps(result)
self.cache.set(cache_key, result_json)
for result_key, result_value in result.items():
if result_key == 'registry' and result_value == 'other':
continue
event.add(key % result_key,
result_value,
overwrite=self.overwrite)
self.send_message(event)
self.acknowledge_message()
class DeduplicatorExpertBot(Bot):
def init(self):
self.cache = Cache(
self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
)
def process(self):
message = self.receive_message()
ignore_keys = set(k.strip()
for k in self.parameters.ignore_keys.split(','))
message_hash = message.hash(ignore_keys)
old_hash = hash(int(message_hash, 16))
if not (self.cache.exists(message_hash)
or self.cache.exists(old_hash)):
self.cache.set(message_hash, 'hash')
self.send_message(message)
else:
self.logger.debug('Dropped message.')
self.acknowledge_message()
class DeduplicatorExpertBot(Bot):
_message_processed_verb = 'Forwarded'
def init(self):
self.cache = Cache(self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
getattr(self.parameters, "redis_cache_password",
None)
)
self.filter_keys = {k.strip() for k in
self.parameters.filter_keys.split(',')}
self.bypass = getattr(self.parameters, "bypass", False)
def process(self):
message = self.receive_message()
if self.bypass:
self.send_message(message)
else:
message_hash = message.hash(filter_keys=self.filter_keys,
filter_type=self.parameters.filter_type)
if not self.cache.exists(message_hash):
self.cache.set(message_hash, 'hash')
self.send_message(message)
else:
self.logger.debug('Dropped message.')
self.acknowledge_message()
def __initialize_cache(self):
cache_host = getattr(self.parameters, 'redis_cache_host')
cache_port = getattr(self.parameters, 'redis_cache_port')
cache_db = getattr(self.parameters, 'redis_cache_db')
cache_ttl = getattr(self.parameters, 'redis_cache_ttl')
if cache_host and cache_port and cache_db and cache_ttl:
self.__cache = Cache(cache_host, cache_port, cache_db, cache_ttl,
getattr(self.parameters, "redis_cache_password", None))
def init(self):
print("joooo")
self.cache = Cache(
self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
)
def tearDownClass(cls):
cache = Cache(
test.BOT_CONFIG["redis_cache_host"],
test.BOT_CONFIG["redis_cache_port"],
test.BOT_CONFIG["redis_cache_db"],
test.BOT_CONFIG["redis_cache_ttl"],
)
cache.flush()
def init(self):
self.cache = Cache(self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
getattr(self.parameters, "redis_cache_password",
None)
)
class CymruExpertBot(Bot):
def init(self):
self.cache = Cache(
self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
)
def process(self):
event = self.receive_message()
if event is None:
self.acknowledge_message()
return
keys = ["source.%s", "destination.%s"]
for key in keys:
ip_key = key % "ip"
if not event.contains(ip_key):
continue
ip = event.value(ip_key)
ip_version = IPAddress.version(ip)
ip_integer = IPAddress.to_int(ip)
if ip_version == 4:
minimum = MINIMUM_BGP_PREFIX_IPV4
elif ip_version == 6:
minimum = MINIMUM_BGP_PREFIX_IPV6
else:
raise ValueError('Unexpected IP version '
'{!r}.'.format(ip_version))
cache_key = bin(ip_integer)[2:minimum + 2]
result_json = self.cache.get(cache_key)
if result_json:
result = json.loads(result_json)
else:
result = Cymru.query(ip)
result_json = json.dumps(result)
self.cache.set(cache_key, result_json)
for result_key, result_value in result.items():
event.add(key % result_key,
result_value,
sanitize=True,
force=True)
self.send_message(event)
self.acknowledge_message()
class CymruExpertBot(Bot):
def init(self):
self.cache = Cache(self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
getattr(self.parameters, "redis_cache_password",
None)
)
def process(self):
event = self.receive_message()
keys = ["source.%s", "destination.%s"]
for key in keys:
ip_key = key % "ip"
if ip_key not in event:
continue
ip = event.get(ip_key)
ip_version = IPAddress.version(ip)
ip_integer = IPAddress.to_int(ip)
if ip_version == 4:
minimum = MINIMUM_BGP_PREFIX_IPV4
elif ip_version == 6:
minimum = MINIMUM_BGP_PREFIX_IPV6
else:
raise ValueError('Unexpected IP version '
'{!r}.'.format(ip_version))
cache_key = bin(ip_integer)[2: minimum + 2]
result_json = self.cache.get(cache_key)
if result_json:
result = json.loads(result_json)
else:
result = Cymru.query(ip)
if not result:
continue
result_json = json.dumps(result)
self.cache.set(cache_key, result_json)
for result_key, result_value in result.items():
if result_key == 'registry' and result_value == 'other':
continue
event.add(key % result_key, result_value, overwrite=True)
self.send_message(event)
self.acknowledge_message()
class CymruExpertBot(Bot):
def init(self):
self.cache = Cache(self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
)
def process(self):
event = self.receive_message()
if event is None:
self.acknowledge_message()
return
keys = ["source.%s", "destination.%s"]
for key in keys:
ip_key = key % "ip"
if not event.contains(ip_key):
continue
ip = event.value(ip_key)
ip_version = IPAddress.version(ip)
ip_integer = IPAddress.to_int(ip)
if ip_version == 4:
minimum = MINIMUM_BGP_PREFIX_IPV4
elif ip_version == 6:
minimum = MINIMUM_BGP_PREFIX_IPV6
else:
raise ValueError('Unexpected IP version '
'{!r}.'.format(ip_version))
cache_key = bin(ip_integer)[2: minimum + 2]
result_json = self.cache.get(cache_key)
if result_json:
result = json.loads(result_json)
else:
result = Cymru.query(ip)
result_json = json.dumps(result)
self.cache.set(cache_key, result_json)
for result_key, result_value in result.items():
event.add(key % result_key, result_value, sanitize=True,
force=True)
self.send_message(event)
self.acknowledge_message()
def init(self):
self.query_db_asn = getattr(self.parameters, 'query_ripe_db_asn', True)
self.query_db_ip = getattr(self.parameters, 'query_ripe_db_ip', True)
self.query_stat_asn = getattr(self.parameters, 'query_ripe_stat', True)
self.query_stat_ip = getattr(self.parameters, 'query_ripe_stat', True)
self.cache = Cache(
self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
)
def init(self):
self.cache = Cache(
self.parameters.redis_cache_host, self.parameters.redis_cache_port,
self.parameters.redis_cache_db, self.parameters.redis_cache_ttl,
getattr(self.parameters, "redis_cache_password", None))
if not hasattr(self.parameters, 'overwrite'):
self.logger.warning(
"Parameter 'overwrite' is not given, assuming 'True'. "
"Please set it explicitly, default will change to "
"'False' in version 3.0.0'.")
self.overwrite = getattr(self.parameters, 'overwrite', True)
class CymruExpertBot(Bot):
def init(self):
self.cache = Cache(
self.parameters.redis_cache_host, self.parameters.redis_cache_port,
self.parameters.redis_cache_db, self.parameters.redis_cache_ttl,
getattr(self.parameters, "redis_cache_password", None))
if not hasattr(self.parameters, 'overwrite'):
self.logger.warning(
"Parameter 'overwrite' is not given, assuming 'True'. "
"Please set it explicitly, default will change to "
"'False' in version 3.0.0'.")
self.overwrite = getattr(self.parameters, 'overwrite', True)
def process(self):
event = self.receive_message()
keys = ["source.%s", "destination.%s"]
for key in keys:
ip_key = key % "ip"
if ip_key not in event:
continue
address = event.get(ip_key)
cache_key = CACHE_KEY % (IPAddress.version(address), address)
result_json = self.cache.get(cache_key)
if result_json:
result = json.loads(result_json)
else:
result = Cymru.query(address)
if not result:
self.logger.info(
'Got no result from Cymru for IP address %r.', address)
result_json = json.dumps(result)
self.cache.set(cache_key, result_json)
if not result:
continue
for result_key, result_value in result.items():
if result_key == 'registry' and result_value == 'other':
continue
event.add(key % result_key,
result_value,
overwrite=self.overwrite)
self.send_message(event)
self.acknowledge_message()
class ForwardDnsExpertBot(Bot):
def init(self):
print("joooo")
self.cache = Cache(
self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
)
def process(self):
event = self.receive_message()
if event is None:
self.acknowledge_message()
return
messageSent = False
for key in ["source.%s", "destination.%s"]:
ip_key = key % "ip"
fqdn_key = key % "fqdn"
if event.contains(ip_key) or not event.contains(fqdn_key):
continue
fqdn = event[fqdn_key]
cache_key = hash(fqdn)
cachevalue = self.cache.get(cache_key)
result = None
if cachevalue:
result = cachevalue
else:
soc = socket.getaddrinfo(fqdn, 0)
try:
result = set([address[4][0] for address in soc])
except socket.error as msg:
print(msg)
continue
else:
self.cache.set(cache_key, result)
print("VEEOF", result)
for ip in result:
print("ZDEE!!", ip, ip_key, fqdn_key)
event.add(ip_key, ip, sanitize=True, force=True)
self.send_message(event)
messageSent = True
if not messageSent:
self.send_message(event)
self.acknowledge_message()
class ForwardDnsExpertBot(Bot):
def init(self):
print("joooo")
self.cache = Cache(self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
)
def process(self):
event = self.receive_message()
if event is None:
self.acknowledge_message()
return
messageSent = False
for key in ["source.%s", "destination.%s"]:
ip_key = key % "ip"
fqdn_key = key % "fqdn"
if event.contains(ip_key) or not event.contains(fqdn_key):
continue
fqdn = event[fqdn_key]
cache_key = hash(fqdn)
cachevalue = self.cache.get(cache_key)
result = None
if cachevalue:
result = cachevalue
else:
soc = socket.getaddrinfo(fqdn, 0)
try:
result = set([address[4][0] for address in soc])
except socket.error as msg:
print(msg)
continue
else:
self.cache.set(cache_key, result)
print("VEEOF",result)
for ip in result:
print("ZDEE!!",ip,ip_key,fqdn_key)
event.add(ip_key, ip, sanitize=True, force=True)
self.send_message(event)
messageSent = True
if not messageSent:
self.send_message(event)
self.acknowledge_message()
class ThresholdExpertBot(Bot):
_message_processed_verb = 'Forwarded'
is_multithreadable = False
def init(self):
self.cache = Cache(
self.parameters.redis_cache_host, self.parameters.redis_cache_port,
self.parameters.redis_cache_db, self.parameters.timeout,
getattr(self.parameters, "redis_cache_password", None))
self.filter_keys = getattr(self.parameters, "filter_keys", [])
self.filter_type = getattr(self.parameters, "filter_type", "whitelist")
self.bypass = getattr(self.parameters, "bypass", False)
self.timeout = getattr(self.parameters, "timeout", -1)
if self.timeout <= 0:
raise ConfigurationError(
'Timeout',
'Invalid timeout specified, use positive integer seconds.')
self.threshold = getattr(self.parameters, "threshold", -1)
if self.threshold <= 0:
raise ConfigurationError(
'Threshold',
'Invalid threshold specified, use positive integer count.')
self.add_keys = getattr(self.parameters, "add_keys", {})
def process(self):
message = self.receive_message()
if self.bypass:
self.send_message(message)
else:
message_hash = message.hash(filter_keys=self.filter_keys,
filter_type=self.filter_type)
old_count = int(self.cache.get(message_hash) or 0)
self.logger.debug('Message %s has been seen %i times before.',
message_hash, old_count)
# Use Redis "set" instead of "incr" to reset the timeout
# every time
self.cache.set(message_hash, str(old_count + 1))
if old_count + 1 == self.threshold:
self.logger.debug('Threshold reached, forwarding message.')
message.update(self.add_keys)
message.add('extra.count', old_count + 1, overwrite=True)
self.send_message(message)
else:
self.logger.debug('Dropped message.')
self.acknowledge_message()
def init(self):
print("joooo")
self.cache = Cache(self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
)
def init(self):
self.cache = Cache(
self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl
)
def init(self):
self.set_request_parameters()
self.http_header['Ocp-Apim-Subscription-Key'] = self.parameters.api_key
if self.parameters.file_match:
self.file_match = re.compile(self.parameters.file_match)
else:
self.file_match = None
if self.parameters.not_older_than:
try:
self.time_match = timedelta(minutes=parse_relative(self.parameters.not_older_than))
except ValueError:
self.time_match = parser.parse(self.parameters.not_older_than)
self.logger.info("Filtering files absolute %r.", self.time_match)
else:
self.logger.info("Filtering files relative %r.", self.time_match)
else:
self.time_match = None
self.cache = Cache(self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
getattr(self.parameters, "redis_cache_password",
None)
)
def init(self):
self.cache = Cache(
self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl
)
self.logger.warning("Bot init succesful.")
def __initialize_cache(self):
cache_host = getattr(self.parameters, 'redis_cache_host')
cache_port = getattr(self.parameters, 'redis_cache_port')
cache_db = getattr(self.parameters, 'redis_cache_db')
cache_ttl = getattr(self.parameters, 'redis_cache_ttl')
if cache_host and cache_port and cache_db and cache_ttl:
self.__cache = Cache(cache_host, cache_port, cache_db, cache_ttl,
getattr(self.parameters, "redis_cache_password", None))
def init(self):
self.cache = Cache(self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
getattr(self.parameters, "redis_cache_password",
None)
)
def init(self):
if requests is None:
raise ValueError('Could not import requests. Please install it.')
self.set_request_parameters()
self.http_header['Ocp-Apim-Subscription-Key'] = self.parameters.api_key
if self.parameters.file_match:
self.file_match = re.compile(self.parameters.file_match)
else:
self.file_match = None
if self.parameters.not_older_than:
try:
self.time_match = timedelta(
minutes=parse_relative(self.parameters.not_older_than))
except ValueError:
if sys.version_info >= (3, 6):
self.time_match = parser.parse(
self.parameters.not_older_than).astimezone(pytz.utc)
else: # "astimezone() cannot be applied to a naive datetime" otherwise
if '+' not in self.parameters.not_older_than:
self.parameters.not_older_than += '+00:00'
self.time_match = parser.parse(
self.parameters.not_older_than)
self.logger.info("Filtering files absolute %r.",
self.time_match)
self.check_ttl_time()
else:
self.logger.info("Filtering files relative %r.",
self.time_match)
if timedelta(seconds=self.parameters.redis_cache_ttl
) < self.time_match:
raise ValueError(
"The cache's TTL must be higher than 'not_older_than', "
"otherwise the bot is processing the same data over and over again."
)
else:
self.time_match = None
self.session = create_request_session_from_bot(self)
self.cache = Cache(
self.parameters.redis_cache_host, self.parameters.redis_cache_port,
self.parameters.redis_cache_db, self.parameters.redis_cache_ttl,
getattr(self.parameters, "redis_cache_password", None))
def init(self):
if ContainerClient is None or create_configuration is None:
raise MissingDependencyError("azure.storage")
self.config = create_configuration(storage_sdk='blob')
if hasattr(self.parameters, 'https_proxy'):
# Create a storage configuration object and update the proxy policy
self.config.proxy_policy.proxies = {
'http': self.parameters.http_proxy,
'https': self.parameters.https_proxy,
}
self.cache = Cache(
self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
getattr(self.parameters, 'redis_cache_ttl', 864000), # 10 days
getattr(self.parameters, "redis_cache_password", None))
def init(self):
self.cache = Cache(self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
getattr(self.parameters, "redis_cache_password",
None)
)
self.filter_keys = set(k.strip() for k in
self.parameters.filter_keys.split(','))
class DeduplicatorBot(Bot):
def init(self):
self.cache = Cache(
self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl
)
def process(self):
message = self.receive_message()
message_hash = hash(message)
if not self.cache.exists(message_hash):
self.cache.set(message_hash, 'hash')
self.send_message(message)
self.acknowledge_message()
def init(self):
self.query_db_asn = getattr(self.parameters, 'query_ripe_db_asn', True)
self.query_db_ip = getattr(self.parameters, 'query_ripe_db_ip', True)
self.query_stat_asn = getattr(self.parameters, 'query_ripe_stat', True)
self.query_stat_ip = getattr(self.parameters, 'query_ripe_stat', True)
self.cache = Cache(self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
)
def init(self):
self.cache = Cache(
self.parameters.redis_cache_host, self.parameters.redis_cache_port,
self.parameters.redis_cache_db, self.parameters.timeout,
getattr(self.parameters, "redis_cache_password", None))
self.filter_keys = getattr(self.parameters, "filter_keys", [])
self.filter_type = getattr(self.parameters, "filter_type", "whitelist")
self.bypass = getattr(self.parameters, "bypass", False)
self.timeout = getattr(self.parameters, "timeout", -1)
if self.timeout <= 0:
raise ConfigurationError(
'Timeout',
'Invalid timeout specified, use positive integer seconds.')
self.threshold = getattr(self.parameters, "threshold", -1)
if self.threshold <= 0:
raise ConfigurationError(
'Threshold',
'Invalid threshold specified, use positive integer count.')
self.add_keys = getattr(self.parameters, "add_keys", {})
def init(self):
if requests is None:
raise MissingDependencyError("requests")
self.set_request_parameters()
self.http_header['Ocp-Apim-Subscription-Key'] = self.parameters.api_key
if self.parameters.file_match:
self.file_match = re.compile(self.parameters.file_match)
else:
self.file_match = None
if self.parameters.not_older_than:
try:
self.time_match = timedelta(
minutes=parse_relative(self.parameters.not_older_than))
except ValueError:
self.time_match = parser.parse(
self.parameters.not_older_than).astimezone(pytz.utc)
self.logger.info("Filtering files absolute %r.",
self.time_match)
self.check_ttl_time()
else:
self.logger.info("Filtering files relative %r.",
self.time_match)
if timedelta(seconds=self.parameters.redis_cache_ttl
) < self.time_match:
raise ValueError(
"The cache's TTL must be higher than 'not_older_than', "
"otherwise the bot is processing the same data over and over again."
)
else:
self.time_match = None
self.session = create_request_session(self)
self.cache = Cache(
self.parameters.redis_cache_host, self.parameters.redis_cache_port,
self.parameters.redis_cache_db, self.parameters.redis_cache_ttl,
getattr(self.parameters, "redis_cache_password", None))
class MicrosoftAzureCollectorBot(CollectorBot):
def init(self):
if ContainerClient is None or create_configuration is None:
raise MissingDependencyError("azure.storage")
self.config = create_configuration(storage_sdk='blob')
if hasattr(self.parameters, 'https_proxy'):
# Create a storage configuration object and update the proxy policy
self.config.proxy_policy.proxies = {
'http': self.parameters.http_proxy,
'https': self.parameters.https_proxy,
}
self.cache = Cache(
self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
getattr(self.parameters, 'redis_cache_ttl', 864000), # 10 days
getattr(self.parameters, "redis_cache_password", None))
def process(self):
container_client = ContainerClient.from_connection_string(
conn_str=self.parameters.connection_string,
container_name=self.parameters.container_name,
_configuration=self.config)
for blob in container_client.list_blobs():
if self.cache.get(blob.name):
self.logger.debug('Processed file %r already.', blob.name)
continue
self.logger.debug('Processing blob %r.', blob.name)
blob_obj = io.BytesIO()
container_client.download_blob(blob).readinto(blob_obj)
blob_obj.seek(0)
report = self.new_report()
report.add('raw', gzip.GzipFile(fileobj=blob_obj).read().decode())
self.send_message(report)
self.cache.set(blob.name,
1) # Redis-py >= 3.0.0 does not allow True
def init(self):
if requests is None:
raise ValueError('Could not import requests. Please install it.')
self.set_request_parameters()
self.http_header['Ocp-Apim-Subscription-Key'] = self.parameters.api_key
if self.parameters.file_match:
self.file_match = re.compile(self.parameters.file_match)
else:
self.file_match = None
if self.parameters.not_older_than:
try:
self.time_match = timedelta(minutes=parse_relative(self.parameters.not_older_than))
except ValueError:
if sys.version_info >= (3, 6):
self.time_match = parser.parse(self.parameters.not_older_than).astimezone(pytz.utc)
else: # "astimezone() cannot be applied to a naive datetime" otherwise
if '+' not in self.parameters.not_older_than:
self.parameters.not_older_than += '+00:00'
self.time_match = parser.parse(self.parameters.not_older_than)
self.logger.info("Filtering files absolute %r.", self.time_match)
self.check_ttl_time()
else:
self.logger.info("Filtering files relative %r.", self.time_match)
if timedelta(seconds=self.parameters.redis_cache_ttl) < self.time_match:
raise ValueError("The cache's TTL must be higher than 'not_older_than', "
"otherwise the bot is processing the same data over and over again.")
else:
self.time_match = None
self.cache = Cache(self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
getattr(self.parameters, "redis_cache_password",
None)
)
class RIPENCCExpertBot(Bot):
def init(self):
self.query_db_asn = getattr(self.parameters, 'query_ripe_db_asn', True)
self.query_db_ip = getattr(self.parameters, 'query_ripe_db_ip', True)
self.query_stat_asn = getattr(self.parameters, 'query_ripe_stat', True)
self.query_stat_ip = getattr(self.parameters, 'query_ripe_stat', True)
self.cache = Cache(self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
)
def process(self):
event = self.receive_message()
if event is None:
self.acknowledge_message()
return
for key in ['source.', 'destination.']:
ip_key = key + "ip"
abuse_key = key + "abuse_contact"
asn_key = key + "asn"
ip = event.get(ip_key, None)
if not ip:
continue
ip_version = IPAddress.version(ip)
ip_integer = IPAddress.to_int(ip)
if ip_version == 4:
minimum = MINIMUM_BGP_PREFIX_IPV4
elif ip_version == 6:
minimum = MINIMUM_BGP_PREFIX_IPV6
else:
raise ValueError('Unexpected IP version '
'{!r}.'.format(ip_version))
cache_key = bin(ip_integer)[2: minimum + 2]
cache_result = self.cache.get(cache_key)
abuse = (event.get(abuse_key).split(',') if abuse_key in event
else [])
if cache_result:
cache_result = ast.literal_eval(cache_result)
cache_result = [n.strip() for n in cache_result]
abuse.extend(cache_result)
else:
asn = event.get(asn_key, None)
if self.query_db_asn and asn:
abuse.extend(lib.query_asn(asn))
if self.query_db_ip and ip:
abuse.extend(lib.query_ripedb(ip))
if self.query_stat_asn and asn:
abuse.extend(lib.query_ripestat(asn))
if self.query_stat_ip and ip:
abuse.extend(lib.query_ripestat(ip))
self.cache.set(cache_key,abuse)
event.add(abuse_key, ','.join(filter(None, set(abuse))), force=True)
self.send_message(event)
self.acknowledge_message()
class RIPEExpertBot(Bot):
QUERY = {
'db_ip': 'https://rest.db.ripe.net/abuse-contact/{}.json',
'db_asn': 'https://rest.db.ripe.net/abuse-contact/as{}.json',
'stat': 'https://stat.ripe.net/data/abuse-contact-finder/data.json?resource={}',
'stat_geolocation': 'https://stat.ripe.net/data/maxmind-geo-lite/data.json?resource={}',
}
REPLY_TO_DATA = {
'db_ip': lambda x: clean_string(x['abuse-contacts']['email']),
'db_asn': lambda x: clean_string(x['abuse-contacts']['email']),
'stat': lambda x: clean_string(x['data']['anti_abuse_contacts']['abuse_c'][0]['email']),
'stat_geolocation': lambda x: clean_geo(x['data']['located_resources'][0]['locations'][0]),
}
GEOLOCATION_REPLY_TO_INTERNAL = {
('cc', 'country'),
('latitude', 'latitude'),
('longitude', 'longitude'),
('city', 'city')
}
def init(self):
if requests is None:
raise ValueError("Could not import 'requests'. Please install the package.")
self.__mode = getattr(self.parameters, 'mode', 'append')
self.__query = {
"db_asn": getattr(self.parameters, 'query_ripe_db_asn', True),
"db_ip": getattr(self.parameters, 'query_ripe_db_ip', True),
"stat_asn": getattr(self.parameters, 'query_ripe_stat_asn', True),
"stat_ip": getattr(self.parameters, 'query_ripe_stat_ip', True),
"stat_geo": getattr(self.parameters, 'query_ripe_stat_geolocation', True)
}
self.__initialize_http_session()
self.__initialize_cache()
def __initialize_http_session(self):
self.http_session = requests.Session()
self.set_request_parameters()
self.http_session.proxies.update(self.proxy)
self.http_session.headers.update(self.http_header)
self.http_session.verify = self.http_verify_cert
self.http_session.cert = self.ssl_client_cert
def __initialize_cache(self):
cache_host = getattr(self.parameters, 'redis_cache_host')
cache_port = getattr(self.parameters, 'redis_cache_port')
cache_db = getattr(self.parameters, 'redis_cache_db')
cache_ttl = getattr(self.parameters, 'redis_cache_ttl')
if cache_host and cache_port and cache_db and cache_ttl:
self.__cache = Cache(cache_host, cache_port, cache_db, cache_ttl,
getattr(self.parameters, "redis_cache_password", None))
def process(self):
with self.event_context() as event:
for target in {'source.', 'destination.'}:
abuse_key = target + "abuse_contact"
abuse = set(event.get(abuse_key).split(',')) if self.__mode == 'append' and abuse_key in event else set()
asn = event.get(target + "asn", None)
if asn:
if self.__query['stat_asn']:
abuse.update(self.__perform_cached_query('stat', asn))
if self.__query['db_asn']:
abuse.update(self.__perform_cached_query('db_asn', asn))
ip = event.get(target + "ip", None)
if ip:
if self.__query['stat_ip']:
abuse.update(self.__perform_cached_query('stat', ip))
if self.__query['db_ip']:
abuse.update(self.__perform_cached_query('db_ip', ip))
if self.__query['stat_geo']:
info = self.__perform_cached_query('stat_geolocation', ip)
should_overwrite = self.__mode == 'replace'
for local_key, ripe_key in self.GEOLOCATION_REPLY_TO_INTERNAL:
if ripe_key in info:
event.add(target + "geolocation." + local_key, info[ripe_key], overwrite=should_overwrite)
event.add(abuse_key, ','.join(abuse), overwrite=True)
@contextmanager
def event_context(self):
event = self.receive_message()
try:
yield event
finally:
self.send_message(event)
self.acknowledge_message()
def __perform_cached_query(self, type, resource):
cached_value = self.__cache.get('{}:{}'.format(type, resource))
if cached_value:
if cached_value == CACHE_NO_VALUE:
return {}
else:
return json.loads(cached_value)
else:
response = self.http_session.get(self.QUERY[type].format(resource), data="", timeout=self.http_timeout_sec)
if response.status_code != 200:
if type == 'db_asn' and response.status_code == 404:
""" If no abuse contact could be found, a 404 is given. """
try:
if response.json()['message'].startswith('No abuse contact found for '):
self.__cache.set('{}:{}'.format(type, resource), CACHE_NO_VALUE)
return {}
except ValueError:
pass
raise ValueError(STATUS_CODE_ERROR.format(response.status_code))
try:
data = self.REPLY_TO_DATA[type](response.json())
self.__cache.set('{}:{}'.format(type, resource),
(json.dumps(list(data) if isinstance(data, set) else data) if data else CACHE_NO_VALUE))
return data
except (KeyError, IndexError):
self.__cache.set('{}:{}'.format(type, resource), CACHE_NO_VALUE)
return {}
class CymruExpertBot(Bot):
def init(self):
self.cache = Cache(
self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl
)
def process(self):
event = self.receive_message()
if not event:
self.acknowledge_message()
return
keys = ["source_%s", "destination_%s"]
for key in keys:
ip = event.value(key % "ip")
if not ip:
self.send_message(event)
self.acknowledge_message()
return
elif utils.is_ipv4(ip):
ip_version = 4
ip_integer = utils.ip_to_int(ip)
cache_key = bin(ip_integer)[2 : MINIMUM_BGP_PREFIX_IPV4 + 2]
elif utils.is_ipv6(ip):
ip_version = 6
ip_integer = utils.ip_to_int(ip)
cache_key = bin(ip_integer)[2 : MINIMUM_BGP_PREFIX_IPV6 + 2]
else:
self.send_message(event)
self.acknowledge_message()
return
result_json = self.cache.get(cache_key)
if result_json:
result = json.loads(result_json)
else:
result = Cymru.query(ip, ip_version)
result_json = json.dumps(result)
self.cache.set(cache_key, result_json)
if "asn" in result:
event.clear(key % 'asn')
event.add(key % 'asn', result['asn'])
if "bgp_prefix" in result:
event.clear(key % 'bgp_prefix')
event.add(key % 'bgp_prefix', result['bgp_prefix'])
if "registry" in result:
event.clear(key % 'registry')
event.add(key % 'registry', result['registry'])
if "allocated" in result:
event.clear(key % 'allocated')
event.add(key % 'allocated', result['allocated'])
if "as_name" in result:
event.clear(key % 'as_name')
event.add(key % 'as_name', result['as_name'])
if "cc" in result:
event.clear(key % 'cymru_cc')
event.add(key % 'cymru_cc', result['cc'])
self.send_message(event)
self.acknowledge_message()
class MicrosoftInterflowCollectorBot(CollectorBot):
def init(self):
self.set_request_parameters()
self.http_header['Ocp-Apim-Subscription-Key'] = self.parameters.api_key
if self.parameters.file_match:
self.file_match = re.compile(self.parameters.file_match)
else:
self.file_match = None
if self.parameters.not_older_than:
try:
self.time_match = timedelta(minutes=parse_relative(self.parameters.not_older_than))
except ValueError:
self.time_match = parser.parse(self.parameters.not_older_than)
self.logger.info("Filtering files absolute %r.", self.time_match)
else:
self.logger.info("Filtering files relative %r.", self.time_match)
else:
self.time_match = None
self.cache = Cache(self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
getattr(self.parameters, "redis_cache_password",
None)
)
def process(self):
self.logger.debug('Downloading file list.')
files = requests.get(URL_LIST,
auth=self.auth,
proxies=self.proxy,
headers=self.http_header,
verify=self.http_verify_cert,
cert=self.ssl_client_cert,
timeout=self.http_timeout_sec)
files.raise_for_status()
self.logger.debug('Downloaded file list, %s entries.', len(files.json()))
for file in files.json():
if self.cache.get(file['Name']):
self.logger.debug('Processed file %s already.', file['Name'])
continue
if self.file_match and not self.file_match.match(file['Name']):
self.logger.debug('File %r does not match filename filter.', file['Name'])
continue
filetime = parser.parse(file['LastModified'])
if isinstance(self.time_match, datetime) and filetime < self.time_match:
self.logger.debug('File %r does not match absolute time filter.', file['Name'])
continue
else:
now = datetime.now(tz=pytz.timezone('UTC'))
if isinstance(self.time_match, timedelta) and filetime < (now - self.time_match):
self.logger.debug('File %r does not match relative time filter.', file['Name'])
continue
self.logger.debug('Processing file %r.', file['Name'])
download_url = URL_DOWNLOAD % file['Name']
download = requests.get(download_url,
auth=self.auth,
proxies=self.proxy,
headers=self.http_header,
verify=self.http_verify_cert,
cert=self.ssl_client_cert,
timeout=self.http_timeout_sec)
download.raise_for_status()
if download_url.endswith('.gz'):
raw = gzip.open(io.BytesIO(download.content)).read().decode()
else:
raw = download.text
report = self.new_report()
report.add('feed.url', download_url)
report.add('raw', raw)
self.send_message(report)
self.cache.set(file['Name'], True)
class ShadowServerAPICollectorBot(CollectorBot):
"""
Shadowserver Reports API Collector Bot
Parameters
----------
api_key: str
Your Shadowserver API key
secret: str
Your Shadowserver API secret
country: str
The country you want to download reports for (i.e. 'austria')
types: list
A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined').
"""
def init(self):
self.apikey = getattr(self.parameters, "api_key", None)
if self.apikey is None:
raise ValueError('No api_key provided.')
self.secret = getattr(self.parameters, "secret", None)
if self.secret is None:
raise ValueError('No secret provided.')
self.country = getattr(self.parameters, "country", None)
if self.country is None:
raise ValueError('No country provided.')
self.types = getattr(self.parameters, 'types', None)
if isinstance(self.types, str):
self.types = self.types.split(',')
self.preamble = '{{ "apikey": "{}" '.format(self.apikey)
self.set_request_parameters()
self.session = create_request_session(self)
self.cache = Cache(
self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
getattr(self.parameters, 'redis_cache_ttl', 864000), # 10 days
getattr(self.parameters, "redis_cache_password", None))
def _headers(self, data):
return {
'HMAC2':
hmac.new(self.secret.encode(),
data.encode('utf-8'),
digestmod=hashlib.sha256).hexdigest()
}
def _reports_list(self, date=None):
"""
Get a list of all the reports shadowserver has for a specific country
via the reports/list endpoint. If a list of types is set in the
parameters, we only process reports with those types.
To be on the safe side regarding different calculations of timestamps,
we request reports over a timespan of four days: two days in the past
until one day in the future.
The names of processed reports are cached and therefore not processed
again.
"""
if date is None:
date = datetime.today().date()
daybefore = date - timedelta(2)
dayafter = date + timedelta(1)
data = self.preamble
data += ',"report": ["{}"] '.format(self.country)
data += ',"date": "{}:{}" '.format(daybefore.isoformat(),
dayafter.isoformat())
data += '}'
self.logger.debug('Downloading report list with data: %s.', data)
response = self.session.post(APIROOT + 'reports/list',
data=data,
headers=self._headers(data))
response.raise_for_status()
reports = response.json()
self.logger.debug('Downloaded report list, %s entries.', len(reports))
if 'error' in reports:
self.logger.debug('There was an error downloading the reports: %s',
reports['error'])
return None
if self.types is not None:
reports = [
report for report in reports
if any(rtype in report['file'] for rtype in self.types)
]
return reports
def _report_download(self, reportid: str):
"""
Download one report from the shadowserver API via the reports/download endpoint
"""
data = self.preamble
data += ',"id": "{}"}}'.format(reportid)
self.logger.debug('Downloading report with data: %s.', data)
response = self.session.post(APIROOT + 'reports/download',
data=data,
headers=self._headers(data))
response.raise_for_status()
return response.text
def process(self):
"""
Download reports and send them.
Cache the filename of the report to not download the same report again.
"""
reportslist = self._reports_list()
self.logger.debug('Reports list contains %s entries after filtering.',
len(reportslist))
for item in reportslist:
filename = item['file']
if self.cache.get(filename):
self.logger.debug('Processed file %r already.', filename)
continue
self.logger.debug('Processing file %s.', filename)
reportdata = self._report_download(item['id'])
report = self.new_report()
report.add('extra.file_name', filename)
report.add('raw', str(reportdata))
self.send_message(report)
self.cache.set(filename, 1)
self.logger.debug('Sent report: %s.', filename)
class ReverseDnsExpertBot(Bot):
def init(self):
self.cache = Cache(
self.parameters.redis_cache_host, self.parameters.redis_cache_port,
self.parameters.redis_cache_db, self.parameters.redis_cache_ttl,
getattr(self.parameters, "redis_cache_password", None))
def process(self):
event = self.receive_message()
keys = ["source.%s", "destination.%s"]
for key in keys:
ip_key = key % "ip"
if ip_key not in event:
continue
ip = event.get(ip_key)
ip_version = IPAddress.version(ip)
ip_integer = IPAddress.to_int(ip)
if ip_version == 4:
minimum = MINIMUM_BGP_PREFIX_IPV4
elif ip_version == 6:
minimum = MINIMUM_BGP_PREFIX_IPV6
cache_key = bin(ip_integer)[2:minimum + 2]
cachevalue = self.cache.get(cache_key)
result = None
if cachevalue == DNS_EXCEPTION_VALUE:
continue
elif cachevalue:
result = cachevalue
else:
rev_name = reversename.from_address(ip)
try:
results = resolver.query(rev_name, "PTR")
expiration = results.expiration
for result in results:
# use first valid result
if event.is_valid('source.reverse_dns', str(result)):
break
else:
raise InvalidPTRResult
except (dns.exception.DNSException, InvalidPTRResult) as e:
# Set default TTL for 'DNS query name does not exist' error
ttl = None if isinstance(e, dns.resolver.NXDOMAIN) else \
getattr(self.parameters, "cache_ttl_invalid_response",
60)
self.cache.set(cache_key, DNS_EXCEPTION_VALUE, ttl)
result = None
else:
ttl = datetime.fromtimestamp(expiration) - datetime.now()
self.cache.set(cache_key,
str(result),
ttl=int(ttl.total_seconds()))
if result is not None:
event.add(key % 'reverse_dns', str(result), overwrite=True)
self.send_message(event)
self.acknowledge_message()
class MicrosoftInterflowCollectorBot(CollectorBot):
def check_ttl_time(self):
"""
Checks if the cache's TTL is big enough compared to the chosen
time frame so that the bot does not process the same data over and
over.
"""
if isinstance(self.time_match, datetime): # absolute
now = datetime.now(tz=pytz.timezone('UTC'))
if now - timedelta(
seconds=self.parameters.redis_cache_ttl) > self.time_match:
raise ValueError(
"The cache's TTL must be higher than 'not_older_than', "
"otherwise the bot is processing the same data over and over again."
)
def init(self):
if requests is None:
raise MissingDependencyError("requests")
self.set_request_parameters()
self.http_header['Ocp-Apim-Subscription-Key'] = self.parameters.api_key
if self.parameters.file_match:
self.file_match = re.compile(self.parameters.file_match)
else:
self.file_match = None
if self.parameters.not_older_than:
try:
self.time_match = timedelta(
minutes=parse_relative(self.parameters.not_older_than))
except ValueError:
self.time_match = parser.parse(
self.parameters.not_older_than).astimezone(pytz.utc)
self.logger.info("Filtering files absolute %r.",
self.time_match)
self.check_ttl_time()
else:
self.logger.info("Filtering files relative %r.",
self.time_match)
if timedelta(seconds=self.parameters.redis_cache_ttl
) < self.time_match:
raise ValueError(
"The cache's TTL must be higher than 'not_older_than', "
"otherwise the bot is processing the same data over and over again."
)
else:
self.time_match = None
self.session = create_request_session(self)
self.cache = Cache(
self.parameters.redis_cache_host, self.parameters.redis_cache_port,
self.parameters.redis_cache_db, self.parameters.redis_cache_ttl,
getattr(self.parameters, "redis_cache_password", None))
def process(self):
self.check_ttl_time()
self.logger.debug('Downloading file list.')
files = self.session.get(URL_LIST)
files.raise_for_status()
self.logger.debug('Downloaded file list, %s entries.',
len(files.json()))
for file in files.json():
if self.cache.get(file['Name']):
self.logger.debug('Processed file %s already.', file['Name'])
continue
if self.file_match and not self.file_match.match(file['Name']):
self.logger.debug('File %r does not match filename filter.',
file['Name'])
continue
filetime = parser.parse(file['LastModified'])
if isinstance(self.time_match,
datetime) and filetime < self.time_match:
self.logger.debug(
'File %r does not match absolute time filter.',
file['Name'])
continue
else:
now = datetime.now(tz=pytz.timezone('UTC'))
if isinstance(
self.time_match,
timedelta) and filetime < (now - self.time_match):
self.logger.debug(
'File %r does not match relative time filter.',
file['Name'])
continue
self.logger.debug('Processing file %r.', file['Name'])
download_url = URL_DOWNLOAD % file['Name']
download = self.session.get(download_url)
download.raise_for_status()
if download_url.endswith('.gz'):
raw = gzip.open(io.BytesIO(download.content)).read().decode()
else:
raw = download.text
report = self.new_report()
report.add('feed.url', download_url)
report.add('raw', raw)
self.send_message(report)
# redis-py >= 3.0.0 does no longer support boolean values, cast to string explicitly, also for backwards compatibility
self.cache.set(file['Name'], "True")
def print_filelist(self):
""" Can be called from the debugger for example. """
self.logger.debug('Downloading file list.')
files = self.session.get(URL_LIST)
files.raise_for_status()
self.logger.debug('Downloaded file list, %s entries.',
len(files.json()))
print(files.text)
class ReverseDnsExpertBot(Bot):
def init(self):
self.cache = Cache(self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
getattr(self.parameters, "redis_cache_password",
None)
)
def process(self):
event = self.receive_message()
keys = ["source.%s", "destination.%s"]
for key in keys:
ip_key = key % "ip"
if ip_key not in event:
continue
ip = event.get(ip_key)
ip_version = IPAddress.version(ip)
ip_integer = IPAddress.to_int(ip)
if ip_version == 4:
minimum = MINIMUM_BGP_PREFIX_IPV4
elif ip_version == 6:
minimum = MINIMUM_BGP_PREFIX_IPV6
cache_key = bin(ip_integer)[2: minimum + 2]
cachevalue = self.cache.get(cache_key)
result = None
if cachevalue == DNS_EXCEPTION_VALUE:
continue
elif cachevalue:
result = cachevalue
else:
rev_name = reversename.from_address(ip)
try:
results = resolver.query(rev_name, "PTR")
expiration = results.expiration
for result in results:
# use first valid result
if event.is_valid('source.reverse_dns', str(result)):
break
else:
raise InvalidPTRResult
except (dns.exception.DNSException, InvalidPTRResult) as e:
# Set default TTL for 'DNS query name does not exist' error
ttl = None if isinstance(e, dns.resolver.NXDOMAIN) else \
getattr(self.parameters, "cache_ttl_invalid_response",
60)
self.cache.set(cache_key, DNS_EXCEPTION_VALUE, ttl)
result = None
else:
ttl = datetime.fromtimestamp(expiration) - datetime.now()
self.cache.set(cache_key, str(result),
ttl=int(ttl.total_seconds()))
if result is not None:
event.add(key % 'reverse_dns', str(result), overwrite=True)
self.send_message(event)
self.acknowledge_message()
class MicrosoftInterflowCollectorBot(CollectorBot):
def check_ttl_time(self):
"""
Checks if the cache's TTL is big enough compared to the chosen
time frame so that the bot does not process the same data over and
over.
"""
if isinstance(self.time_match, datetime): # absolute
now = datetime.now(tz=pytz.timezone('UTC'))
if now - timedelta(seconds=self.parameters.redis_cache_ttl) > self.time_match:
raise ValueError("The cache's TTL must be higher than 'not_older_than', "
"otherwise the bot is processing the same data over and over again.")
def init(self):
if requests is None:
raise ValueError('Could not import requests. Please install it.')
self.set_request_parameters()
self.http_header['Ocp-Apim-Subscription-Key'] = self.parameters.api_key
if self.parameters.file_match:
self.file_match = re.compile(self.parameters.file_match)
else:
self.file_match = None
if self.parameters.not_older_than:
try:
self.time_match = timedelta(minutes=parse_relative(self.parameters.not_older_than))
except ValueError:
if sys.version_info >= (3, 6):
self.time_match = parser.parse(self.parameters.not_older_than).astimezone(pytz.utc)
else: # "astimezone() cannot be applied to a naive datetime" otherwise
if '+' not in self.parameters.not_older_than:
self.parameters.not_older_than += '+00:00'
self.time_match = parser.parse(self.parameters.not_older_than)
self.logger.info("Filtering files absolute %r.", self.time_match)
self.check_ttl_time()
else:
self.logger.info("Filtering files relative %r.", self.time_match)
if timedelta(seconds=self.parameters.redis_cache_ttl) < self.time_match:
raise ValueError("The cache's TTL must be higher than 'not_older_than', "
"otherwise the bot is processing the same data over and over again.")
else:
self.time_match = None
self.cache = Cache(self.parameters.redis_cache_host,
self.parameters.redis_cache_port,
self.parameters.redis_cache_db,
self.parameters.redis_cache_ttl,
getattr(self.parameters, "redis_cache_password",
None)
)
def process(self):
self.check_ttl_time()
self.logger.debug('Downloading file list.')
files = requests.get(URL_LIST,
auth=self.auth,
proxies=self.proxy,
headers=self.http_header,
verify=self.http_verify_cert,
cert=self.ssl_client_cert,
timeout=self.http_timeout_sec)
files.raise_for_status()
self.logger.debug('Downloaded file list, %s entries.', len(files.json()))
for file in files.json():
if self.cache.get(file['Name']):
self.logger.debug('Processed file %s already.', file['Name'])
continue
if self.file_match and not self.file_match.match(file['Name']):
self.logger.debug('File %r does not match filename filter.', file['Name'])
continue
filetime = parser.parse(file['LastModified'])
if isinstance(self.time_match, datetime) and filetime < self.time_match:
self.logger.debug('File %r does not match absolute time filter.', file['Name'])
continue
else:
now = datetime.now(tz=pytz.timezone('UTC'))
if isinstance(self.time_match, timedelta) and filetime < (now - self.time_match):
self.logger.debug('File %r does not match relative time filter.', file['Name'])
continue
self.logger.debug('Processing file %r.', file['Name'])
download_url = URL_DOWNLOAD % file['Name']
download = requests.get(download_url,
auth=self.auth,
proxies=self.proxy,
headers=self.http_header,
verify=self.http_verify_cert,
cert=self.ssl_client_cert,
timeout=self.http_timeout_sec)
download.raise_for_status()
if download_url.endswith('.gz'):
raw = gzip.open(io.BytesIO(download.content)).read().decode()
else:
raw = download.text
report = self.new_report()
report.add('feed.url', download_url)
report.add('raw', raw)
self.send_message(report)
self.cache.set(file['Name'], True)
class RIPEExpertBot(Bot):
QUERY = {
'db_ip':
'https://rest.db.ripe.net/abuse-contact/{}.json',
'db_asn':
'https://rest.db.ripe.net/abuse-contact/as{}.json',
'stat':
'https://stat.ripe.net/data/abuse-contact-finder/data.json?resource={}',
'stat_geolocation':
'https://stat.ripe.net/data/maxmind-geo-lite/data.json?resource={}',
}
REPLY_TO_DATA = {
'db_ip':
lambda x: clean_string(x['abuse-contacts']['email']),
'db_asn':
lambda x: clean_string(x['abuse-contacts']['email']),
'stat':
lambda x: clean_string(x['data']['anti_abuse_contacts']['abuse_c'][0][
'email']),
'stat_geolocation':
lambda x: clean_geo(x['data']['located_resources'][0]['locations'][0]),
}
GEOLOCATION_REPLY_TO_INTERNAL = {('cc', 'country'),
('latitude', 'latitude'),
('longitude', 'longitude'),
('city', 'city')}
def init(self):
if requests is None:
raise ValueError(
"Could not import 'requests'. Please install the package.")
self.__mode = getattr(self.parameters, 'mode', 'append')
self.__query = {
"db_asn":
getattr(self.parameters, 'query_ripe_db_asn', True),
"db_ip":
getattr(self.parameters, 'query_ripe_db_ip', True),
"stat_asn":
getattr(self.parameters, 'query_ripe_stat_asn', True),
"stat_ip":
getattr(self.parameters, 'query_ripe_stat_ip', True),
"stat_geo":
getattr(self.parameters, 'query_ripe_stat_geolocation', True)
}
self.__initialize_http_session()
self.__initialize_cache()
def __initialize_http_session(self):
self.set_request_parameters()
self.http_session = utils.create_request_session_from_bot(self)
def __initialize_cache(self):
cache_host = getattr(self.parameters, 'redis_cache_host')
cache_port = getattr(self.parameters, 'redis_cache_port')
cache_db = getattr(self.parameters, 'redis_cache_db')
cache_ttl = getattr(self.parameters, 'redis_cache_ttl')
if cache_host and cache_port and cache_db and cache_ttl:
self.__cache = Cache(
cache_host, cache_port, cache_db, cache_ttl,
getattr(self.parameters, "redis_cache_password", None))
def process(self):
event = self.receive_message()
for target in {'source.', 'destination.'}:
abuse_key = target + "abuse_contact"
abuse = set(
event.get(abuse_key).split(',')
) if self.__mode == 'append' and abuse_key in event else set()
asn = event.get(target + "asn", None)
if asn:
if self.__query['stat_asn']:
abuse.update(self.__perform_cached_query('stat', asn))
if self.__query['db_asn']:
abuse.update(self.__perform_cached_query('db_asn', asn))
ip = event.get(target + "ip", None)
if ip:
if self.__query['stat_ip']:
abuse.update(self.__perform_cached_query('stat', ip))
if self.__query['db_ip']:
abuse.update(self.__perform_cached_query('db_ip', ip))
if self.__query['stat_geo']:
info = self.__perform_cached_query('stat_geolocation', ip)
should_overwrite = self.__mode == 'replace'
for local_key, ripe_key in self.GEOLOCATION_REPLY_TO_INTERNAL:
if ripe_key in info:
event.add(target + "geolocation." + local_key,
info[ripe_key],
overwrite=should_overwrite)
event.add(abuse_key, ','.join(abuse), overwrite=True)
self.send_message(event)
self.acknowledge_message()
def __perform_cached_query(self, type, resource):
cached_value = self.__cache.get('{}:{}'.format(type, resource))
if cached_value:
if cached_value == CACHE_NO_VALUE:
return {}
else:
return json.loads(cached_value)
else:
response = self.http_session.get(self.QUERY[type].format(resource),
data="",
timeout=self.http_timeout_sec)
if response.status_code != 200:
if type == 'db_asn' and response.status_code == 404:
""" If no abuse contact could be found, a 404 is given. """
try:
if response.json()['message'].startswith(
'No abuse contact found for '):
self.__cache.set('{}:{}'.format(type, resource),
CACHE_NO_VALUE)
return {}
except ValueError:
pass
raise ValueError(STATUS_CODE_ERROR.format(
response.status_code))
try:
response_data = response.json()
# geolocation was marked as under maintenance by this, see
# https://lists.cert.at/pipermail/intelmq-users/2020-March/000140.html
status = response_data.get('data_call_status', '')
if status.startswith('maintenance'):
warnings.warn(
'The API call %s is currently under maintenance. '
'Response: %r. This warning is only given once per bot run.'
'' % (type, status))
data = self.REPLY_TO_DATA[type](response_data)
self.__cache.set(
'{}:{}'.format(type, resource),
(json.dumps(list(data) if isinstance(data, set) else data)
if data else CACHE_NO_VALUE))
return data
except (KeyError, IndexError):
self.__cache.set('{}:{}'.format(type, resource),
CACHE_NO_VALUE)
return {}
