def process(self):
event = self.receive_message()
keys = ["source.%s", "destination.%s"]
for key in keys:
ip_key = key % "ip"
if ip_key not in event:
continue
ip = event.get(ip_key)
ip_version = IPAddress.version(ip)
ip_integer = IPAddress.to_int(ip)
if ip_version == 4:
minimum = MINIMUM_BGP_PREFIX_IPV4
elif ip_version == 6:
minimum = MINIMUM_BGP_PREFIX_IPV6
cache_key = bin(ip_integer)[2:minimum + 2]
cachevalue = self.cache.get(cache_key)
result = None
if cachevalue == DNS_EXCEPTION_VALUE:
continue
elif cachevalue:
result = cachevalue
else:
rev_name = reversename.from_address(ip)
try:
results = resolver.query(rev_name, "PTR")
expiration = results.expiration
for result in results:
# use first valid result
if event.is_valid('source.reverse_dns', str(result)):
break
else:
raise InvalidPTRResult
except (dns.exception.DNSException, InvalidPTRResult) as e:
# Set default TTL for 'DNS query name does not exist' error
ttl = None if isinstance(e, dns.resolver.NXDOMAIN) else \
getattr(self.parameters, "cache_ttl_invalid_response",
60)
self.cache.set(cache_key, DNS_EXCEPTION_VALUE, ttl)
result = None
else:
ttl = datetime.fromtimestamp(expiration) - datetime.now()
self.cache.set(cache_key,
str(result),
ttl=int(ttl.total_seconds()))
if result is not None:
event.add(key % 'reverse_dns', str(result), overwrite=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
event = self.receive_message()
if event is None:
self.acknowledge_message()
return
keys = ["source.%s", "destination.%s"]
for key in keys:
ip_key = key % "ip"
if not event.contains(ip_key):
continue
ip = event.value(ip_key)
ip_version = IPAddress.version(ip)
ip_integer = IPAddress.to_int(ip)
if ip_version == 4:
minimum = MINIMUM_BGP_PREFIX_IPV4
elif ip_version == 6:
minimum = MINIMUM_BGP_PREFIX_IPV6
else:
self.logger.error("Invalid IP version {!r}".format(ip_version))
self.send_message(event)
self.acknowledge_message()
cache_key = bin(ip_integer)[2:minimum + 2]
cachevalue = self.cache.get(cache_key)
result = None
if cachevalue:
result = cachevalue
else:
rev_name = reversename.from_address(ip)
try:
result = resolver.query(rev_name, "PTR")
expiration = result.expiration
result = result[0]
except dns.exception.DNSException as e:
if isinstance(e, dns.resolver.NXDOMAIN):
continue
else:
ttl = datetime.fromtimestamp(expiration) - datetime.now()
self.cache.set(cache_key,
str(result),
ttl=int(ttl.total_seconds()))
if result is not None:
event.add(key % 'reverse_dns',
str(result),
sanitize=True,
force=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
event = self.receive_message()
keys = ["source.%s", "destination.%s"]
for key in keys:
ip_key = key % "ip"
if ip_key not in event:
continue
ip = event.get(ip_key)
ip_version = IPAddress.version(ip)
ip_integer = IPAddress.to_int(ip)
if ip_version == 4:
minimum = MINIMUM_BGP_PREFIX_IPV4
elif ip_version == 6:
minimum = MINIMUM_BGP_PREFIX_IPV6
cache_key = bin(ip_integer)[2: minimum + 2]
cachevalue = self.cache.get(cache_key)
result = None
if cachevalue == DNS_EXCEPTION_VALUE:
continue
elif cachevalue:
result = cachevalue
else:
rev_name = reversename.from_address(ip)
try:
results = resolver.query(rev_name, "PTR")
expiration = results.expiration
for result in results:
# use first valid result
if event.is_valid('source.reverse_dns', str(result)):
break
else:
raise InvalidPTRResult
except (dns.exception.DNSException, InvalidPTRResult) as e:
# Set default TTL for 'DNS query name does not exist' error
ttl = None if isinstance(e, dns.resolver.NXDOMAIN) else \
getattr(self.parameters, "cache_ttl_invalid_response",
60)
self.cache.set(cache_key, DNS_EXCEPTION_VALUE, ttl)
result = None
else:
ttl = datetime.fromtimestamp(expiration) - datetime.now()
self.cache.set(cache_key, str(result),
ttl=int(ttl.total_seconds()))
if result is not None:
event.add(key % 'reverse_dns', str(result), overwrite=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
event = self.receive_message()
if event is None:
self.acknowledge_message()
return
for key in ['source.', 'destination.']:
ip_key = key + "ip"
abuse_key = key + "abuse_contact"
asn_key = key + "asn"
ip = event.get(ip_key, None)
if not ip:
continue
ip_version = IPAddress.version(ip)
ip_integer = IPAddress.to_int(ip)
if ip_version == 4:
minimum = MINIMUM_BGP_PREFIX_IPV4
elif ip_version == 6:
minimum = MINIMUM_BGP_PREFIX_IPV6
else:
raise ValueError('Unexpected IP version '
'{!r}.'.format(ip_version))
cache_key = bin(ip_integer)[2:minimum + 2]
cache_result = self.cache.get(cache_key)
abuse = (event.get(abuse_key).split(',')
if abuse_key in event else [])
if cache_result:
cache_result = ast.literal_eval(cache_result)
cache_result = [n.strip() for n in cache_result]
abuse.extend(cache_result)
else:
asn = event.get(asn_key, None)
if self.query_db_asn and asn:
abuse.extend(lib.query_asn(asn))
if self.query_db_ip and ip:
abuse.extend(lib.query_ripedb(ip))
if self.query_stat_asn and asn:
abuse.extend(lib.query_ripestat(asn))
if self.query_stat_ip and ip:
abuse.extend(lib.query_ripestat(ip))
self.cache.set(cache_key, abuse)
event.add(abuse_key,
','.join(filter(None, set(abuse))),
force=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
event = self.receive_message()
if event is None:
self.acknowledge_message()
return
keys = ["source.%s", "destination.%s"]
for key in keys:
ip_key = key % "ip"
if not event.contains(ip_key):
continue
ip = event.value(ip_key)
ip_version = IPAddress.version(ip)
ip_integer = IPAddress.to_int(ip)
if ip_version == 4:
minimum = MINIMUM_BGP_PREFIX_IPV4
elif ip_version == 6:
minimum = MINIMUM_BGP_PREFIX_IPV6
else:
self.logger.error("Invalid IP version {!r}".format(ip_version))
self.send_message(event)
self.acknowledge_message()
cache_key = bin(ip_integer)[2: minimum + 2]
cachevalue = self.cache.get(cache_key)
result = None
if cachevalue:
result = cachevalue
else:
rev_name = reversename.from_address(ip)
try:
result = resolver.query(rev_name, "PTR")
expiration = result.expiration
result = result[0]
except dns.exception.DNSException as e:
if isinstance(e, dns.resolver.NXDOMAIN):
continue
else:
ttl = datetime.fromtimestamp(expiration) - datetime.now()
self.cache.set(cache_key, str(result),
ttl=int(ttl.total_seconds()))
if result is not None:
event.add(key % 'reverse_dns',
str(result), sanitize=True, force=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
event = self.receive_message()
if event is None:
self.acknowledge_message()
return
for key in ['source.', 'destination.']:
ip_key = key + "ip"
abuse_key = key + "abuse_contact"
asn_key = key + "asn"
ip = event.get(ip_key, None)
if not ip:
continue
ip_version = IPAddress.version(ip)
ip_integer = IPAddress.to_int(ip)
if ip_version == 4:
minimum = MINIMUM_BGP_PREFIX_IPV4
elif ip_version == 6:
minimum = MINIMUM_BGP_PREFIX_IPV6
else:
raise ValueError('Unexpected IP version '
'{!r}.'.format(ip_version))
cache_key = bin(ip_integer)[2: minimum + 2]
cache_result = self.cache.get(cache_key)
abuse = (event.get(abuse_key).split(',') if abuse_key in event
else [])
if cache_result:
cache_result = ast.literal_eval(cache_result)
cache_result = [n.strip() for n in cache_result]
abuse.extend(cache_result)
else:
asn = event.get(asn_key, None)
if self.query_db_asn and asn:
abuse.extend(lib.query_asn(asn))
if self.query_db_ip and ip:
abuse.extend(lib.query_ripedb(ip))
if self.query_stat_asn and asn:
abuse.extend(lib.query_ripestat(asn))
if self.query_stat_ip and ip:
abuse.extend(lib.query_ripestat(ip))
self.cache.set(cache_key,abuse)
event.add(abuse_key, ','.join(filter(None, set(abuse))), force=True)
self.send_message(event)
self.acknowledge_message()
def __ip_query(ip):
ip_version = IPAddress.version(ip)
reverse_ip = IPAddress.to_reverse(ip)
if ip_version == 4:
reverse = reverse_ip.split('.in-addr.arpa.')[0]
version = ""
else:
reverse = reverse_ip.split('.ip6.arpa.')[0]
version = "6"
query = IP_QUERY % (reverse, version)
return Cymru.__query(query)
def __ip_query(ip):
ip_version = IPAddress.version(ip)
reverse_ip = IPAddress.to_reverse(ip)
if ip_version == 4:
reverse = reverse_ip.split('.in-addr.arpa.')[0]
version = ""
else:
reverse = reverse_ip.split('.ip6.arpa.')[0]
version = "6"
query = IP_QUERY % (reverse, version)
return Cymru.__query(query)
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.strip()
if len(row) == 0 or not row.startswith('http'):
continue
url_object = urlparse(row)
if not url_object:
continue
url = url_object.geturl()
hostname = url_object.hostname
port = url_object.port
event = Event(report)
if IPAddress.is_valid(hostname):
event.add("source.ip", hostname)
else:
event.add("source.fqdn", hostname)
event.add('classification.type', 'malware')
event.add("source.url", url)
if port:
event.add("source.port", port)
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw")).strip()
row = raw_report.splitlines()[0]
time_str = row[row.find('(')+1:row.find(')')]
time = dateutil.parser.parse(time_str).isoformat()
for row in raw_report.split('\n'):
val = row.strip()
if not len(val) or val.startswith('#') or val.startswith('//'):
continue
event = Event(report)
if IPAddress.is_valid(val):
event.add('source.ip', val)
else:
event.add('source.network', val)
event.add('time.source', time)
event.add('classification.type', u'blacklist')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.strip()
if len(row) == 0 or row.startswith('#'):
continue
row = row.replace('\r', '')
values = row.split('\t')
# if special char is in string should not be allowed
if "#" in values[1]:
continue
# if domain name is localhost we are not interested
if values[1].lower().strip() == "localhost":
continue
event = self.new_event(report)
if IPAddress.is_valid(values[1]):
event.add("source.ip", values[1])
else:
event.add("source.fqdn", values[1])
event.add('classification.type', 'blacklist')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
event = self.receive_message()
keys = ["source.%s", "destination.%s"]
for key in keys:
ip_key = key % "ip"
if ip_key not in event:
continue
address = event.get(ip_key)
cache_key = CACHE_KEY % (IPAddress.version(address), address)
result_json = self.cache_get(cache_key)
if result_json:
result = json.loads(result_json)
else:
result = Cymru.query(address)
if not result:
self.logger.info('Got no result from Cymru for IP address %r.',
address)
result_json = json.dumps(result)
self.cache_set(cache_key, result_json)
if not result:
continue
for result_key, result_value in result.items():
if result_key == 'registry' and result_value == 'other':
continue
event.add(key % result_key, result_value, overwrite=self.overwrite)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if row == "" or row.startswith("#"):
continue
event = Event(report)
if IPAddress.is_valid(row):
event.add('source.ip', row)
else:
event.add('source.fqdn', row)
event.add('classification.type', u'malware')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if row == "" or row.startswith("#"):
continue
event = Event(report)
if IPAddress.is_valid(row, sanitize=True):
event.add('source.ip', row, sanitize=True)
else:
event.add('source.fqdn', row, sanitize=True)
event.add('classification.type', u'malware')
event.add('raw', row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
event = self.receive_message()
if event is None:
self.acknowledge_message()
return
for key in ["source.", "destination."]:
ip_key = key + "ip"
asn_key = key + "asn"
bgp_key = key + "network"
if not event.contains(ip_key):
continue
ip = event.get(ip_key)
if IPAddress.version(ip) == 6:
# Currently not supported by pyasn, fix will come soon
continue
info = self.database.lookup(ip)
if info:
if info[0]:
event.add(asn_key, str(info[0]), force=True)
if info[1]:
event.add(bgp_key, str(info[1]), force=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
event = self.receive_message()
for key in ["source.", "destination."]:
ip_key = key + "ip"
asn_key = key + "asn"
bgp_key = key + "network"
if not event.contains(ip_key):
continue
ip = event.get(ip_key)
if IPAddress.version(ip) == 6:
# Currently not supported by pyasn, fix will come soon
continue
info = self.database.lookup(ip)
if info:
if info[0]:
event.add(asn_key, str(info[0]), force=True)
if info[1]:
event.add(bgp_key, str(info[1]), force=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in csv.DictReader(io.StringIO(raw_report)):
event = Event(report)
for key, value in row.items():
if not value:
continue
if key is None:
self.logger.warning('Value without key found, skipping the'
' value: {!r}'.format(value))
continue
key = COLUMNS[key]
if key == "__IGNORE__" or key == "__TDB__":
continue
if key == "source.fqdn" and IPAddress.is_valid(value,
sanitize=True):
continue
if key == "time.source":
value = value + " UTC"
event.add(key, value)
event.add('classification.type', 'phishing')
event.add("raw", ",".join(row))
self.send_message(event)
self.acknowledge_message()
def parse_line(self, row, report):
if not row.startswith('http'):
return
url_object = urlparse(row)
if not url_object:
return
url = url_object.geturl()
hostname = url_object.hostname
port = url_object.port
event = Event(report)
if IPAddress.is_valid(hostname):
event.add("source.ip", hostname)
else:
event.add("source.fqdn", hostname)
event.add('classification.type', 'malware')
event.add("source.url", url)
if port:
event.add("source.port", port)
event.add("raw", row)
self.send_message(event)
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in csv.DictReader(io.StringIO(raw_report)):
event = self.new_event(report)
for key, value in row.items():
if not value:
continue
if key is None:
self.logger.warning('Value without key found, skipping the'
' value: {!r}'.format(value))
continue
key = COLUMNS[key]
if key == "__IGNORE__" or key == "__TDB__":
continue
if key == "source.fqdn" and IPAddress.is_valid(value,
sanitize=True):
continue
if key == "time.source":
value = value + " UTC"
event.add(key, value)
event.add('classification.type', 'phishing')
event.add("raw", ",".join(row))
self.send_message(event)
self.acknowledge_message()
def parse_line(self, line, report):
lastgenerated = None
if line.startswith('#') or len(line) == 0:
self.tempdata.append(line)
if '#Updated on' in line:
self.lastgenerated = line.strip('#Updated on ')
self.lastgenerated = dateutil.parser.parse(self.lastgenerated + ' -04:00').isoformat()
else:
event = self.new_event(report)
value = line.strip()
if self.lastgenerated:
event.add('time.source', self.lastgenerated)
event.add('raw', line)
event.add('classification.type', 'malware')
if report['feed.url'] in URLVirParserBot.IP_FEED:
event.add('source.ip', value)
event.add('event_description.text', 'Active Malicious IP Addresses Hosting Malware')
event.add('event_description.url', 'http://www.urlvir.com/search-ip-address/' + value + '/')
elif report['feed.url'] in URLVirParserBot.HOST_FEED:
if IPAddress.is_valid(value):
event.add('source.ip', value)
event.add('event_description.url', 'http://www.urlvir.com/search-ip-address/' + value + '/')
else:
event.add('source.fqdn', value)
event.add('event_description.url', 'http://www.urlvir.com/search-host/' + value + '/')
event.add('event_description.text', 'Active Malicious Hosts')
else:
raise ValueError('Unknown data feed %s.' % report['feed.url'])
yield event
def parse_line(self, row, report):
if not row.startswith('http'):
return []
url_object = urlparse(row)
if not url_object:
return []
url = url_object.geturl()
hostname = url_object.hostname
port = url_object.port
event = self.new_event(report)
if IPAddress.is_valid(hostname):
event.add("source.ip", hostname)
else:
event.add("source.fqdn", hostname)
event.add('classification.type', 'malware')
event.add("source.url", url)
if port:
event.add("source.port", port)
event.add("raw", row)
event.add("time.source", self.tempdata[1])
yield event
def parse_line(self, row, report):
if not row.startswith('http'):
return []
url_object = urlparse(row)
if not url_object:
return []
url = url_object.geturl()
hostname = url_object.hostname
port = url_object.port
event = self.new_event(report)
if IPAddress.is_valid(hostname):
event.add("source.ip", hostname)
else:
event.add("source.fqdn", hostname)
event.add('classification.type', 'malware')
event.add("source.url", url)
if port:
event.add("source.port", port)
event.add("raw", row)
event.add("time.source", self.tempdata[1])
yield event
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw")).strip()
row = raw_report.splitlines()[0]
time_str = row[row.find('(') + 1:row.find(')')]
time = dateutil.parser.parse(time_str).isoformat()
for row in raw_report.split('\n'):
val = row.strip()
if not len(val) or val.startswith('#') or val.startswith('//'):
continue
event = Event(report)
if IPAddress.is_valid(val):
event.add('source.ip', val)
else:
event.add('source.network', val)
event.add('time.source', time)
event.add('classification.type', u'blacklist')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
val = row.strip()
if not len(val) or val.startswith('#') or val.startswith('//'):
continue
event = Event(report)
if IPAddress.is_valid(val, sanitize=True):
event.add('source.ip', val, sanitize=True)
else:
event.add('source.network', val, sanitize=True)
event.add('classification.type', u'blacklist')
event.add('raw', row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, row, report):
if not row.startswith('http'):
return
url_object = urlparse(row)
if not url_object:
return
url = url_object.geturl()
hostname = url_object.hostname
port = url_object.port
event = Event(report)
if IPAddress.is_valid(hostname):
event.add("source.ip", hostname)
else:
event.add("source.fqdn", hostname)
event.add('classification.type', 'malware')
event.add("source.url", url)
if port:
event.add("source.port", port)
event.add("raw", row)
self.send_message(event)
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
val = row.strip()
if not len(val) or val.startswith('#') or val.startswith('//'):
continue
event = Event(report)
if IPAddress.is_valid(val, sanitize=True):
event.add('source.ip', val, sanitize=True)
else:
event.add('source.network', val, sanitize=True)
event.add('classification.type', u'blacklist')
event.add('raw', row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.strip()
if len(row) == 0 or row.startswith('#'):
continue
row = row.replace('\r', '')
values = row.split('\t')
# if special char is in string should not be allowed
if "#" in values[1]:
continue
# if domain name is localhost we are not interested
if values[1].lower().strip() == "localhost":
continue
event = self.new_event(report)
if IPAddress.is_valid(values[1]):
event.add("source.ip", values[1])
else:
event.add("source.fqdn", values[1])
event.add('classification.type', 'blacklist')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
event = self.receive_message()
if event is None:
self.acknowledge_message()
return
keys = ["source.%s", "destination.%s"]
for key in keys:
ip_key = key % "ip"
if not event.contains(ip_key):
continue
ip = event.value(ip_key)
ip_version = IPAddress.version(ip)
ip_integer = IPAddress.to_int(ip)
if ip_version == 4:
minimum = MINIMUM_BGP_PREFIX_IPV4
elif ip_version == 6:
minimum = MINIMUM_BGP_PREFIX_IPV6
else:
raise ValueError('Unexpected IP version '
'{!r}.'.format(ip_version))
cache_key = bin(ip_integer)[2:minimum + 2]
result_json = self.cache.get(cache_key)
if result_json:
result = json.loads(result_json)
else:
result = Cymru.query(ip)
result_json = json.dumps(result)
self.cache.set(cache_key, result_json)
for result_key, result_value in result.items():
event.add(key % result_key,
result_value,
sanitize=True,
force=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
event = self.receive_message()
keys = ["source.%s", "destination.%s"]
for key in keys:
ip_key = key % "ip"
if ip_key not in event:
continue
ip = event.get(ip_key)
ip_version = IPAddress.version(ip)
ip_integer = IPAddress.to_int(ip)
if ip_version == 4:
minimum = MINIMUM_BGP_PREFIX_IPV4
elif ip_version == 6:
minimum = MINIMUM_BGP_PREFIX_IPV6
else:
raise ValueError('Unexpected IP version '
'{!r}.'.format(ip_version))
cache_key = bin(ip_integer)[2:minimum + 2]
result_json = self.cache.get(cache_key)
if result_json:
result = json.loads(result_json)
else:
result = Cymru.query(ip)
if not result:
continue
result_json = json.dumps(result)
self.cache.set(cache_key, result_json)
for result_key, result_value in result.items():
if result_key == 'registry' and result_value == 'other':
continue
event.add(key % result_key,
result_value,
overwrite=self.overwrite)
self.send_message(event)
self.acknowledge_message()
def process(self):
event = self.receive_message()
if event is None:
self.acknowledge_message()
return
keys = ["source.%s", "destination.%s"]
for key in keys:
ip_key = key % "ip"
if not event.contains(ip_key):
continue
ip = event.value(ip_key)
ip_version = IPAddress.version(ip)
ip_integer = IPAddress.to_int(ip)
if ip_version == 4:
minimum = MINIMUM_BGP_PREFIX_IPV4
elif ip_version == 6:
minimum = MINIMUM_BGP_PREFIX_IPV6
else:
raise ValueError('Unexpected IP version '
'{!r}.'.format(ip_version))
cache_key = bin(ip_integer)[2: minimum + 2]
result_json = self.cache.get(cache_key)
if result_json:
result = json.loads(result_json)
else:
result = Cymru.query(ip)
result_json = json.dumps(result)
self.cache.set(cache_key, result_json)
for result_key, result_value in result.items():
event.add(key % result_key, result_value, sanitize=True,
force=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
event = self.receive_message()
keys = ["source.%s", "destination.%s"]
for key in keys:
ip_key = key % "ip"
if ip_key not in event:
continue
ip = event.get(ip_key)
ip_version = IPAddress.version(ip)
ip_integer = IPAddress.to_int(ip)
if ip_version == 4:
minimum = MINIMUM_BGP_PREFIX_IPV4
elif ip_version == 6:
minimum = MINIMUM_BGP_PREFIX_IPV6
else:
raise ValueError('Unexpected IP version '
'{!r}.'.format(ip_version))
cache_key = bin(ip_integer)[2: minimum + 2]
result_json = self.cache.get(cache_key)
if result_json:
result = json.loads(result_json)
else:
result = Cymru.query(ip)
if not result:
continue
result_json = json.dumps(result)
self.cache.set(cache_key, result_json)
for result_key, result_value in result.items():
if result_key == 'registry' and result_value == 'other':
continue
event.add(key % result_key, result_value, overwrite=True)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, row, report):
event = self.new_event(report)
extra = {}
for key, value in row.items():
if not value:
continue
if key is None:
self.logger.warning('Value without key found, skipping the'
' value: {!r}'.format(value))
continue
key_orig = key
key = self.csv_fieldnames[key]
if key == "__IGNORE__":
continue
if key == "source.fqdn" and IPAddress.is_valid(value,
sanitize=True):
continue
if key == "time.source":
value = value + " UTC"
if key == "source.asn":
if value.upper().startswith("ASNA"):
continue
for asn in value.upper().split(','):
if asn.startswith("AS"):
value = asn.split("AS")[1]
break
if key == "status":
if value == 'down':
value = 'offline'
elif value == 'up':
value = 'online'
if key_orig == 'scanner' and value == 'undef':
continue
if key == 'extra':
extra[key_orig] = value
continue
event.add(key, value)
if extra:
event.add('extra', extra)
event.add('classification.type', self.type)
event.add("raw", self.recover_line_csv_dict(row))
yield event
def parse_line(self, row, report):
event = self.new_event(report)
extra = {}
for key, value in row.items():
if not value:
continue
if key is None:
self.logger.warning('Value without key found, skipping the'
' value: {!r}'.format(value))
continue
key_orig = key
key = self.csv_fieldnames[key]
if key == "__IGNORE__":
continue
if key == "source.fqdn" and IPAddress.is_valid(value,
sanitize=True):
continue
if key == "time.source":
value = value + " UTC"
if key == "source.asn":
if value.upper().startswith("ASNA"):
continue
for asn in value.upper().split(','):
if asn.startswith("AS"):
value = asn.split("AS")[1]
break
if key == "status":
if value == 'down':
value = 'offline'
elif value == 'up':
value = 'online'
if key_orig == 'scanner' and value == 'undef':
continue
if key == 'extra':
extra[key_orig] = value
continue
event.add(key, value)
if extra:
event.add('extra', extra)
event.add('classification.type', self.type)
event.add("raw", self.recover_line_csv_dict(row))
yield event
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
fp = io.StringIO(raw_report)
rows = csv.DictReader(fp)
for row in rows:
event = Event(report)
for key, value in row.items():
if not value:
continue
if key is None:
self.logger.warning('Value without key found, skipping the'
' value: {!r}'.format(value))
continue
key = COLUMNS[key]
if key == "__IGNORE__" or key == "__TDB__":
continue
if key == "source.fqdn" and IPAddress.is_valid(value,
sanitize=True):
continue
if key == "time.source":
value = value + " UTC"
if key == "source.asn" and value.startswith("ASNA"):
continue
if key == "source.asn":
for asn in value.split(','):
if asn.startswith("AS"):
value = asn.split("AS")[1]
break
event.add(key, value, sanitize=True)
event.add('classification.type', u'malware')
event.add("raw", ",".join(row), sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw"))
for row in csv.DictReader(io.StringIO(raw_report)):
event = Event(report)
for key, value in row.items():
if not value:
continue
if key is None:
self.logger.warning('Value without key found, skipping the'
' value: {!r}'.format(value))
continue
key = COLUMNS[key]
if key == "__IGNORE__" or key == "__TDB__":
continue
if key == "source.fqdn" and IPAddress.is_valid(value,
sanitize=True):
continue
if key == "time.source":
value = value + " UTC"
if key == "source.asn" and value.startswith("ASNA"):
continue
if key == "source.asn":
for asn in value.split(','):
if asn.startswith("AS"):
value = asn.split("AS")[1]
break
event.add(key, value)
event.add('classification.type', u'malware')
event.add("raw", ",".join(row))
self.send_message(event)
self.acknowledge_message()
def process(self):
"""
The Ranswomware Tracker has comments in it.
The IP address field can also have more than one address.
The ASN and Country code are being ignored, an expert parser can get those added.
"""
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in csv.reader(io.StringIO(raw_report)):
if row[0].startswith('#'):
continue
if '|' in row[7]:
for ipaddr in row[7].split('|'):
new_row = '"' + row[0] + '","' + row[1] + '","' + row[2] + '","' + row[3] \
+ '","' + row[4] + '","' + row[5] + '","' + row[6] + '","' + ipaddr \
+ '","' + row[8] + '","' + row[9] + '"'
for nrow in csv.reader(io.StringIO(new_row)):
ev = Event(report)
ev.add('classification.identifier', nrow[2].lower())
ev.add('classification.type', 'c&c')
ev.add('time.source', nrow[0] + ' UTC', force=True)
ev.add('status', nrow[5])
ev.add('source.ip', nrow[7])
ev.add('raw', ','.join(nrow))
if FQDN.is_valid(nrow[3]):
ev.add('source.fqdn', nrow[3])
if URL.is_valid(nrow[4]):
ev.add('source.url', nrow[4])
self.send_message(ev)
else:
event = Event(report)
event.add('classification.identifier', row[2].lower())
event.add('classification.type', 'c&c')
event.add('time.source', row[0] + ' UTC')
event.add('status', row[5])
event.add('raw', ','.join(row))
if IPAddress.is_valid(row[7]):
event.add('source.ip', row[7])
if FQDN.is_valid(row[3]):
event.add('source.fqdn', row[3])
if URL.is_valid(row[4]):
event.add('source.url', row[4])
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.strip()
if row == "" or row.startswith("#"):
continue
event = Event(report)
if IPAddress.is_valid(row):
event.add('source.ip', row)
else:
event.add('source.fqdn', row)
event.add('classification.type', 'malware')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, line, report):
if line.startswith('#') or len(line) == 0:
self.tempdata.append(line)
else:
lvalue = line.split('\t')
event = Event(report)
event.add('classification.identifier', lvalue[0].lower())
event.add('time.source', DateTime.from_timestamp(int(lvalue[1])))
if IPAddress.is_valid(lvalue[2]):
event.add('source.ip', lvalue[2])
if FQDN.is_valid(lvalue[3]):
event.add('source.fqdn', lvalue[3])
if URL.is_valid(lvalue[4]):
event.add('source.url', lvalue[4])
event.add('raw', line)
event.add('classification.type', 'exploit')
event.add('event_description.url', 'http://data.netlab.360.com/ek')
yield event
def upload_file():
success = False
filename = os.path.join(VAR_STATE_PATH, '../webinput_csv.csv')
if 'file' in request.files and request.files['file'].filename:
request.files['file'].save(filename)
request.files['file'].stream.seek(0)
total_lines = request.files['file'].stream.read().count(b'\n') # we don't care about headers here
success = True
elif 'text' in request.form and request.form['text']:
with open(filename, mode='w', encoding='utf8') as handle:
handle.write(request.form['text'])
success = True
total_lines = len(request.form['text'].splitlines())
if not success and request.form.get('use_last_file', False):
success = True
filename, total_lines = get_temp_file()
elif success:
write_temp_file((filename, total_lines))
if not success:
return create_response('no file or text')
parameters = handle_parameters(request.form)
if parameters['has_header']:
total_lines -= 1
preview = []
valid_ip_addresses = None
valid_date_times = None
try:
with open(filename, encoding='utf8') as handle:
reader = csv.reader(handle, delimiter=parameters['delimiter'],
quotechar=parameters['quotechar'],
skipinitialspace=parameters['skipInitialSpace'],
escapechar=parameters['escapechar'],
)
for lineindex, line in enumerate(reader):
line = [col.replace(parameters['escapechar']*2, parameters['escapechar']) for col in line]
if parameters['skipInitialLines']:
if parameters['has_header'] and lineindex == 1:
for _ in range(parameters['skipInitialLines']):
line = next(reader)
elif not parameters['has_header'] and lineindex == 0:
for _ in range(parameters['skipInitialLines']):
line = next(reader)
if lineindex >= parameters['loadLinesMax'] + parameters['has_header']:
break
if valid_ip_addresses is None: # first data line
valid_ip_addresses = [0] * len(line)
valid_date_times = [0] * len(line)
for columnindex, value in enumerate(line):
if IPAddress.is_valid(value, sanitize=True):
valid_ip_addresses[columnindex] += 1
if DateTime.is_valid(value, sanitize=True):
valid_date_times[columnindex] += 1
preview.append(line)
except Exception as exc:
preview = [['Parse Error'], ['Is the number of columns consistent?']] + \
[[x] for x in traceback.format_exc().splitlines()]
column_types = ["IPAddress" if x/(total_lines if total_lines else 1) > 0.7 else None for x in valid_ip_addresses]
column_types = ["DateTime" if valid_date_times[i]/(total_lines if total_lines else 1) > 0.7 else x for i, x in enumerate(column_types)]
return create_response({"column_types": column_types,
"use_column": [bool(x) for x in column_types],
"preview": preview,
})
def process(self):
"""
The Ranswomware Tracker has comments in it.
The IP address field can also have more than one address.
The ASN and Country code are being ignored, an expert parser can get those added.
"""
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in csv.reader(io.StringIO(raw_report)):
if row[0].startswith("#"):
continue
if "|" in row[7]:
for ipaddr in row[7].split("|"):
new_row = (
'"'
+ row[0]
+ '","'
+ row[1]
+ '","'
+ row[2]
+ '","'
+ row[3]
+ '","'
+ row[4]
+ '","'
+ row[5]
+ '","'
+ row[6]
+ '","'
+ ipaddr
+ '","'
+ row[8]
+ '","'
+ row[9]
+ '"'
)
for nrow in csv.reader(io.StringIO(new_row)):
ev = Event(report)
ev.add("classification.identifier", nrow[2].lower())
ev.add("classification.type", "c&c")
ev.add("time.source", nrow[0] + " UTC", force=True)
ev.add("status", nrow[5])
ev.add("source.ip", nrow[7])
ev.add("raw", ",".join(nrow))
if FQDN.is_valid(nrow[3]):
ev.add("source.fqdn", nrow[3])
if URL.is_valid(nrow[4]):
ev.add("source.url", nrow[4])
self.send_message(ev)
else:
event = Event(report)
event.add("classification.identifier", row[2].lower())
event.add("classification.type", "c&c")
event.add("time.source", row[0] + " UTC")
event.add("status", row[5])
event.add("raw", ",".join(row))
if IPAddress.is_valid(row[7]):
event.add("source.ip", row[7])
if FQDN.is_valid(row[3]):
event.add("source.fqdn", row[3])
if URL.is_valid(row[4]):
event.add("source.url", row[4])
self.send_message(event)
self.acknowledge_message()
