def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report["raw"])
for row in raw_report.splitlines():
row = row.strip()
if not len(row) or row.startswith(';'):
continue
row_splitted = [field.strip() for field in row.split(',')]
event = self.new_event(report)
event.add('source.ip', row_splitted[0])
event.add('source.asn', row_splitted[1].replace('AS', ''))
event.add('source.geolocation.cc', row_splitted[2])
event.add('time.source',
DateTime.from_timestamp(int(row_splitted[3])))
event.add('malware.name', row_splitted[4].lower())
try:
event.add('destination.fqdn', row_splitted[5])
except InvalidValue:
pass # otherwise the same ip, ignore
event.add('destination.ip', row_splitted[6])
event.add('destination.port', row_splitted[7])
if row_splitted[8] and row_splitted[8] != '-':
event.add('extra', {'destination.local_port':
int(row_splitted[8])})
event.add('protocol.transport', row_splitted[9])
event.add('classification.type', 'botnet drone')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, line, report):
if line.startswith('#') or len(line) == 0:
self.tempdata.append(line)
else:
value = line.split('\t')
event = self.new_event(report)
event.add('classification.identifier', value[0].lower())
event.add('raw', line)
if report['feed.url'] in Netlab360ParserBot.DGA_FEED:
event.add('source.fqdn', value[1])
event.add('time.source', value[3] + ' UTC')
event.add('classification.type', 'c&c')
event.add('event_description.url', 'http://data.netlab.360.com/dga')
elif report['feed.url'] in Netlab360ParserBot.MAGNITUDE_FEED:
event.add('time.source', DateTime.from_timestamp(int(value[1])))
event.add('source.ip', value[2])
# ignore ips as fqdns
event.add('source.fqdn', value[3], raise_failure=False)
if value[4] != 'N/A':
event.add('source.url', value[4])
event.add('classification.type', 'exploit')
event.add('event_description.url', 'http://data.netlab.360.com/ek')
elif report['feed.url'] in Netlab360ParserBot.MIRAI_SCANNER_FEED:
event.add('time.source', value[0] + ' UTC')
event.add('source.ip', value[1].replace('sip=', ''))
event.add('destination.port', value[2].replace('dport=', ''))
event.add('classification.type', 'scanner')
event.add('classification.identifier', 'mirai', overwrite=True)
else:
raise ValueError('Unknown data feed %s.' % report['feed.url'])
yield event
def parse_line(self, line, report):
if line.startswith('#') or len(line) == 0:
self.tempdata.append(line)
else:
value = line.split('\t')
event = self.new_event(report)
event.add('classification.identifier', value[0].lower())
event.add('raw', line)
if report['feed.url'] in Netlab360ParserBot.DGA_FEED:
event.add('source.fqdn', value[1])
event.add('time.source', value[3] + ' UTC')
event.add('classification.type', 'c&c')
event.add('event_description.url', 'http://data.netlab.360.com/dga')
elif report['feed.url'] in Netlab360ParserBot.MAGNITUDE_FEED:
event.add('time.source', DateTime.from_timestamp(int(value[1])))
event.add('source.ip', value[2])
event.add('source.fqdn', value[3])
if value[4] != 'N/A':
event.add('source.url', value[4])
event.add('classification.type', 'exploit')
event.add('event_description.url', 'http://data.netlab.360.com/ek')
else:
raise ValueError('Unknown data feed %s.' % report['feed.url'])
yield event
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report["raw"])
for row in raw_report.splitlines():
row = row.strip()
if not len(row) or row.startswith(';'):
continue
row_splitted = [field.strip() for field in row.split(',')]
event = self.new_event(report)
event.add('source.ip', row_splitted[0])
event.add('source.asn', row_splitted[1].replace('AS', ''))
event.add('source.geolocation.cc', row_splitted[2])
event.add('time.source',
DateTime.from_timestamp(int(row_splitted[3])))
event.add('malware.name', row_splitted[4].lower())
try:
event.add('destination.fqdn', row_splitted[5])
except InvalidValue:
pass # otherwise the same ip, ignore
event.add('destination.ip', row_splitted[6])
event.add('destination.port', row_splitted[7])
if row_splitted[8] and row_splitted[8] != '-':
event.add('extra',
{'destination.local_port': int(row_splitted[8])})
event.add('protocol.transport', row_splitted[9])
event.add('classification.type', 'botnet drone')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, line, report):
if line.startswith('#') or len(line) == 0:
self.tempdata.append(line)
else:
value = line.split('\t')
event = self.new_event(report)
event.add('classification.identifier', value[0].lower())
event.add('raw', line)
if report['feed.url'] in Netlab360ParserBot.DGA_FEED:
event.add('source.fqdn', value[1])
event.add('time.source', value[3] + ' UTC')
event.add('classification.type', 'c&c')
event.add('event_description.url',
'http://data.netlab.360.com/dga')
elif report['feed.url'] in Netlab360ParserBot.MAGNITUDE_FEED:
event.add('time.source',
DateTime.from_timestamp(int(value[1])))
event.add('source.ip', value[2])
event.add('source.fqdn', value[3])
if value[4] != 'N/A':
event.add('source.url', value[4])
event.add('classification.type', 'exploit')
event.add('event_description.url',
'http://data.netlab.360.com/ek')
else:
raise ValueError('Unknown data feed %s.' % report['feed.url'])
yield event
def parse_line(self, line, report):
if line.startswith('#') or not line.strip():
self.tempdata.append(line)
else:
value = line.split('\t')
event = self.new_event(report)
event.add('classification.identifier', value[0].lower())
event.add('raw', line)
if report['feed.url'] in Netlab360ParserBot.DGA_FEED:
event.add('source.fqdn', value[1])
# DGA Feed format is
# DGA family, Domain, Start and end of valid time(UTC)
event.add('time.source', value[2] + ' UTC')
if event['time.source'] > event['time.observation']:
event.change('time.source', event['time.observation'])
event.add('classification.type', 'c2-server')
event.add('event_description.url',
'http://data.netlab.360.com/dga')
elif report['feed.url'] in Netlab360ParserBot.MAGNITUDE_FEED:
event.add('time.source',
DateTime.from_timestamp(int(value[1])))
event.add('source.ip', value[2])
# ignore ips as fqdns
event.add('source.fqdn', value[3], raise_failure=False)
if value[4] != 'N/A':
event.add('source.url', value[4])
event.add('classification.type', 'exploit')
event.add('event_description.url',
'http://data.netlab.360.com/ek')
elif report['feed.url'] in Netlab360ParserBot.MIRAI_SCANNER_FEED:
event.add('time.source', value[0] + ' UTC')
event.add('source.ip', value[1].replace('sip=', ''))
event.add('destination.port', value[2].replace('dport=', ''))
event.add('classification.type', 'scanner')
event.add('classification.identifier', 'mirai', overwrite=True)
elif report['feed.url'] in Netlab360ParserBot.HAJIME_SCANNER_FEED:
event.add('time.source', value[0] + 'T00:00:00 UTC')
event.add('source.ip', value[1].replace('ip=', ''))
event.add('classification.type', 'scanner')
event.add('classification.identifier',
'hajime',
overwrite=True)
else:
raise ValueError('Unknown data feed %s.' % report['feed.url'])
yield event
def parse_azure(self, line, report):
raw = self.recover_line(line)
event = self.new_event(report)
for key, value in line.copy().items():
if key == 'Payload':
if value == 'AA==': # NULL
del line[key]
continue
try:
value = json.loads(utils.base64_decode(value))
# continue unpacking in next loop
except json.decoder.JSONDecodeError:
line[key] = utils.base64_decode(value)
elif key == 'TLP' and value.lower() == 'unknown':
del line[key]
if isinstance(value, dict):
for subkey, subvalue in value.items():
line['%s.%s' % (key, subkey)] = subvalue
del line[key]
for key, value in line.items():
if key == 'ThreatConfidence':
if value == 'None':
continue
value = event.get('feed.accuracy',
100) * CONFIDENCE[value] / 100
elif key == 'DateTimeReceivedUtc':
value = DateTime.from_windows_nt(value)
elif key == 'Payload.ts':
value = DateTime.from_timestamp(value)
elif key == 'Payload.Protocol':
payload_protocol = value[:value.find('/')]
if payload_protocol:
# needs to overwrite a field previously parsed and written
event.add('protocol.application',
payload_protocol,
overwrite=True) # "HTTP/1.1", save additionally
elif not value:
continue
if AZURE[key] != '__IGNORE__':
# feed.accuracy is calculated newly and always needs to be overwritten
event.add(AZURE[key],
value,
overwrite=self.overwrite
or AZURE[key] == "feed.accuracy")
event.add('classification.type', 'infected-system')
event.add('raw', raw)
yield event
def process(self):
report = self.receive_message()
raw_report = json.loads(utils.base64_decode(report.get('raw')))
extra = {}
event = self.new_event(report)
event.change("feed.url", event["feed.url"].split("?key=")[0])
event.add("raw", report.get('raw'), sanitize=False)
event.add('classification.type', 'malware')
event.add('event_description.text', 'Sinkhole attempted connection')
for key, value in raw_report.items():
if key == "_ts":
event.add('time.source',
DateTime.from_timestamp(int(value))) # Source is UTC
if key == "trojanfamily":
event.add('malware.name', value)
if key == "env":
if "remote_addr" in value:
event.add('source.ip', value["remote_addr"])
if "remote_port" in value:
event.add('source.port', value["remote_port"])
if "server_addr" in value:
event.add('destination.ip', value["server_addr"])
if "server_port" in value:
event.add('destination.port', value["server_port"])
if "server_name" in value:
event.add('destination.fqdn',
value["server_name"],
raise_failure=False)
for k in [
"request_method", "cookies", "path_info",
"http_referer"
]:
if k in value:
extra[k] = value[k]
if key == "_geo_env_remote_addr":
for k, v in MAP_geo_env_remote_addr.items():
if k in value:
event[v] = value[k]
if "ip" in value and "netmask" in value:
event.add('source.network',
'%s/%s' % (value["ip"], value["netmask"]))
if key in ["_origin", "_provider", "pattern_verified"]:
extra[key] = value
if extra:
event.add('extra', extra)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report["raw"])
for row in raw_report.splitlines():
row = row.strip()
if not len(row) or row.startswith(';'):
continue
row_splitted = [field.strip() for field in row.split(',')]
event = self.new_event(report)
event.change("feed.url", event["feed.url"].split("key=")[0])
event.add('source.ip', row_splitted[0])
source_asn = row_splitted[1].replace('AS', '')
if source_asn != '?':
event.add('source.asn', source_asn)
event.add('source.geolocation.cc', row_splitted[2])
event.add('time.source',
DateTime.from_timestamp(int(row_splitted[3])))
event.add('malware.name', row_splitted[4].lower())
# otherwise the same ip, ignore
event.add('destination.fqdn', row_splitted[5], raise_failure=False)
event.add('destination.ip', row_splitted[6], raise_failure=False)
event.add('destination.port', row_splitted[7], raise_failure=False)
if row_splitted[8] and row_splitted[8] not in ('-', '?'):
try:
port = int(row_splitted[8])
except ValueError:
event.add('destination.fqdn',
row_splitted[8],
raise_failure=False)
else:
event.add('extra', {'destination.local_port': port})
event.add('protocol.transport',
row_splitted[9],
raise_failure=False)
event.add('classification.type', 'botnet drone')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = json.loads(utils.base64_decode(report.get('raw')))
extra = {}
event = self.new_event(report)
event.change("feed.url", event["feed.url"].split("?key=")[0])
event.add("raw", report.get('raw'), sanitize=False)
event.add('classification.type', 'malware')
event.add('event_description.text', 'Sinkhole attempted connection')
for key, value in raw_report.items():
if key == "_ts":
event.add('time.source', DateTime.from_timestamp(int(value))) # Source is UTC
if key == "trojanfamily":
event.add('malware.name', value)
if key == "env":
if "remote_addr" in value:
event.add('source.ip', value["remote_addr"])
if "remote_port" in value:
event.add('source.port', value["remote_port"])
if "server_addr" in value:
event.add('destination.ip', value["server_addr"])
if "server_port" in value:
event.add('destination.port', value["server_port"])
if "server_name" in value:
event.add('destination.fqdn', value["server_name"],
raise_failure=False)
for k in ["request_method", "cookies", "path_info", "http_referer"]:
if k in value:
extra[k] = value[k]
if key == "_geo_env_remote_addr":
for k, v in MAP_geo_env_remote_addr.items():
if k in value:
event[v] = value[k]
if "ip" in value and "netmask" in value:
event.add('source.network', '%s/%s' % (value["ip"], value["netmask"]))
if key in ["_origin", "_provider", "pattern_verified"]:
extra[key] = value
if extra:
event.add('extra', extra)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains('raw'):
self.acknowledge_message()
return
raw_report = json.loads(utils.base64_decode(report.get('raw')))
extra = {}
event = self.new_event(report)
event.add("raw", report.get('raw'), sanitize=False)
event.add('classification.type', 'malware')
event.add('event_description.text', 'Sinkhole attempted connection')
for key, value in raw_report.items():
if key == "_ts":
event.add('time.source',
DateTime.from_timestamp(int(value))) # Source is UTC
if key == "trojanfamily":
event.add('malware.name', value)
if key == "env":
if "remote_addr" in value:
event.add('source.ip', value["remote_addr"])
if "remote_port" in value:
event.add('source.port', value["remote_port"])
if "server_addr" in value:
event.add('destination.ip', value["server_addr"])
if "server_port" in value:
event.add('destination.port', value["server_port"])
if "server_name" in value:
try:
event.add('destination.fqdn', value["server_name"])
except InvalidValue:
pass
if "request_method" in value:
extra['request_method'] = value["request_method"]
if extra:
event.add('extra', extra)
if key == "_geo_env_remote_addr":
event.add('source.geolocation.country', value["country_name"])
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains('raw'):
self.acknowledge_message()
return
raw_report = json.loads(utils.base64_decode(report.get('raw')))
extra = {}
event = self.new_event(report)
event.add("raw", report.get('raw'), sanitize=False)
event.add('classification.type', 'malware')
event.add('event_description.text', 'Sinkhole attempted connection')
for key, value in raw_report.items():
if key == "_ts":
event.add('time.source', DateTime.from_timestamp(int(value))) # Source is UTC
if key == "trojanfamily":
event.add('malware.name', value)
if key == "env":
if "remote_addr" in value:
event.add('source.ip', value["remote_addr"])
if "remote_port" in value:
event.add('source.port', value["remote_port"])
if "server_addr" in value:
event.add('destination.ip', value["server_addr"])
if "server_port" in value:
event.add('destination.port', value["server_port"])
if "server_name" in value:
try:
event.add('destination.fqdn', value["server_name"])
except InvalidValue:
pass
if "request_method" in value:
extra['request_method'] = value["request_method"]
if extra:
event.add('extra', extra)
if key == "_geo_env_remote_addr":
event.add('source.geolocation.country', value["country_name"])
self.send_message(event)
self.acknowledge_message()
def parse_line(self, line, report):
if line.startswith('#') or len(line) == 0:
self.tempdata.append(line)
else:
value = line.split('\t')
event = self.new_event(report)
event.add('classification.identifier', value[0].lower())
event.add('raw', line)
if report['feed.url'] in Netlab360ParserBot.DGA_FEED:
event.add('source.fqdn', value[1])
event.add('time.source', value[3] + ' UTC')
event.add('classification.type', 'c&c')
event.add('event_description.url',
'http://data.netlab.360.com/dga')
elif report['feed.url'] in Netlab360ParserBot.MAGNITUDE_FEED:
event.add('time.source',
DateTime.from_timestamp(int(value[1])))
event.add('source.ip', value[2])
# ignore ips as fqdns
event.add('source.fqdn', value[3], raise_failure=False)
if value[4] != 'N/A':
event.add('source.url', value[4])
event.add('classification.type', 'exploit')
event.add('event_description.url',
'http://data.netlab.360.com/ek')
elif report['feed.url'] in Netlab360ParserBot.MIRAI_SCANNER_FEED:
event.add('time.source', value[0] + ' UTC')
event.add('source.ip', value[1].replace('sip=', ''))
event.add('destination.port', value[2].replace('dport=', ''))
event.add('classification.type', 'scanner')
event.add('classification.identifier', 'mirai', overwrite=True)
else:
raise ValueError('Unknown data feed %s.' % report['feed.url'])
yield event
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report["raw"])
for row in raw_report.splitlines():
row = row.strip()
if not len(row) or row.startswith(';'):
continue
row_splitted = [field.strip() for field in row.split(',')]
event = self.new_event(report)
event.change("feed.url", event["feed.url"].split("key=")[0])
event.add('source.ip', row_splitted[0])
event.add('source.asn', row_splitted[1].replace('AS', ''))
event.add('source.geolocation.cc', row_splitted[2])
event.add('time.source',
DateTime.from_timestamp(int(row_splitted[3])))
event.add('malware.name', row_splitted[4].lower())
# otherwise the same ip, ignore
event.add('destination.fqdn', row_splitted[5], raise_failure=False)
event.add('destination.ip', row_splitted[6], raise_failure=False)
event.add('destination.port', row_splitted[7], raise_failure=False)
if row_splitted[8] and row_splitted[8] not in ('-', '?'):
try:
port = int(row_splitted[8])
except ValueError:
event.add('destination.fqdn', row_splitted[8], raise_failure=False)
else:
event.add('extra', {'destination.local_port': port})
event.add('protocol.transport', row_splitted[9], raise_failure=False)
event.add('classification.type', 'botnet drone')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, line, report):
if line.startswith('#') or len(line) == 0:
self.tempdata.append(line)
else:
lvalue = line.split('\t')
event = Event(report)
event.add('classification.identifier', lvalue[0].lower())
event.add('time.source', DateTime.from_timestamp(int(lvalue[1])))
if IPAddress.is_valid(lvalue[2]):
event.add('source.ip', lvalue[2])
if FQDN.is_valid(lvalue[3]):
event.add('source.fqdn', lvalue[3])
if URL.is_valid(lvalue[4]):
event.add('source.url', lvalue[4])
event.add('raw', line)
event.add('classification.type', 'exploit')
event.add('event_description.url', 'http://data.netlab.360.com/ek')
yield event
def parse_line(self, row, report):
event = self.new_event(report)
for kv_pair in row.split(self.pair_separator):
(key, _, value) = kv_pair.rpartition(self.kv_separator)
if not key:
continue
if self.strip_quotes and value.startswith('"') and value.endswith(
'"'):
value = value[1:-1]
if self.keys.get(key) == 'time.source':
try:
if value.isnumeric():
value = DateTime.from_timestamp(int(value))
else:
value = parse(value, fuzzy=True).isoformat() + " UTC"
except ValueError:
value = None # Will be ignored by event.add()
self.logger.warning(
"Could not parse key %r for 'time.source'."
" Ignoring this key in line %r.", (value, row))
if key in self.keys:
event.add(self.keys[key], value, raise_failure=False)
event.add("raw", self.recover_line(row))
yield event
def process(self):
report = self.receive_message()
raw = utils.base64_decode(report.get('raw')).strip()
if not raw:
self.acknowledge_message()
return
raw_report = json.loads(raw)
del raw
event = self.new_event(report)
event.change("feed.url", event["feed.url"].split("?key=")[0])
event.add("raw", report.get('raw'), sanitize=False)
event.add('classification.type', 'malware')
event.add('classification.taxonomy', 'malicious code')
event.add('event_description.text', 'Sinkhole attempted connection')
for key, value in raw_report.items():
if key == "_ts":
event.add('time.source',
DateTime.from_timestamp(int(value))) # Source is UTC
elif key == "trojanfamily":
event.add('malware.name', value)
elif key == "env":
for subkey, subvalue in value.items():
if subkey == "remote_addr":
event.add('source.ip', subvalue)
elif subkey == "remote_port":
event.add('source.port', subvalue)
elif subkey == "server_addr":
event.add('destination.ip', subvalue)
elif subkey == "server_port":
event.add('destination.port', subvalue)
elif subkey == "server_name":
event.add('destination.fqdn',
subvalue,
raise_failure=False)
elif subkey in [
"request_method", "cookies", "path_info",
"http_referer"
]:
event['extra.%s' % subkey] = subvalue
else:
raise ValueError(
"Unable to parse data field env.%r. Please report this as bug."
% subkey)
elif key == "src" or key == 'dst':
identity = 'source' if key == 'src' else 'destination'
for subkey, subvalue in value.items():
if subkey == "ip":
event.add('%s.ip' % identity, subvalue)
elif subkey == "port":
event.add('%s.port' % identity, subvalue)
else:
raise ValueError(
"Unable to parse data field env.%r. Please report this as bug."
% subkey)
elif key == "_geo_env_remote_addr":
for k, v in MAP_geo_env_remote_addr.items():
if k in value:
event[v] = value[k]
if "ip" in value and "netmask" in value:
event.add('source.network',
'%s/%s' % (value["ip"], value["netmask"]))
elif key == 'qtype':
event['extra.dns_query_type'] = value
elif key == 'app_proto':
event.add('protocol.application', value, overwrite=True)
elif key == 'malw':
for subkey, subvalue in value.items():
if subkey == "severity":
event.add('extra.malware.severity', subvalue)
elif subkey == "family":
if self.malware_as_identifier:
event.add('classification.identifier', subvalue)
else:
if subvalue == value['variant']:
pass
else:
event.add('extra.malware.family', subvalue)
elif subkey == "variant":
event.add('malware.name', subvalue)
elif subkey == "categories":
event.add('extra.malware.categories', subvalue)
elif subkey in [
"request_method", "cookies", "path_info",
"http_referer"
]:
event['extra.%s' % subkey] = subvalue
else:
raise ValueError(
"Unable to parse data field malw.%r. Please report this as bug."
% subkey)
elif key == 'comm':
for subkey, subvalue in value.items():
if subkey == "proto":
event.add('protocol.application',
subvalue,
overwrite=True)
elif subkey == "method":
event.add('extra.communication.type', subvalue)
elif subkey == "http":
for subsubkey, subsubvalue in subvalue.items():
if subsubkey == 'method':
event.add('extra.request_method', subsubvalue)
elif subsubkey == 'host':
if not event.add('destination.fqdn',
subsubvalue,
raise_failure=False):
# event.add('destination.ip', subsubvalue)
assert raw_report['dst'][
'ip'] == subsubvalue
elif subsubkey == 'path':
event.add('destination.urlpath', subsubvalue)
elif subsubkey == 'user_agent':
event.add('extra.user_agent', subsubvalue)
elif subsubkey == 'more_headers':
event.add('extra.communication.headers',
subsubvalue)
elif subsubkey in ('cookies', 'unverified_domain',
'x_forwarded_for'):
event.add('extra.communication.%s' % subsubkey,
subsubvalue)
else:
raise ValueError(
"Unable to parse data field comm.http.%r. Please report this as bug."
% subsubkey)
try:
event.add(
'destination.url', '%s://%s%s' %
(value['proto'], subvalue['host'],
subvalue['path']))
except KeyError:
pass
elif subkey == 'dns':
for subsubkey, subsubvalue in subvalue.items():
if subsubkey == 'name':
event.add('destination.fqdn', subsubvalue)
elif subsubkey == 'qtype':
event['extra.dns_query_type'] = subsubvalue
else:
raise ValueError(
"Unable to parse data field comm.dns.%r. Please report this as bug."
% subsubkey)
elif subkey == "categories":
event.add('extra.malware.categories', subvalue)
elif subkey in [
"request_method", "cookies", "path_info",
"http_referer"
]:
event['extra.%s' % subkey] = subvalue
else:
raise ValueError(
"Unable to parse data field comm.%r. Please report this as bug."
% subkey)
elif key == 'tracking':
for subkey, subvalue in value.items():
if subkey == "id":
event.add('extra.tracking.id', subvalue)
elif subkey == 'last_ip':
event.add('extra.tracking.last.ip', subvalue)
elif subkey == 'first':
event.add('extra.first_seen', subvalue)
elif subkey == 'seen':
event.add('extra.last_seen', subvalue)
elif subkey == 'changes':
event.add('extra.tracking.changes', subvalue)
elif subkey == 'checkins':
event.add('extra.tracking.checkins', subvalue)
elif subkey == 'days':
event.add('extra.days_seen', subvalue)
elif subkey == 'same_ip':
event.add('extra.tracking.same_ip', subvalue)
elif subkey == 'tr':
event.add('extra.tracking.tr', subvalue)
else:
raise ValueError(
"Unable to parse data field tracking.%r. Please report this as bug."
% subkey)
elif key == '_geo_src_ip':
event = self.parse_geo(event, value, 'source', raw_report, key)
elif key == '_geo_tracking_last_ip':
event = self.parse_geo(event, value, 'tracking.last',
raw_report, key)
if value["path"] != 'tracking.last_ip':
raise ValueError(
'_geo_tracking_last_ip.path is not \'tracking.last_ip\' (%r).'
'' % subvalue)
elif key == '_geo_comm_http_host':
event = self.parse_geo(event, value, 'communication.http.host',
raw_report, key)
if value["path"] != 'comm.http.host':
raise ValueError(
'_geo_tracking_last_ip.path is not \'comm.http.host\' (%r).'
'' % subvalue)
elif key.startswith('_geo_comm_http_x_forwarded_for_'):
event = self.parse_geo(
event, value, 'extra.communication.http.%s' % key[15:],
raw_report, '_geo_comm_http_x_forwarded_for_')
elif key in ["_origin", "_provider", "pattern_verified"]:
event['extra.%s' % key] = value
elif key == "metadata":
for subkey, subvalue in value.items():
event['extra.metadata.%s' % subkey] = subvalue
else:
raise ValueError(
"Unable to parse data field %r. Please report this as bug."
% key)
if event.get("malware.name", None) != 'testsinkholingloss':
# used for internal tests, should actually not be part of the feed
self.logger.debug("Ignoring 'TestSinkholingLoss' event.")
self.send_message(event)
self.acknowledge_message()
def parse_line(self, row, report):
if not len(row) or row.startswith(';'):
self.tempdata.append(row)
else:
row_splitted = [field.strip() for field in row.strip().split(',')]
event = self.new_event(report)
event.change("feed.url", event["feed.url"].split("key=")[0])
event.add('source.ip', row_splitted[0])
source_asn = row_splitted[1].replace('AS', '')
if source_asn != '?':
event.add('source.asn', source_asn)
event.add('source.geolocation.cc', row_splitted[2])
event.add('time.source',
DateTime.from_timestamp(int(row_splitted[3])))
malware = row_splitted[4].lower()
if malware == 'openrelay':
event.add('classification.type', 'vulnerable service')
event.add('classification.identifier', 'openrelay')
event.add('protocol.application', 'smtp')
elif malware == 'sshauth':
event.add('classification.type', 'brute-force')
event.add('classification.identifier', 'ssh')
event.add('protocol.application', 'ssh')
elif malware == 'telnetauth':
event.add('classification.type', 'brute-force')
event.add('classification.identifier', 'telnet')
event.add('protocol.application', 'telnet')
elif malware == 'smtpauth':
event.add('classification.type', 'brute-force')
event.add('classification.identifier', 'smtp')
event.add('protocol.application', 'smtp')
elif malware in ['iotscan', 'iotuser']:
event.add('classification.type', 'scanner')
event.add('event_description.text', 'The possibly infected IoT device scanned for other vulnerable IoT devices.')
if row_splitted[7] in ['23', '2323']:
event.add('protocol.application', 'telnet')
event.add('classification.identifier', 'telnet')
else:
event.add('classification.identifier', 'scanner-generic')
elif malware == 'wpscanner':
event.add('classification.type', 'scanner')
event.add('classification.identifier', 'wordpress-vulnerabilities')
event.add('event_description.text', 'scanning for wordpress vulnerabilities')
event.add('protocol.application', 'http')
elif malware == 'w_wplogin':
event.add('classification.type', 'scanner')
event.add('classification.identifier', 'wordpress-login')
event.add('event_description.text', 'scanning for wordpress login pages')
event.add('protocol.application', 'http')
elif malware == 'l_spamlink':
event.add('classification.type', 'spam')
event.add('classification.identifier', 'spamlink')
event.add('event_description.text', 'The URL appeared in a spam email sent by extra.spam_ip.')
# event.add('protocol.application', 'http')
ip, malware_version, malware_name = row_splitted[8].split(':')
event.add('malware.name', malware_name)
event.add('malware.version', malware_version)
event.add('source.url', row_splitted[6])
event.add('extra.spam_ip', ip)
elif malware in ['pop', 'imap']:
event.add('classification.type', 'brute-force')
event.add('classification.identifier', malware)
event.add('protocol.application', malware)
elif malware in ['smb', 'rdp', 'iotrdp', 'iotmicrosoftds']:
if malware.startswith('iot'):
malware = malware[3:]
event.add('classification.type', 'scanner')
event.add('classification.identifier', malware)
event.add('protocol.application', malware)
elif malware == 'proxyget':
event.add('classification.type', 'other')
event.add('classification.identifier', malware)
event.add('event_description.text', 'The malicous client used a honeypot as proxy.')
elif malware == 'iotlogin':
event.add('classification.type', 'unauthorized-login')
event.add('classification.identifier', 'iot')
event.add('event_description.text', 'The infected iot device logged in to a honeypot.')
elif malware == 'iotcmd':
event.add('classification.type', 'unauthorized-command')
event.add('classification.identifier', 'iot')
event.add('event_description.text', 'The infected iot device logged in to a honeypot and issued malicous commands.')
elif malware == 'iotmirai':
event.add('classification.type', 'infected-system')
event.add('classification.identifier', 'mirai')
event.add('malware.name', 'mirai')
elif malware == 'ioturl':
event.add('classification.type', 'c2server')
event.add('classification.identifier', 'malware-generic')
elif malware == 'automatedtest':
event.add('classification.type', 'brute-force')
event.add('classification.identifier', 'lookup-captcha')
event.add('event_description.text', 'The device automatically brute-forced the Spamhaus CBL lookup.')
elif malware == 'authspoofbadehlo':
event.add('classification.type', 'brute-force')
event.add('classification.identifier', 'authentication-spoof')
event.add('protocol.application', 'smtp')
event.add('event_description.text', 'The device spoofed SMTP authentication with a bad EHLO.')
elif malware == 'extortion':
event.add('classification.type', 'spam')
event.add('classification.identifier', 'extortion')
if row_splitted[7] == '25':
event.add('protocol.application', 'smtp')
event.add('event_description.text', 'This device sent extortion mails.')
else:
if malware == 'auto':
malware = 's_other'
event.add('malware.name', malware)
event.add('classification.type', 'infected-system')
event.add('source.url', row_splitted[5], raise_failure=False)
# otherwise the same ip, ignore
if not (malware == 'iotscan' or # the data is wrong according to the feed provider 2018-06-15
':' in row_splitted[5]): # IP or Port in this field: also broken according to provider 2018-06-15
event.add('destination.fqdn', row_splitted[5], raise_failure=False)
event.add('destination.ip', row_splitted[6], raise_failure=False)
event.add('destination.port', row_splitted[7], raise_failure=False)
if row_splitted[8] and row_splitted[8] not in ('-', '?') and malware != 'l_spamlink':
event.add('extra.source.local_port', int(row_splitted[8]))
event.add('protocol.transport', row_splitted[9], raise_failure=False)
event.add('raw', self.recover_line(row))
yield event
def parse_line(self, row, report):
if not len(row) or row.startswith(';'):
self.tempdata.append(row)
else:
row_splitted = [field.strip() for field in row.strip().split(',')]
event = self.new_event(report)
event.change("feed.url", event["feed.url"].split("key=")[0])
event.add('source.ip', row_splitted[0])
source_asn = row_splitted[1].replace('AS', '')
if source_asn != '?':
event.add('source.asn', source_asn)
event.add('source.geolocation.cc', row_splitted[2])
event.add('time.source',
DateTime.from_timestamp(int(row_splitted[3])))
malware = row_splitted[4].lower()
if malware == 'openrelay':
event.add('classification.type', 'vulnerable service')
event.add('classification.identifier', 'openrelay')
event.add('protocol.application', 'smtp')
elif malware == 'iotrdp':
event.add('classification.type', 'brute-force')
event.add('classification.identifier', 'rdp')
event.add('protocol.application', 'rdp')
elif malware == 'sshauth':
event.add('classification.type', 'brute-force')
event.add('classification.identifier', 'ssh')
event.add('protocol.application', 'ssh')
elif malware in ('telnetauth', 'iotcmd', 'iotuser'):
event.add('classification.type', 'brute-force')
event.add('classification.identifier', 'telnet')
event.add('protocol.application', 'telnet')
elif malware == 'iotscan':
event.add('classification.type', 'scanner')
event.add('event_description.text', 'infected IoT device scanning for other vulnerable IoT devices')
if row_splitted[7] == '23':
event.add('protocol.application', 'telnet')
event.add('classification.identifier', 'telnet')
else:
event.add('classification.identifier', 'scanner-generic')
elif malware == 'wpscanner':
event.add('classification.type', 'scanner')
event.add('classification.identifier', 'wordpress-vulnerabilities')
event.add('event_description.text', 'scanning for wordpress vulnerabilities')
event.add('protocol.application', 'http')
elif malware == 'w_wplogin':
event.add('classification.type', 'scanner')
event.add('classification.identifier', 'wordpress-login')
event.add('event_description.text', 'scanning for wordpress login pages')
event.add('protocol.application', 'http')
else:
if malware == 'auto':
malware = 's_other'
event.add('malware.name', malware)
event.add('classification.type', 'botnet drone')
# otherwise the same ip, ignore
event.add('destination.fqdn', row_splitted[5], raise_failure=False)
event.add('destination.ip', row_splitted[6], raise_failure=False)
event.add('destination.port', row_splitted[7], raise_failure=False)
if row_splitted[8] and row_splitted[8] not in ('-', '?'):
try:
port = int(row_splitted[8])
except ValueError:
event.add('destination.fqdn', row_splitted[8], raise_failure=False)
else:
event.add('extra', {'destination.local_port': port})
event.add('protocol.transport', row_splitted[9], raise_failure=False)
event.add('raw', self.recover_line(row))
yield event
