def parse_azure(self, line, report):
raw = self.recover_line(line)
event = self.new_event(report)
for key, value in line.copy().items():
if key == 'Payload':
if value == 'AA==': # NULL
del line[key]
continue
try:
value = json.loads(utils.base64_decode(value))
# continue unpacking in next loop
except json.decoder.JSONDecodeError:
line[key] = utils.base64_decode(value)
elif key == 'TLP' and value.lower() == 'unknown':
del line[key]
if isinstance(value, dict):
for subkey, subvalue in value.items():
line['%s.%s' % (key, subkey)] = subvalue
del line[key]
for key, value in line.items():
if key == 'ThreatConfidence':
if value == 'None':
continue
value = event.get('feed.accuracy',
100) * CONFIDENCE[value] / 100
elif key == 'DateTimeReceivedUtc':
value = DateTime.from_windows_nt(value)
elif key == 'Payload.ts':
value = DateTime.from_timestamp(value)
elif key == 'Payload.Protocol':
payload_protocol = value[:value.find('/')]
if payload_protocol:
# needs to overwrite a field previously parsed and written
event.add('protocol.application',
payload_protocol,
overwrite=True) # "HTTP/1.1", save additionally
elif not value:
continue
if AZURE[key] != '__IGNORE__':
# feed.accuracy is calculated newly and always needs to be overwritten
event.add(AZURE[key],
value,
overwrite=self.overwrite
or AZURE[key] == "feed.accuracy")
event.add('classification.type', 'infected-system')
event.add('raw', raw)
yield event
def parse_line(self, line, report):
if line.startswith('#') or len(line) == 0:
self.tempdata.append(line)
else:
value = line.split('\t')
event = self.new_event(report)
event.add('classification.identifier', value[0].lower())
event.add('raw', line)
if report['feed.url'] in Netlab360ParserBot.DGA_FEED:
event.add('source.fqdn', value[1])
event.add('time.source', value[3] + ' UTC')
event.add('classification.type', 'c&c')
event.add('event_description.url', 'http://data.netlab.360.com/dga')
elif report['feed.url'] in Netlab360ParserBot.MAGNITUDE_FEED:
event.add('time.source', DateTime.from_timestamp(int(value[1])))
event.add('source.ip', value[2])
# ignore ips as fqdns
event.add('source.fqdn', value[3], raise_failure=False)
if value[4] != 'N/A':
event.add('source.url', value[4])
event.add('classification.type', 'exploit')
event.add('event_description.url', 'http://data.netlab.360.com/ek')
elif report['feed.url'] in Netlab360ParserBot.MIRAI_SCANNER_FEED:
event.add('time.source', value[0] + ' UTC')
event.add('source.ip', value[1].replace('sip=', ''))
event.add('destination.port', value[2].replace('dport=', ''))
event.add('classification.type', 'scanner')
event.add('classification.identifier', 'mirai', overwrite=True)
else:
raise ValueError('Unknown data feed %s.' % report['feed.url'])
yield event
def parse_line(self, line, report):
if line.startswith('#') or len(line) == 0:
self.tempdata.append(line)
else:
value = line.split('\t')
event = self.new_event(report)
event.add('classification.identifier', value[0].lower())
event.add('raw', line)
if report['feed.url'] in Netlab360ParserBot.DGA_FEED:
event.add('source.fqdn', value[1])
event.add('time.source', value[3] + ' UTC')
event.add('classification.type', 'c&c')
event.add('event_description.url',
'http://data.netlab.360.com/dga')
elif report['feed.url'] in Netlab360ParserBot.MAGNITUDE_FEED:
event.add('time.source',
DateTime.from_timestamp(int(value[1])))
event.add('source.ip', value[2])
event.add('source.fqdn', value[3])
if value[4] != 'N/A':
event.add('source.url', value[4])
event.add('classification.type', 'exploit')
event.add('event_description.url',
'http://data.netlab.360.com/ek')
else:
raise ValueError('Unknown data feed %s.' % report['feed.url'])
yield event
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report["raw"])
for row in raw_report.splitlines():
row = row.strip()
if not len(row) or row.startswith(';'):
continue
row_splitted = [field.strip() for field in row.split(',')]
event = self.new_event(report)
event.add('source.ip', row_splitted[0])
event.add('source.asn', row_splitted[1].replace('AS', ''))
event.add('source.geolocation.cc', row_splitted[2])
event.add('time.source',
DateTime.from_timestamp(int(row_splitted[3])))
event.add('malware.name', row_splitted[4].lower())
try:
event.add('destination.fqdn', row_splitted[5])
except InvalidValue:
pass # otherwise the same ip, ignore
event.add('destination.ip', row_splitted[6])
event.add('destination.port', row_splitted[7])
if row_splitted[8] and row_splitted[8] != '-':
event.add('extra', {'destination.local_port':
int(row_splitted[8])})
event.add('protocol.transport', row_splitted[9])
event.add('classification.type', 'botnet drone')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, line, report):
if line.startswith('#') or len(line) == 0:
self.tempdata.append(line)
else:
value = line.split('\t')
event = self.new_event(report)
event.add('classification.identifier', value[0].lower())
event.add('raw', line)
if report['feed.url'] in Netlab360ParserBot.DGA_FEED:
event.add('source.fqdn', value[1])
event.add('time.source', value[3] + ' UTC')
event.add('classification.type', 'c&c')
event.add('event_description.url', 'http://data.netlab.360.com/dga')
elif report['feed.url'] in Netlab360ParserBot.MAGNITUDE_FEED:
event.add('time.source', DateTime.from_timestamp(int(value[1])))
event.add('source.ip', value[2])
event.add('source.fqdn', value[3])
if value[4] != 'N/A':
event.add('source.url', value[4])
event.add('classification.type', 'exploit')
event.add('event_description.url', 'http://data.netlab.360.com/ek')
else:
raise ValueError('Unknown data feed %s.' % report['feed.url'])
yield event
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report["raw"])
for row in raw_report.splitlines():
row = row.strip()
if not len(row) or row.startswith(';'):
continue
row_splitted = [field.strip() for field in row.split(',')]
event = self.new_event(report)
event.add('source.ip', row_splitted[0])
event.add('source.asn', row_splitted[1].replace('AS', ''))
event.add('source.geolocation.cc', row_splitted[2])
event.add('time.source',
DateTime.from_timestamp(int(row_splitted[3])))
event.add('malware.name', row_splitted[4].lower())
try:
event.add('destination.fqdn', row_splitted[5])
except InvalidValue:
pass # otherwise the same ip, ignore
event.add('destination.ip', row_splitted[6])
event.add('destination.port', row_splitted[7])
if row_splitted[8] and row_splitted[8] != '-':
event.add('extra',
{'destination.local_port': int(row_splitted[8])})
event.add('protocol.transport', row_splitted[9])
event.add('classification.type', 'botnet drone')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def __process_fields(event, line, feed_url):
for field, value in zip(FEEDS[feed_url]['format'], line.split(',')):
if value and field in ('extra.first_seen', 'extra.last_online'):
if ':' in value:
event.add(field, DateTime.sanitize(value + '+00:00'))
else:
event.add(field, value + 'T00:00:00+00:00')
else:
event.add(field, value)
def parse_line(self, line, report):
domain, data, raw = line
event = self.new_event(report)
event.add('time.source', DateTime.from_epoch_millis(int(data['seen'])))
event.add('classification.type', 'other')
event.add('raw', raw)
if not event.add('source.fqdn', domain, raise_failure=False):
event.add('source.ip', domain)
yield event
def parse_line(self, line, report):
domain, data, raw = line
event = self.new_event(report)
event.add('time.source', DateTime.from_epoch_millis(int(data['seen'])))
event.add('classification.type', 'other')
event.add('raw', raw)
if not event.add('source.fqdn', domain, raise_failure=False):
event.add('source.ip', domain)
yield event
def process(self):
RT = rt.Rt(self.parameters.uri, self.parameters.user,
self.parameters.password)
if not RT.login():
raise ValueError('Login failed.')
query = RT.search(Queue=self.parameters.search_queue,
Subject__like=self.parameters.search_subject_like,
Owner=self.parameters.search_owner,
Status=self.parameters.search_status)
self.logger.info('{} results on search query.'.format(len(query)))
for ticket in query:
ticket_id = int(ticket['id'].split('/')[1])
self.logger.debug('Process ticket {}.'.format(ticket_id))
for (att_id, att_name, _, _) in RT.get_attachments(ticket_id):
if re.search(self.parameters.attachment_regex, att_name):
self.logger.debug('Found attachment {}: {!r}.'
''.format(att_id, att_name))
break
else:
self.logger.debug('No matching attachement name found.')
continue
attachment = RT.get_attachment_content(ticket_id, att_id)
if self.parameters.unzip_attachment:
file_obj = io.BytesIO(attachment)
zipped = zipfile.ZipFile(file_obj)
raw = zipped.read(zipped.namelist()[0])
else:
raw = attachment
if self.parameters.gnupg_decrypt:
raw = str(
self.gpg.decrypt(
raw,
always_trust=self.parameters.gnupg_trust,
passphrase=self.parameters.gnupg_passphrase))
self.logger.info('Successfully decrypted attachment.')
self.logger.debug(raw)
report = Report()
report.add("raw", raw, sanitize=True)
report.add("rtir_id", ticket_id, sanitize=True)
report.add("feed.name", self.parameters.feed, sanitize=True)
report.add("feed.accuracy",
self.parameters.accuracy,
sanitize=True)
time_observation = DateTime().generate_datetime_now()
report.add('time.observation', time_observation, sanitize=True)
self.send_message(report)
if self.parameters.take_ticket:
RT.edit_ticket(ticket_id, Owner=self.parameters.user)
def process(self):
self.logger.info("Downloading report through API")
otx = OTXv2(self.parameters.api_key)
pulses = otx.getall()
self.logger.info("Report downloaded.")
report = Report()
report.add("raw", json.dumps(pulses), sanitize=True)
report.add("feed.name", self.parameters.feed, sanitize=True)
report.add("feed.accuracy", self.parameters.accuracy, sanitize=True)
time_observation = DateTime().generate_datetime_now()
report.add('time.observation', time_observation, sanitize=True)
self.send_message(report)
def process(self):
mailbox = imbox.Imbox(self.parameters.mail_host,
self.parameters.mail_user,
self.parameters.mail_password,
self.parameters.mail_ssl)
emails = mailbox.messages(folder=self.parameters.mail_folder,
unread=True)
if emails:
for uid, message in emails:
if (self.parameters.mail_subject_regex and not re.search(
self.parameters.mail_subject_regex, message.subject)):
continue
self.logger.info("Reading email report")
for attach in message.attachments:
if not attach:
continue
# remove quote marks from filename
attach_name = attach['filename'][1:len(attach['filename']
) - 1]
if re.search(self.parameters.mail_attach_regex,
attach_name):
if self.parameters.mail_attach_unzip:
zipped = zipfile.ZipFile(attach['content'])
raw_report = zipped.read(zipped.namelist()[0])
else:
raw_report = attach['content'].read()
report = Report()
report.add("raw", raw_report, sanitize=True)
report.add("feed.name",
self.parameters.feed,
sanitize=True)
report.add("feed.accuracy",
self.parameters.accuracy,
sanitize=True)
time_observation = DateTime().generate_datetime_now()
report.add('time.observation',
time_observation,
sanitize=True)
self.send_message(report)
mailbox.mark_seen(uid)
self.logger.info("Email report read")
def parse_line(self, line, report):
event = self.new_event(report)
if line["version"] not in ("1.5", ''):
raise ValueError(
"Unknown version %r. Please report this with an example."
"" % line["version"])
for unknown in self._unknown_fields:
if line[unknown]:
raise ValueError(
"Unable to parse field %r. Please report this with an example"
"" % unknown)
event["extra.datasource"] = line["feed code"]
event.add("source.ip", line["source ip"])
event.add("source.network", line["source bgp prefix"])
event.add("extra.cert_eu_time_observation",
DateTime.sanitize(line["observation time"]))
event.add("tlp", line["tlp"])
event.add("event_description.text", line["description"])
event.add("classification.type", self.ABUSE_TO_INTELMQ[line["type"]])
if line['type'] == 'dropzone':
event.add("classification.identifier", 'dropzone')
if line["count"]:
event["extra.count"] = int(line["count"])
event.add("time.source", line["source time"])
event.add("source.geolocation.country", line["source country"])
event.add("protocol.application", line["protocol"])
event.add("destination.port", line["destination port"])
event.add("source.geolocation.latitude", line["source latitude"])
event.add("source.geolocation.city", line["source city"])
event.add("source.geolocation.geoip_cc", line["source cc"])
event.add("source.geolocation.longitude", line["source longitude"])
event.add("extra.source.geolocation.geohash", line["source geohash"])
event["extra.first_seen"] = line["first seen"]
event.add('feed.accuracy',
event.get('feed.accuracy', 100) *
int(line["confidence level"]) / 100,
overwrite=True)
event["extra.last_seen"] = line["last seen"]
event["extra.expiration_date"] = line["expiration date"]
if line["status"]:
event["status"] = line["status"]
event.add("event_description.target", line["target"])
event.add("source.url", line["url"])
event.add("source.abuse_contact", line["abuse contact"])
event.add("source.asn", line["source asn"])
event.add("source.as_name", line["source as name"])
event.add("source.fqdn", line["domain name"])
event.add("raw", self.recover_line(line))
yield event
def parse_line(self, line, report):
if line.startswith('#') or not line.strip():
self.tempdata.append(line)
else:
value = line.split('\t')
event = self.new_event(report)
event.add('classification.identifier', value[0].lower())
event.add('raw', line)
if report['feed.url'] in Netlab360ParserBot.DGA_FEED:
event.add('source.fqdn', value[1])
# DGA Feed format is
# DGA family, Domain, Start and end of valid time(UTC)
event.add('time.source', value[2] + ' UTC')
if event['time.source'] > event['time.observation']:
event.change('time.source', event['time.observation'])
event.add('classification.type', 'c2-server')
event.add('event_description.url',
'http://data.netlab.360.com/dga')
elif report['feed.url'] in Netlab360ParserBot.MAGNITUDE_FEED:
event.add('time.source',
DateTime.from_timestamp(int(value[1])))
event.add('source.ip', value[2])
# ignore ips as fqdns
event.add('source.fqdn', value[3], raise_failure=False)
if value[4] != 'N/A':
event.add('source.url', value[4])
event.add('classification.type', 'exploit')
event.add('event_description.url',
'http://data.netlab.360.com/ek')
elif report['feed.url'] in Netlab360ParserBot.MIRAI_SCANNER_FEED:
event.add('time.source', value[0] + ' UTC')
event.add('source.ip', value[1].replace('sip=', ''))
event.add('destination.port', value[2].replace('dport=', ''))
event.add('classification.type', 'scanner')
event.add('classification.identifier', 'mirai', overwrite=True)
elif report['feed.url'] in Netlab360ParserBot.HAJIME_SCANNER_FEED:
event.add('time.source', value[0] + 'T00:00:00 UTC')
event.add('source.ip', value[1].replace('ip=', ''))
event.add('classification.type', 'scanner')
event.add('classification.identifier',
'hajime',
overwrite=True)
else:
raise ValueError('Unknown data feed %s.' % report['feed.url'])
yield event
def on_message(self, headers, message):
self.n6stomper.logger.debug('Receive message '
'{!r}...'.format(message[:500]))
report = Report()
report.add("raw", message.rstrip(), sanitize=True)
report.add("feed.name", self.n6stomper.parameters.feed, sanitize=True)
report.add("feed.url",
"stomp://" + self.n6stomper.parameters.server + ":" +
str(self.n6stomper.parameters.port) + "/" +
self.n6stomper.parameters.exchange,
sanitize=True)
time_observation = DateTime().generate_datetime_now()
report.add('time.observation', time_observation, sanitize=True)
self.n6stomper.send_message(report)
self.n6stomper.logger.debug('Receiving Message.')
def process(self):
report = self.receive_message()
raw_report = json.loads(utils.base64_decode(report.get('raw')))
extra = {}
event = self.new_event(report)
event.change("feed.url", event["feed.url"].split("?key=")[0])
event.add("raw", report.get('raw'), sanitize=False)
event.add('classification.type', 'malware')
event.add('event_description.text', 'Sinkhole attempted connection')
for key, value in raw_report.items():
if key == "_ts":
event.add('time.source',
DateTime.from_timestamp(int(value))) # Source is UTC
if key == "trojanfamily":
event.add('malware.name', value)
if key == "env":
if "remote_addr" in value:
event.add('source.ip', value["remote_addr"])
if "remote_port" in value:
event.add('source.port', value["remote_port"])
if "server_addr" in value:
event.add('destination.ip', value["server_addr"])
if "server_port" in value:
event.add('destination.port', value["server_port"])
if "server_name" in value:
event.add('destination.fqdn',
value["server_name"],
raise_failure=False)
for k in [
"request_method", "cookies", "path_info",
"http_referer"
]:
if k in value:
extra[k] = value[k]
if key == "_geo_env_remote_addr":
for k, v in MAP_geo_env_remote_addr.items():
if k in value:
event[v] = value[k]
if "ip" in value and "netmask" in value:
event.add('source.network',
'%s/%s' % (value["ip"], value["netmask"]))
if key in ["_origin", "_provider", "pattern_verified"]:
extra[key] = value
if extra:
event.add('extra', extra)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report["raw"])
for row in raw_report.splitlines():
row = row.strip()
if not len(row) or row.startswith(';'):
continue
row_splitted = [field.strip() for field in row.split(',')]
event = self.new_event(report)
event.change("feed.url", event["feed.url"].split("key=")[0])
event.add('source.ip', row_splitted[0])
source_asn = row_splitted[1].replace('AS', '')
if source_asn != '?':
event.add('source.asn', source_asn)
event.add('source.geolocation.cc', row_splitted[2])
event.add('time.source',
DateTime.from_timestamp(int(row_splitted[3])))
event.add('malware.name', row_splitted[4].lower())
# otherwise the same ip, ignore
event.add('destination.fqdn', row_splitted[5], raise_failure=False)
event.add('destination.ip', row_splitted[6], raise_failure=False)
event.add('destination.port', row_splitted[7], raise_failure=False)
if row_splitted[8] and row_splitted[8] not in ('-', '?'):
try:
port = int(row_splitted[8])
except ValueError:
event.add('destination.fqdn',
row_splitted[8],
raise_failure=False)
else:
event.add('extra', {'destination.local_port': port})
event.add('protocol.transport',
row_splitted[9],
raise_failure=False)
event.add('classification.type', 'botnet drone')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = json.loads(utils.base64_decode(report.get('raw')))
extra = {}
event = self.new_event(report)
event.change("feed.url", event["feed.url"].split("?key=")[0])
event.add("raw", report.get('raw'), sanitize=False)
event.add('classification.type', 'malware')
event.add('event_description.text', 'Sinkhole attempted connection')
for key, value in raw_report.items():
if key == "_ts":
event.add('time.source', DateTime.from_timestamp(int(value))) # Source is UTC
if key == "trojanfamily":
event.add('malware.name', value)
if key == "env":
if "remote_addr" in value:
event.add('source.ip', value["remote_addr"])
if "remote_port" in value:
event.add('source.port', value["remote_port"])
if "server_addr" in value:
event.add('destination.ip', value["server_addr"])
if "server_port" in value:
event.add('destination.port', value["server_port"])
if "server_name" in value:
event.add('destination.fqdn', value["server_name"],
raise_failure=False)
for k in ["request_method", "cookies", "path_info", "http_referer"]:
if k in value:
extra[k] = value[k]
if key == "_geo_env_remote_addr":
for k, v in MAP_geo_env_remote_addr.items():
if k in value:
event[v] = value[k]
if "ip" in value and "netmask" in value:
event.add('source.network', '%s/%s' % (value["ip"], value["netmask"]))
if key in ["_origin", "_provider", "pattern_verified"]:
extra[key] = value
if extra:
event.add('extra', extra)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains('raw'):
self.acknowledge_message()
return
raw_report = json.loads(utils.base64_decode(report.get('raw')))
extra = {}
event = self.new_event(report)
event.add("raw", report.get('raw'), sanitize=False)
event.add('classification.type', 'malware')
event.add('event_description.text', 'Sinkhole attempted connection')
for key, value in raw_report.items():
if key == "_ts":
event.add('time.source',
DateTime.from_timestamp(int(value))) # Source is UTC
if key == "trojanfamily":
event.add('malware.name', value)
if key == "env":
if "remote_addr" in value:
event.add('source.ip', value["remote_addr"])
if "remote_port" in value:
event.add('source.port', value["remote_port"])
if "server_addr" in value:
event.add('destination.ip', value["server_addr"])
if "server_port" in value:
event.add('destination.port', value["server_port"])
if "server_name" in value:
try:
event.add('destination.fqdn', value["server_name"])
except InvalidValue:
pass
if "request_method" in value:
extra['request_method'] = value["request_method"]
if extra:
event.add('extra', extra)
if key == "_geo_env_remote_addr":
event.add('source.geolocation.country', value["country_name"])
self.send_message(event)
self.acknowledge_message()
def process(self):
mailbox = imbox.Imbox(self.parameters.mail_host,
self.parameters.mail_user,
self.parameters.mail_password,
self.parameters.mail_ssl)
emails = mailbox.messages(folder=self.parameters.mail_folder, unread=True)
if emails:
for uid, message in emails:
if (self.parameters.mail_subject_regex and
not re.search(self.parameters.mail_subject_regex,
message.subject)):
continue
self.logger.info("Reading email report")
for body in message.body['plain']:
match = re.search(self.parameters.mail_url_regex, body)
if match:
url = match.group()
self.logger.info("Downloading report from %s" % url)
raw_report = fetch_url(url, timeout=60.0,
chunk_size=16384)
self.logger.info("Report downloaded.")
report = Report()
report.add("raw", raw_report, sanitize=True)
report.add("feed.name",
self.parameters.feed, sanitize=True)
report.add("feed.accuracy", self.parameters.accuracy, sanitize=True)
time_observation = DateTime().generate_datetime_now()
report.add('time.observation', time_observation,
sanitize=True)
self.send_message(report)
mailbox.mark_seen(uid)
self.logger.info("Email report read")
def process(self):
report = self.receive_message()
if report is None or not report.contains('raw'):
self.acknowledge_message()
return
raw_report = json.loads(utils.base64_decode(report.get('raw')))
extra = {}
event = self.new_event(report)
event.add("raw", report.get('raw'), sanitize=False)
event.add('classification.type', 'malware')
event.add('event_description.text', 'Sinkhole attempted connection')
for key, value in raw_report.items():
if key == "_ts":
event.add('time.source', DateTime.from_timestamp(int(value))) # Source is UTC
if key == "trojanfamily":
event.add('malware.name', value)
if key == "env":
if "remote_addr" in value:
event.add('source.ip', value["remote_addr"])
if "remote_port" in value:
event.add('source.port', value["remote_port"])
if "server_addr" in value:
event.add('destination.ip', value["server_addr"])
if "server_port" in value:
event.add('destination.port', value["server_port"])
if "server_name" in value:
try:
event.add('destination.fqdn', value["server_name"])
except InvalidValue:
pass
if "request_method" in value:
extra['request_method'] = value["request_method"]
if extra:
event.add('extra', extra)
if key == "_geo_env_remote_addr":
event.add('source.geolocation.country', value["country_name"])
self.send_message(event)
self.acknowledge_message()
def parse_line(self, line, report):
if line.startswith('#') or len(line) == 0:
self.tempdata.append(line)
else:
value = line.split('\t')
event = self.new_event(report)
event.add('classification.identifier', value[0].lower())
event.add('raw', line)
if report['feed.url'] in Netlab360ParserBot.DGA_FEED:
event.add('source.fqdn', value[1])
event.add('time.source', value[3] + ' UTC')
event.add('classification.type', 'c&c')
event.add('event_description.url',
'http://data.netlab.360.com/dga')
elif report['feed.url'] in Netlab360ParserBot.MAGNITUDE_FEED:
event.add('time.source',
DateTime.from_timestamp(int(value[1])))
event.add('source.ip', value[2])
# ignore ips as fqdns
event.add('source.fqdn', value[3], raise_failure=False)
if value[4] != 'N/A':
event.add('source.url', value[4])
event.add('classification.type', 'exploit')
event.add('event_description.url',
'http://data.netlab.360.com/ek')
elif report['feed.url'] in Netlab360ParserBot.MIRAI_SCANNER_FEED:
event.add('time.source', value[0] + ' UTC')
event.add('source.ip', value[1].replace('sip=', ''))
event.add('destination.port', value[2].replace('dport=', ''))
event.add('classification.type', 'scanner')
event.add('classification.identifier', 'mirai', overwrite=True)
else:
raise ValueError('Unknown data feed %s.' % report['feed.url'])
yield event
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report["raw"])
for row in raw_report.splitlines():
row = row.strip()
if not len(row) or row.startswith(';'):
continue
row_splitted = [field.strip() for field in row.split(',')]
event = self.new_event(report)
event.change("feed.url", event["feed.url"].split("key=")[0])
event.add('source.ip', row_splitted[0])
event.add('source.asn', row_splitted[1].replace('AS', ''))
event.add('source.geolocation.cc', row_splitted[2])
event.add('time.source',
DateTime.from_timestamp(int(row_splitted[3])))
event.add('malware.name', row_splitted[4].lower())
# otherwise the same ip, ignore
event.add('destination.fqdn', row_splitted[5], raise_failure=False)
event.add('destination.ip', row_splitted[6], raise_failure=False)
event.add('destination.port', row_splitted[7], raise_failure=False)
if row_splitted[8] and row_splitted[8] not in ('-', '?'):
try:
port = int(row_splitted[8])
except ValueError:
event.add('destination.fqdn', row_splitted[8], raise_failure=False)
else:
event.add('extra', {'destination.local_port': port})
event.add('protocol.transport', row_splitted[9], raise_failure=False)
event.add('classification.type', 'botnet drone')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, line, report):
if line.startswith('#') or len(line) == 0:
self.tempdata.append(line)
else:
lvalue = line.split('\t')
event = Event(report)
event.add('classification.identifier', lvalue[0].lower())
event.add('time.source', DateTime.from_timestamp(int(lvalue[1])))
if IPAddress.is_valid(lvalue[2]):
event.add('source.ip', lvalue[2])
if FQDN.is_valid(lvalue[3]):
event.add('source.fqdn', lvalue[3])
if URL.is_valid(lvalue[4]):
event.add('source.url', lvalue[4])
event.add('raw', line)
event.add('classification.type', 'exploit')
event.add('event_description.url', 'http://data.netlab.360.com/ek')
yield event
def process(self):
self.logger.info("Downloading report from %s" %
self.parameters.http_url)
resp = requests.get(url=self.parameters.http_url,
auth=self.auth,
proxies=self.proxy,
headers=self.http_header,
verify=self.http_verify_cert)
if resp.status_code // 100 != 2:
raise ValueError('HTTP response status code was {}.'
''.format(resp.status_code))
self.logger.info("Report downloaded.")
report = Report()
report.add("raw", resp.text, sanitize=True)
report.add("feed.name", self.parameters.feed, sanitize=True)
report.add("feed.url", self.parameters.http_url, sanitize=True)
report.add("feed.accuracy", self.parameters.accuracy, sanitize=True)
time_observation = DateTime().generate_datetime_now()
report.add('time.observation', time_observation, sanitize=True)
self.send_message(report)
def parse_line(self, row, report):
event = self.new_event(report)
for kv_pair in row.split(self.pair_separator):
(key, _, value) = kv_pair.rpartition(self.kv_separator)
if not key:
continue
if self.strip_quotes and value.startswith('"') and value.endswith(
'"'):
value = value[1:-1]
if self.keys.get(key) == 'time.source':
try:
if value.isnumeric():
value = DateTime.from_timestamp(int(value))
else:
value = parse(value, fuzzy=True).isoformat() + " UTC"
except ValueError:
value = None # Will be ignored by event.add()
self.logger.warning(
"Could not parse key %r for 'time.source'."
" Ignoring this key in line %r.", (value, row))
if key in self.keys:
event.add(self.keys[key], value, raise_failure=False)
event.add("raw", self.recover_line(row))
yield event
def process(self):
report = self.receive_message()
peek = utils.base64_decode(report.get("raw"))
self.logger.debug("Peeking at event %r.", peek)
if "TEST MESSAGE" in peek:
self.logger.debug("Ignoring test message/event.")
self.acknowledge_message()
return
# try to parse a JSON object
event = self.new_event(report)
dict_report = json.loads(peek)
event.add("raw", report.get("raw"), sanitize=False)
if "time" in dict_report:
event.add("time.source", dict_report["time"])
if "dip" in dict_report:
event.add("destination.ip", dict_report["dip"])
if "dport" in dict_report:
event.add("destination.port", dict_report["dport"])
if "md5" in dict_report:
event.add("malware.hash.md5", dict_report["md5"])
if "sha1" in dict_report:
event.add("malware.hash.sha1", dict_report["sha1"])
if "fqdn" in dict_report:
if dict_report["fqdn"] == 'unknown':
del dict_report["fqdn"]
else:
event.add("source.fqdn", dict_report["fqdn"])
if "id" in dict_report:
event["extra.feed_id"] = dict_report["id"]
if "adip" in dict_report:
event["extra.adip"] = dict_report["adip"]
if "proto" in dict_report:
event.add("protocol.transport", dict_report["proto"])
if "sport" in dict_report:
event.add("source.port", dict_report["sport"])
if "url" in dict_report:
event.add("source.url", dict_report["url"])
if "confidence" in dict_report:
event.add("extra.confidence", dict_report["confidence"])
if "expires" in dict_report:
event.add("extra.expires", DateTime.sanitize(dict_report["expires"]))
if "source" in dict_report:
event.add("extra.feed_source", dict_report["source"])
if ("category" in dict_report and "name" in dict_report and
dict_report["category"] == 'bots'):
event.add("malware.name", dict_report["name"])
if ("name" in dict_report):
mapping['bots']['identifier'] = dict_report["name"]
else:
mapping['bots']['identifier'] = "generic-n6-drone"
if dict_report["type"] == "bl-update":
event.add("classification.taxonomy", "other")
event.add("classification.type", "blacklist")
elif dict_report["category"] is not None:
event.add("classification.taxonomy",
mapping[dict_report["category"]]["taxonomy"],
overwrite=True)
event.add("classification.type",
mapping[dict_report["category"]]["type"],
overwrite=True)
event.add("classification.identifier",
mapping[dict_report["category"]]["identifier"],
overwrite=True)
# split up the event into multiple ones, one for each address
for addr in dict_report.get('address', []):
ev = self.new_event(event)
ev.add("source.ip", addr["ip"])
if ("asn" in addr):
ev.add("source.asn", addr["asn"])
if ("rdns" in addr):
ev.add("source.reverse_dns", addr["rdns"])
# XXX ignore for now, only relevant for flows
# ev.add("source.dir", addr["dir"])
if ("cc" in addr):
ev.add("source.geolocation.cc", addr["cc"])
self.send_message(ev)
else: # no address
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report["raw"])
soup = bs(raw_report, self.parser)
if self.attr_name:
table = soup.find_all('table',
attrs={self.attr_name: self.attr_value})
self.logger.debug('Found %d table(s) by attribute %r: %r.',
(len(table), self.attr_name, self.attr_value))
else:
table = soup.find_all('table')
self.logger.debug('Found %d table(s).', len(table))
table = table[self.table_index]
rows = table.find_all('tr')[self.skip_row:]
self.logger.debug('Handling %d row(s).', len(rows))
for feed in rows:
event = self.new_event(report)
tdata = [data.text for data in feed.find_all('td')]
data_added = False
for key, data, ignore_value in zip(self.columns, tdata,
self.ignore_values):
keys = key.split('|') if '|' in key else [
key,
]
data = data.strip()
if data == ignore_value:
continue
for key in keys:
if isinstance(
data,
str) and not data: # empty string is never valid
break
if key in ["__IGNORE__", ""]:
break
if self.split_column and key == self.split_column:
data = data.split(self.split_separator)[int(
self.split_index)]
data = data.strip()
if key in ["time.source", "time.destination"]:
try:
data = int(data)
except ValueError:
pass
data = DateTime.convert(data, format=self.time_format)
elif key.endswith('.url'):
if not data:
continue
if '://' not in data:
data = self.default_url_protocol + data
if event.add(key, data, raise_failure=False):
data_added = True
break
else:
raise ValueError(
"Could not add value %r to %s, all invalid."
"" % (data, keys))
if not data_added:
# we added nothing from this row, so skip it
continue
if hasattr(self.parameters, 'type')\
and "classification.type" not in event:
event.add('classification.type', self.parameters.type)
event.add('raw', feed)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, row, report):
if not len(row) or row.startswith(';'):
self.tempdata.append(row)
else:
row_splitted = [field.strip() for field in row.strip().split(',')]
event = self.new_event(report)
event.change("feed.url", event["feed.url"].split("key=")[0])
event.add('source.ip', row_splitted[0])
source_asn = row_splitted[1].replace('AS', '')
if source_asn != '?':
event.add('source.asn', source_asn)
event.add('source.geolocation.cc', row_splitted[2])
event.add('time.source',
DateTime.from_timestamp(int(row_splitted[3])))
malware = row_splitted[4].lower()
if malware == 'openrelay':
event.add('classification.type', 'vulnerable service')
event.add('classification.identifier', 'openrelay')
event.add('protocol.application', 'smtp')
elif malware == 'iotrdp':
event.add('classification.type', 'brute-force')
event.add('classification.identifier', 'rdp')
event.add('protocol.application', 'rdp')
elif malware == 'sshauth':
event.add('classification.type', 'brute-force')
event.add('classification.identifier', 'ssh')
event.add('protocol.application', 'ssh')
elif malware in ('telnetauth', 'iotcmd', 'iotuser'):
event.add('classification.type', 'brute-force')
event.add('classification.identifier', 'telnet')
event.add('protocol.application', 'telnet')
elif malware == 'iotscan':
event.add('classification.type', 'scanner')
event.add('event_description.text', 'infected IoT device scanning for other vulnerable IoT devices')
if row_splitted[7] == '23':
event.add('protocol.application', 'telnet')
event.add('classification.identifier', 'telnet')
else:
event.add('classification.identifier', 'scanner-generic')
elif malware == 'wpscanner':
event.add('classification.type', 'scanner')
event.add('classification.identifier', 'wordpress-vulnerabilities')
event.add('event_description.text', 'scanning for wordpress vulnerabilities')
event.add('protocol.application', 'http')
elif malware == 'w_wplogin':
event.add('classification.type', 'scanner')
event.add('classification.identifier', 'wordpress-login')
event.add('event_description.text', 'scanning for wordpress login pages')
event.add('protocol.application', 'http')
else:
if malware == 'auto':
malware = 's_other'
event.add('malware.name', malware)
event.add('classification.type', 'botnet drone')
# otherwise the same ip, ignore
event.add('destination.fqdn', row_splitted[5], raise_failure=False)
event.add('destination.ip', row_splitted[6], raise_failure=False)
event.add('destination.port', row_splitted[7], raise_failure=False)
if row_splitted[8] and row_splitted[8] not in ('-', '?'):
try:
port = int(row_splitted[8])
except ValueError:
event.add('destination.fqdn', row_splitted[8], raise_failure=False)
else:
event.add('extra', {'destination.local_port': port})
event.add('protocol.transport', row_splitted[9], raise_failure=False)
event.add('raw', self.recover_line(row))
yield event
def process(self):
report = self.receive_message()
raw = utils.base64_decode(report.get('raw')).strip()
if not raw:
self.acknowledge_message()
return
raw_report = json.loads(raw)
del raw
event = self.new_event(report)
event.change("feed.url", event["feed.url"].split("?key=")[0])
event.add("raw", report.get('raw'), sanitize=False)
event.add('classification.type', 'malware')
event.add('classification.taxonomy', 'malicious code')
event.add('event_description.text', 'Sinkhole attempted connection')
for key, value in raw_report.items():
if key == "_ts":
event.add('time.source',
DateTime.from_timestamp(int(value))) # Source is UTC
elif key == "trojanfamily":
event.add('malware.name', value)
elif key == "env":
for subkey, subvalue in value.items():
if subkey == "remote_addr":
event.add('source.ip', subvalue)
elif subkey == "remote_port":
event.add('source.port', subvalue)
elif subkey == "server_addr":
event.add('destination.ip', subvalue)
elif subkey == "server_port":
event.add('destination.port', subvalue)
elif subkey == "server_name":
event.add('destination.fqdn',
subvalue,
raise_failure=False)
elif subkey in [
"request_method", "cookies", "path_info",
"http_referer"
]:
event['extra.%s' % subkey] = subvalue
else:
raise ValueError(
"Unable to parse data field env.%r. Please report this as bug."
% subkey)
elif key == "src" or key == 'dst':
identity = 'source' if key == 'src' else 'destination'
for subkey, subvalue in value.items():
if subkey == "ip":
event.add('%s.ip' % identity, subvalue)
elif subkey == "port":
event.add('%s.port' % identity, subvalue)
else:
raise ValueError(
"Unable to parse data field env.%r. Please report this as bug."
% subkey)
elif key == "_geo_env_remote_addr":
for k, v in MAP_geo_env_remote_addr.items():
if k in value:
event[v] = value[k]
if "ip" in value and "netmask" in value:
event.add('source.network',
'%s/%s' % (value["ip"], value["netmask"]))
elif key == 'qtype':
event['extra.dns_query_type'] = value
elif key == 'app_proto':
event.add('protocol.application', value, overwrite=True)
elif key == 'malw':
for subkey, subvalue in value.items():
if subkey == "severity":
event.add('extra.malware.severity', subvalue)
elif subkey == "family":
if self.malware_as_identifier:
event.add('classification.identifier', subvalue)
else:
if subvalue == value['variant']:
pass
else:
event.add('extra.malware.family', subvalue)
elif subkey == "variant":
event.add('malware.name', subvalue)
elif subkey == "categories":
event.add('extra.malware.categories', subvalue)
elif subkey in [
"request_method", "cookies", "path_info",
"http_referer"
]:
event['extra.%s' % subkey] = subvalue
else:
raise ValueError(
"Unable to parse data field malw.%r. Please report this as bug."
% subkey)
elif key == 'comm':
for subkey, subvalue in value.items():
if subkey == "proto":
event.add('protocol.application',
subvalue,
overwrite=True)
elif subkey == "method":
event.add('extra.communication.type', subvalue)
elif subkey == "http":
for subsubkey, subsubvalue in subvalue.items():
if subsubkey == 'method':
event.add('extra.request_method', subsubvalue)
elif subsubkey == 'host':
if not event.add('destination.fqdn',
subsubvalue,
raise_failure=False):
# event.add('destination.ip', subsubvalue)
assert raw_report['dst'][
'ip'] == subsubvalue
elif subsubkey == 'path':
event.add('destination.urlpath', subsubvalue)
elif subsubkey == 'user_agent':
event.add('extra.user_agent', subsubvalue)
elif subsubkey == 'more_headers':
event.add('extra.communication.headers',
subsubvalue)
elif subsubkey in ('cookies', 'unverified_domain',
'x_forwarded_for'):
event.add('extra.communication.%s' % subsubkey,
subsubvalue)
else:
raise ValueError(
"Unable to parse data field comm.http.%r. Please report this as bug."
% subsubkey)
try:
event.add(
'destination.url', '%s://%s%s' %
(value['proto'], subvalue['host'],
subvalue['path']))
except KeyError:
pass
elif subkey == 'dns':
for subsubkey, subsubvalue in subvalue.items():
if subsubkey == 'name':
event.add('destination.fqdn', subsubvalue)
elif subsubkey == 'qtype':
event['extra.dns_query_type'] = subsubvalue
else:
raise ValueError(
"Unable to parse data field comm.dns.%r. Please report this as bug."
% subsubkey)
elif subkey == "categories":
event.add('extra.malware.categories', subvalue)
elif subkey in [
"request_method", "cookies", "path_info",
"http_referer"
]:
event['extra.%s' % subkey] = subvalue
else:
raise ValueError(
"Unable to parse data field comm.%r. Please report this as bug."
% subkey)
elif key == 'tracking':
for subkey, subvalue in value.items():
if subkey == "id":
event.add('extra.tracking.id', subvalue)
elif subkey == 'last_ip':
event.add('extra.tracking.last.ip', subvalue)
elif subkey == 'first':
event.add('extra.first_seen', subvalue)
elif subkey == 'seen':
event.add('extra.last_seen', subvalue)
elif subkey == 'changes':
event.add('extra.tracking.changes', subvalue)
elif subkey == 'checkins':
event.add('extra.tracking.checkins', subvalue)
elif subkey == 'days':
event.add('extra.days_seen', subvalue)
elif subkey == 'same_ip':
event.add('extra.tracking.same_ip', subvalue)
elif subkey == 'tr':
event.add('extra.tracking.tr', subvalue)
else:
raise ValueError(
"Unable to parse data field tracking.%r. Please report this as bug."
% subkey)
elif key == '_geo_src_ip':
event = self.parse_geo(event, value, 'source', raw_report, key)
elif key == '_geo_tracking_last_ip':
event = self.parse_geo(event, value, 'tracking.last',
raw_report, key)
if value["path"] != 'tracking.last_ip':
raise ValueError(
'_geo_tracking_last_ip.path is not \'tracking.last_ip\' (%r).'
'' % subvalue)
elif key == '_geo_comm_http_host':
event = self.parse_geo(event, value, 'communication.http.host',
raw_report, key)
if value["path"] != 'comm.http.host':
raise ValueError(
'_geo_tracking_last_ip.path is not \'comm.http.host\' (%r).'
'' % subvalue)
elif key.startswith('_geo_comm_http_x_forwarded_for_'):
event = self.parse_geo(
event, value, 'extra.communication.http.%s' % key[15:],
raw_report, '_geo_comm_http_x_forwarded_for_')
elif key in ["_origin", "_provider", "pattern_verified"]:
event['extra.%s' % key] = value
elif key == "metadata":
for subkey, subvalue in value.items():
event['extra.metadata.%s' % subkey] = subvalue
else:
raise ValueError(
"Unable to parse data field %r. Please report this as bug."
% key)
if event.get("malware.name", None) != 'testsinkholingloss':
# used for internal tests, should actually not be part of the feed
self.logger.debug("Ignoring 'TestSinkholingLoss' event.")
self.send_message(event)
self.acknowledge_message()
def process(self):
mailbox = imbox.Imbox(self.parameters.mail_host, self.parameters.mail_user, utils.base64_decode(self.parameters.mail_password), self.parameters.mail_ssl)
self.logger.info("Connected to mail server")
emails = mailbox.messages(folder=self.parameters.folder, unread=True)
try:
if emails:
self.logger.info("Parsing emails in mailbox")
for uid, message in emails:
if self.parameters.subject_regex and not re.search(self.parameters.subject_regex, message.subject):
continue
self.logger.info("Reading email report")
if hasattr(message,'attachments') and message.attachments:
for attach in message.attachments:
if not attach:
continue
attach_name = attach['filename'][1:len(attach['filename'])-1] # remove quote marks from filename
if re.search(self.parameters.attach_regex, attach_name):
self.logger.info("Parsing attachment")
if self.parameters.attach_unzip:
zipped = zipfile.ZipFile(attach['content'])
raw_report = zipped.read(zipped.namelist()[0])
else:
raw_report = attach['content'].read()
self.logger.info('content read')
report = Report()
report.add("raw", raw_report, sanitize=True)
report.add("feed.name", self.parameters.feed,sanitize=True)
report.add("feed.accuracy", self.parameters.accuracy, sanitize=True)
time_observation = DateTime().generate_datetime_now()
#report.add('time.observation', time_observation)
report.add('feed.reportname', message.subject, sanitize=True)
self.logger.info('rocking in a free world')
self.send_message(report)
self.logger.info('just some administration left')
mailbox.mark_seen(uid)
self.logger.info("Email report read")
else:
# If no attachment, read from url
# update way of fetching from url to new way in http/
self.logger.info("No attachment found, trying collecting from URL")
for body in message.body['plain']:
self.logger.info("Parsing message body")
match = re.search(self.parameters.url_regex, body)
if match:
url = match.group()
self.logger.info("Downloading report from %s" % url)
resp = requests.get(url=url)
if resp.status_code // 100 != 2:
raise ValueError('HTTP response status code was {}.' ''.format(resp.status_code))
raw_report = resp.content
self.logger.info("Report downloaded.")
report = Report()
report.add("raw", raw_report, sanitize=True)
report.add("feed.name", self.parameters.feed, sanitize=True)
report.add("feed.accuracy", self.parameters.accuracy, sanitize=True)
self.logger.info("all is well sir")
time_observation = DateTime().generate_datetime_now()
#report.add('time.observation', time_observation, sanitize=True)
report.add('feed.reportname', message.subject, sanitize=True)
self.send_message(report)
mailbox.mark_seen(uid)
self.logger.info("Email report read")
except:
self.logger.info("ERROR with the collector ---")
def compute_basic_math(self, action, event) -> str:
date = DateTime.parse_utc_isoformat(event[action.key], True)
delta = datetime.timedelta(minutes=parse_relative(action.value))
return self._basic_math_op_map[action.operator](date, delta).isoformat()
def upload_file():
success = False
filename = os.path.join(VAR_STATE_PATH, '../webinput_csv.csv')
if 'file' in request.files and request.files['file'].filename:
request.files['file'].save(filename)
request.files['file'].stream.seek(0)
total_lines = request.files['file'].stream.read().count(b'\n') # we don't care about headers here
success = True
elif 'text' in request.form and request.form['text']:
with open(filename, mode='w', encoding='utf8') as handle:
handle.write(request.form['text'])
success = True
total_lines = len(request.form['text'].splitlines())
if not success and request.form.get('use_last_file', False):
success = True
filename, total_lines = get_temp_file()
elif success:
write_temp_file((filename, total_lines))
if not success:
return create_response('no file or text')
parameters = handle_parameters(request.form)
if parameters['has_header']:
total_lines -= 1
preview = []
valid_ip_addresses = None
valid_date_times = None
try:
with open(filename, encoding='utf8') as handle:
reader = csv.reader(handle, delimiter=parameters['delimiter'],
quotechar=parameters['quotechar'],
skipinitialspace=parameters['skipInitialSpace'],
escapechar=parameters['escapechar'],
)
for lineindex, line in enumerate(reader):
line = [col.replace(parameters['escapechar']*2, parameters['escapechar']) for col in line]
if parameters['skipInitialLines']:
if parameters['has_header'] and lineindex == 1:
for _ in range(parameters['skipInitialLines']):
line = next(reader)
elif not parameters['has_header'] and lineindex == 0:
for _ in range(parameters['skipInitialLines']):
line = next(reader)
if lineindex >= parameters['loadLinesMax'] + parameters['has_header']:
break
if valid_ip_addresses is None: # first data line
valid_ip_addresses = [0] * len(line)
valid_date_times = [0] * len(line)
for columnindex, value in enumerate(line):
if IPAddress.is_valid(value, sanitize=True):
valid_ip_addresses[columnindex] += 1
if DateTime.is_valid(value, sanitize=True):
valid_date_times[columnindex] += 1
preview.append(line)
except Exception as exc:
preview = [['Parse Error'], ['Is the number of columns consistent?']] + \
[[x] for x in traceback.format_exc().splitlines()]
column_types = ["IPAddress" if x/(total_lines if total_lines else 1) > 0.7 else None for x in valid_ip_addresses]
column_types = ["DateTime" if valid_date_times[i]/(total_lines if total_lines else 1) > 0.7 else x for i, x in enumerate(column_types)]
return create_response({"column_types": column_types,
"use_column": [bool(x) for x in column_types],
"preview": preview,
})
def parse_line(self, row, report):
if not len(row) or row.startswith(';'):
self.tempdata.append(row)
else:
row_splitted = [field.strip() for field in row.strip().split(',')]
event = self.new_event(report)
event.change("feed.url", event["feed.url"].split("key=")[0])
event.add('source.ip', row_splitted[0])
source_asn = row_splitted[1].replace('AS', '')
if source_asn != '?':
event.add('source.asn', source_asn)
event.add('source.geolocation.cc', row_splitted[2])
event.add('time.source',
DateTime.from_timestamp(int(row_splitted[3])))
malware = row_splitted[4].lower()
if malware == 'openrelay':
event.add('classification.type', 'vulnerable service')
event.add('classification.identifier', 'openrelay')
event.add('protocol.application', 'smtp')
elif malware == 'sshauth':
event.add('classification.type', 'brute-force')
event.add('classification.identifier', 'ssh')
event.add('protocol.application', 'ssh')
elif malware == 'telnetauth':
event.add('classification.type', 'brute-force')
event.add('classification.identifier', 'telnet')
event.add('protocol.application', 'telnet')
elif malware == 'smtpauth':
event.add('classification.type', 'brute-force')
event.add('classification.identifier', 'smtp')
event.add('protocol.application', 'smtp')
elif malware in ['iotscan', 'iotuser']:
event.add('classification.type', 'scanner')
event.add('event_description.text', 'The possibly infected IoT device scanned for other vulnerable IoT devices.')
if row_splitted[7] in ['23', '2323']:
event.add('protocol.application', 'telnet')
event.add('classification.identifier', 'telnet')
else:
event.add('classification.identifier', 'scanner-generic')
elif malware == 'wpscanner':
event.add('classification.type', 'scanner')
event.add('classification.identifier', 'wordpress-vulnerabilities')
event.add('event_description.text', 'scanning for wordpress vulnerabilities')
event.add('protocol.application', 'http')
elif malware == 'w_wplogin':
event.add('classification.type', 'scanner')
event.add('classification.identifier', 'wordpress-login')
event.add('event_description.text', 'scanning for wordpress login pages')
event.add('protocol.application', 'http')
elif malware == 'l_spamlink':
event.add('classification.type', 'spam')
event.add('classification.identifier', 'spamlink')
event.add('event_description.text', 'The URL appeared in a spam email sent by extra.spam_ip.')
# event.add('protocol.application', 'http')
ip, malware_version, malware_name = row_splitted[8].split(':')
event.add('malware.name', malware_name)
event.add('malware.version', malware_version)
event.add('source.url', row_splitted[6])
event.add('extra.spam_ip', ip)
elif malware in ['pop', 'imap']:
event.add('classification.type', 'brute-force')
event.add('classification.identifier', malware)
event.add('protocol.application', malware)
elif malware in ['smb', 'rdp', 'iotrdp', 'iotmicrosoftds']:
if malware.startswith('iot'):
malware = malware[3:]
event.add('classification.type', 'scanner')
event.add('classification.identifier', malware)
event.add('protocol.application', malware)
elif malware == 'proxyget':
event.add('classification.type', 'other')
event.add('classification.identifier', malware)
event.add('event_description.text', 'The malicous client used a honeypot as proxy.')
elif malware == 'iotlogin':
event.add('classification.type', 'unauthorized-login')
event.add('classification.identifier', 'iot')
event.add('event_description.text', 'The infected iot device logged in to a honeypot.')
elif malware == 'iotcmd':
event.add('classification.type', 'unauthorized-command')
event.add('classification.identifier', 'iot')
event.add('event_description.text', 'The infected iot device logged in to a honeypot and issued malicous commands.')
elif malware == 'iotmirai':
event.add('classification.type', 'infected-system')
event.add('classification.identifier', 'mirai')
event.add('malware.name', 'mirai')
elif malware == 'ioturl':
event.add('classification.type', 'c2server')
event.add('classification.identifier', 'malware-generic')
elif malware == 'automatedtest':
event.add('classification.type', 'brute-force')
event.add('classification.identifier', 'lookup-captcha')
event.add('event_description.text', 'The device automatically brute-forced the Spamhaus CBL lookup.')
elif malware == 'authspoofbadehlo':
event.add('classification.type', 'brute-force')
event.add('classification.identifier', 'authentication-spoof')
event.add('protocol.application', 'smtp')
event.add('event_description.text', 'The device spoofed SMTP authentication with a bad EHLO.')
elif malware == 'extortion':
event.add('classification.type', 'spam')
event.add('classification.identifier', 'extortion')
if row_splitted[7] == '25':
event.add('protocol.application', 'smtp')
event.add('event_description.text', 'This device sent extortion mails.')
else:
if malware == 'auto':
malware = 's_other'
event.add('malware.name', malware)
event.add('classification.type', 'infected-system')
event.add('source.url', row_splitted[5], raise_failure=False)
# otherwise the same ip, ignore
if not (malware == 'iotscan' or # the data is wrong according to the feed provider 2018-06-15
':' in row_splitted[5]): # IP or Port in this field: also broken according to provider 2018-06-15
event.add('destination.fqdn', row_splitted[5], raise_failure=False)
event.add('destination.ip', row_splitted[6], raise_failure=False)
event.add('destination.port', row_splitted[7], raise_failure=False)
if row_splitted[8] and row_splitted[8] not in ('-', '?') and malware != 'l_spamlink':
event.add('extra.source.local_port', int(row_splitted[8]))
event.add('protocol.transport', row_splitted[9], raise_failure=False)
event.add('raw', self.recover_line(row))
yield event
def submit():
parameters = handle_parameters(request.form)
temp_file = get_temp_file()
if not temp_file:
return create_response('No file')
destination_pipeline = PipelineFactory.create(PipelineParameters(),
logger=app.logger,
direction='destination')
if not CONFIG.get('destination_pipeline_queue_formatted', False):
destination_pipeline.set_queues(CONFIG['destination_pipeline_queue'], "destination")
destination_pipeline.connect()
time_observation = DateTime().generate_datetime_now()
successful_lines = 0
with open(temp_file[0], encoding='utf8') as handle:
reader = csv.reader(handle, delimiter=parameters['delimiter'],
quotechar=parameters['quotechar'],
skipinitialspace=parameters['skipInitialSpace'],
escapechar=parameters['escapechar'],
)
if parameters['has_header']:
next(reader)
for _ in range(parameters['skipInitialLines']):
next(reader)
for lineindex, line in enumerate(reader):
event = Event()
try:
for columnindex, (column, value) in \
enumerate(zip(parameters['columns'], line)):
if not column or not value:
continue
if column.startswith('time.'):
parsed = dateutil.parser.parse(value)
if not parsed.tzinfo:
value += parameters['timezone']
parsed = dateutil.parser.parse(value)
value = parsed.isoformat()
if column == 'extra':
value = handle_extra(value)
event.add(column, value)
for key, value in parameters.get('constant_fields', {}).items():
if key not in event:
event.add(key, value)
for key, value in request.form.items():
if not key.startswith('custom_'):
continue
key = key[7:]
if key not in event:
event.add(key, value)
if CONFIG.get('destination_pipeline_queue_formatted', False):
queue_name = CONFIG['destination_pipeline_queue'].format(ev=event)
destination_pipeline.set_queues(queue_name, "destination")
destination_pipeline.connect()
except Exception:
continue
if 'classification.type' not in event:
event.add('classification.type', parameters['classification.type'])
if 'classification.identifier' not in event:
event.add('classification.identifier', parameters['classification.identifier'])
if 'feed.code' not in event:
event.add('feed.code', parameters['feed.code'])
if 'time.observation' not in event:
event.add('time.observation', time_observation, sanitize=False)
raw_message = MessageFactory.serialize(event)
destination_pipeline.send(raw_message)
successful_lines += 1
return create_response('Successfully processed %s lines.' % successful_lines)
def compute_basic_math(action, event):
date = DateTime.parse_utc_isoformat(event[action.key], True)
if action.operator == '+=':
return (date + datetime.timedelta(minutes=parse_relative(action.value))).isoformat()
elif action.operator == '-=':
return (date - datetime.timedelta(minutes=parse_relative(action.value))).isoformat()
def process(self):
report = self.receive_message()
peek = utils.base64_decode(report.get("raw"))
self.logger.debug("Peeking at event %r.", peek)
if "TEST MESSAGE" in peek:
self.logger.debug("Ignoring test message/event.")
self.acknowledge_message()
return
# try to parse a JSON object
event = self.new_event(report)
dict_report = json.loads(peek)
event.add("raw", report.get("raw"), sanitize=False)
if "time" in dict_report:
event.add("time.source", dict_report["time"])
if "dip" in dict_report:
event.add("destination.ip", dict_report["dip"])
if "dport" in dict_report:
event.add("destination.port", dict_report["dport"])
if "md5" in dict_report:
event.add("malware.hash.md5", dict_report["md5"])
if "sha1" in dict_report:
event.add("malware.hash.sha1", dict_report["sha1"])
if "fqdn" in dict_report:
if dict_report["fqdn"] == 'unknown':
del dict_report["fqdn"]
else:
event.add("source.fqdn", dict_report["fqdn"])
if "id" in dict_report:
event["extra.feed_id"] = dict_report["id"]
if "adip" in dict_report:
event["extra.adip"] = dict_report["adip"]
if "proto" in dict_report:
event.add("protocol.transport", dict_report["proto"])
if "sport" in dict_report:
event.add("source.port", dict_report["sport"])
if "url" in dict_report:
event.add("source.url", dict_report["url"])
if "confidence" in dict_report:
event.add("extra.confidence", dict_report["confidence"])
if "expires" in dict_report:
event.add("extra.expires",
DateTime.sanitize(dict_report["expires"]))
if "source" in dict_report:
event.add("extra.feed_source", dict_report["source"])
if "name" in dict_report:
mapping['bots']['identifier'] = dict_report["name"]
try:
event.add("malware.name", dict_report["name"])
except InvalidValue:
event.add("malware.name",
re.sub("[^ -~]", '', dict_report["name"]))
event.add("event_description.text", dict_report["name"])
else:
mapping['bots']['identifier'] = "malware-generic"
if dict_report["type"] == "bl-update":
event.add("classification.taxonomy", "other")
event.add("classification.type", "blacklist")
elif dict_report["category"] is not None:
event.add("classification.taxonomy",
mapping[dict_report["category"]]["taxonomy"],
overwrite=True)
event.add("classification.type",
mapping[dict_report["category"]]["type"],
overwrite=True)
event.add("classification.identifier",
mapping[dict_report["category"]]["identifier"],
overwrite=True)
# split up the event into multiple ones, one for each address
for addr in dict_report.get('address', []):
ev = self.new_event(event)
ev.add("source.ip", addr["ip"])
if ("asn" in addr):
ev.add("source.asn", addr["asn"])
if ("rdns" in addr):
ev.add("source.reverse_dns", addr["rdns"])
# XXX ignore for now, only relevant for flows
# ev.add("source.dir", addr["dir"])
if ("cc" in addr):
ev.add("source.geolocation.cc", addr["cc"])
self.send_message(ev)
else: # no address
self.send_message(event)
self.acknowledge_message()
