def __process_fields(event, line, feed_url):
for field, value in zip(FEEDS[feed_url]['format'], line.split(',')):
if value and field in ('extra.first_seen', 'extra.last_online'):
if ':' in value:
event.add(field, DateTime.sanitize(value + '+00:00'))
else:
event.add(field, value + 'T00:00:00+00:00')
else:
event.add(field, value)
def parse_line(self, line, report):
event = self.new_event(report)
if line["version"] not in ("1.5", ''):
raise ValueError(
"Unknown version %r. Please report this with an example."
"" % line["version"])
for unknown in self._unknown_fields:
if line[unknown]:
raise ValueError(
"Unable to parse field %r. Please report this with an example"
"" % unknown)
event["extra.datasource"] = line["feed code"]
event.add("source.ip", line["source ip"])
event.add("source.network", line["source bgp prefix"])
event.add("extra.cert_eu_time_observation",
DateTime.sanitize(line["observation time"]))
event.add("tlp", line["tlp"])
event.add("event_description.text", line["description"])
event.add("classification.type", self.ABUSE_TO_INTELMQ[line["type"]])
if line['type'] == 'dropzone':
event.add("classification.identifier", 'dropzone')
if line["count"]:
event["extra.count"] = int(line["count"])
event.add("time.source", line["source time"])
event.add("source.geolocation.country", line["source country"])
event.add("protocol.application", line["protocol"])
event.add("destination.port", line["destination port"])
event.add("source.geolocation.latitude", line["source latitude"])
event.add("source.geolocation.city", line["source city"])
event.add("source.geolocation.geoip_cc", line["source cc"])
event.add("source.geolocation.longitude", line["source longitude"])
event.add("extra.source.geolocation.geohash", line["source geohash"])
event["extra.first_seen"] = line["first seen"]
event.add('feed.accuracy',
event.get('feed.accuracy', 100) *
int(line["confidence level"]) / 100,
overwrite=True)
event["extra.last_seen"] = line["last seen"]
event["extra.expiration_date"] = line["expiration date"]
if line["status"]:
event["status"] = line["status"]
event.add("event_description.target", line["target"])
event.add("source.url", line["url"])
event.add("source.abuse_contact", line["abuse contact"])
event.add("source.asn", line["source asn"])
event.add("source.as_name", line["source as name"])
event.add("source.fqdn", line["domain name"])
event.add("raw", self.recover_line(line))
yield event
def process(self):
report = self.receive_message()
peek = utils.base64_decode(report.get("raw"))
self.logger.debug("Peeking at event %r.", peek)
if "TEST MESSAGE" in peek:
self.logger.debug("Ignoring test message/event.")
self.acknowledge_message()
return
# try to parse a JSON object
event = self.new_event(report)
dict_report = json.loads(peek)
event.add("raw", report.get("raw"), sanitize=False)
if "time" in dict_report:
event.add("time.source", dict_report["time"])
if "dip" in dict_report:
event.add("destination.ip", dict_report["dip"])
if "dport" in dict_report:
event.add("destination.port", dict_report["dport"])
if "md5" in dict_report:
event.add("malware.hash.md5", dict_report["md5"])
if "sha1" in dict_report:
event.add("malware.hash.sha1", dict_report["sha1"])
if "fqdn" in dict_report:
if dict_report["fqdn"] == 'unknown':
del dict_report["fqdn"]
else:
event.add("source.fqdn", dict_report["fqdn"])
if "id" in dict_report:
event["extra.feed_id"] = dict_report["id"]
if "adip" in dict_report:
event["extra.adip"] = dict_report["adip"]
if "proto" in dict_report:
event.add("protocol.transport", dict_report["proto"])
if "sport" in dict_report:
event.add("source.port", dict_report["sport"])
if "url" in dict_report:
event.add("source.url", dict_report["url"])
if "confidence" in dict_report:
event.add("extra.confidence", dict_report["confidence"])
if "expires" in dict_report:
event.add("extra.expires",
DateTime.sanitize(dict_report["expires"]))
if "source" in dict_report:
event.add("extra.feed_source", dict_report["source"])
if "name" in dict_report:
mapping['bots']['identifier'] = dict_report["name"]
try:
event.add("malware.name", dict_report["name"])
except InvalidValue:
event.add("malware.name",
re.sub("[^ -~]", '', dict_report["name"]))
event.add("event_description.text", dict_report["name"])
else:
mapping['bots']['identifier'] = "malware-generic"
if dict_report["type"] == "bl-update":
event.add("classification.taxonomy", "other")
event.add("classification.type", "blacklist")
elif dict_report["category"] is not None:
event.add("classification.taxonomy",
mapping[dict_report["category"]]["taxonomy"],
overwrite=True)
event.add("classification.type",
mapping[dict_report["category"]]["type"],
overwrite=True)
event.add("classification.identifier",
mapping[dict_report["category"]]["identifier"],
overwrite=True)
# split up the event into multiple ones, one for each address
for addr in dict_report.get('address', []):
ev = self.new_event(event)
ev.add("source.ip", addr["ip"])
if ("asn" in addr):
ev.add("source.asn", addr["asn"])
if ("rdns" in addr):
ev.add("source.reverse_dns", addr["rdns"])
# XXX ignore for now, only relevant for flows
# ev.add("source.dir", addr["dir"])
if ("cc" in addr):
ev.add("source.geolocation.cc", addr["cc"])
self.send_message(ev)
else: # no address
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
peek = utils.base64_decode(report.get("raw"))
self.logger.debug("Peeking at event %r.", peek)
if "TEST MESSAGE" in peek:
self.logger.debug("Ignoring test message/event.")
self.acknowledge_message()
return
# try to parse a JSON object
event = self.new_event(report)
dict_report = json.loads(peek)
event.add("raw", report.get("raw"), sanitize=False)
if "time" in dict_report:
event.add("time.source", dict_report["time"])
if "dip" in dict_report:
event.add("destination.ip", dict_report["dip"])
if "dport" in dict_report:
event.add("destination.port", dict_report["dport"])
if "md5" in dict_report:
event.add("malware.hash.md5", dict_report["md5"])
if "sha1" in dict_report:
event.add("malware.hash.sha1", dict_report["sha1"])
if "fqdn" in dict_report:
if dict_report["fqdn"] == 'unknown':
del dict_report["fqdn"]
else:
event.add("source.fqdn", dict_report["fqdn"])
if "id" in dict_report:
event["extra.feed_id"] = dict_report["id"]
if "adip" in dict_report:
event["extra.adip"] = dict_report["adip"]
if "proto" in dict_report:
event.add("protocol.transport", dict_report["proto"])
if "sport" in dict_report:
event.add("source.port", dict_report["sport"])
if "url" in dict_report:
event.add("source.url", dict_report["url"])
if "confidence" in dict_report:
event.add("extra.confidence", dict_report["confidence"])
if "expires" in dict_report:
event.add("extra.expires", DateTime.sanitize(dict_report["expires"]))
if "source" in dict_report:
event.add("extra.feed_source", dict_report["source"])
if ("category" in dict_report and "name" in dict_report and
dict_report["category"] == 'bots'):
event.add("malware.name", dict_report["name"])
if ("name" in dict_report):
mapping['bots']['identifier'] = dict_report["name"]
else:
mapping['bots']['identifier'] = "generic-n6-drone"
if dict_report["type"] == "bl-update":
event.add("classification.taxonomy", "other")
event.add("classification.type", "blacklist")
elif dict_report["category"] is not None:
event.add("classification.taxonomy",
mapping[dict_report["category"]]["taxonomy"],
overwrite=True)
event.add("classification.type",
mapping[dict_report["category"]]["type"],
overwrite=True)
event.add("classification.identifier",
mapping[dict_report["category"]]["identifier"],
overwrite=True)
# split up the event into multiple ones, one for each address
for addr in dict_report.get('address', []):
ev = self.new_event(event)
ev.add("source.ip", addr["ip"])
if ("asn" in addr):
ev.add("source.asn", addr["asn"])
if ("rdns" in addr):
ev.add("source.reverse_dns", addr["rdns"])
# XXX ignore for now, only relevant for flows
# ev.add("source.dir", addr["dir"])
if ("cc" in addr):
ev.add("source.geolocation.cc", addr["cc"])
self.send_message(ev)
else: # no address
self.send_message(event)
self.acknowledge_message()
