def __process_fields(event, line, feed_url): for field, value in zip(FEEDS[feed_url]['format'], line.split(',')): if value and field in ('extra.first_seen', 'extra.last_online'): if ':' in value: event.add(field, DateTime.sanitize(value + '+00:00')) else: event.add(field, value + 'T00:00:00+00:00') else: event.add(field, value)
def parse_line(self, line, report): event = self.new_event(report) if line["version"] not in ("1.5", ''): raise ValueError( "Unknown version %r. Please report this with an example." "" % line["version"]) for unknown in self._unknown_fields: if line[unknown]: raise ValueError( "Unable to parse field %r. Please report this with an example" "" % unknown) event["extra.datasource"] = line["feed code"] event.add("source.ip", line["source ip"]) event.add("source.network", line["source bgp prefix"]) event.add("extra.cert_eu_time_observation", DateTime.sanitize(line["observation time"])) event.add("tlp", line["tlp"]) event.add("event_description.text", line["description"]) event.add("classification.type", self.ABUSE_TO_INTELMQ[line["type"]]) if line['type'] == 'dropzone': event.add("classification.identifier", 'dropzone') if line["count"]: event["extra.count"] = int(line["count"]) event.add("time.source", line["source time"]) event.add("source.geolocation.country", line["source country"]) event.add("protocol.application", line["protocol"]) event.add("destination.port", line["destination port"]) event.add("source.geolocation.latitude", line["source latitude"]) event.add("source.geolocation.city", line["source city"]) event.add("source.geolocation.geoip_cc", line["source cc"]) event.add("source.geolocation.longitude", line["source longitude"]) event.add("extra.source.geolocation.geohash", line["source geohash"]) event["extra.first_seen"] = line["first seen"] event.add('feed.accuracy', event.get('feed.accuracy', 100) * int(line["confidence level"]) / 100, overwrite=True) event["extra.last_seen"] = line["last seen"] event["extra.expiration_date"] = line["expiration date"] if line["status"]: event["status"] = line["status"] event.add("event_description.target", line["target"]) event.add("source.url", line["url"]) event.add("source.abuse_contact", line["abuse contact"]) event.add("source.asn", line["source asn"]) event.add("source.as_name", line["source as name"]) event.add("source.fqdn", line["domain name"]) event.add("raw", self.recover_line(line)) yield event
def process(self): report = self.receive_message() peek = utils.base64_decode(report.get("raw")) self.logger.debug("Peeking at event %r.", peek) if "TEST MESSAGE" in peek: self.logger.debug("Ignoring test message/event.") self.acknowledge_message() return # try to parse a JSON object event = self.new_event(report) dict_report = json.loads(peek) event.add("raw", report.get("raw"), sanitize=False) if "time" in dict_report: event.add("time.source", dict_report["time"]) if "dip" in dict_report: event.add("destination.ip", dict_report["dip"]) if "dport" in dict_report: event.add("destination.port", dict_report["dport"]) if "md5" in dict_report: event.add("malware.hash.md5", dict_report["md5"]) if "sha1" in dict_report: event.add("malware.hash.sha1", dict_report["sha1"]) if "fqdn" in dict_report: if dict_report["fqdn"] == 'unknown': del dict_report["fqdn"] else: event.add("source.fqdn", dict_report["fqdn"]) if "id" in dict_report: event["extra.feed_id"] = dict_report["id"] if "adip" in dict_report: event["extra.adip"] = dict_report["adip"] if "proto" in dict_report: event.add("protocol.transport", dict_report["proto"]) if "sport" in dict_report: event.add("source.port", dict_report["sport"]) if "url" in dict_report: event.add("source.url", dict_report["url"]) if "confidence" in dict_report: event.add("extra.confidence", dict_report["confidence"]) if "expires" in dict_report: event.add("extra.expires", DateTime.sanitize(dict_report["expires"])) if "source" in dict_report: event.add("extra.feed_source", dict_report["source"]) if "name" in dict_report: mapping['bots']['identifier'] = dict_report["name"] try: event.add("malware.name", dict_report["name"]) except InvalidValue: event.add("malware.name", re.sub("[^ -~]", '', dict_report["name"])) event.add("event_description.text", dict_report["name"]) else: mapping['bots']['identifier'] = "malware-generic" if dict_report["type"] == "bl-update": event.add("classification.taxonomy", "other") event.add("classification.type", "blacklist") elif dict_report["category"] is not None: event.add("classification.taxonomy", mapping[dict_report["category"]]["taxonomy"], overwrite=True) event.add("classification.type", mapping[dict_report["category"]]["type"], overwrite=True) event.add("classification.identifier", mapping[dict_report["category"]]["identifier"], overwrite=True) # split up the event into multiple ones, one for each address for addr in dict_report.get('address', []): ev = self.new_event(event) ev.add("source.ip", addr["ip"]) if ("asn" in addr): ev.add("source.asn", addr["asn"]) if ("rdns" in addr): ev.add("source.reverse_dns", addr["rdns"]) # XXX ignore for now, only relevant for flows # ev.add("source.dir", addr["dir"]) if ("cc" in addr): ev.add("source.geolocation.cc", addr["cc"]) self.send_message(ev) else: # no address self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() peek = utils.base64_decode(report.get("raw")) self.logger.debug("Peeking at event %r.", peek) if "TEST MESSAGE" in peek: self.logger.debug("Ignoring test message/event.") self.acknowledge_message() return # try to parse a JSON object event = self.new_event(report) dict_report = json.loads(peek) event.add("raw", report.get("raw"), sanitize=False) if "time" in dict_report: event.add("time.source", dict_report["time"]) if "dip" in dict_report: event.add("destination.ip", dict_report["dip"]) if "dport" in dict_report: event.add("destination.port", dict_report["dport"]) if "md5" in dict_report: event.add("malware.hash.md5", dict_report["md5"]) if "sha1" in dict_report: event.add("malware.hash.sha1", dict_report["sha1"]) if "fqdn" in dict_report: if dict_report["fqdn"] == 'unknown': del dict_report["fqdn"] else: event.add("source.fqdn", dict_report["fqdn"]) if "id" in dict_report: event["extra.feed_id"] = dict_report["id"] if "adip" in dict_report: event["extra.adip"] = dict_report["adip"] if "proto" in dict_report: event.add("protocol.transport", dict_report["proto"]) if "sport" in dict_report: event.add("source.port", dict_report["sport"]) if "url" in dict_report: event.add("source.url", dict_report["url"]) if "confidence" in dict_report: event.add("extra.confidence", dict_report["confidence"]) if "expires" in dict_report: event.add("extra.expires", DateTime.sanitize(dict_report["expires"])) if "source" in dict_report: event.add("extra.feed_source", dict_report["source"]) if ("category" in dict_report and "name" in dict_report and dict_report["category"] == 'bots'): event.add("malware.name", dict_report["name"]) if ("name" in dict_report): mapping['bots']['identifier'] = dict_report["name"] else: mapping['bots']['identifier'] = "generic-n6-drone" if dict_report["type"] == "bl-update": event.add("classification.taxonomy", "other") event.add("classification.type", "blacklist") elif dict_report["category"] is not None: event.add("classification.taxonomy", mapping[dict_report["category"]]["taxonomy"], overwrite=True) event.add("classification.type", mapping[dict_report["category"]]["type"], overwrite=True) event.add("classification.identifier", mapping[dict_report["category"]]["identifier"], overwrite=True) # split up the event into multiple ones, one for each address for addr in dict_report.get('address', []): ev = self.new_event(event) ev.add("source.ip", addr["ip"]) if ("asn" in addr): ev.add("source.asn", addr["asn"]) if ("rdns" in addr): ev.add("source.reverse_dns", addr["rdns"]) # XXX ignore for now, only relevant for flows # ev.add("source.dir", addr["dir"]) if ("cc" in addr): ev.add("source.geolocation.cc", addr["cc"]) self.send_message(ev) else: # no address self.send_message(event) self.acknowledge_message()