def parse_azure(self, line, report): raw = self.recover_line(line) event = self.new_event(report) for key, value in line.copy().items(): if key == 'Payload': if value == 'AA==': # NULL del line[key] continue try: value = json.loads(utils.base64_decode(value)) # continue unpacking in next loop except json.decoder.JSONDecodeError: line[key] = utils.base64_decode(value) elif key == 'TLP' and value.lower() == 'unknown': del line[key] if isinstance(value, dict): for subkey, subvalue in value.items(): line['%s.%s' % (key, subkey)] = subvalue del line[key] for key, value in line.items(): if key == 'ThreatConfidence': if value == 'None': continue value = event.get('feed.accuracy', 100) * CONFIDENCE[value] / 100 elif key == 'DateTimeReceivedUtc': value = DateTime.from_windows_nt(value) elif key == 'Payload.ts': value = DateTime.from_timestamp(value) elif key == 'Payload.Protocol': payload_protocol = value[:value.find('/')] if payload_protocol: # needs to overwrite a field previously parsed and written event.add('protocol.application', payload_protocol, overwrite=True) # "HTTP/1.1", save additionally elif not value: continue if AZURE[key] != '__IGNORE__': # feed.accuracy is calculated newly and always needs to be overwritten event.add(AZURE[key], value, overwrite=self.overwrite or AZURE[key] == "feed.accuracy") event.add('classification.type', 'infected-system') event.add('raw', raw) yield event
def parse_line(self, line, report): if line.startswith('#') or len(line) == 0: self.tempdata.append(line) else: value = line.split('\t') event = self.new_event(report) event.add('classification.identifier', value[0].lower()) event.add('raw', line) if report['feed.url'] in Netlab360ParserBot.DGA_FEED: event.add('source.fqdn', value[1]) event.add('time.source', value[3] + ' UTC') event.add('classification.type', 'c&c') event.add('event_description.url', 'http://data.netlab.360.com/dga') elif report['feed.url'] in Netlab360ParserBot.MAGNITUDE_FEED: event.add('time.source', DateTime.from_timestamp(int(value[1]))) event.add('source.ip', value[2]) # ignore ips as fqdns event.add('source.fqdn', value[3], raise_failure=False) if value[4] != 'N/A': event.add('source.url', value[4]) event.add('classification.type', 'exploit') event.add('event_description.url', 'http://data.netlab.360.com/ek') elif report['feed.url'] in Netlab360ParserBot.MIRAI_SCANNER_FEED: event.add('time.source', value[0] + ' UTC') event.add('source.ip', value[1].replace('sip=', '')) event.add('destination.port', value[2].replace('dport=', '')) event.add('classification.type', 'scanner') event.add('classification.identifier', 'mirai', overwrite=True) else: raise ValueError('Unknown data feed %s.' % report['feed.url']) yield event
def parse_line(self, line, report): if line.startswith('#') or len(line) == 0: self.tempdata.append(line) else: value = line.split('\t') event = self.new_event(report) event.add('classification.identifier', value[0].lower()) event.add('raw', line) if report['feed.url'] in Netlab360ParserBot.DGA_FEED: event.add('source.fqdn', value[1]) event.add('time.source', value[3] + ' UTC') event.add('classification.type', 'c&c') event.add('event_description.url', 'http://data.netlab.360.com/dga') elif report['feed.url'] in Netlab360ParserBot.MAGNITUDE_FEED: event.add('time.source', DateTime.from_timestamp(int(value[1]))) event.add('source.ip', value[2]) event.add('source.fqdn', value[3]) if value[4] != 'N/A': event.add('source.url', value[4]) event.add('classification.type', 'exploit') event.add('event_description.url', 'http://data.netlab.360.com/ek') else: raise ValueError('Unknown data feed %s.' % report['feed.url']) yield event
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report["raw"]) for row in raw_report.splitlines(): row = row.strip() if not len(row) or row.startswith(';'): continue row_splitted = [field.strip() for field in row.split(',')] event = self.new_event(report) event.add('source.ip', row_splitted[0]) event.add('source.asn', row_splitted[1].replace('AS', '')) event.add('source.geolocation.cc', row_splitted[2]) event.add('time.source', DateTime.from_timestamp(int(row_splitted[3]))) event.add('malware.name', row_splitted[4].lower()) try: event.add('destination.fqdn', row_splitted[5]) except InvalidValue: pass # otherwise the same ip, ignore event.add('destination.ip', row_splitted[6]) event.add('destination.port', row_splitted[7]) if row_splitted[8] and row_splitted[8] != '-': event.add('extra', {'destination.local_port': int(row_splitted[8])}) event.add('protocol.transport', row_splitted[9]) event.add('classification.type', 'botnet drone') event.add('raw', row) self.send_message(event) self.acknowledge_message()
def __process_fields(event, line, feed_url): for field, value in zip(FEEDS[feed_url]['format'], line.split(',')): if value and field in ('extra.first_seen', 'extra.last_online'): if ':' in value: event.add(field, DateTime.sanitize(value + '+00:00')) else: event.add(field, value + 'T00:00:00+00:00') else: event.add(field, value)
def parse_line(self, line, report): domain, data, raw = line event = self.new_event(report) event.add('time.source', DateTime.from_epoch_millis(int(data['seen']))) event.add('classification.type', 'other') event.add('raw', raw) if not event.add('source.fqdn', domain, raise_failure=False): event.add('source.ip', domain) yield event
def process(self): RT = rt.Rt(self.parameters.uri, self.parameters.user, self.parameters.password) if not RT.login(): raise ValueError('Login failed.') query = RT.search(Queue=self.parameters.search_queue, Subject__like=self.parameters.search_subject_like, Owner=self.parameters.search_owner, Status=self.parameters.search_status) self.logger.info('{} results on search query.'.format(len(query))) for ticket in query: ticket_id = int(ticket['id'].split('/')[1]) self.logger.debug('Process ticket {}.'.format(ticket_id)) for (att_id, att_name, _, _) in RT.get_attachments(ticket_id): if re.search(self.parameters.attachment_regex, att_name): self.logger.debug('Found attachment {}: {!r}.' ''.format(att_id, att_name)) break else: self.logger.debug('No matching attachement name found.') continue attachment = RT.get_attachment_content(ticket_id, att_id) if self.parameters.unzip_attachment: file_obj = io.BytesIO(attachment) zipped = zipfile.ZipFile(file_obj) raw = zipped.read(zipped.namelist()[0]) else: raw = attachment if self.parameters.gnupg_decrypt: raw = str( self.gpg.decrypt( raw, always_trust=self.parameters.gnupg_trust, passphrase=self.parameters.gnupg_passphrase)) self.logger.info('Successfully decrypted attachment.') self.logger.debug(raw) report = Report() report.add("raw", raw, sanitize=True) report.add("rtir_id", ticket_id, sanitize=True) report.add("feed.name", self.parameters.feed, sanitize=True) report.add("feed.accuracy", self.parameters.accuracy, sanitize=True) time_observation = DateTime().generate_datetime_now() report.add('time.observation', time_observation, sanitize=True) self.send_message(report) if self.parameters.take_ticket: RT.edit_ticket(ticket_id, Owner=self.parameters.user)
def process(self): self.logger.info("Downloading report through API") otx = OTXv2(self.parameters.api_key) pulses = otx.getall() self.logger.info("Report downloaded.") report = Report() report.add("raw", json.dumps(pulses), sanitize=True) report.add("feed.name", self.parameters.feed, sanitize=True) report.add("feed.accuracy", self.parameters.accuracy, sanitize=True) time_observation = DateTime().generate_datetime_now() report.add('time.observation', time_observation, sanitize=True) self.send_message(report)
def process(self): mailbox = imbox.Imbox(self.parameters.mail_host, self.parameters.mail_user, self.parameters.mail_password, self.parameters.mail_ssl) emails = mailbox.messages(folder=self.parameters.mail_folder, unread=True) if emails: for uid, message in emails: if (self.parameters.mail_subject_regex and not re.search( self.parameters.mail_subject_regex, message.subject)): continue self.logger.info("Reading email report") for attach in message.attachments: if not attach: continue # remove quote marks from filename attach_name = attach['filename'][1:len(attach['filename'] ) - 1] if re.search(self.parameters.mail_attach_regex, attach_name): if self.parameters.mail_attach_unzip: zipped = zipfile.ZipFile(attach['content']) raw_report = zipped.read(zipped.namelist()[0]) else: raw_report = attach['content'].read() report = Report() report.add("raw", raw_report, sanitize=True) report.add("feed.name", self.parameters.feed, sanitize=True) report.add("feed.accuracy", self.parameters.accuracy, sanitize=True) time_observation = DateTime().generate_datetime_now() report.add('time.observation', time_observation, sanitize=True) self.send_message(report) mailbox.mark_seen(uid) self.logger.info("Email report read")
def parse_line(self, line, report): event = self.new_event(report) if line["version"] not in ("1.5", ''): raise ValueError( "Unknown version %r. Please report this with an example." "" % line["version"]) for unknown in self._unknown_fields: if line[unknown]: raise ValueError( "Unable to parse field %r. Please report this with an example" "" % unknown) event["extra.datasource"] = line["feed code"] event.add("source.ip", line["source ip"]) event.add("source.network", line["source bgp prefix"]) event.add("extra.cert_eu_time_observation", DateTime.sanitize(line["observation time"])) event.add("tlp", line["tlp"]) event.add("event_description.text", line["description"]) event.add("classification.type", self.ABUSE_TO_INTELMQ[line["type"]]) if line['type'] == 'dropzone': event.add("classification.identifier", 'dropzone') if line["count"]: event["extra.count"] = int(line["count"]) event.add("time.source", line["source time"]) event.add("source.geolocation.country", line["source country"]) event.add("protocol.application", line["protocol"]) event.add("destination.port", line["destination port"]) event.add("source.geolocation.latitude", line["source latitude"]) event.add("source.geolocation.city", line["source city"]) event.add("source.geolocation.geoip_cc", line["source cc"]) event.add("source.geolocation.longitude", line["source longitude"]) event.add("extra.source.geolocation.geohash", line["source geohash"]) event["extra.first_seen"] = line["first seen"] event.add('feed.accuracy', event.get('feed.accuracy', 100) * int(line["confidence level"]) / 100, overwrite=True) event["extra.last_seen"] = line["last seen"] event["extra.expiration_date"] = line["expiration date"] if line["status"]: event["status"] = line["status"] event.add("event_description.target", line["target"]) event.add("source.url", line["url"]) event.add("source.abuse_contact", line["abuse contact"]) event.add("source.asn", line["source asn"]) event.add("source.as_name", line["source as name"]) event.add("source.fqdn", line["domain name"]) event.add("raw", self.recover_line(line)) yield event
def parse_line(self, line, report): if line.startswith('#') or not line.strip(): self.tempdata.append(line) else: value = line.split('\t') event = self.new_event(report) event.add('classification.identifier', value[0].lower()) event.add('raw', line) if report['feed.url'] in Netlab360ParserBot.DGA_FEED: event.add('source.fqdn', value[1]) # DGA Feed format is # DGA family, Domain, Start and end of valid time(UTC) event.add('time.source', value[2] + ' UTC') if event['time.source'] > event['time.observation']: event.change('time.source', event['time.observation']) event.add('classification.type', 'c2-server') event.add('event_description.url', 'http://data.netlab.360.com/dga') elif report['feed.url'] in Netlab360ParserBot.MAGNITUDE_FEED: event.add('time.source', DateTime.from_timestamp(int(value[1]))) event.add('source.ip', value[2]) # ignore ips as fqdns event.add('source.fqdn', value[3], raise_failure=False) if value[4] != 'N/A': event.add('source.url', value[4]) event.add('classification.type', 'exploit') event.add('event_description.url', 'http://data.netlab.360.com/ek') elif report['feed.url'] in Netlab360ParserBot.MIRAI_SCANNER_FEED: event.add('time.source', value[0] + ' UTC') event.add('source.ip', value[1].replace('sip=', '')) event.add('destination.port', value[2].replace('dport=', '')) event.add('classification.type', 'scanner') event.add('classification.identifier', 'mirai', overwrite=True) elif report['feed.url'] in Netlab360ParserBot.HAJIME_SCANNER_FEED: event.add('time.source', value[0] + 'T00:00:00 UTC') event.add('source.ip', value[1].replace('ip=', '')) event.add('classification.type', 'scanner') event.add('classification.identifier', 'hajime', overwrite=True) else: raise ValueError('Unknown data feed %s.' % report['feed.url']) yield event
def on_message(self, headers, message): self.n6stomper.logger.debug('Receive message ' '{!r}...'.format(message[:500])) report = Report() report.add("raw", message.rstrip(), sanitize=True) report.add("feed.name", self.n6stomper.parameters.feed, sanitize=True) report.add("feed.url", "stomp://" + self.n6stomper.parameters.server + ":" + str(self.n6stomper.parameters.port) + "/" + self.n6stomper.parameters.exchange, sanitize=True) time_observation = DateTime().generate_datetime_now() report.add('time.observation', time_observation, sanitize=True) self.n6stomper.send_message(report) self.n6stomper.logger.debug('Receiving Message.')
def process(self): report = self.receive_message() raw_report = json.loads(utils.base64_decode(report.get('raw'))) extra = {} event = self.new_event(report) event.change("feed.url", event["feed.url"].split("?key=")[0]) event.add("raw", report.get('raw'), sanitize=False) event.add('classification.type', 'malware') event.add('event_description.text', 'Sinkhole attempted connection') for key, value in raw_report.items(): if key == "_ts": event.add('time.source', DateTime.from_timestamp(int(value))) # Source is UTC if key == "trojanfamily": event.add('malware.name', value) if key == "env": if "remote_addr" in value: event.add('source.ip', value["remote_addr"]) if "remote_port" in value: event.add('source.port', value["remote_port"]) if "server_addr" in value: event.add('destination.ip', value["server_addr"]) if "server_port" in value: event.add('destination.port', value["server_port"]) if "server_name" in value: event.add('destination.fqdn', value["server_name"], raise_failure=False) for k in [ "request_method", "cookies", "path_info", "http_referer" ]: if k in value: extra[k] = value[k] if key == "_geo_env_remote_addr": for k, v in MAP_geo_env_remote_addr.items(): if k in value: event[v] = value[k] if "ip" in value and "netmask" in value: event.add('source.network', '%s/%s' % (value["ip"], value["netmask"])) if key in ["_origin", "_provider", "pattern_verified"]: extra[key] = value if extra: event.add('extra', extra) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report["raw"]) for row in raw_report.splitlines(): row = row.strip() if not len(row) or row.startswith(';'): continue row_splitted = [field.strip() for field in row.split(',')] event = self.new_event(report) event.change("feed.url", event["feed.url"].split("key=")[0]) event.add('source.ip', row_splitted[0]) source_asn = row_splitted[1].replace('AS', '') if source_asn != '?': event.add('source.asn', source_asn) event.add('source.geolocation.cc', row_splitted[2]) event.add('time.source', DateTime.from_timestamp(int(row_splitted[3]))) event.add('malware.name', row_splitted[4].lower()) # otherwise the same ip, ignore event.add('destination.fqdn', row_splitted[5], raise_failure=False) event.add('destination.ip', row_splitted[6], raise_failure=False) event.add('destination.port', row_splitted[7], raise_failure=False) if row_splitted[8] and row_splitted[8] not in ('-', '?'): try: port = int(row_splitted[8]) except ValueError: event.add('destination.fqdn', row_splitted[8], raise_failure=False) else: event.add('extra', {'destination.local_port': port}) event.add('protocol.transport', row_splitted[9], raise_failure=False) event.add('classification.type', 'botnet drone') event.add('raw', row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = json.loads(utils.base64_decode(report.get('raw'))) extra = {} event = self.new_event(report) event.change("feed.url", event["feed.url"].split("?key=")[0]) event.add("raw", report.get('raw'), sanitize=False) event.add('classification.type', 'malware') event.add('event_description.text', 'Sinkhole attempted connection') for key, value in raw_report.items(): if key == "_ts": event.add('time.source', DateTime.from_timestamp(int(value))) # Source is UTC if key == "trojanfamily": event.add('malware.name', value) if key == "env": if "remote_addr" in value: event.add('source.ip', value["remote_addr"]) if "remote_port" in value: event.add('source.port', value["remote_port"]) if "server_addr" in value: event.add('destination.ip', value["server_addr"]) if "server_port" in value: event.add('destination.port', value["server_port"]) if "server_name" in value: event.add('destination.fqdn', value["server_name"], raise_failure=False) for k in ["request_method", "cookies", "path_info", "http_referer"]: if k in value: extra[k] = value[k] if key == "_geo_env_remote_addr": for k, v in MAP_geo_env_remote_addr.items(): if k in value: event[v] = value[k] if "ip" in value and "netmask" in value: event.add('source.network', '%s/%s' % (value["ip"], value["netmask"])) if key in ["_origin", "_provider", "pattern_verified"]: extra[key] = value if extra: event.add('extra', extra) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report is None or not report.contains('raw'): self.acknowledge_message() return raw_report = json.loads(utils.base64_decode(report.get('raw'))) extra = {} event = self.new_event(report) event.add("raw", report.get('raw'), sanitize=False) event.add('classification.type', 'malware') event.add('event_description.text', 'Sinkhole attempted connection') for key, value in raw_report.items(): if key == "_ts": event.add('time.source', DateTime.from_timestamp(int(value))) # Source is UTC if key == "trojanfamily": event.add('malware.name', value) if key == "env": if "remote_addr" in value: event.add('source.ip', value["remote_addr"]) if "remote_port" in value: event.add('source.port', value["remote_port"]) if "server_addr" in value: event.add('destination.ip', value["server_addr"]) if "server_port" in value: event.add('destination.port', value["server_port"]) if "server_name" in value: try: event.add('destination.fqdn', value["server_name"]) except InvalidValue: pass if "request_method" in value: extra['request_method'] = value["request_method"] if extra: event.add('extra', extra) if key == "_geo_env_remote_addr": event.add('source.geolocation.country', value["country_name"]) self.send_message(event) self.acknowledge_message()
def process(self): mailbox = imbox.Imbox(self.parameters.mail_host, self.parameters.mail_user, self.parameters.mail_password, self.parameters.mail_ssl) emails = mailbox.messages(folder=self.parameters.mail_folder, unread=True) if emails: for uid, message in emails: if (self.parameters.mail_subject_regex and not re.search(self.parameters.mail_subject_regex, message.subject)): continue self.logger.info("Reading email report") for body in message.body['plain']: match = re.search(self.parameters.mail_url_regex, body) if match: url = match.group() self.logger.info("Downloading report from %s" % url) raw_report = fetch_url(url, timeout=60.0, chunk_size=16384) self.logger.info("Report downloaded.") report = Report() report.add("raw", raw_report, sanitize=True) report.add("feed.name", self.parameters.feed, sanitize=True) report.add("feed.accuracy", self.parameters.accuracy, sanitize=True) time_observation = DateTime().generate_datetime_now() report.add('time.observation', time_observation, sanitize=True) self.send_message(report) mailbox.mark_seen(uid) self.logger.info("Email report read")
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report["raw"]) for row in raw_report.splitlines(): row = row.strip() if not len(row) or row.startswith(';'): continue row_splitted = [field.strip() for field in row.split(',')] event = self.new_event(report) event.change("feed.url", event["feed.url"].split("key=")[0]) event.add('source.ip', row_splitted[0]) event.add('source.asn', row_splitted[1].replace('AS', '')) event.add('source.geolocation.cc', row_splitted[2]) event.add('time.source', DateTime.from_timestamp(int(row_splitted[3]))) event.add('malware.name', row_splitted[4].lower()) # otherwise the same ip, ignore event.add('destination.fqdn', row_splitted[5], raise_failure=False) event.add('destination.ip', row_splitted[6], raise_failure=False) event.add('destination.port', row_splitted[7], raise_failure=False) if row_splitted[8] and row_splitted[8] not in ('-', '?'): try: port = int(row_splitted[8]) except ValueError: event.add('destination.fqdn', row_splitted[8], raise_failure=False) else: event.add('extra', {'destination.local_port': port}) event.add('protocol.transport', row_splitted[9], raise_failure=False) event.add('classification.type', 'botnet drone') event.add('raw', row) self.send_message(event) self.acknowledge_message()
def parse_line(self, line, report): if line.startswith('#') or len(line) == 0: self.tempdata.append(line) else: lvalue = line.split('\t') event = Event(report) event.add('classification.identifier', lvalue[0].lower()) event.add('time.source', DateTime.from_timestamp(int(lvalue[1]))) if IPAddress.is_valid(lvalue[2]): event.add('source.ip', lvalue[2]) if FQDN.is_valid(lvalue[3]): event.add('source.fqdn', lvalue[3]) if URL.is_valid(lvalue[4]): event.add('source.url', lvalue[4]) event.add('raw', line) event.add('classification.type', 'exploit') event.add('event_description.url', 'http://data.netlab.360.com/ek') yield event
def process(self): self.logger.info("Downloading report from %s" % self.parameters.http_url) resp = requests.get(url=self.parameters.http_url, auth=self.auth, proxies=self.proxy, headers=self.http_header, verify=self.http_verify_cert) if resp.status_code // 100 != 2: raise ValueError('HTTP response status code was {}.' ''.format(resp.status_code)) self.logger.info("Report downloaded.") report = Report() report.add("raw", resp.text, sanitize=True) report.add("feed.name", self.parameters.feed, sanitize=True) report.add("feed.url", self.parameters.http_url, sanitize=True) report.add("feed.accuracy", self.parameters.accuracy, sanitize=True) time_observation = DateTime().generate_datetime_now() report.add('time.observation', time_observation, sanitize=True) self.send_message(report)
def parse_line(self, row, report): event = self.new_event(report) for kv_pair in row.split(self.pair_separator): (key, _, value) = kv_pair.rpartition(self.kv_separator) if not key: continue if self.strip_quotes and value.startswith('"') and value.endswith( '"'): value = value[1:-1] if self.keys.get(key) == 'time.source': try: if value.isnumeric(): value = DateTime.from_timestamp(int(value)) else: value = parse(value, fuzzy=True).isoformat() + " UTC" except ValueError: value = None # Will be ignored by event.add() self.logger.warning( "Could not parse key %r for 'time.source'." " Ignoring this key in line %r.", (value, row)) if key in self.keys: event.add(self.keys[key], value, raise_failure=False) event.add("raw", self.recover_line(row)) yield event
def process(self): report = self.receive_message() peek = utils.base64_decode(report.get("raw")) self.logger.debug("Peeking at event %r.", peek) if "TEST MESSAGE" in peek: self.logger.debug("Ignoring test message/event.") self.acknowledge_message() return # try to parse a JSON object event = self.new_event(report) dict_report = json.loads(peek) event.add("raw", report.get("raw"), sanitize=False) if "time" in dict_report: event.add("time.source", dict_report["time"]) if "dip" in dict_report: event.add("destination.ip", dict_report["dip"]) if "dport" in dict_report: event.add("destination.port", dict_report["dport"]) if "md5" in dict_report: event.add("malware.hash.md5", dict_report["md5"]) if "sha1" in dict_report: event.add("malware.hash.sha1", dict_report["sha1"]) if "fqdn" in dict_report: if dict_report["fqdn"] == 'unknown': del dict_report["fqdn"] else: event.add("source.fqdn", dict_report["fqdn"]) if "id" in dict_report: event["extra.feed_id"] = dict_report["id"] if "adip" in dict_report: event["extra.adip"] = dict_report["adip"] if "proto" in dict_report: event.add("protocol.transport", dict_report["proto"]) if "sport" in dict_report: event.add("source.port", dict_report["sport"]) if "url" in dict_report: event.add("source.url", dict_report["url"]) if "confidence" in dict_report: event.add("extra.confidence", dict_report["confidence"]) if "expires" in dict_report: event.add("extra.expires", DateTime.sanitize(dict_report["expires"])) if "source" in dict_report: event.add("extra.feed_source", dict_report["source"]) if ("category" in dict_report and "name" in dict_report and dict_report["category"] == 'bots'): event.add("malware.name", dict_report["name"]) if ("name" in dict_report): mapping['bots']['identifier'] = dict_report["name"] else: mapping['bots']['identifier'] = "generic-n6-drone" if dict_report["type"] == "bl-update": event.add("classification.taxonomy", "other") event.add("classification.type", "blacklist") elif dict_report["category"] is not None: event.add("classification.taxonomy", mapping[dict_report["category"]]["taxonomy"], overwrite=True) event.add("classification.type", mapping[dict_report["category"]]["type"], overwrite=True) event.add("classification.identifier", mapping[dict_report["category"]]["identifier"], overwrite=True) # split up the event into multiple ones, one for each address for addr in dict_report.get('address', []): ev = self.new_event(event) ev.add("source.ip", addr["ip"]) if ("asn" in addr): ev.add("source.asn", addr["asn"]) if ("rdns" in addr): ev.add("source.reverse_dns", addr["rdns"]) # XXX ignore for now, only relevant for flows # ev.add("source.dir", addr["dir"]) if ("cc" in addr): ev.add("source.geolocation.cc", addr["cc"]) self.send_message(ev) else: # no address self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report["raw"]) soup = bs(raw_report, self.parser) if self.attr_name: table = soup.find_all('table', attrs={self.attr_name: self.attr_value}) self.logger.debug('Found %d table(s) by attribute %r: %r.', (len(table), self.attr_name, self.attr_value)) else: table = soup.find_all('table') self.logger.debug('Found %d table(s).', len(table)) table = table[self.table_index] rows = table.find_all('tr')[self.skip_row:] self.logger.debug('Handling %d row(s).', len(rows)) for feed in rows: event = self.new_event(report) tdata = [data.text for data in feed.find_all('td')] data_added = False for key, data, ignore_value in zip(self.columns, tdata, self.ignore_values): keys = key.split('|') if '|' in key else [ key, ] data = data.strip() if data == ignore_value: continue for key in keys: if isinstance( data, str) and not data: # empty string is never valid break if key in ["__IGNORE__", ""]: break if self.split_column and key == self.split_column: data = data.split(self.split_separator)[int( self.split_index)] data = data.strip() if key in ["time.source", "time.destination"]: try: data = int(data) except ValueError: pass data = DateTime.convert(data, format=self.time_format) elif key.endswith('.url'): if not data: continue if '://' not in data: data = self.default_url_protocol + data if event.add(key, data, raise_failure=False): data_added = True break else: raise ValueError( "Could not add value %r to %s, all invalid." "" % (data, keys)) if not data_added: # we added nothing from this row, so skip it continue if hasattr(self.parameters, 'type')\ and "classification.type" not in event: event.add('classification.type', self.parameters.type) event.add('raw', feed) self.send_message(event) self.acknowledge_message()
def parse_line(self, row, report): if not len(row) or row.startswith(';'): self.tempdata.append(row) else: row_splitted = [field.strip() for field in row.strip().split(',')] event = self.new_event(report) event.change("feed.url", event["feed.url"].split("key=")[0]) event.add('source.ip', row_splitted[0]) source_asn = row_splitted[1].replace('AS', '') if source_asn != '?': event.add('source.asn', source_asn) event.add('source.geolocation.cc', row_splitted[2]) event.add('time.source', DateTime.from_timestamp(int(row_splitted[3]))) malware = row_splitted[4].lower() if malware == 'openrelay': event.add('classification.type', 'vulnerable service') event.add('classification.identifier', 'openrelay') event.add('protocol.application', 'smtp') elif malware == 'iotrdp': event.add('classification.type', 'brute-force') event.add('classification.identifier', 'rdp') event.add('protocol.application', 'rdp') elif malware == 'sshauth': event.add('classification.type', 'brute-force') event.add('classification.identifier', 'ssh') event.add('protocol.application', 'ssh') elif malware in ('telnetauth', 'iotcmd', 'iotuser'): event.add('classification.type', 'brute-force') event.add('classification.identifier', 'telnet') event.add('protocol.application', 'telnet') elif malware == 'iotscan': event.add('classification.type', 'scanner') event.add('event_description.text', 'infected IoT device scanning for other vulnerable IoT devices') if row_splitted[7] == '23': event.add('protocol.application', 'telnet') event.add('classification.identifier', 'telnet') else: event.add('classification.identifier', 'scanner-generic') elif malware == 'wpscanner': event.add('classification.type', 'scanner') event.add('classification.identifier', 'wordpress-vulnerabilities') event.add('event_description.text', 'scanning for wordpress vulnerabilities') event.add('protocol.application', 'http') elif malware == 'w_wplogin': event.add('classification.type', 'scanner') event.add('classification.identifier', 'wordpress-login') event.add('event_description.text', 'scanning for wordpress login pages') event.add('protocol.application', 'http') else: if malware == 'auto': malware = 's_other' event.add('malware.name', malware) event.add('classification.type', 'botnet drone') # otherwise the same ip, ignore event.add('destination.fqdn', row_splitted[5], raise_failure=False) event.add('destination.ip', row_splitted[6], raise_failure=False) event.add('destination.port', row_splitted[7], raise_failure=False) if row_splitted[8] and row_splitted[8] not in ('-', '?'): try: port = int(row_splitted[8]) except ValueError: event.add('destination.fqdn', row_splitted[8], raise_failure=False) else: event.add('extra', {'destination.local_port': port}) event.add('protocol.transport', row_splitted[9], raise_failure=False) event.add('raw', self.recover_line(row)) yield event
def process(self): report = self.receive_message() raw = utils.base64_decode(report.get('raw')).strip() if not raw: self.acknowledge_message() return raw_report = json.loads(raw) del raw event = self.new_event(report) event.change("feed.url", event["feed.url"].split("?key=")[0]) event.add("raw", report.get('raw'), sanitize=False) event.add('classification.type', 'malware') event.add('classification.taxonomy', 'malicious code') event.add('event_description.text', 'Sinkhole attempted connection') for key, value in raw_report.items(): if key == "_ts": event.add('time.source', DateTime.from_timestamp(int(value))) # Source is UTC elif key == "trojanfamily": event.add('malware.name', value) elif key == "env": for subkey, subvalue in value.items(): if subkey == "remote_addr": event.add('source.ip', subvalue) elif subkey == "remote_port": event.add('source.port', subvalue) elif subkey == "server_addr": event.add('destination.ip', subvalue) elif subkey == "server_port": event.add('destination.port', subvalue) elif subkey == "server_name": event.add('destination.fqdn', subvalue, raise_failure=False) elif subkey in [ "request_method", "cookies", "path_info", "http_referer" ]: event['extra.%s' % subkey] = subvalue else: raise ValueError( "Unable to parse data field env.%r. Please report this as bug." % subkey) elif key == "src" or key == 'dst': identity = 'source' if key == 'src' else 'destination' for subkey, subvalue in value.items(): if subkey == "ip": event.add('%s.ip' % identity, subvalue) elif subkey == "port": event.add('%s.port' % identity, subvalue) else: raise ValueError( "Unable to parse data field env.%r. Please report this as bug." % subkey) elif key == "_geo_env_remote_addr": for k, v in MAP_geo_env_remote_addr.items(): if k in value: event[v] = value[k] if "ip" in value and "netmask" in value: event.add('source.network', '%s/%s' % (value["ip"], value["netmask"])) elif key == 'qtype': event['extra.dns_query_type'] = value elif key == 'app_proto': event.add('protocol.application', value, overwrite=True) elif key == 'malw': for subkey, subvalue in value.items(): if subkey == "severity": event.add('extra.malware.severity', subvalue) elif subkey == "family": if self.malware_as_identifier: event.add('classification.identifier', subvalue) else: if subvalue == value['variant']: pass else: event.add('extra.malware.family', subvalue) elif subkey == "variant": event.add('malware.name', subvalue) elif subkey == "categories": event.add('extra.malware.categories', subvalue) elif subkey in [ "request_method", "cookies", "path_info", "http_referer" ]: event['extra.%s' % subkey] = subvalue else: raise ValueError( "Unable to parse data field malw.%r. Please report this as bug." % subkey) elif key == 'comm': for subkey, subvalue in value.items(): if subkey == "proto": event.add('protocol.application', subvalue, overwrite=True) elif subkey == "method": event.add('extra.communication.type', subvalue) elif subkey == "http": for subsubkey, subsubvalue in subvalue.items(): if subsubkey == 'method': event.add('extra.request_method', subsubvalue) elif subsubkey == 'host': if not event.add('destination.fqdn', subsubvalue, raise_failure=False): # event.add('destination.ip', subsubvalue) assert raw_report['dst'][ 'ip'] == subsubvalue elif subsubkey == 'path': event.add('destination.urlpath', subsubvalue) elif subsubkey == 'user_agent': event.add('extra.user_agent', subsubvalue) elif subsubkey == 'more_headers': event.add('extra.communication.headers', subsubvalue) elif subsubkey in ('cookies', 'unverified_domain', 'x_forwarded_for'): event.add('extra.communication.%s' % subsubkey, subsubvalue) else: raise ValueError( "Unable to parse data field comm.http.%r. Please report this as bug." % subsubkey) try: event.add( 'destination.url', '%s://%s%s' % (value['proto'], subvalue['host'], subvalue['path'])) except KeyError: pass elif subkey == 'dns': for subsubkey, subsubvalue in subvalue.items(): if subsubkey == 'name': event.add('destination.fqdn', subsubvalue) elif subsubkey == 'qtype': event['extra.dns_query_type'] = subsubvalue else: raise ValueError( "Unable to parse data field comm.dns.%r. Please report this as bug." % subsubkey) elif subkey == "categories": event.add('extra.malware.categories', subvalue) elif subkey in [ "request_method", "cookies", "path_info", "http_referer" ]: event['extra.%s' % subkey] = subvalue else: raise ValueError( "Unable to parse data field comm.%r. Please report this as bug." % subkey) elif key == 'tracking': for subkey, subvalue in value.items(): if subkey == "id": event.add('extra.tracking.id', subvalue) elif subkey == 'last_ip': event.add('extra.tracking.last.ip', subvalue) elif subkey == 'first': event.add('extra.first_seen', subvalue) elif subkey == 'seen': event.add('extra.last_seen', subvalue) elif subkey == 'changes': event.add('extra.tracking.changes', subvalue) elif subkey == 'checkins': event.add('extra.tracking.checkins', subvalue) elif subkey == 'days': event.add('extra.days_seen', subvalue) elif subkey == 'same_ip': event.add('extra.tracking.same_ip', subvalue) elif subkey == 'tr': event.add('extra.tracking.tr', subvalue) else: raise ValueError( "Unable to parse data field tracking.%r. Please report this as bug." % subkey) elif key == '_geo_src_ip': event = self.parse_geo(event, value, 'source', raw_report, key) elif key == '_geo_tracking_last_ip': event = self.parse_geo(event, value, 'tracking.last', raw_report, key) if value["path"] != 'tracking.last_ip': raise ValueError( '_geo_tracking_last_ip.path is not \'tracking.last_ip\' (%r).' '' % subvalue) elif key == '_geo_comm_http_host': event = self.parse_geo(event, value, 'communication.http.host', raw_report, key) if value["path"] != 'comm.http.host': raise ValueError( '_geo_tracking_last_ip.path is not \'comm.http.host\' (%r).' '' % subvalue) elif key.startswith('_geo_comm_http_x_forwarded_for_'): event = self.parse_geo( event, value, 'extra.communication.http.%s' % key[15:], raw_report, '_geo_comm_http_x_forwarded_for_') elif key in ["_origin", "_provider", "pattern_verified"]: event['extra.%s' % key] = value elif key == "metadata": for subkey, subvalue in value.items(): event['extra.metadata.%s' % subkey] = subvalue else: raise ValueError( "Unable to parse data field %r. Please report this as bug." % key) if event.get("malware.name", None) != 'testsinkholingloss': # used for internal tests, should actually not be part of the feed self.logger.debug("Ignoring 'TestSinkholingLoss' event.") self.send_message(event) self.acknowledge_message()
def process(self): mailbox = imbox.Imbox(self.parameters.mail_host, self.parameters.mail_user, utils.base64_decode(self.parameters.mail_password), self.parameters.mail_ssl) self.logger.info("Connected to mail server") emails = mailbox.messages(folder=self.parameters.folder, unread=True) try: if emails: self.logger.info("Parsing emails in mailbox") for uid, message in emails: if self.parameters.subject_regex and not re.search(self.parameters.subject_regex, message.subject): continue self.logger.info("Reading email report") if hasattr(message,'attachments') and message.attachments: for attach in message.attachments: if not attach: continue attach_name = attach['filename'][1:len(attach['filename'])-1] # remove quote marks from filename if re.search(self.parameters.attach_regex, attach_name): self.logger.info("Parsing attachment") if self.parameters.attach_unzip: zipped = zipfile.ZipFile(attach['content']) raw_report = zipped.read(zipped.namelist()[0]) else: raw_report = attach['content'].read() self.logger.info('content read') report = Report() report.add("raw", raw_report, sanitize=True) report.add("feed.name", self.parameters.feed,sanitize=True) report.add("feed.accuracy", self.parameters.accuracy, sanitize=True) time_observation = DateTime().generate_datetime_now() #report.add('time.observation', time_observation) report.add('feed.reportname', message.subject, sanitize=True) self.logger.info('rocking in a free world') self.send_message(report) self.logger.info('just some administration left') mailbox.mark_seen(uid) self.logger.info("Email report read") else: # If no attachment, read from url # update way of fetching from url to new way in http/ self.logger.info("No attachment found, trying collecting from URL") for body in message.body['plain']: self.logger.info("Parsing message body") match = re.search(self.parameters.url_regex, body) if match: url = match.group() self.logger.info("Downloading report from %s" % url) resp = requests.get(url=url) if resp.status_code // 100 != 2: raise ValueError('HTTP response status code was {}.' ''.format(resp.status_code)) raw_report = resp.content self.logger.info("Report downloaded.") report = Report() report.add("raw", raw_report, sanitize=True) report.add("feed.name", self.parameters.feed, sanitize=True) report.add("feed.accuracy", self.parameters.accuracy, sanitize=True) self.logger.info("all is well sir") time_observation = DateTime().generate_datetime_now() #report.add('time.observation', time_observation, sanitize=True) report.add('feed.reportname', message.subject, sanitize=True) self.send_message(report) mailbox.mark_seen(uid) self.logger.info("Email report read") except: self.logger.info("ERROR with the collector ---")
def compute_basic_math(self, action, event) -> str: date = DateTime.parse_utc_isoformat(event[action.key], True) delta = datetime.timedelta(minutes=parse_relative(action.value)) return self._basic_math_op_map[action.operator](date, delta).isoformat()
def upload_file(): success = False filename = os.path.join(VAR_STATE_PATH, '../webinput_csv.csv') if 'file' in request.files and request.files['file'].filename: request.files['file'].save(filename) request.files['file'].stream.seek(0) total_lines = request.files['file'].stream.read().count(b'\n') # we don't care about headers here success = True elif 'text' in request.form and request.form['text']: with open(filename, mode='w', encoding='utf8') as handle: handle.write(request.form['text']) success = True total_lines = len(request.form['text'].splitlines()) if not success and request.form.get('use_last_file', False): success = True filename, total_lines = get_temp_file() elif success: write_temp_file((filename, total_lines)) if not success: return create_response('no file or text') parameters = handle_parameters(request.form) if parameters['has_header']: total_lines -= 1 preview = [] valid_ip_addresses = None valid_date_times = None try: with open(filename, encoding='utf8') as handle: reader = csv.reader(handle, delimiter=parameters['delimiter'], quotechar=parameters['quotechar'], skipinitialspace=parameters['skipInitialSpace'], escapechar=parameters['escapechar'], ) for lineindex, line in enumerate(reader): line = [col.replace(parameters['escapechar']*2, parameters['escapechar']) for col in line] if parameters['skipInitialLines']: if parameters['has_header'] and lineindex == 1: for _ in range(parameters['skipInitialLines']): line = next(reader) elif not parameters['has_header'] and lineindex == 0: for _ in range(parameters['skipInitialLines']): line = next(reader) if lineindex >= parameters['loadLinesMax'] + parameters['has_header']: break if valid_ip_addresses is None: # first data line valid_ip_addresses = [0] * len(line) valid_date_times = [0] * len(line) for columnindex, value in enumerate(line): if IPAddress.is_valid(value, sanitize=True): valid_ip_addresses[columnindex] += 1 if DateTime.is_valid(value, sanitize=True): valid_date_times[columnindex] += 1 preview.append(line) except Exception as exc: preview = [['Parse Error'], ['Is the number of columns consistent?']] + \ [[x] for x in traceback.format_exc().splitlines()] column_types = ["IPAddress" if x/(total_lines if total_lines else 1) > 0.7 else None for x in valid_ip_addresses] column_types = ["DateTime" if valid_date_times[i]/(total_lines if total_lines else 1) > 0.7 else x for i, x in enumerate(column_types)] return create_response({"column_types": column_types, "use_column": [bool(x) for x in column_types], "preview": preview, })
def parse_line(self, row, report): if not len(row) or row.startswith(';'): self.tempdata.append(row) else: row_splitted = [field.strip() for field in row.strip().split(',')] event = self.new_event(report) event.change("feed.url", event["feed.url"].split("key=")[0]) event.add('source.ip', row_splitted[0]) source_asn = row_splitted[1].replace('AS', '') if source_asn != '?': event.add('source.asn', source_asn) event.add('source.geolocation.cc', row_splitted[2]) event.add('time.source', DateTime.from_timestamp(int(row_splitted[3]))) malware = row_splitted[4].lower() if malware == 'openrelay': event.add('classification.type', 'vulnerable service') event.add('classification.identifier', 'openrelay') event.add('protocol.application', 'smtp') elif malware == 'sshauth': event.add('classification.type', 'brute-force') event.add('classification.identifier', 'ssh') event.add('protocol.application', 'ssh') elif malware == 'telnetauth': event.add('classification.type', 'brute-force') event.add('classification.identifier', 'telnet') event.add('protocol.application', 'telnet') elif malware == 'smtpauth': event.add('classification.type', 'brute-force') event.add('classification.identifier', 'smtp') event.add('protocol.application', 'smtp') elif malware in ['iotscan', 'iotuser']: event.add('classification.type', 'scanner') event.add('event_description.text', 'The possibly infected IoT device scanned for other vulnerable IoT devices.') if row_splitted[7] in ['23', '2323']: event.add('protocol.application', 'telnet') event.add('classification.identifier', 'telnet') else: event.add('classification.identifier', 'scanner-generic') elif malware == 'wpscanner': event.add('classification.type', 'scanner') event.add('classification.identifier', 'wordpress-vulnerabilities') event.add('event_description.text', 'scanning for wordpress vulnerabilities') event.add('protocol.application', 'http') elif malware == 'w_wplogin': event.add('classification.type', 'scanner') event.add('classification.identifier', 'wordpress-login') event.add('event_description.text', 'scanning for wordpress login pages') event.add('protocol.application', 'http') elif malware == 'l_spamlink': event.add('classification.type', 'spam') event.add('classification.identifier', 'spamlink') event.add('event_description.text', 'The URL appeared in a spam email sent by extra.spam_ip.') # event.add('protocol.application', 'http') ip, malware_version, malware_name = row_splitted[8].split(':') event.add('malware.name', malware_name) event.add('malware.version', malware_version) event.add('source.url', row_splitted[6]) event.add('extra.spam_ip', ip) elif malware in ['pop', 'imap']: event.add('classification.type', 'brute-force') event.add('classification.identifier', malware) event.add('protocol.application', malware) elif malware in ['smb', 'rdp', 'iotrdp', 'iotmicrosoftds']: if malware.startswith('iot'): malware = malware[3:] event.add('classification.type', 'scanner') event.add('classification.identifier', malware) event.add('protocol.application', malware) elif malware == 'proxyget': event.add('classification.type', 'other') event.add('classification.identifier', malware) event.add('event_description.text', 'The malicous client used a honeypot as proxy.') elif malware == 'iotlogin': event.add('classification.type', 'unauthorized-login') event.add('classification.identifier', 'iot') event.add('event_description.text', 'The infected iot device logged in to a honeypot.') elif malware == 'iotcmd': event.add('classification.type', 'unauthorized-command') event.add('classification.identifier', 'iot') event.add('event_description.text', 'The infected iot device logged in to a honeypot and issued malicous commands.') elif malware == 'iotmirai': event.add('classification.type', 'infected-system') event.add('classification.identifier', 'mirai') event.add('malware.name', 'mirai') elif malware == 'ioturl': event.add('classification.type', 'c2server') event.add('classification.identifier', 'malware-generic') elif malware == 'automatedtest': event.add('classification.type', 'brute-force') event.add('classification.identifier', 'lookup-captcha') event.add('event_description.text', 'The device automatically brute-forced the Spamhaus CBL lookup.') elif malware == 'authspoofbadehlo': event.add('classification.type', 'brute-force') event.add('classification.identifier', 'authentication-spoof') event.add('protocol.application', 'smtp') event.add('event_description.text', 'The device spoofed SMTP authentication with a bad EHLO.') elif malware == 'extortion': event.add('classification.type', 'spam') event.add('classification.identifier', 'extortion') if row_splitted[7] == '25': event.add('protocol.application', 'smtp') event.add('event_description.text', 'This device sent extortion mails.') else: if malware == 'auto': malware = 's_other' event.add('malware.name', malware) event.add('classification.type', 'infected-system') event.add('source.url', row_splitted[5], raise_failure=False) # otherwise the same ip, ignore if not (malware == 'iotscan' or # the data is wrong according to the feed provider 2018-06-15 ':' in row_splitted[5]): # IP or Port in this field: also broken according to provider 2018-06-15 event.add('destination.fqdn', row_splitted[5], raise_failure=False) event.add('destination.ip', row_splitted[6], raise_failure=False) event.add('destination.port', row_splitted[7], raise_failure=False) if row_splitted[8] and row_splitted[8] not in ('-', '?') and malware != 'l_spamlink': event.add('extra.source.local_port', int(row_splitted[8])) event.add('protocol.transport', row_splitted[9], raise_failure=False) event.add('raw', self.recover_line(row)) yield event
def submit(): parameters = handle_parameters(request.form) temp_file = get_temp_file() if not temp_file: return create_response('No file') destination_pipeline = PipelineFactory.create(PipelineParameters(), logger=app.logger, direction='destination') if not CONFIG.get('destination_pipeline_queue_formatted', False): destination_pipeline.set_queues(CONFIG['destination_pipeline_queue'], "destination") destination_pipeline.connect() time_observation = DateTime().generate_datetime_now() successful_lines = 0 with open(temp_file[0], encoding='utf8') as handle: reader = csv.reader(handle, delimiter=parameters['delimiter'], quotechar=parameters['quotechar'], skipinitialspace=parameters['skipInitialSpace'], escapechar=parameters['escapechar'], ) if parameters['has_header']: next(reader) for _ in range(parameters['skipInitialLines']): next(reader) for lineindex, line in enumerate(reader): event = Event() try: for columnindex, (column, value) in \ enumerate(zip(parameters['columns'], line)): if not column or not value: continue if column.startswith('time.'): parsed = dateutil.parser.parse(value) if not parsed.tzinfo: value += parameters['timezone'] parsed = dateutil.parser.parse(value) value = parsed.isoformat() if column == 'extra': value = handle_extra(value) event.add(column, value) for key, value in parameters.get('constant_fields', {}).items(): if key not in event: event.add(key, value) for key, value in request.form.items(): if not key.startswith('custom_'): continue key = key[7:] if key not in event: event.add(key, value) if CONFIG.get('destination_pipeline_queue_formatted', False): queue_name = CONFIG['destination_pipeline_queue'].format(ev=event) destination_pipeline.set_queues(queue_name, "destination") destination_pipeline.connect() except Exception: continue if 'classification.type' not in event: event.add('classification.type', parameters['classification.type']) if 'classification.identifier' not in event: event.add('classification.identifier', parameters['classification.identifier']) if 'feed.code' not in event: event.add('feed.code', parameters['feed.code']) if 'time.observation' not in event: event.add('time.observation', time_observation, sanitize=False) raw_message = MessageFactory.serialize(event) destination_pipeline.send(raw_message) successful_lines += 1 return create_response('Successfully processed %s lines.' % successful_lines)
def compute_basic_math(action, event): date = DateTime.parse_utc_isoformat(event[action.key], True) if action.operator == '+=': return (date + datetime.timedelta(minutes=parse_relative(action.value))).isoformat() elif action.operator == '-=': return (date - datetime.timedelta(minutes=parse_relative(action.value))).isoformat()
def process(self): report = self.receive_message() peek = utils.base64_decode(report.get("raw")) self.logger.debug("Peeking at event %r.", peek) if "TEST MESSAGE" in peek: self.logger.debug("Ignoring test message/event.") self.acknowledge_message() return # try to parse a JSON object event = self.new_event(report) dict_report = json.loads(peek) event.add("raw", report.get("raw"), sanitize=False) if "time" in dict_report: event.add("time.source", dict_report["time"]) if "dip" in dict_report: event.add("destination.ip", dict_report["dip"]) if "dport" in dict_report: event.add("destination.port", dict_report["dport"]) if "md5" in dict_report: event.add("malware.hash.md5", dict_report["md5"]) if "sha1" in dict_report: event.add("malware.hash.sha1", dict_report["sha1"]) if "fqdn" in dict_report: if dict_report["fqdn"] == 'unknown': del dict_report["fqdn"] else: event.add("source.fqdn", dict_report["fqdn"]) if "id" in dict_report: event["extra.feed_id"] = dict_report["id"] if "adip" in dict_report: event["extra.adip"] = dict_report["adip"] if "proto" in dict_report: event.add("protocol.transport", dict_report["proto"]) if "sport" in dict_report: event.add("source.port", dict_report["sport"]) if "url" in dict_report: event.add("source.url", dict_report["url"]) if "confidence" in dict_report: event.add("extra.confidence", dict_report["confidence"]) if "expires" in dict_report: event.add("extra.expires", DateTime.sanitize(dict_report["expires"])) if "source" in dict_report: event.add("extra.feed_source", dict_report["source"]) if "name" in dict_report: mapping['bots']['identifier'] = dict_report["name"] try: event.add("malware.name", dict_report["name"]) except InvalidValue: event.add("malware.name", re.sub("[^ -~]", '', dict_report["name"])) event.add("event_description.text", dict_report["name"]) else: mapping['bots']['identifier'] = "malware-generic" if dict_report["type"] == "bl-update": event.add("classification.taxonomy", "other") event.add("classification.type", "blacklist") elif dict_report["category"] is not None: event.add("classification.taxonomy", mapping[dict_report["category"]]["taxonomy"], overwrite=True) event.add("classification.type", mapping[dict_report["category"]]["type"], overwrite=True) event.add("classification.identifier", mapping[dict_report["category"]]["identifier"], overwrite=True) # split up the event into multiple ones, one for each address for addr in dict_report.get('address', []): ev = self.new_event(event) ev.add("source.ip", addr["ip"]) if ("asn" in addr): ev.add("source.asn", addr["asn"]) if ("rdns" in addr): ev.add("source.reverse_dns", addr["rdns"]) # XXX ignore for now, only relevant for flows # ev.add("source.dir", addr["dir"]) if ("cc" in addr): ev.add("source.geolocation.cc", addr["cc"]) self.send_message(ev) else: # no address self.send_message(event) self.acknowledge_message()