def process(self): report = self.receive_message() if report: for row in report.split('\n'): row = row.strip() if len(row) == 0 or row.startswith('#'): continue row = row.split() event = Event() columns = ["source_ip", "source_time"] for key, value in zip(columns, row): if key == "source_time": value = datetime.utcfromtimestamp(int(value)).strftime('%Y-%m-%d %H:%M:%S') + " UTC" event.add(key, value.strip()) event.add('feed', 'openbl') event.add('feed_url', 'http://www.openbl.org/lists/date_all.txt') event.add('type', 'blacklist') event = utils.parse_source_time(event, "source_time") event = utils.generate_observation_time(event, "observation_time") event = utils.generate_reported_fields(event) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) for row in raw_report.splitlines(): row = row.strip() if len(row) == 0 or row.startswith('#'): continue row = row.replace('\r', '') values = row.split('\t') # if special char is in string should not be allowed if "#" in values[1]: continue # if domain name is localhost we are not interested if values[1].lower().strip() == "localhost": continue event = Event(report) if IPAddress.is_valid(values[1]): event.add("source.ip", values[1]) else: event.add("source.fqdn", values[1]) event.add('classification.type', 'blacklist') event.add("raw", row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if not report: self.acknowledge_message() return if not report.contains("raw"): self.acknowledge_message() raw_report = utils.base64_decode(report.get("raw")) for row in raw_report.split('\n'): row = row.strip() if row.startswith("#") or len(row) == 0: continue event = Event(report) event.add('classification.type', u'c&c') event.add('source.fqdn', row) event.add("raw", row) event.add("malware.name", SOURCE_FEEDS[report.get("feed.url")]) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.get("raw")) for row in raw_report.split('\n'): if row.startswith("#") or len(row) == 0 or row == "Site": if 'updated' in row: time_str = row[row.find(': ') + 2:] time = dateutil.parser.parse(time_str).isoformat() continue event = Event(report) event.add('classification.type', u'malware') event.add('source.fqdn', row.strip()) event.add('time.source', time) event.add("raw", row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report: report = report.strip() headers = lib.dcu_headers() rows = report.split("\n") for row in rows: try: columns = row.strip().split("\t") fields = dict(zip(headers, columns)) event = Event(lib.convert_dcu_fields(fields)) event.add("feed", "microsoft-dcu") event = utils.generate_observation_time(event, "observation_time") event = utils.generate_reported_fields(event) self.send_message(event) except lib.ParsingError as exc: msg = "Got a parsing problem: %s affected row '%s' IGNORING AND CONTINUING" % (exc.message, row.strip()) self.logger.warning(msg, exc_info=True) continue self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) for row in raw_report.splitlines(): if row.startswith('#'): continue event = Event(report) match = re.search(REGEX_IP, row) if match: ip = match.group() match = re.search(REGEX_TIMESTAMP, row) if match: timestamp = match.group(1) + " UTC" event.add('time.source', timestamp) event.add('source.ip', ip) event.add('classification.type', 'brute-force') event.add("raw", row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.value("raw")) for row in raw_report.split('\n'): val = row.strip() if not len(val) or val.startswith('#') or val.startswith('//'): continue event = Event(report) if IPAddress.is_valid(val, sanitize=True): event.add('source.ip', val, sanitize=True) else: event.add('source.network', val, sanitize=True) event.add('classification.type', u'blacklist') event.add('raw', row, sanitize=True) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if not report: self.acknowledge_message() return if not report.contains("raw"): self.acknowledge_message() raw_report = utils.base64_decode(report.value("raw")) for row in raw_report.split('\n'): row = row.strip() if row == "" or row.startswith("#"): continue event = Event(report) if IPAddress.is_valid(row, sanitize=True): event.add('source.ip', row, sanitize=True) else: event.add('source.fqdn', row, sanitize=True) event.add('classification.type', u'malware') event.add('raw', row, sanitize=True) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.value("raw")) for row in raw_report.split('\n'): if row.startswith('#'): continue event = Event(report) match = re.search(REGEX_IP, row) if match: ip = match.group() match = re.search(REGEX_TIMESTAMP, row) if match: timestamp = match.group(1) + " UTC" event.add('time.source', timestamp, sanitize=True) event.add('source.ip', ip, sanitize=True) event.add('classification.type', u'brute-force') event.add("raw", row, sanitize=True) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) lastgenerated = None for row in raw_report.splitlines(): event = Event(report) row = row.strip() if len(row) == 0: continue elif row.startswith("#"): if 'Generated on' in row: row = row.strip('# ')[13:] lastgenerated = dateutil.parser.parse(row).isoformat() continue event.add('time.source', lastgenerated) event.add('classification.type', 'c&c') event.add('source.fqdn', row) event.add("raw", row) event.add("malware.name", SOURCE_FEEDS[report.get("feed.url")]) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) raw_report = raw_report.split("<tbody>")[1] raw_report = raw_report.split("</tbody>")[0] raw_report_splitted = raw_report.split("<tr>") for row in raw_report_splitted: row = row.strip() if row == "": continue row_splitted = row.split("<td>") ip = row_splitted[1].split('">')[1].split("<")[0].strip() time_source = row_splitted[6].replace("</td></tr>", "").strip() time_source = time_source + " 00:00:00 UTC" event = Event(report) event.add('time.source', time_source) event.add('classification.type', 'malware') event.add('source.ip', ip) event.add('raw', row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.get("raw")) raw_report = raw_report.strip() url = report.get('feed.url') path = urlparse(url).path filename = posixpath.basename(path) for row in raw_report.split('\n'): event = Event(report) event.add('source.ip', row.strip()) if filename in MAPPING: for key, value in MAPPING[filename].items(): event.add(key, value) else: event.add('classification.type', 'blacklist') event.add("raw", row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report: for row in report.split('\n'): row = row.strip() if len(row) == 0: continue row = row.split('|') event = Event() columns = ["source_url", "source_asn", "source_ip", "source_time", "source_reverse_dns", "source_cc", "__IGNORE__", "additional_information"] for key, value in zip(columns, row): value = value.strip() if key == "source_time": value += " UTC" if value != "N/A" and key != "__IGNORE__": event.add(key, value) event.add('feed', 'cert-eu') event.add('type', 'malware') event = utils.parse_source_time(event, "source_time") event = utils.generate_observation_time(event, "observation_time") event = utils.generate_reported_fields(event) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report: report = report.strip() columns = { "timestamp": "source_time", "ip": "source_ip", "protocol": "transport_protocol", "port": "source_port", "hostname": "source_reverse_dns", "sysdesc": "__TDB__", "sysname": "__TDB__", "asn": "source_asn", "geo": "source_cc", "region": "source_region", "city": "source_city", "version": "__IGNORE__" } rows = csv.DictReader(StringIO.StringIO(report)) for row in rows: event = Event() for key, value in row.items(): key = columns[key] if not value: continue value = value.strip() if key is "__IGNORE__" or key is "__TDB__": continue # set timezone explicitly to UTC as it is absent in the input if key == "source_time": value += " UTC" event.add(key, value) event.add('feed', 'shadowserver-snmp') event.add('type', 'vulnerable service') event.add('application_protocol', 'snmp') event = utils.parse_source_time(event, "source_time") event = utils.generate_observation_time( event, "observation_time") event = utils.generate_reported_fields(event) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report: report = encode(report) columns = ["source_time", "source_url", "source_ip", "source_reverse_dns", "malware", "__IGNORE__", "source_asn"] for row in unicodecsv.reader(StringIO(report), encoding='utf-8'): event = Event() for key, value in zip(columns, row): if key is "__IGNORE__": continue if key is "source_time": value = value.replace('_',' ') value += " UTC" if key is "malware": value = value.lower() event.add(key, value) event.add('feed', 'malwaredomainslist') event.add('feed_url', 'http://www.malwaredomainlist.com/updatescsv.php') event.add('type', 'malware') # FIXME event = utils.parse_source_time(event, "source_time") event = utils.generate_observation_time(event, "observation_time") event = utils.generate_reported_fields(event) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report: for row in report.split('\n'): row = row.strip() if len(row) == 0 or not row.startswith('<td style'): continue m = re.search("color: black;\">(\d+.\d+.\d+.\d+)</span></td><td>(.*)</td>", row) if m: event = Event() event.add("source_ip", m.group(1)) event.add('feed', 'netflowhtml') event.add('feed_url', 'https://tc.edu.tw/net/netflow/lkout/recent/1') event.add('type', m.group(2)) event = utils.generate_source_time(event, "source_time") event = utils.generate_observation_time(event, "observation_time") event = utils.generate_reported_fields(event) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() self.logger.debug("Will apply regex %s" % self.parameters.regex) if report: rowcount = 0 for row in report.split('\n'): # For each line event = Event() match = re.search(self.parameters.regex, row) if match: for key in match.groupdict(): event.add(key, match.groupdict()[key]) else: continue # skip lines without matching regex rowcount += 1 # Get detail from parser parameters, will be nice to have it by # source parameters.. Avoid adding if parsed if not 'feed' in match.groupdict(): event.add('feed', self.parameters.feed) if not 'feed_url' in match.groupdict(): event.add('feed_url', self.parameters.feed_url) if not 'type' in match.groupdict(): event.add('type', self.parameters.type) event = utils.parse_source_time(event, "source_time") event = utils.generate_observation_time(event, "observation_time") event = utils.generate_reported_fields(event) self.send_message(event) self.logger.info("Processed %d event" % rowcount) self.acknowledge_message()
def process(self): report = self.receive_message() if report: for row in report.split('\n'): row = row.strip() if len(row) == 0 or row.startswith('other'): continue row = row.split() event = Event() columns = ["source_ip"] for key, value in zip(columns, row): event.add(key, value) event.add('feed', 'arbor') event.add('feed_url', 'http://atlas-public.ec2.arbor.net/public/ssh_attackers') event.add('type', 'brute-force') event = utils.generate_source_time(event, "source_time") event = utils.generate_observation_time(event, "observation_time") event = utils.generate_reported_fields(event) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if not report: self.acknowledge_message() return if not report.contains("raw"): self.acknowledge_message() raw_report = utils.base64_decode(report.value("raw")) for row in raw_report.split('\n'): row = row.strip() if row == "" or row.startswith("#"): continue event = Event(report) if IPAddress.is_valid(row): event.add('source.ip', row) else: event.add('source.fqdn', row) event.add('classification.type', u'malware') event.add('raw', row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) for row in raw_report.splitlines(): if not row or row.startswith('#'): continue event = Event(report) match = re.search(REGEX_IP, row) if match: ip = match.group() match = re.search(REGEX_TIMESTAMP, row) if match: timestamp = match.group(1) + " UTC" if not timestamp: raise ValueError('No timestamp found.') event.add('time.source', timestamp) event.add('source.ip', ip) event.add('classification.type', 'brute-force') event.add("raw", row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if not report: self.acknowledge_message() return if not report.contains("raw"): self.acknowledge_message() raw_report = utils.base64_decode(report.value("raw")) for row in raw_report.split('\n'): row = row.strip() if row.startswith("#") or len(row) == 0: continue event = Event(report) event.add('classification.type', u'c&c') event.add('source.fqdn', row, sanitize=True) event.add("raw", row, sanitize=True) event.add("malware.name", SOURCE_FEEDS[report.value("feed.url")], sanitize=True) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() self.columns = { "# timestamp": None, "ip": "source.ip", "asn": "source.asn", "old_category": None, "malware": "classification.identifier", "geoip": None, "comment": None, "addtime": None, "category": None, "rir": None, "cc": "source.geolocation.country", "loadindex": None, "dstip": "destination.ip", "srcport": "source.port", "dstport": "destination.port", } self.categories = { "bots": "botnet drone", # TODO: more types ? } raw_report = utils.base64_decode(report.value("raw")) for row in csv.DictReader(StringIO(raw_report), delimiter="\t"): # Skip unknown categories if row['old_category'] not in self.categories: continue event = Event(report) # Iterate through mapped fields sanitizing them if needed for key, field in self.columns.items(): if not field: continue event.add(field, row[key], sanitize=True) event.add('classification.type', self.categories[row['old_category']]) event.add('time.source', row['# timestamp'] + ' UTC', sanitize=True) event.add("raw", ",".join(row), sanitize=True) self.logger.debug('event -> %r', event) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) for row in csv.DictReader(io.StringIO(raw_report)): event = Event(report) for key, value in row.items(): if not value: continue if key is None: self.logger.warning('Value without key found, skipping the' ' value: {!r}'.format(value)) continue key = COLUMNS[key] if key == "__IGNORE__" or key == "__TDB__": continue if key == "source.fqdn" and IPAddress.is_valid(value, sanitize=True): continue if key == "time.source": value = value + " UTC" event.add(key, value) event.add('classification.type', 'phishing') event.add("raw", ",".join(row)) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if not report: self.acknowledge_message() return if not report.contains("raw"): self.acknowledge_message() raw_report = utils.base64_decode(report.get("raw")) for row in raw_report.split('\n'): row = row.strip() if row == "" or row.startswith("#"): continue event = Event(report) event.add('source.ip', row) event.add('classification.type', u'malware') event.add('raw', row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report: report = report.strip() columns = { "timestamp": "source_time", "ip": "source_ip", "protocol" : "transport_protocol", "port" : "source_port", "hostname": "source_reverse_dns", "tag" : "__IGNORE__", "quote" : "__IGNORE__", "asn": "source_asn", "geo": "source_cc", "region" : "source_region", "city" : "source_city", "naics": "__IGNORE__", "sic": "__IGNORE__" } rows = csv.DictReader(StringIO.StringIO(report)) for row in rows: event = Event() for key, value in row.items(): key = columns[key] if not value: continue value = value.strip() if key is "__IGNORE__" or key is "__TDB__": continue # set timezone explicitly to UTC as it is absent in the input if key == "source_time": value += " UTC" event.add(key, value) event.add('feed', 'shadowserver-qotd') event.add('type', 'vulnerable service') event.add('application_protocol', 'qotd') event = utils.parse_source_time(event, "source_time") event = utils.generate_observation_time(event, "observation_time") event = utils.generate_reported_fields(event) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.value("raw")) for row in raw_report.split('\n'): row = row.strip() if row.startswith("#") or len(row) == 0 or row.startswith('Start'): continue values = row.split("\t") if len(values) < 3: continue # raise an error network_ip = values[0] network_mask = values[2] network = '%s/%s' % (network_ip, network_mask) event = Event(report) event.add('source.network', network, sanitize=True) event.add('classification.type', u'blacklist') event.add("raw", row, sanitize=True) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return columns = [ "source.ip", "__IGNORE__", "event_description.text", "__IGNORE__" ] headers = True raw_report = utils.base64_decode(report.get("raw")) for row in utils.csv_reader(raw_report): # ignore headers if headers: headers = False continue event = Event(report) for key, value in zip(columns, row): if key == "__IGNORE__": continue event.add(key, value) event.add('classification.type', u'scanner') event.add("raw", ",".join(row)) self.send_message(event) self.acknowledge_message()
def parse_line(self, line, report): if line.startswith('#') or len(line) == 0: self.tempdata.append(line) else: event = Event(report) line_contents = line.split('|') feed_name = line_contents[-1].strip() file_format = FILE_FORMATS.get(feed_name) or FILE_FORMATS['_default'] if len(line_contents) != len(file_format) + 1: raise ValueError(f'Incorrect format for feed {event.get("feed.url")}, found line: "{line}"') if feed_name not in CATEGORY: raise ValueError(f'Unknown data feed {feed_name}.') event.update(CATEGORY[feed_name]) for field, (field_name, converter) in zip(line_contents, file_format): value = converter(field.strip()) if value is not None: event.add(field_name, value) event.add('raw', line) yield event
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return columns = [ "__IGNORE__", "source.url", "event_description.url", "time.source", "__IGNORE__", "__IGNORE__", "__IGNORE__", "event_description.target" ] raw_report = utils.base64_decode(report.get("raw")) for row in csv.reader(io.StringIO(raw_report)): # ignore headers if "phish_id" in row: continue event = Event(report) for key, value in zip(columns, row): if key == "__IGNORE__": continue event.add(key, value) event.add('classification.type', u'phishing') event.add("raw", ",".join(row)) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.value("raw")) for row in raw_report.split('\n'): row = row.strip() if len(row) == 0 or row.startswith('other'): continue event = Event(report) event.add('classification.type', u'brute-force') event.add("raw", row, sanitize=True) columns = ["source.ip"] row = row.split() for key, value in zip(columns, row): event.add(key, value, sanitize=True) self.send_message(event) self.acknowledge_message()
def parse_line(self, line, report): path = urlparse(report['feed.url']).path filename = posixpath.basename(path) event = Event(report) event.add('source.ip', line) if filename in MAPPING: for key, value in MAPPING[filename].items(): event.add(key, value) else: event.add('classification.type', 'blacklist') event.add("raw", line) yield event
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) for row in raw_report.splitlines(): row = row.strip() if len(row) == 0 or row.startswith('#'): continue splitted_row = row.split() event = Event(report) columns = ["source.ip", "time.source"] for key, value in zip(columns, splitted_row): if key == "time.source": value = datetime.utcfromtimestamp( int(value)).strftime('%Y-%m-%d %H:%M:%S') + " UTC" event.add(key, value.strip()) event.add('classification.type', 'blacklist') event.add("raw", row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.get("raw")) for row in raw_report.split('\n'): row = row.strip() if len(row) == 0 or row.startswith('#'): continue event = Event(report) splitted_row = row.split() columns = ["source.url"] for key, value in zip(columns, splitted_row): event.add(key, value) event.add('classification.type', u'malware') event.add("raw", row) self.send_message(event) self.acknowledge_message()
def parse_line(self, row, report): event = Event(report) for key, value in zip(self.parameters.columns, row): if key in ["__IGNORE__", ""]: continue if key in ["time.source", "time.destination"]: value = parse(value, fuzzy=True).isoformat() value += " UTC" # regex from http://stackoverflow.com/a/23483979 # matching ipv4/ipv6 IP within string elif key in ["source.ip", "destination.ip"]: value = re.compile( '(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])' '\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0' '-5])|(([a-zA-Z]|[a-zA-Z][a-zA-Z0-9\-]*[a-zA-Z0-9])' '\.)*([A-Za-z]|[A-Za-z][A-Za-z0-9\-]*[A-Za-z0-9])|' '\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(' '([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]' '|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d' '\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[' '0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[' '1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|' ':))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1' ',3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d' '\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){' '3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,' '4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-' '4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-' '9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-' 'Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0' '-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1' '\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}((' '(:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4' '}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[' '0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]' '{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2' '[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|' '[1-9]?\d)){3}))|:)))(%.+)?').match(value).group() elif key.endswith('.url') and '://' not in value: value = self.parameters.default_url_protocol + value elif key in ["classification.type"] and self.type_translation: if value in self.type_translation: value = self.type_translation[value] elif not hasattr(self.parameters, 'type'): continue event.add(key, value) if hasattr(self.parameters, 'type')\ and not event.contains("classification.type"): event.add('classification.type', self.parameters.type) event.add("raw", ",".join(row)) yield event
def process(self): report = self.receive_message() if report: report = report.strip() columns = { "timestamp": "source_time", "ip": "source_ip", "asn": "source_asn", "geo": "source_cc", "md5hash": "artifact_hash", "url" : "reported_destination_url", "user_agent" : "user_agent", "host": "reported_destination_reverse_dns", "method": "comment" } rows = csv.DictReader(StringIO.StringIO(report)) for row in rows: event = Event() for key, value in row.items(): key = columns[key] if not value: continue value = value.strip() if key is "__IGNORE__" or key is "__TDB__": continue # set timezone explicitly to UTC as it is absent in the input if key == "source_time": value += " UTC" if key== "comment": value ="HTTP Method ->"+value event.add(key, value) event.add('feed', 'shadowserver-Sandbox-Url') event.add('type', 'malware') event.add('artifact_hash_type','MD5') event = utils.parse_source_time(event, "source_time") event = utils.generate_observation_time(event, "observation_time") event = utils.generate_reported_fields(event) self.send_message(event) self.acknowledge_message()
def process(self): event = self.receive_message() if self.type == 'whitelist': new_event = Event() for key in self.keys: if key in event: new_event.add(key, event[key], sanitize=False) event = new_event else: for key in self.keys: if key in event: del event[key] self.send_message(event) self.acknowledge_message()
def test_multiple_bots(self): """ Let's simulate multiple IntelMQ instances are pushing the events at once! Every message must be queued. Note that if too much clients want connect, connections are refused. """ client_count = 5 msg_count = 300 for _ in range(client_count): bot = TestTCPOutputBot() bot.setUpClass() # bot.bot_id = "test-client-{}".format(_) bot.input_message = [] for i in range(msg_count): bot.input_message.append(Event(INPUT1, harmonization=self.harmonization)) Process(target=bot._delayed_start).start() thread = threading.Thread(target=Client().random_client) thread.start() self.input_message = None # can't use standard .bot_run(iteration) or .start() because shutdown() would be called # and we need to handle multiple connections self.prepare_bot() self.bot._Bot__source_pipeline = self.pipe self.bot._Bot__destination_pipeline = self.pipe for _ in range(client_count + 1): # every single calling of process() method will serve to a single connection with mock.patch('intelmq.lib.utils.load_configuration', new=self.mocked_config): with mock.patch('intelmq.lib.utils.log', self.mocked_log): self.bot.process() self.bot.stop() # let's call shutdown() and free up bound address self.assertOutputQueueLen(client_count * msg_count + 2)
def test_intelmq_exchange(self): """ Test if correct Events have been produced, sent from a TCP Output of another IntelMQ instance. We spawn independent process of the TCPOutput bot that sends a bunch of messages. """ bot = TestTCPOutputBot() bot.setUpClass() bot.input_message = [] msg_count = 100 for i in range(msg_count): bot.input_message.append( Event(INPUT1, harmonization=self.harmonization)) (Process(target=bot._delayed_start)).start() self.run_bot() self.assertOutputQueueLen(msg_count) for i, msg in enumerate(self.get_output_queue()): report = MessageFactory.unserialize( msg, harmonization=self.harmonization, default_type='Event') output = MessageFactory.unserialize( utils.base64_decode(report["raw"]), harmonization=self.harmonization, default_type='Event') self.assertDictEqual(output, INPUT1) del report['time.observation'] del report['raw'] self.assertDictEqual(report, REPORT1)
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.value("raw")) for row in raw_report.split('\n'): row = row.strip() if len(row) == 0 or row.startswith('#'): continue splitted_row = row.split() event = Event(report) columns = ["source.ip", "time.source"] for key, value in zip(columns, splitted_row): if key == "time.source": value = datetime.utcfromtimestamp( int(value)).strftime('%Y-%m-%d %H:%M:%S') + " UTC" event.add(key, value.strip(), sanitize=True) event.add('classification.type', u'blacklist') event.add("raw", row, sanitize=True) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report: event = Event() report = encode(report) # colums according to https://www.phishtank.com/developer_info.php as of 2015/04/30: # phish_id,url,phish_detail_url,submission_time,verified,verification_time,online,target # example: # 123456,http://www.example.com/,http://www.phishtank.com/phish_detail.php?phish_id=123456,2009-06-19T15:15:47+00:00,yes,2009-06-19T15:37:31+00:00,yes,1st National Example Bank columns = ["__IGNORE__", "source_url", "description_url", "source_time", "__IGNORE__", "__IGNORE__", "__IGNORE__", "target"] for row in unicodecsv.reader(StringIO(report), encoding='utf-8'): if "phish_id" in row: continue # skip header for key, value in zip(columns, row): if key == "__IGNORE__": continue event.add(key, value.strip()) event.add('feed', 'phishtank') event.add('type', 'phishing') event = utils.parse_source_time(event, "source_time") event = utils.generate_observation_time(event, "observation_time") event = utils.generate_reported_fields(event) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if not report: self.acknowledge_message() return if not report.contains("raw"): self.acknowledge_message() raw_report = utils.base64_decode(report.value("raw")) for row in raw_report.split("\n"): row = row.strip() if row == "" or row.startswith("#"): continue event = Event(report) event.add("source.ip", row, sanitize=True) event.add("classification.type", "malware") event.add("raw", row, sanitize=True) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.value("raw")) for row in raw_report.split('\n'): row = row.strip() if len(row) == 0 or row.startswith('#'): continue event = Event(report) splitted_row = row.split() columns = ["source.url"] for key, value in zip(columns, splitted_row): event.add(key, value, sanitize=True) event.add('classification.type', u'malware') event.add("raw", row, sanitize=True) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.get("raw")) for row in raw_report.split('\n'): row = row.strip() if len(row) == 0 or row.startswith('other'): continue event = Event(report) event.add('classification.type', u'brute-force') event.add("raw", row) columns = ["source.ip"] row = row.split() for key, value in zip(columns, row): event.add(key, value) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report: for row in report.split('\n'): row = row.strip() if len(row) == 0 or row.startswith('#'): continue row = row.split() event = Event() columns = ["source_url"] for key, value in zip(columns, row): event.add(key, value) event.add('feed', 'malwarepatrol-dansguardian') event.add('type', 'malware') event = utils.parse_source_time(event, "source_time") event = utils.generate_observation_time(event, "observation_time") event = utils.generate_reported_fields(event) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report: event = Event() columns = ["__IGNORE__", "source_url", "description_url", "source_time", "__IGNORE__", "__IGNORE__", "__IGNORE__", "target"] for row in unicodecsv.reader(StringIO(report), encoding='utf-8'): if "phish_id" in row: continue for key, value in zip(columns, row): if key == "__IGNORE__": continue event.add(key, value.strip()) event.add('feed', 'phishtank') event.add('type', 'phishing') event = utils.parse_source_time(event, "source_time") event = utils.generate_observation_time(event, "observation_time") event = utils.generate_reported_fields(event) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() dict_report = json.loads(utils.base64_decode(report.get("raw"))) # add all lists together, only one loop needed for row in sum(dict_report.values(), []): event = Event(report) event.add('classification.type', 'c&c') try: event.add('source.ip', row) except InvalidValue: event.add('source.fqdn', row) event.add("raw", row) self.send_message(event) self.acknowledge_message()
def process(self): """ Passing through all information from Report to Event. Also logs a sample line, which will be tested afterwards. """ report = self.receive_message() if not report: self.acknowledge_message() return event = Event() self.logger.info('Lorem ipsum dolor sit amet') for key in report.keys(): event.add(key, report[key], sanitize=False) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() columns = self.parameters.columns if not report or not report.contains("raw"): self.acknowledge_message() return if report: raw_report = utils.base64_decode(report.get("raw")) rows = csv.DictReader(StringIO(raw_report)) for row in rows: event = Event(report) for key, value in row.items(): key = columns[key] if not value: continue value = value.strip() if key == u'__IGNORE__' or key == u'__TBD__': continue event.add(key, value, sanitize=True) event.add('classification.type', u'vulnerable service') self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) for row in raw_report.splitlines(): row = row.strip() if row == "" or row.startswith("#"): continue event = Event(report) if IPAddress.is_valid(row): event.add('source.ip', row) else: event.add('source.fqdn', row) event.add('classification.type', 'malware') event.add('raw', row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.get("raw")) for row in csv.DictReader(io.StringIO(raw_report)): event = Event(report) for key, value in row.items(): if not value: continue if key is None: self.logger.warning('Value without key found, skipping the' ' value: {!r}'.format(value)) continue key = COLUMNS[key] if key == "__IGNORE__" or key == "__TDB__": continue if key == "source.fqdn" and IPAddress.is_valid(value, sanitize=True): continue if key == "time.source": value = value + " UTC" if key == "source.asn" and value.startswith("ASNA"): continue if key == "source.asn": for asn in value.split(','): if asn.startswith("AS"): value = asn.split("AS")[1] break event.add(key, value) event.add('classification.type', u'malware') event.add("raw", ",".join(row)) self.send_message(event) self.acknowledge_message()