def is_valid(value, sanitize=False): if sanitize: value = Base64().sanitize(value) try: utils.base64_decode(value) except TypeError: return False if not GenericType().is_valid(value): return False return True
def is_valid(value, sanitize=False): if sanitize: value = GenericType().sanitize(value) value = Base64().sanitize(value) try: utils.base64_decode(value) except TypeError: return False if not GenericType().is_valid(value): return False return True
def parse_azure(self, line, report): raw = self.recover_line(line) event = self.new_event(report) for key, value in line.copy().items(): if key == 'Payload': if value == 'AA==': # NULL del line[key] continue try: value = json.loads(utils.base64_decode(value)) # continue unpacking in next loop except json.decoder.JSONDecodeError: line[key] = utils.base64_decode(value) elif key == 'TLP' and value.lower() == 'unknown': del line[key] if isinstance(value, dict): for subkey, subvalue in value.items(): line['%s.%s' % (key, subkey)] = subvalue del line[key] for key, value in line.items(): if key == 'ThreatConfidence': if value == 'None': continue value = event.get('feed.accuracy', 100) * CONFIDENCE[value] / 100 elif key == 'DateTimeReceivedUtc': value = DateTime.from_windows_nt(value) elif key == 'Payload.ts': value = DateTime.from_timestamp(value) elif key == 'Payload.Protocol': payload_protocol = value[:value.find('/')] if payload_protocol: # needs to overwrite a field previously parsed and written event.add('protocol.application', payload_protocol, overwrite=True) # "HTTP/1.1", save additionally elif not value: continue if AZURE[key] != '__IGNORE__': # feed.accuracy is calculated newly and always needs to be overwritten event.add(AZURE[key], value, overwrite=self.overwrite or AZURE[key] == "feed.accuracy") event.add('classification.type', 'infected-system') event.add('raw', raw) yield event
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) lastgenerated = None for row in raw_report.splitlines(): event = Event(report) row = row.strip() if len(row) == 0: continue elif row.startswith("#"): if 'Generated on' in row: row = row.strip('# ')[13:] lastgenerated = dateutil.parser.parse(row).isoformat() continue event.add('time.source', lastgenerated) event.add('classification.type', 'c&c') event.add('source.fqdn', row) event.add("raw", row) event.add("malware.name", SOURCE_FEEDS[report.get("feed.url")]) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() columns = [ "source.ip", "source.geolocation.cc", "event_description.text", "source.asn" ] headers = True raw_report = utils.base64_decode(report.get("raw")) raw_report = raw_report.translate({0: None}) for row in csv.reader(io.StringIO(raw_report)): # ignore headers if headers: headers = False continue event = self.new_event(report) for key, value in zip(columns, row): if key == "__IGNORE__": continue event.add(key, value) event.add('classification.type', 'scanner') event.add("raw", ",".join(row)) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw = base64_decode(report['raw']) decoded = json.loads(raw) event = self.new_event(report) event['raw'] = raw if self.minimal_mode: for intelmqkey, shodankey in MAPPING_MINIMAL.items(): try: if decoded[shodankey]: event[intelmqkey] = decoded[shodankey] except KeyError: pass try: event['source.geolocation.cc'] = decoded["location"]["country_code"] except KeyError: pass event['time.source'] = CONVERSIONS['timestamp'](decoded["timestamp"]) event['extra.shodan'] = decoded event['classification.type'] = 'other' else: event.update(self.apply_mapping(MAPPING, decoded)) event.add('classification.type', 'other') event.add('classification.identifier', 'shodan-scan') for protocol in PROTOCOLS: if protocol in decoded: event.add('protocol.application', protocol) self.send_message(event) self.acknowledge_message()
def parse_json(self, report: dict): """ A basic JSON parser """ raw_report = utils.base64_decode(report.get("raw")) for line in json.loads(raw_report): yield line
def process(self): report = self.receive_message() if not report or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.value("raw")) for row in utils.csv_reader(raw_report): event = Event(report) self.logger.debug(repr(row)) event.add("time.source", row[0].replace('_', ' ') + " UTC") if row[1] != '-': event.add("source.url", self.add_http(row[1])) try: event.add("source.ip", row[2]) except InvalidValue: event.add("source.url", self.add_http(row[2])) event.add('source.ip', urlparse(row[2]).netloc) event.add("source.reverse_dns", row[3]) event.add("event_description.text", row[4]) # TODO: ignore abuse contact for now event.add("source.asn", int(row[6])) event.add('classification.type', u'malware') event.add("raw", ",".join(row)) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report["raw"]) for row in csv.DictReader(io.StringIO(raw_report), dictreader=True): event = Event(report) extra = {} event.add('time.source', row['timestamp']+' UTC') event.add('source.ip', row['ip']) event.add('protocol.transport', row['protocol']) event.add('source.port', row['port']) event.add('source.reverse_dns', row['hostname']) extra['sysdesc'] = row['sysdesc'] extra['sysname'] = row['sysname'] event.add('source.asn', row['asn']) event.add('source.geolocation.cc', row['geo']) event.add('source.geolocation.region', row['region']) event.add('source.geolocation.city', row['city']) if int(row['naics']): extra['naics'] = int(row['naics']) if int(row['sic']): extra['sic'] = int(row['sic']) if row['sector']: extra['sector'] = row['sector'] event.add('extra', extra) event.add('protocol.application', 'snmp') event.add('classification.type', 'vulnerable service') event.add('classification.identifier', 'snmp') event.add('raw', '"'+','.join(map(str, row.items()))+'"') self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if not report or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.value("raw")) for row in utils.csv_reader(raw_report): event = Event(report) self.logger.debug(repr(row)) event.add("time.source", row[0].replace("_", " ") + " UTC", sanitize=True) if row[1] != "-": event.add("source.url", row[1], sanitize=True) try: event.add("source.ip", row[2], sanitize=True) except InvalidValue: event.add("source.url", row[2], sanitize=True) event.add("source.ip", urlparse(row[2]).netloc, sanitize=True) event.add("source.reverse_dns", row[3], sanitize=True) event.add("event_description.text", row[4], sanitize=True) # TODO: ignore abuse contact for now event.add("source.asn", int(row[6])) event.add("classification.type", "malware") event.add("raw", ",".join(row), sanitize=True) self.send_message(event) self.acknowledge_message()
def parse(self, report: dict): feed = report['feed.url'] raw_lines = utils.base64_decode(report.get("raw")).splitlines() self.comments = [] data_lines = [] for r in raw_lines: if self.__is_comment_line_regex.search(r): self.comments.append(r) else: data_lines.append(self.__sanitize_csv_lines(r)) self.header_line = data_lines.pop(0) # remove CSV header line fields = [self.__sanitize_csv_lines(f) for f in self.header_line.split(',')] # First line is the CSV header file if len(fields) != len(FEEDS[feed]['format']): self.logger.warning("Feed '{}' has not the expected fields: {} != {}".format(feed, len(fields), len(FEEDS[feed]['format']))) raise ValueError("Abusech ip parser is not up to date with the format online") for line in self.comments: if 'Last updated' in line: self.__last_generated_date = dateutil.parser.parse(self.__date_regex.search(line).group(0)).isoformat() for line in data_lines: yield line.strip()
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.value("raw")) for row in raw_report.split('\n'): if row.startswith('#'): continue event = Event(report) match = re.search(REGEX_IP, row) if match: ip = match.group() match = re.search(REGEX_TIMESTAMP, row) if match: timestamp = match.group(1) + " UTC" event.add('time.source', timestamp, sanitize=True) event.add('source.ip', ip, sanitize=True) event.add('classification.type', u'brute-force') event.add("raw", row, sanitize=True) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.value("raw")) for row in raw_report.split('\n'): row = row.strip() if len(row) == 0 or row.startswith('#'): continue splitted_row = row.split() event = Event(report) columns = ["source.ip", "time.source"] for key, value in zip(columns, splitted_row): if key == "time.source": value = datetime.utcfromtimestamp( int(value)).strftime('%Y-%m-%d %H:%M:%S') + " UTC" event.add(key, value.strip(), sanitize=True) event.add('classification.type', u'blacklist') event.add("raw", row, sanitize=True) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.get("raw")) for row in raw_report.split('\n'): if row.startswith("#") or len(row) == 0 or row == "Site": if 'updated' in row: time_str = row[row.find(': ') + 2:] time = dateutil.parser.parse(time_str).isoformat() continue event = Event(report) event.add('classification.type', u'malware') event.add('source.fqdn', row.strip()) event.add('time.source', time) event.add("raw", row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if not report: self.acknowledge_message() return if not report.contains("raw"): self.acknowledge_message() raw_report = utils.base64_decode(report.value("raw")) for row in raw_report.split('\n'): row = row.strip() if row.startswith("#") or len(row) == 0: continue event = Event(report) event.add('classification.type', u'c&c') event.add('source.fqdn', row, sanitize=True) event.add("raw", row, sanitize=True) event.add("malware.name", SOURCE_FEEDS[report.value("feed.url")], sanitize=True) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() event = MessageFactory.unserialize(base64_decode(report['raw'])) self.send_message(event) self.acknowledge_message()
def parse(self, report): raw_report = utils.base64_decode(report.get("raw")) split = raw_report.split("</tr>") self.tempdata = ['</tr>'.join(split[:2])] # TODO: save ending line for line in split[2:]: yield line.strip()
def process(self): report = self.receive_message() if not report: self.acknowledge_message() return if not report.contains("raw"): self.acknowledge_message() raw_report = utils.base64_decode(report.get("raw")) lastgenerated = None for row in raw_report.split('\n'): event = Event(report) row = row.strip() if len(row) == 0: continue elif row.startswith("#"): if 'Generated on' in row: row = row.strip('# ')[13:] lastgenerated = dateutil.parser.parse(row).isoformat() continue event.add('time.source', lastgenerated) event.add('classification.type', u'c&c') event.add('source.fqdn', row) event.add("raw", row) event.add("malware.name", SOURCE_FEEDS[report.get("feed.url")]) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report["raw"]) report_list = [row.strip() for row in raw_report.splitlines()] index = 0 actual_line = report_list[index] while parser.lsData != "IPs": index += 1 actual_line = report_list[index] parser.feed(actual_line) count = 0 while actual_line[:8] != "</TBODY>": index += 1 actual_line = report_list[index] parser.feed(actual_line) if parser.lsTag == "input": if count % 2 == 0: url = parser.lsValue url_raw_line = actual_line else: ip = parser.lsValue event = self.new_event(report) if harmonization.FQDN.is_valid(url, sanitize=True): event.add("source.fqdn", url) if harmonization.IPAddress.is_valid(ip, sanitize=True): event.add("source.ip", ip) event.add("raw", url_raw_line + actual_line) event.add("classification.type", "phishing") self.send_message(event) count += 1 self.acknowledge_message()
def parse(self, report): raw_report = utils.base64_decode(report.get("raw")) raw_report = raw_report.translate({0: None}) # ignore lines starting with #. # can have leading spaces/tabs raw_report = re.sub(r'(?m)^[ \t]*#.*\n?', '', raw_report) # ignore null bytes raw_report = re.sub(r'(?m)\0', '', raw_report) # ignore lines having mix of spaces and tabs only raw_report = re.sub(r'(?m)^[ \t]*\n?', '', raw_report) # skip header if getattr(self.parameters, 'skip_header', False): self.tempdata.append(raw_report[:raw_report.find('\n')]) raw_report = raw_report[raw_report.find('\n') + 1:] for row in csv.reader(io.StringIO(raw_report), delimiter=str(self.parameters.delimiter)): if self.filter_text and self.filter_type: text_in_row = self.filter_text in self.parameters.delimiter.join( row) if text_in_row and self.filter_type == 'whitelist': yield row elif not text_in_row and self.filter_type == 'blacklist': yield row else: continue else: yield row
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) for row in raw_report.splitlines(): if not row or row.startswith('#'): continue event = self.new_event(report) match = re.search(REGEX_IP, row) ip = None if match: ip = match.group() match = re.search(REGEX_TIMESTAMP, row) timestamp = None if match: timestamp = match.group(1) + " UTC" if not timestamp: raise ValueError('No timestamp found.') elif not ip: raise ValueError('No ip found.') event.add('time.source', timestamp) event.add('source.ip', ip) event.add('classification.type', 'brute-force') event.add("raw", row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) for row in raw_report.splitlines(): row = row.strip() if len(row) == 0 or row.startswith('#'): continue splitted_row = row.split() event = Event(report) columns = ["source.ip", "time.source"] for key, value in zip(columns, splitted_row): if key == "time.source": value = datetime.utcfromtimestamp( int(value)).strftime('%Y-%m-%d %H:%M:%S') + " UTC" event.add(key, value.strip()) event.add('classification.type', 'blacklist') event.add("raw", row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if not report.contains("raw"): self.acknowledge_message() raw_report = utils.base64_decode(report.value("raw")) for row in raw_report.split('\n'): row = row.strip() if len(row) == 0 or row.startswith('other'): continue event = Event() time_observation = DateTime().generate_datetime_now() event.add('time.observation', time_observation, sanitize=True) event.add('feed.name', u'arbor') event.add('feed.url', u'http://atlas-public.ec2.arbor.net/public/ssh_attackers') event.add('classification.type', u'brute-force') event.add("raw", row, sanitize=True) columns = ["source.ip"] row = row.split() for key, value in zip(columns, row): event.add(key, value, sanitize=True) self.send_message(event) self.acknowledge_message()
def parse_csv(self, report): """ A basic CSV parser. """ raw_report = utils.base64_decode(report.get("raw")) for line in csv.reader(io.StringIO(raw_report)): yield line
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) for row in raw_report.splitlines(): row = row.strip() if len(row) == 0 or row.startswith('#'): continue row = row.replace('\r', '') values = row.split('\t') # if special char is in string should not be allowed if "#" in values[1]: continue # if domain name is localhost we are not interested if values[1].lower().strip() == "localhost": continue event = self.new_event(report) if IPAddress.is_valid(values[1]): event.add("source.ip", values[1]) else: event.add("source.fqdn", values[1]) event.add('classification.type', 'blacklist') event.add("raw", row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) for row in raw_report.splitlines(): if not row or row.startswith('#'): continue event = Event(report) match = re.search(REGEX_IP, row) if match: ip = match.group() match = re.search(REGEX_TIMESTAMP, row) if match: timestamp = match.group(1) + " UTC" if not timestamp: raise ValueError('No timestamp found.') event.add('time.source', timestamp) event.add('source.ip', ip) event.add('classification.type', 'brute-force') event.add("raw", row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")).strip() if not len(raw_report): # We depend on first line = date self.acknowledge_message() return row = raw_report.splitlines()[0] time_str = row[row.find('(') + 1:row.find(')')] time = dateutil.parser.parse(time_str).isoformat() for row in raw_report.splitlines(): val = row.strip() if not len(val) or val.startswith('#') or val.startswith('//'): continue event = self.new_event(report) if not event.add('source.ip', val, raise_failure=False): event.add('source.network', val) event.add('time.source', time) event.add('classification.type', 'blacklist') event.add('raw', row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report["raw"]) report_list = [row.strip() for row in raw_report.splitlines()] index = 0 actual_line = report_list[index] while parser.lsData != "IPs": index += 1 actual_line = report_list[index] parser.feed(actual_line) count = 0 while actual_line[:8] != "</TBODY>": index += 1 actual_line = report_list[index] parser.feed(actual_line) if parser.lsTag == "input": if count % 2 == 0: url = parser.lsValue url_raw_line = actual_line else: ip = parser.lsValue event = self.new_event(report) event.add("source.fqdn", url) event.add("source.ip", ip) event.add("raw", url_raw_line + actual_line) event.add("classification.type", "phishing") self.send_message(event) count += 1 self.acknowledge_message()
def process(self): event = self.receive_message() if self.format_filename: ev = defaultdict(None) ev.update(event) # remove once #671 is done if 'time.observation' in ev: ev['time.observation'] = datetime.datetime.strptime(ev['time.observation'], '%Y-%m-%dT%H:%M:%S+00:00') if 'time.source' in ev: ev['time.source'] = datetime.datetime.strptime(ev['time.source'], '%Y-%m-%dT%H:%M:%S+00:00') filename = self.parameters.file.format(event=ev) if not self.file or filename != self.file.name: self.open_file(filename) if self.single_key: event_data = str(event.get(self.single_key)) if self.single_key == 'raw': event_data = base64_decode(event_data) else: event_data = event.to_json(hierarchical=self.parameters.hierarchical_output) try: self.file.write(event_data) self.file.write("\n") self.file.flush() except FileNotFoundError: self.init() else: self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report["raw"]) report_list = [row.strip() for row in raw_report.splitlines()] index = 0 while parser.lsData != "Details": index += 1 parser.feed(report_list[index]) while report_list[index] != "</table>": index += 1 parser.feed(report_list[index]) if parser.lsCSP: index += 1 parser.feed(report_list[index]) event = self.new_event(report) raw_url_line = report_list[index] event.add("source.url", parser.lsData) event.add("classification.type", "blacklist") index += 1 parser.feed(report_list[index]) event.add("classification.identifier", parser.lsData) event.add("classification.taxonomy", "other") index += 1 parser.feed(report_list[index]) event.add("time.source", parser.lsData) event.add("raw", raw_url_line + report_list[index]) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if not report: self.acknowledge_message() return if not report.contains("raw"): self.acknowledge_message() raw_report = utils.base64_decode(report.get("raw")) for row in raw_report.split('\n'): row = row.strip() if row == "" or row.startswith("#"): continue event = Event(report) event.add('source.ip', row) event.add('classification.type', u'malware') event.add('raw', row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) for row in csv.DictReader(io.StringIO(raw_report)): event = self.new_event(report) for key, value in row.items(): if not value: continue if key is None: self.logger.warning('Value without key found, skipping the' ' value: {!r}'.format(value)) continue key = COLUMNS[key] if key == "__IGNORE__" or key == "__TDB__": continue if key == "source.fqdn" and IPAddress.is_valid(value, sanitize=True): continue if key == "time.source": value = value + " UTC" event.add(key, value) event.add('classification.type', 'phishing') event.add("raw", ",".join(row)) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) raw_report = raw_report.split("<tbody>")[1] raw_report = raw_report.split("</tbody>")[0] raw_report_splitted = raw_report.split("<tr>") for row in raw_report_splitted: row = row.strip() if row == "": continue row_splitted = row.split("<td>") ip = row_splitted[1].split('">')[1].split("<")[0].strip() time_source = row_splitted[6].replace("</td></tr>", "").strip() time_source = time_source + " 00:00:00 UTC" event = Event(report) event.add('time.source', time_source) event.add('classification.type', 'malware') event.add('source.ip', ip) event.add('raw', row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) for row in raw_report.split('<tr>'): # Get IP and Type info1 = re.search( ">[\ ]*(\d+\.\d+\.\d+\.\d+)[\ ]*<.*</td><td>([^<]+)</td>", row) if not info1: continue # Get Timestamp info2 = re.search( "<td>[\ ]*(\d{4}-\d{2}-\d{2}\ \d{2}:\d{2}:\d{2})[\ ]*</td>", row) event = self.new_event(report) description = info1.group(2) description = utils.decode(description) event_type = self.get_type(description) time_source = info2.group(1) + " UTC-8" event.add("time.source", time_source) event.add("source.ip", info1.group(1)) event.add('classification.type', event_type) event.add('event_description.text', description) event.add("raw", row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return columns = [ "__IGNORE__", "source.url", "event_description.url", "time.source", "__IGNORE__", "__IGNORE__", "__IGNORE__", "event_description.target" ] raw_report = utils.base64_decode(report.get("raw")) for row in csv.reader(io.StringIO(raw_report)): # ignore headers if "phish_id" in row: continue event = Event(report) for key, value in zip(columns, row): if key == "__IGNORE__": continue event.add(key, value) event.add('classification.type', u'phishing') event.add("raw", ",".join(row)) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw = base64_decode(report['raw']) decoded = json.loads(raw) event = self.new_event(report) event['raw'] = raw if self.minimal_mode: for intelmqkey, shodankey in MAPPING_MINIMAL.items(): try: if decoded[shodankey]: event[intelmqkey] = decoded[shodankey] except KeyError: pass try: event['source.geolocation.cc'] = decoded["location"][ "country_code"] except KeyError: pass event['time.source'] = CONVERSIONS['timestamp']( decoded["timestamp"]) event['extra.shodan'] = decoded event['classification.type'] = 'other' else: event.update(self.apply_mapping(MAPPING, decoded)) event.add('classification.type', 'other') event.add('classification.identifier', 'shodan-scan') for protocol in PROTOCOLS: if protocol in decoded: event.add('protocol.application', protocol) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() columns = self.parameters.columns if not report or not report.contains("raw"): self.acknowledge_message() return if report: raw_report = utils.base64_decode(report.get("raw")) rows = csv.DictReader(StringIO(raw_report)) for row in rows: event = Event(report) for key, value in row.items(): key = columns[key] if not value: continue value = value.strip() if key == u'__IGNORE__' or key == u'__TBD__': continue event.add(key, value, sanitize=True) event.add('classification.type', u'vulnerable service') self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report["raw"]) for row in csv.DictReader(io.StringIO(raw_report), dictreader=True): event = Event(report) extra = {} event.add("time.source", row["timestamp"] + " UTC") event.add("source.ip", row["ip"]) event.add("protocol.transport", row["protocol"]) event.add("source.port", row["port"]) event.add("source.reverse_dns", row["hostname"]) event.add("protocol.application", row["tag"]) extra["quote"] = row["quote"] event.add("source.asn", row["asn"]) event.add("source.geolocation.cc", row["geo"]) event.add("source.geolocation.region", row["region"]) event.add("source.geolocation.city", row["city"]) if int(row["naics"]): extra["naics"] = int(row["naics"]) if int(row["sic"]): extra["sic"] = int(row["sic"]) if row["sector"]: extra["sector"] = row["sector"] event.add("extra", extra) event.add("classification.type", "vulnerable service") event.add("classification.identifier", "qotd") event.add("raw", '"' + ",".join(map(str, row.items())) + '"') self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if not report: self.acknowledge_message() return if not report.contains("raw"): self.acknowledge_message() raw_report = utils.base64_decode(report.value("raw")) for row in raw_report.split('\n'): row = row.strip() if row == "" or row.startswith("#"): continue event = Event(report) if IPAddress.is_valid(row, sanitize=True): event.add('source.ip', row, sanitize=True) else: event.add('source.fqdn', row, sanitize=True) event.add('classification.type', u'malware') event.add('raw', row, sanitize=True) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.value("raw")) for row in raw_report.split('\n'): row = row.rstrip() if row.startswith("#") or len(row) == 0: continue values = row.split('\t')[1:] event = Event(report) event.add('source.fqdn', values[1], sanitize=True) event.add('event_description.text', values[2], sanitize=True) for i in range(4, len(values)): if is_valid_date(values[i]): event.add('time.source', # TODO: verify timezone values[i] + "T00:00:00+00:00", sanitize=True) break event.add('classification.type', u'malware') event.add('raw', row, sanitize=True) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report["raw"]) report_list = [row.strip() for row in raw_report.splitlines()] index = 0 actual_line = report_list[index] while actual_line != "Recent domains": index += 1 actual_line = report_list[index] parser.set_empty_data while actual_line != "Recent hosts": index += 1 actual_line = report_list[index] count1 = len(parser.lsData) parser.feed(actual_line) count2 = len(parser.lsData) if count1 != count2: parser.raw_lines.append(actual_line) parser.process_data for item in range(len(parser.lsData)): event = self.new_event(report) event.add("source.fqdn", parser.lsData[item]) event.add("classification.type", "blacklist") event.add("raw", parser.raw_lines[item]) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.get("raw")) raw_report = raw_report.strip() url = report.get('feed.url') path = urlparse(url).path filename = posixpath.basename(path) for row in raw_report.split('\n'): event = Event(report) event.add('source.ip', row.strip()) if filename in MAPPING: for key, value in MAPPING[filename].items(): event.add(key, value) else: event.add('classification.type', 'blacklist') event.add("raw", row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() self.event_date = None raw_report = utils.base64_decode(report.get("raw")) for row in raw_report.splitlines(): row = row.strip() if row.startswith('; Last-Modified:'): self.event_date = row.split('; Last-Modified: ')[1].strip() self.event_date = dateparser(self.event_date) if row == "" or row.startswith(';'): continue row_splitted = row.split(';') network = row_splitted[0].strip() event = Event(report) event.add('source.network', network) event.add('extra', {'blocklist': row_splitted[1].strip()}) if self.event_date: event.add('time.source', self.event_date.isoformat()) event.add('classification.type', 'spam') event.add('raw', row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get('raw')) misp_event = json.loads(raw_report) # Set the classifier based on the ecsirt tag classifier = None if misp_event.get('Tag'): for tag in misp_event['Tag']: if tag['name'] in self.MISP_TAXONOMY_MAPPING: classifier = self.MISP_TAXONOMY_MAPPING[tag['name']] break # get the attributes from the event event_attributes = misp_event['Attribute'] # payload type - get malware variant for the event malware_variant = None for attribute in event_attributes: if attribute['category'] == 'Payload type': value = attribute['value'].lower() # TODO: use misp galaxies if value and harmonization.LowercaseString.is_valid(value): malware_variant = value # MISP event URL url_path = 'event/view/{}'.format(misp_event['id']) misp_event_url = urljoin(report['feed.url'], url_path) # Process MISP event attributes as separate IntelMQ events for attribute in event_attributes: # get details of attribute value = attribute['value'] uuid = attribute['uuid'] comment = attribute['comment'] timestamp = attribute['timestamp'] category = attribute['category'] type_ = attribute['type'] # create intelmq events based on the category if (category in self.SUPPORTED_MISP_CATEGORIES and type_ in self.MISP_TYPE_MAPPING): # Create and send the intelmq event event = Event(report) event.add('raw', json.dumps(attribute, sort_keys=True)) event.add(self.MISP_TYPE_MAPPING[type_], value) event.add('misp.event_uuid', misp_event['uuid']) event.add('misp.attribute_uuid', uuid) event.add('comment', comment) event.add('event_description.text', category) event.add('event_description.url', misp_event_url) event.add('malware.name', malware_variant) event.add('classification.type', classifier) event.add('time.source', '{} UTC'.format( datetime.utcfromtimestamp(float(timestamp)))) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) for row in raw_report.splitlines(): row = row.strip() if len(row) == 0 or row.startswith('#'): continue splitted_row = row.split() event = self.new_event(report) columns = ["source.ip", "time.source"] for key, value in zip(columns, splitted_row): if key == "time.source": value = datetime.utcfromtimestamp( int(value)).strftime('%Y-%m-%d %H:%M:%S') + " UTC" event.add(key, value.strip()) event.add('classification.type', 'blacklist') event.add("raw", row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.get("raw")).strip() row = raw_report.splitlines()[0] time_str = row[row.find('(') + 1:row.find(')')] time = dateutil.parser.parse(time_str).isoformat() for row in raw_report.split('\n'): val = row.strip() if not len(val) or val.startswith('#') or val.startswith('//'): continue event = Event(report) if IPAddress.is_valid(val): event.add('source.ip', val) else: event.add('source.network', val) event.add('time.source', time) event.add('classification.type', u'blacklist') event.add('raw', row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.value("raw")) for row in raw_report.split('\n'): row = row.strip() if len(row) == 0 or row.startswith('#'): continue event = Event(report) splitted_row = row.split() columns = ["source.url"] for key, value in zip(columns, splitted_row): event.add(key, value, sanitize=True) event.add('classification.type', u'malware') event.add("raw", row, sanitize=True) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if not report: self.acknowledge_message() return if not report.contains("raw"): self.acknowledge_message() raw_report = utils.base64_decode(report.value("raw")) for row in raw_report.split("\n"): row = row.strip() if row == "" or row.startswith("#"): continue event = Event(report) event.add("source.ip", row, sanitize=True) event.add("classification.type", "malware") event.add("raw", row, sanitize=True) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return columns = [ "source.ip", "__IGNORE__", "event_description.text", "__IGNORE__" ] headers = True raw_report = utils.base64_decode(report.get("raw")) for row in utils.csv_reader(raw_report): # ignore headers if headers: headers = False continue event = Event(report) for key, value in zip(columns, row): if key == "__IGNORE__": continue event.add(key, value) event.add('classification.type', u'scanner') event.add("raw", ",".join(row)) self.send_message(event) self.acknowledge_message()