def process(self): event = self.receive_message() keys = ["source.%s", "destination.%s"] for key in keys: ip_key = key % "ip" if ip_key not in event: continue ip = event.get(ip_key) ip_version = IPAddress.version(ip) ip_integer = IPAddress.to_int(ip) if ip_version == 4: minimum = MINIMUM_BGP_PREFIX_IPV4 elif ip_version == 6: minimum = MINIMUM_BGP_PREFIX_IPV6 cache_key = bin(ip_integer)[2:minimum + 2] cachevalue = self.cache.get(cache_key) result = None if cachevalue == DNS_EXCEPTION_VALUE: continue elif cachevalue: result = cachevalue else: rev_name = reversename.from_address(ip) try: results = resolver.query(rev_name, "PTR") expiration = results.expiration for result in results: # use first valid result if event.is_valid('source.reverse_dns', str(result)): break else: raise InvalidPTRResult except (dns.exception.DNSException, InvalidPTRResult) as e: # Set default TTL for 'DNS query name does not exist' error ttl = None if isinstance(e, dns.resolver.NXDOMAIN) else \ getattr(self.parameters, "cache_ttl_invalid_response", 60) self.cache.set(cache_key, DNS_EXCEPTION_VALUE, ttl) result = None else: ttl = datetime.fromtimestamp(expiration) - datetime.now() self.cache.set(cache_key, str(result), ttl=int(ttl.total_seconds())) if result is not None: event.add(key % 'reverse_dns', str(result), overwrite=True) self.send_message(event) self.acknowledge_message()
def process(self): event = self.receive_message() if event is None: self.acknowledge_message() return keys = ["source.%s", "destination.%s"] for key in keys: ip_key = key % "ip" if not event.contains(ip_key): continue ip = event.value(ip_key) ip_version = IPAddress.version(ip) ip_integer = IPAddress.to_int(ip) if ip_version == 4: minimum = MINIMUM_BGP_PREFIX_IPV4 elif ip_version == 6: minimum = MINIMUM_BGP_PREFIX_IPV6 else: self.logger.error("Invalid IP version {!r}".format(ip_version)) self.send_message(event) self.acknowledge_message() cache_key = bin(ip_integer)[2:minimum + 2] cachevalue = self.cache.get(cache_key) result = None if cachevalue: result = cachevalue else: rev_name = reversename.from_address(ip) try: result = resolver.query(rev_name, "PTR") expiration = result.expiration result = result[0] except dns.exception.DNSException as e: if isinstance(e, dns.resolver.NXDOMAIN): continue else: ttl = datetime.fromtimestamp(expiration) - datetime.now() self.cache.set(cache_key, str(result), ttl=int(ttl.total_seconds())) if result is not None: event.add(key % 'reverse_dns', str(result), sanitize=True, force=True) self.send_message(event) self.acknowledge_message()
def process(self): event = self.receive_message() keys = ["source.%s", "destination.%s"] for key in keys: ip_key = key % "ip" if ip_key not in event: continue ip = event.get(ip_key) ip_version = IPAddress.version(ip) ip_integer = IPAddress.to_int(ip) if ip_version == 4: minimum = MINIMUM_BGP_PREFIX_IPV4 elif ip_version == 6: minimum = MINIMUM_BGP_PREFIX_IPV6 cache_key = bin(ip_integer)[2: minimum + 2] cachevalue = self.cache.get(cache_key) result = None if cachevalue == DNS_EXCEPTION_VALUE: continue elif cachevalue: result = cachevalue else: rev_name = reversename.from_address(ip) try: results = resolver.query(rev_name, "PTR") expiration = results.expiration for result in results: # use first valid result if event.is_valid('source.reverse_dns', str(result)): break else: raise InvalidPTRResult except (dns.exception.DNSException, InvalidPTRResult) as e: # Set default TTL for 'DNS query name does not exist' error ttl = None if isinstance(e, dns.resolver.NXDOMAIN) else \ getattr(self.parameters, "cache_ttl_invalid_response", 60) self.cache.set(cache_key, DNS_EXCEPTION_VALUE, ttl) result = None else: ttl = datetime.fromtimestamp(expiration) - datetime.now() self.cache.set(cache_key, str(result), ttl=int(ttl.total_seconds())) if result is not None: event.add(key % 'reverse_dns', str(result), overwrite=True) self.send_message(event) self.acknowledge_message()
def process(self): event = self.receive_message() if event is None: self.acknowledge_message() return for key in ['source.', 'destination.']: ip_key = key + "ip" abuse_key = key + "abuse_contact" asn_key = key + "asn" ip = event.get(ip_key, None) if not ip: continue ip_version = IPAddress.version(ip) ip_integer = IPAddress.to_int(ip) if ip_version == 4: minimum = MINIMUM_BGP_PREFIX_IPV4 elif ip_version == 6: minimum = MINIMUM_BGP_PREFIX_IPV6 else: raise ValueError('Unexpected IP version ' '{!r}.'.format(ip_version)) cache_key = bin(ip_integer)[2:minimum + 2] cache_result = self.cache.get(cache_key) abuse = (event.get(abuse_key).split(',') if abuse_key in event else []) if cache_result: cache_result = ast.literal_eval(cache_result) cache_result = [n.strip() for n in cache_result] abuse.extend(cache_result) else: asn = event.get(asn_key, None) if self.query_db_asn and asn: abuse.extend(lib.query_asn(asn)) if self.query_db_ip and ip: abuse.extend(lib.query_ripedb(ip)) if self.query_stat_asn and asn: abuse.extend(lib.query_ripestat(asn)) if self.query_stat_ip and ip: abuse.extend(lib.query_ripestat(ip)) self.cache.set(cache_key, abuse) event.add(abuse_key, ','.join(filter(None, set(abuse))), force=True) self.send_message(event) self.acknowledge_message()
def process(self): event = self.receive_message() if event is None: self.acknowledge_message() return keys = ["source.%s", "destination.%s"] for key in keys: ip_key = key % "ip" if not event.contains(ip_key): continue ip = event.value(ip_key) ip_version = IPAddress.version(ip) ip_integer = IPAddress.to_int(ip) if ip_version == 4: minimum = MINIMUM_BGP_PREFIX_IPV4 elif ip_version == 6: minimum = MINIMUM_BGP_PREFIX_IPV6 else: self.logger.error("Invalid IP version {!r}".format(ip_version)) self.send_message(event) self.acknowledge_message() cache_key = bin(ip_integer)[2: minimum + 2] cachevalue = self.cache.get(cache_key) result = None if cachevalue: result = cachevalue else: rev_name = reversename.from_address(ip) try: result = resolver.query(rev_name, "PTR") expiration = result.expiration result = result[0] except dns.exception.DNSException as e: if isinstance(e, dns.resolver.NXDOMAIN): continue else: ttl = datetime.fromtimestamp(expiration) - datetime.now() self.cache.set(cache_key, str(result), ttl=int(ttl.total_seconds())) if result is not None: event.add(key % 'reverse_dns', str(result), sanitize=True, force=True) self.send_message(event) self.acknowledge_message()
def process(self): event = self.receive_message() if event is None: self.acknowledge_message() return for key in ['source.', 'destination.']: ip_key = key + "ip" abuse_key = key + "abuse_contact" asn_key = key + "asn" ip = event.get(ip_key, None) if not ip: continue ip_version = IPAddress.version(ip) ip_integer = IPAddress.to_int(ip) if ip_version == 4: minimum = MINIMUM_BGP_PREFIX_IPV4 elif ip_version == 6: minimum = MINIMUM_BGP_PREFIX_IPV6 else: raise ValueError('Unexpected IP version ' '{!r}.'.format(ip_version)) cache_key = bin(ip_integer)[2: minimum + 2] cache_result = self.cache.get(cache_key) abuse = (event.get(abuse_key).split(',') if abuse_key in event else []) if cache_result: cache_result = ast.literal_eval(cache_result) cache_result = [n.strip() for n in cache_result] abuse.extend(cache_result) else: asn = event.get(asn_key, None) if self.query_db_asn and asn: abuse.extend(lib.query_asn(asn)) if self.query_db_ip and ip: abuse.extend(lib.query_ripedb(ip)) if self.query_stat_asn and asn: abuse.extend(lib.query_ripestat(asn)) if self.query_stat_ip and ip: abuse.extend(lib.query_ripestat(ip)) self.cache.set(cache_key,abuse) event.add(abuse_key, ','.join(filter(None, set(abuse))), force=True) self.send_message(event) self.acknowledge_message()
def __ip_query(ip): ip_version = IPAddress.version(ip) reverse_ip = IPAddress.to_reverse(ip) if ip_version == 4: reverse = reverse_ip.split('.in-addr.arpa.')[0] version = "" else: reverse = reverse_ip.split('.ip6.arpa.')[0] version = "6" query = IP_QUERY % (reverse, version) return Cymru.__query(query)
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) for row in raw_report.splitlines(): row = row.strip() if len(row) == 0 or not row.startswith('http'): continue url_object = urlparse(row) if not url_object: continue url = url_object.geturl() hostname = url_object.hostname port = url_object.port event = Event(report) if IPAddress.is_valid(hostname): event.add("source.ip", hostname) else: event.add("source.fqdn", hostname) event.add('classification.type', 'malware') event.add("source.url", url) if port: event.add("source.port", port) event.add("raw", row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.value("raw")).strip() row = raw_report.splitlines()[0] time_str = row[row.find('(')+1:row.find(')')] time = dateutil.parser.parse(time_str).isoformat() for row in raw_report.split('\n'): val = row.strip() if not len(val) or val.startswith('#') or val.startswith('//'): continue event = Event(report) if IPAddress.is_valid(val): event.add('source.ip', val) else: event.add('source.network', val) event.add('time.source', time) event.add('classification.type', u'blacklist') event.add('raw', row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) for row in raw_report.splitlines(): row = row.strip() if len(row) == 0 or row.startswith('#'): continue row = row.replace('\r', '') values = row.split('\t') # if special char is in string should not be allowed if "#" in values[1]: continue # if domain name is localhost we are not interested if values[1].lower().strip() == "localhost": continue event = self.new_event(report) if IPAddress.is_valid(values[1]): event.add("source.ip", values[1]) else: event.add("source.fqdn", values[1]) event.add('classification.type', 'blacklist') event.add("raw", row) self.send_message(event) self.acknowledge_message()
def process(self): event = self.receive_message() keys = ["source.%s", "destination.%s"] for key in keys: ip_key = key % "ip" if ip_key not in event: continue address = event.get(ip_key) cache_key = CACHE_KEY % (IPAddress.version(address), address) result_json = self.cache_get(cache_key) if result_json: result = json.loads(result_json) else: result = Cymru.query(address) if not result: self.logger.info('Got no result from Cymru for IP address %r.', address) result_json = json.dumps(result) self.cache_set(cache_key, result_json) if not result: continue for result_key, result_value in result.items(): if result_key == 'registry' and result_value == 'other': continue event.add(key % result_key, result_value, overwrite=self.overwrite) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if not report: self.acknowledge_message() return if not report.contains("raw"): self.acknowledge_message() raw_report = utils.base64_decode(report.value("raw")) for row in raw_report.split('\n'): row = row.strip() if row == "" or row.startswith("#"): continue event = Event(report) if IPAddress.is_valid(row): event.add('source.ip', row) else: event.add('source.fqdn', row) event.add('classification.type', u'malware') event.add('raw', row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if not report: self.acknowledge_message() return if not report.contains("raw"): self.acknowledge_message() raw_report = utils.base64_decode(report.value("raw")) for row in raw_report.split('\n'): row = row.strip() if row == "" or row.startswith("#"): continue event = Event(report) if IPAddress.is_valid(row, sanitize=True): event.add('source.ip', row, sanitize=True) else: event.add('source.fqdn', row, sanitize=True) event.add('classification.type', u'malware') event.add('raw', row, sanitize=True) self.send_message(event) self.acknowledge_message()
def process(self): event = self.receive_message() if event is None: self.acknowledge_message() return for key in ["source.", "destination."]: ip_key = key + "ip" asn_key = key + "asn" bgp_key = key + "network" if not event.contains(ip_key): continue ip = event.get(ip_key) if IPAddress.version(ip) == 6: # Currently not supported by pyasn, fix will come soon continue info = self.database.lookup(ip) if info: if info[0]: event.add(asn_key, str(info[0]), force=True) if info[1]: event.add(bgp_key, str(info[1]), force=True) self.send_message(event) self.acknowledge_message()
def process(self): event = self.receive_message() for key in ["source.", "destination."]: ip_key = key + "ip" asn_key = key + "asn" bgp_key = key + "network" if not event.contains(ip_key): continue ip = event.get(ip_key) if IPAddress.version(ip) == 6: # Currently not supported by pyasn, fix will come soon continue info = self.database.lookup(ip) if info: if info[0]: event.add(asn_key, str(info[0]), force=True) if info[1]: event.add(bgp_key, str(info[1]), force=True) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) for row in csv.DictReader(io.StringIO(raw_report)): event = Event(report) for key, value in row.items(): if not value: continue if key is None: self.logger.warning('Value without key found, skipping the' ' value: {!r}'.format(value)) continue key = COLUMNS[key] if key == "__IGNORE__" or key == "__TDB__": continue if key == "source.fqdn" and IPAddress.is_valid(value, sanitize=True): continue if key == "time.source": value = value + " UTC" event.add(key, value) event.add('classification.type', 'phishing') event.add("raw", ",".join(row)) self.send_message(event) self.acknowledge_message()
def parse_line(self, row, report): if not row.startswith('http'): return url_object = urlparse(row) if not url_object: return url = url_object.geturl() hostname = url_object.hostname port = url_object.port event = Event(report) if IPAddress.is_valid(hostname): event.add("source.ip", hostname) else: event.add("source.fqdn", hostname) event.add('classification.type', 'malware') event.add("source.url", url) if port: event.add("source.port", port) event.add("raw", row) self.send_message(event)
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) for row in csv.DictReader(io.StringIO(raw_report)): event = self.new_event(report) for key, value in row.items(): if not value: continue if key is None: self.logger.warning('Value without key found, skipping the' ' value: {!r}'.format(value)) continue key = COLUMNS[key] if key == "__IGNORE__" or key == "__TDB__": continue if key == "source.fqdn" and IPAddress.is_valid(value, sanitize=True): continue if key == "time.source": value = value + " UTC" event.add(key, value) event.add('classification.type', 'phishing') event.add("raw", ",".join(row)) self.send_message(event) self.acknowledge_message()
def parse_line(self, line, report): lastgenerated = None if line.startswith('#') or len(line) == 0: self.tempdata.append(line) if '#Updated on' in line: self.lastgenerated = line.strip('#Updated on ') self.lastgenerated = dateutil.parser.parse(self.lastgenerated + ' -04:00').isoformat() else: event = self.new_event(report) value = line.strip() if self.lastgenerated: event.add('time.source', self.lastgenerated) event.add('raw', line) event.add('classification.type', 'malware') if report['feed.url'] in URLVirParserBot.IP_FEED: event.add('source.ip', value) event.add('event_description.text', 'Active Malicious IP Addresses Hosting Malware') event.add('event_description.url', 'http://www.urlvir.com/search-ip-address/' + value + '/') elif report['feed.url'] in URLVirParserBot.HOST_FEED: if IPAddress.is_valid(value): event.add('source.ip', value) event.add('event_description.url', 'http://www.urlvir.com/search-ip-address/' + value + '/') else: event.add('source.fqdn', value) event.add('event_description.url', 'http://www.urlvir.com/search-host/' + value + '/') event.add('event_description.text', 'Active Malicious Hosts') else: raise ValueError('Unknown data feed %s.' % report['feed.url']) yield event
def parse_line(self, row, report): if not row.startswith('http'): return [] url_object = urlparse(row) if not url_object: return [] url = url_object.geturl() hostname = url_object.hostname port = url_object.port event = self.new_event(report) if IPAddress.is_valid(hostname): event.add("source.ip", hostname) else: event.add("source.fqdn", hostname) event.add('classification.type', 'malware') event.add("source.url", url) if port: event.add("source.port", port) event.add("raw", row) event.add("time.source", self.tempdata[1]) yield event
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.get("raw")).strip() row = raw_report.splitlines()[0] time_str = row[row.find('(') + 1:row.find(')')] time = dateutil.parser.parse(time_str).isoformat() for row in raw_report.split('\n'): val = row.strip() if not len(val) or val.startswith('#') or val.startswith('//'): continue event = Event(report) if IPAddress.is_valid(val): event.add('source.ip', val) else: event.add('source.network', val) event.add('time.source', time) event.add('classification.type', u'blacklist') event.add('raw', row) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.value("raw")) for row in raw_report.split('\n'): val = row.strip() if not len(val) or val.startswith('#') or val.startswith('//'): continue event = Event(report) if IPAddress.is_valid(val, sanitize=True): event.add('source.ip', val, sanitize=True) else: event.add('source.network', val, sanitize=True) event.add('classification.type', u'blacklist') event.add('raw', row, sanitize=True) self.send_message(event) self.acknowledge_message()
def process(self): event = self.receive_message() if event is None: self.acknowledge_message() return keys = ["source.%s", "destination.%s"] for key in keys: ip_key = key % "ip" if not event.contains(ip_key): continue ip = event.value(ip_key) ip_version = IPAddress.version(ip) ip_integer = IPAddress.to_int(ip) if ip_version == 4: minimum = MINIMUM_BGP_PREFIX_IPV4 elif ip_version == 6: minimum = MINIMUM_BGP_PREFIX_IPV6 else: raise ValueError('Unexpected IP version ' '{!r}.'.format(ip_version)) cache_key = bin(ip_integer)[2:minimum + 2] result_json = self.cache.get(cache_key) if result_json: result = json.loads(result_json) else: result = Cymru.query(ip) result_json = json.dumps(result) self.cache.set(cache_key, result_json) for result_key, result_value in result.items(): event.add(key % result_key, result_value, sanitize=True, force=True) self.send_message(event) self.acknowledge_message()
def process(self): event = self.receive_message() keys = ["source.%s", "destination.%s"] for key in keys: ip_key = key % "ip" if ip_key not in event: continue ip = event.get(ip_key) ip_version = IPAddress.version(ip) ip_integer = IPAddress.to_int(ip) if ip_version == 4: minimum = MINIMUM_BGP_PREFIX_IPV4 elif ip_version == 6: minimum = MINIMUM_BGP_PREFIX_IPV6 else: raise ValueError('Unexpected IP version ' '{!r}.'.format(ip_version)) cache_key = bin(ip_integer)[2:minimum + 2] result_json = self.cache.get(cache_key) if result_json: result = json.loads(result_json) else: result = Cymru.query(ip) if not result: continue result_json = json.dumps(result) self.cache.set(cache_key, result_json) for result_key, result_value in result.items(): if result_key == 'registry' and result_value == 'other': continue event.add(key % result_key, result_value, overwrite=self.overwrite) self.send_message(event) self.acknowledge_message()
def process(self): event = self.receive_message() if event is None: self.acknowledge_message() return keys = ["source.%s", "destination.%s"] for key in keys: ip_key = key % "ip" if not event.contains(ip_key): continue ip = event.value(ip_key) ip_version = IPAddress.version(ip) ip_integer = IPAddress.to_int(ip) if ip_version == 4: minimum = MINIMUM_BGP_PREFIX_IPV4 elif ip_version == 6: minimum = MINIMUM_BGP_PREFIX_IPV6 else: raise ValueError('Unexpected IP version ' '{!r}.'.format(ip_version)) cache_key = bin(ip_integer)[2: minimum + 2] result_json = self.cache.get(cache_key) if result_json: result = json.loads(result_json) else: result = Cymru.query(ip) result_json = json.dumps(result) self.cache.set(cache_key, result_json) for result_key, result_value in result.items(): event.add(key % result_key, result_value, sanitize=True, force=True) self.send_message(event) self.acknowledge_message()
def process(self): event = self.receive_message() keys = ["source.%s", "destination.%s"] for key in keys: ip_key = key % "ip" if ip_key not in event: continue ip = event.get(ip_key) ip_version = IPAddress.version(ip) ip_integer = IPAddress.to_int(ip) if ip_version == 4: minimum = MINIMUM_BGP_PREFIX_IPV4 elif ip_version == 6: minimum = MINIMUM_BGP_PREFIX_IPV6 else: raise ValueError('Unexpected IP version ' '{!r}.'.format(ip_version)) cache_key = bin(ip_integer)[2: minimum + 2] result_json = self.cache.get(cache_key) if result_json: result = json.loads(result_json) else: result = Cymru.query(ip) if not result: continue result_json = json.dumps(result) self.cache.set(cache_key, result_json) for result_key, result_value in result.items(): if result_key == 'registry' and result_value == 'other': continue event.add(key % result_key, result_value, overwrite=True) self.send_message(event) self.acknowledge_message()
def parse_line(self, row, report): event = self.new_event(report) extra = {} for key, value in row.items(): if not value: continue if key is None: self.logger.warning('Value without key found, skipping the' ' value: {!r}'.format(value)) continue key_orig = key key = self.csv_fieldnames[key] if key == "__IGNORE__": continue if key == "source.fqdn" and IPAddress.is_valid(value, sanitize=True): continue if key == "time.source": value = value + " UTC" if key == "source.asn": if value.upper().startswith("ASNA"): continue for asn in value.upper().split(','): if asn.startswith("AS"): value = asn.split("AS")[1] break if key == "status": if value == 'down': value = 'offline' elif value == 'up': value = 'online' if key_orig == 'scanner' and value == 'undef': continue if key == 'extra': extra[key_orig] = value continue event.add(key, value) if extra: event.add('extra', extra) event.add('classification.type', self.type) event.add("raw", self.recover_line_csv_dict(row)) yield event
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.value("raw")) fp = io.StringIO(raw_report) rows = csv.DictReader(fp) for row in rows: event = Event(report) for key, value in row.items(): if not value: continue if key is None: self.logger.warning('Value without key found, skipping the' ' value: {!r}'.format(value)) continue key = COLUMNS[key] if key == "__IGNORE__" or key == "__TDB__": continue if key == "source.fqdn" and IPAddress.is_valid(value, sanitize=True): continue if key == "time.source": value = value + " UTC" if key == "source.asn" and value.startswith("ASNA"): continue if key == "source.asn": for asn in value.split(','): if asn.startswith("AS"): value = asn.split("AS")[1] break event.add(key, value, sanitize=True) event.add('classification.type', u'malware') event.add("raw", ",".join(row), sanitize=True) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.get("raw")) for row in csv.DictReader(io.StringIO(raw_report)): event = Event(report) for key, value in row.items(): if not value: continue if key is None: self.logger.warning('Value without key found, skipping the' ' value: {!r}'.format(value)) continue key = COLUMNS[key] if key == "__IGNORE__" or key == "__TDB__": continue if key == "source.fqdn" and IPAddress.is_valid(value, sanitize=True): continue if key == "time.source": value = value + " UTC" if key == "source.asn" and value.startswith("ASNA"): continue if key == "source.asn": for asn in value.split(','): if asn.startswith("AS"): value = asn.split("AS")[1] break event.add(key, value) event.add('classification.type', u'malware') event.add("raw", ",".join(row)) self.send_message(event) self.acknowledge_message()
def process(self): """ The Ranswomware Tracker has comments in it. The IP address field can also have more than one address. The ASN and Country code are being ignored, an expert parser can get those added. """ report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) for row in csv.reader(io.StringIO(raw_report)): if row[0].startswith('#'): continue if '|' in row[7]: for ipaddr in row[7].split('|'): new_row = '"' + row[0] + '","' + row[1] + '","' + row[2] + '","' + row[3] \ + '","' + row[4] + '","' + row[5] + '","' + row[6] + '","' + ipaddr \ + '","' + row[8] + '","' + row[9] + '"' for nrow in csv.reader(io.StringIO(new_row)): ev = Event(report) ev.add('classification.identifier', nrow[2].lower()) ev.add('classification.type', 'c&c') ev.add('time.source', nrow[0] + ' UTC', force=True) ev.add('status', nrow[5]) ev.add('source.ip', nrow[7]) ev.add('raw', ','.join(nrow)) if FQDN.is_valid(nrow[3]): ev.add('source.fqdn', nrow[3]) if URL.is_valid(nrow[4]): ev.add('source.url', nrow[4]) self.send_message(ev) else: event = Event(report) event.add('classification.identifier', row[2].lower()) event.add('classification.type', 'c&c') event.add('time.source', row[0] + ' UTC') event.add('status', row[5]) event.add('raw', ','.join(row)) if IPAddress.is_valid(row[7]): event.add('source.ip', row[7]) if FQDN.is_valid(row[3]): event.add('source.fqdn', row[3]) if URL.is_valid(row[4]): event.add('source.url', row[4]) self.send_message(event) self.acknowledge_message()
def process(self): report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) for row in raw_report.splitlines(): row = row.strip() if row == "" or row.startswith("#"): continue event = Event(report) if IPAddress.is_valid(row): event.add('source.ip', row) else: event.add('source.fqdn', row) event.add('classification.type', 'malware') event.add('raw', row) self.send_message(event) self.acknowledge_message()
def parse_line(self, line, report): if line.startswith('#') or len(line) == 0: self.tempdata.append(line) else: lvalue = line.split('\t') event = Event(report) event.add('classification.identifier', lvalue[0].lower()) event.add('time.source', DateTime.from_timestamp(int(lvalue[1]))) if IPAddress.is_valid(lvalue[2]): event.add('source.ip', lvalue[2]) if FQDN.is_valid(lvalue[3]): event.add('source.fqdn', lvalue[3]) if URL.is_valid(lvalue[4]): event.add('source.url', lvalue[4]) event.add('raw', line) event.add('classification.type', 'exploit') event.add('event_description.url', 'http://data.netlab.360.com/ek') yield event
def upload_file(): success = False filename = os.path.join(VAR_STATE_PATH, '../webinput_csv.csv') if 'file' in request.files and request.files['file'].filename: request.files['file'].save(filename) request.files['file'].stream.seek(0) total_lines = request.files['file'].stream.read().count(b'\n') # we don't care about headers here success = True elif 'text' in request.form and request.form['text']: with open(filename, mode='w', encoding='utf8') as handle: handle.write(request.form['text']) success = True total_lines = len(request.form['text'].splitlines()) if not success and request.form.get('use_last_file', False): success = True filename, total_lines = get_temp_file() elif success: write_temp_file((filename, total_lines)) if not success: return create_response('no file or text') parameters = handle_parameters(request.form) if parameters['has_header']: total_lines -= 1 preview = [] valid_ip_addresses = None valid_date_times = None try: with open(filename, encoding='utf8') as handle: reader = csv.reader(handle, delimiter=parameters['delimiter'], quotechar=parameters['quotechar'], skipinitialspace=parameters['skipInitialSpace'], escapechar=parameters['escapechar'], ) for lineindex, line in enumerate(reader): line = [col.replace(parameters['escapechar']*2, parameters['escapechar']) for col in line] if parameters['skipInitialLines']: if parameters['has_header'] and lineindex == 1: for _ in range(parameters['skipInitialLines']): line = next(reader) elif not parameters['has_header'] and lineindex == 0: for _ in range(parameters['skipInitialLines']): line = next(reader) if lineindex >= parameters['loadLinesMax'] + parameters['has_header']: break if valid_ip_addresses is None: # first data line valid_ip_addresses = [0] * len(line) valid_date_times = [0] * len(line) for columnindex, value in enumerate(line): if IPAddress.is_valid(value, sanitize=True): valid_ip_addresses[columnindex] += 1 if DateTime.is_valid(value, sanitize=True): valid_date_times[columnindex] += 1 preview.append(line) except Exception as exc: preview = [['Parse Error'], ['Is the number of columns consistent?']] + \ [[x] for x in traceback.format_exc().splitlines()] column_types = ["IPAddress" if x/(total_lines if total_lines else 1) > 0.7 else None for x in valid_ip_addresses] column_types = ["DateTime" if valid_date_times[i]/(total_lines if total_lines else 1) > 0.7 else x for i, x in enumerate(column_types)] return create_response({"column_types": column_types, "use_column": [bool(x) for x in column_types], "preview": preview, })
def process(self): """ The Ranswomware Tracker has comments in it. The IP address field can also have more than one address. The ASN and Country code are being ignored, an expert parser can get those added. """ report = self.receive_message() raw_report = utils.base64_decode(report.get("raw")) for row in csv.reader(io.StringIO(raw_report)): if row[0].startswith("#"): continue if "|" in row[7]: for ipaddr in row[7].split("|"): new_row = ( '"' + row[0] + '","' + row[1] + '","' + row[2] + '","' + row[3] + '","' + row[4] + '","' + row[5] + '","' + row[6] + '","' + ipaddr + '","' + row[8] + '","' + row[9] + '"' ) for nrow in csv.reader(io.StringIO(new_row)): ev = Event(report) ev.add("classification.identifier", nrow[2].lower()) ev.add("classification.type", "c&c") ev.add("time.source", nrow[0] + " UTC", force=True) ev.add("status", nrow[5]) ev.add("source.ip", nrow[7]) ev.add("raw", ",".join(nrow)) if FQDN.is_valid(nrow[3]): ev.add("source.fqdn", nrow[3]) if URL.is_valid(nrow[4]): ev.add("source.url", nrow[4]) self.send_message(ev) else: event = Event(report) event.add("classification.identifier", row[2].lower()) event.add("classification.type", "c&c") event.add("time.source", row[0] + " UTC") event.add("status", row[5]) event.add("raw", ",".join(row)) if IPAddress.is_valid(row[7]): event.add("source.ip", row[7]) if FQDN.is_valid(row[3]): event.add("source.fqdn", row[3]) if URL.is_valid(row[4]): event.add("source.url", row[4]) self.send_message(event) self.acknowledge_message()