def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
val = row.strip()
if not len(val) or val.startswith('#') or val.startswith('//'):
continue
event = Event(report)
if IPAddress.is_valid(val, sanitize=True):
event.add('source.ip', val, sanitize=True)
else:
event.add('source.network', val, sanitize=True)
event.add('classification.type', u'blacklist')
event.add('raw', row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, row, report):
if not row.startswith('http'):
return
url_object = urlparse(row)
if not url_object:
return
url = url_object.geturl()
hostname = url_object.hostname
port = url_object.port
event = Event(report)
if IPAddress.is_valid(hostname):
event.add("source.ip", hostname)
else:
event.add("source.fqdn", hostname)
event.add('classification.type', 'malware')
event.add("source.url", url)
if port:
event.add("source.port", port)
event.add("raw", row)
self.send_message(event)
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
val = row.strip()
if not len(val) or val.startswith('#') or val.startswith('//'):
continue
event = Event(report)
if IPAddress.is_valid(val, sanitize=True):
event.add('source.ip', val, sanitize=True)
else:
event.add('source.network', val, sanitize=True)
event.add('classification.type', u'blacklist')
event.add('raw', row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw")).strip()
row = raw_report.splitlines()[0]
time_str = row[row.find('(')+1:row.find(')')]
time = dateutil.parser.parse(time_str).isoformat()
for row in raw_report.split('\n'):
val = row.strip()
if not len(val) or val.startswith('#') or val.startswith('//'):
continue
event = Event(report)
if IPAddress.is_valid(val):
event.add('source.ip', val)
else:
event.add('source.network', val)
event.add('time.source', time)
event.add('classification.type', u'blacklist')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, row, report):
if not row.startswith('http'):
return
url_object = urlparse(row)
if not url_object:
return
url = url_object.geturl()
hostname = url_object.hostname
port = url_object.port
event = Event(report)
if IPAddress.is_valid(hostname):
event.add("source.ip", hostname)
else:
event.add("source.fqdn", hostname)
event.add('classification.type', 'malware')
event.add("source.url", url)
if port:
event.add("source.port", port)
event.add("raw", row)
self.send_message(event)
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in csv.DictReader(io.StringIO(raw_report)):
event = Event(report)
for key, value in row.items():
if not value:
continue
if key is None:
self.logger.warning('Value without key found, skipping the'
' value: {!r}'.format(value))
continue
key = COLUMNS[key]
if key == "__IGNORE__" or key == "__TDB__":
continue
if key == "source.fqdn" and IPAddress.is_valid(value,
sanitize=True):
continue
if key == "time.source":
value = value + " UTC"
event.add(key, value)
event.add('classification.type', 'phishing')
event.add("raw", ",".join(row))
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.strip()
if len(row) == 0 or row.startswith('#'):
continue
row = row.replace('\r', '')
values = row.split('\t')
# if special char is in string should not be allowed
if "#" in values[1]:
continue
# if domain name is localhost we are not interested
if values[1].lower().strip() == "localhost":
continue
event = self.new_event(report)
if IPAddress.is_valid(values[1]):
event.add("source.ip", values[1])
else:
event.add("source.fqdn", values[1])
event.add('classification.type', 'blacklist')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, row, report):
if not row.startswith('http'):
return []
url_object = urlparse(row)
if not url_object:
return []
url = url_object.geturl()
hostname = url_object.hostname
port = url_object.port
event = self.new_event(report)
if IPAddress.is_valid(hostname):
event.add("source.ip", hostname)
else:
event.add("source.fqdn", hostname)
event.add('classification.type', 'malware')
event.add("source.url", url)
if port:
event.add("source.port", port)
event.add("raw", row)
event.add("time.source", self.tempdata[1])
yield event
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.strip()
if len(row) == 0 or row.startswith('#'):
continue
row = row.replace('\r', '')
values = row.split('\t')
# if special char is in string should not be allowed
if "#" in values[1]:
continue
# if domain name is localhost we are not interested
if values[1].lower().strip() == "localhost":
continue
event = self.new_event(report)
if IPAddress.is_valid(values[1]):
event.add("source.ip", values[1])
else:
event.add("source.fqdn", values[1])
event.add('classification.type', 'blacklist')
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, line, report):
lastgenerated = None
if line.startswith('#') or len(line) == 0:
self.tempdata.append(line)
if '#Updated on' in line:
self.lastgenerated = line.strip('#Updated on ')
self.lastgenerated = dateutil.parser.parse(self.lastgenerated + ' -04:00').isoformat()
else:
event = self.new_event(report)
value = line.strip()
if self.lastgenerated:
event.add('time.source', self.lastgenerated)
event.add('raw', line)
event.add('classification.type', 'malware')
if report['feed.url'] in URLVirParserBot.IP_FEED:
event.add('source.ip', value)
event.add('event_description.text', 'Active Malicious IP Addresses Hosting Malware')
event.add('event_description.url', 'http://www.urlvir.com/search-ip-address/' + value + '/')
elif report['feed.url'] in URLVirParserBot.HOST_FEED:
if IPAddress.is_valid(value):
event.add('source.ip', value)
event.add('event_description.url', 'http://www.urlvir.com/search-ip-address/' + value + '/')
else:
event.add('source.fqdn', value)
event.add('event_description.url', 'http://www.urlvir.com/search-host/' + value + '/')
event.add('event_description.text', 'Active Malicious Hosts')
else:
raise ValueError('Unknown data feed %s.' % report['feed.url'])
yield event
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if row == "" or row.startswith("#"):
continue
event = Event(report)
if IPAddress.is_valid(row):
event.add('source.ip', row)
else:
event.add('source.fqdn', row)
event.add('classification.type', u'malware')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, row, report):
if not row.startswith('http'):
return []
url_object = urlparse(row)
if not url_object:
return []
url = url_object.geturl()
hostname = url_object.hostname
port = url_object.port
event = self.new_event(report)
if IPAddress.is_valid(hostname):
event.add("source.ip", hostname)
else:
event.add("source.fqdn", hostname)
event.add('classification.type', 'malware')
event.add("source.url", url)
if port:
event.add("source.port", port)
event.add("raw", row)
event.add("time.source", self.tempdata[1])
yield event
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in csv.DictReader(io.StringIO(raw_report)):
event = self.new_event(report)
for key, value in row.items():
if not value:
continue
if key is None:
self.logger.warning('Value without key found, skipping the'
' value: {!r}'.format(value))
continue
key = COLUMNS[key]
if key == "__IGNORE__" or key == "__TDB__":
continue
if key == "source.fqdn" and IPAddress.is_valid(value,
sanitize=True):
continue
if key == "time.source":
value = value + " UTC"
event.add(key, value)
event.add('classification.type', 'phishing')
event.add("raw", ",".join(row))
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.strip()
if len(row) == 0 or not row.startswith('http'):
continue
url_object = urlparse(row)
if not url_object:
continue
url = url_object.geturl()
hostname = url_object.hostname
port = url_object.port
event = Event(report)
if IPAddress.is_valid(hostname):
event.add("source.ip", hostname)
else:
event.add("source.fqdn", hostname)
event.add('classification.type', 'malware')
event.add("source.url", url)
if port:
event.add("source.port", port)
event.add("raw", row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw")).strip()
row = raw_report.splitlines()[0]
time_str = row[row.find('(') + 1:row.find(')')]
time = dateutil.parser.parse(time_str).isoformat()
for row in raw_report.split('\n'):
val = row.strip()
if not len(val) or val.startswith('#') or val.startswith('//'):
continue
event = Event(report)
if IPAddress.is_valid(val):
event.add('source.ip', val)
else:
event.add('source.network', val)
event.add('time.source', time)
event.add('classification.type', u'blacklist')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if not report:
self.acknowledge_message()
return
if not report.contains("raw"):
self.acknowledge_message()
raw_report = utils.base64_decode(report.value("raw"))
for row in raw_report.split('\n'):
row = row.strip()
if row == "" or row.startswith("#"):
continue
event = Event(report)
if IPAddress.is_valid(row, sanitize=True):
event.add('source.ip', row, sanitize=True)
else:
event.add('source.fqdn', row, sanitize=True)
event.add('classification.type', u'malware')
event.add('raw', row, sanitize=True)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, row, report):
event = self.new_event(report)
extra = {}
for key, value in row.items():
if not value:
continue
if key is None:
self.logger.warning('Value without key found, skipping the'
' value: {!r}'.format(value))
continue
key_orig = key
key = self.csv_fieldnames[key]
if key == "__IGNORE__":
continue
if key == "source.fqdn" and IPAddress.is_valid(value,
sanitize=True):
continue
if key == "time.source":
value = value + " UTC"
if key == "source.asn":
if value.upper().startswith("ASNA"):
continue
for asn in value.upper().split(','):
if asn.startswith("AS"):
value = asn.split("AS")[1]
break
if key == "status":
if value == 'down':
value = 'offline'
elif value == 'up':
value = 'online'
if key_orig == 'scanner' and value == 'undef':
continue
if key == 'extra':
extra[key_orig] = value
continue
event.add(key, value)
if extra:
event.add('extra', extra)
event.add('classification.type', self.type)
event.add("raw", self.recover_line_csv_dict(row))
yield event
def parse_line(self, row, report):
event = self.new_event(report)
extra = {}
for key, value in row.items():
if not value:
continue
if key is None:
self.logger.warning('Value without key found, skipping the'
' value: {!r}'.format(value))
continue
key_orig = key
key = self.csv_fieldnames[key]
if key == "__IGNORE__":
continue
if key == "source.fqdn" and IPAddress.is_valid(value,
sanitize=True):
continue
if key == "time.source":
value = value + " UTC"
if key == "source.asn":
if value.upper().startswith("ASNA"):
continue
for asn in value.upper().split(','):
if asn.startswith("AS"):
value = asn.split("AS")[1]
break
if key == "status":
if value == 'down':
value = 'offline'
elif value == 'up':
value = 'online'
if key_orig == 'scanner' and value == 'undef':
continue
if key == 'extra':
extra[key_orig] = value
continue
event.add(key, value)
if extra:
event.add('extra', extra)
event.add('classification.type', self.type)
event.add("raw", self.recover_line_csv_dict(row))
yield event
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.value("raw"))
fp = io.StringIO(raw_report)
rows = csv.DictReader(fp)
for row in rows:
event = Event(report)
for key, value in row.items():
if not value:
continue
if key is None:
self.logger.warning('Value without key found, skipping the'
' value: {!r}'.format(value))
continue
key = COLUMNS[key]
if key == "__IGNORE__" or key == "__TDB__":
continue
if key == "source.fqdn" and IPAddress.is_valid(value,
sanitize=True):
continue
if key == "time.source":
value = value + " UTC"
if key == "source.asn" and value.startswith("ASNA"):
continue
if key == "source.asn":
for asn in value.split(','):
if asn.startswith("AS"):
value = asn.split("AS")[1]
break
event.add(key, value, sanitize=True)
event.add('classification.type', u'malware')
event.add("raw", ",".join(row), sanitize=True)
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
if report is None or not report.contains("raw"):
self.acknowledge_message()
return
raw_report = utils.base64_decode(report.get("raw"))
for row in csv.DictReader(io.StringIO(raw_report)):
event = Event(report)
for key, value in row.items():
if not value:
continue
if key is None:
self.logger.warning('Value without key found, skipping the'
' value: {!r}'.format(value))
continue
key = COLUMNS[key]
if key == "__IGNORE__" or key == "__TDB__":
continue
if key == "source.fqdn" and IPAddress.is_valid(value,
sanitize=True):
continue
if key == "time.source":
value = value + " UTC"
if key == "source.asn" and value.startswith("ASNA"):
continue
if key == "source.asn":
for asn in value.split(','):
if asn.startswith("AS"):
value = asn.split("AS")[1]
break
event.add(key, value)
event.add('classification.type', u'malware')
event.add("raw", ",".join(row))
self.send_message(event)
self.acknowledge_message()
def process(self):
"""
The Ranswomware Tracker has comments in it.
The IP address field can also have more than one address.
The ASN and Country code are being ignored, an expert parser can get those added.
"""
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in csv.reader(io.StringIO(raw_report)):
if row[0].startswith('#'):
continue
if '|' in row[7]:
for ipaddr in row[7].split('|'):
new_row = '"' + row[0] + '","' + row[1] + '","' + row[2] + '","' + row[3] \
+ '","' + row[4] + '","' + row[5] + '","' + row[6] + '","' + ipaddr \
+ '","' + row[8] + '","' + row[9] + '"'
for nrow in csv.reader(io.StringIO(new_row)):
ev = Event(report)
ev.add('classification.identifier', nrow[2].lower())
ev.add('classification.type', 'c&c')
ev.add('time.source', nrow[0] + ' UTC', force=True)
ev.add('status', nrow[5])
ev.add('source.ip', nrow[7])
ev.add('raw', ','.join(nrow))
if FQDN.is_valid(nrow[3]):
ev.add('source.fqdn', nrow[3])
if URL.is_valid(nrow[4]):
ev.add('source.url', nrow[4])
self.send_message(ev)
else:
event = Event(report)
event.add('classification.identifier', row[2].lower())
event.add('classification.type', 'c&c')
event.add('time.source', row[0] + ' UTC')
event.add('status', row[5])
event.add('raw', ','.join(row))
if IPAddress.is_valid(row[7]):
event.add('source.ip', row[7])
if FQDN.is_valid(row[3]):
event.add('source.fqdn', row[3])
if URL.is_valid(row[4]):
event.add('source.url', row[4])
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in raw_report.splitlines():
row = row.strip()
if row == "" or row.startswith("#"):
continue
event = Event(report)
if IPAddress.is_valid(row):
event.add('source.ip', row)
else:
event.add('source.fqdn', row)
event.add('classification.type', 'malware')
event.add('raw', row)
self.send_message(event)
self.acknowledge_message()
def parse_line(self, line, report):
if line.startswith('#') or len(line) == 0:
self.tempdata.append(line)
else:
lvalue = line.split('\t')
event = Event(report)
event.add('classification.identifier', lvalue[0].lower())
event.add('time.source', DateTime.from_timestamp(int(lvalue[1])))
if IPAddress.is_valid(lvalue[2]):
event.add('source.ip', lvalue[2])
if FQDN.is_valid(lvalue[3]):
event.add('source.fqdn', lvalue[3])
if URL.is_valid(lvalue[4]):
event.add('source.url', lvalue[4])
event.add('raw', line)
event.add('classification.type', 'exploit')
event.add('event_description.url', 'http://data.netlab.360.com/ek')
yield event
def process(self):
"""
The Ranswomware Tracker has comments in it.
The IP address field can also have more than one address.
The ASN and Country code are being ignored, an expert parser can get those added.
"""
report = self.receive_message()
raw_report = utils.base64_decode(report.get("raw"))
for row in csv.reader(io.StringIO(raw_report)):
if row[0].startswith("#"):
continue
if "|" in row[7]:
for ipaddr in row[7].split("|"):
new_row = (
'"'
+ row[0]
+ '","'
+ row[1]
+ '","'
+ row[2]
+ '","'
+ row[3]
+ '","'
+ row[4]
+ '","'
+ row[5]
+ '","'
+ row[6]
+ '","'
+ ipaddr
+ '","'
+ row[8]
+ '","'
+ row[9]
+ '"'
)
for nrow in csv.reader(io.StringIO(new_row)):
ev = Event(report)
ev.add("classification.identifier", nrow[2].lower())
ev.add("classification.type", "c&c")
ev.add("time.source", nrow[0] + " UTC", force=True)
ev.add("status", nrow[5])
ev.add("source.ip", nrow[7])
ev.add("raw", ",".join(nrow))
if FQDN.is_valid(nrow[3]):
ev.add("source.fqdn", nrow[3])
if URL.is_valid(nrow[4]):
ev.add("source.url", nrow[4])
self.send_message(ev)
else:
event = Event(report)
event.add("classification.identifier", row[2].lower())
event.add("classification.type", "c&c")
event.add("time.source", row[0] + " UTC")
event.add("status", row[5])
event.add("raw", ",".join(row))
if IPAddress.is_valid(row[7]):
event.add("source.ip", row[7])
if FQDN.is_valid(row[3]):
event.add("source.fqdn", row[3])
if URL.is_valid(row[4]):
event.add("source.url", row[4])
self.send_message(event)
self.acknowledge_message()
def upload_file():
success = False
filename = os.path.join(VAR_STATE_PATH, '../webinput_csv.csv')
if 'file' in request.files and request.files['file'].filename:
request.files['file'].save(filename)
request.files['file'].stream.seek(0)
total_lines = request.files['file'].stream.read().count(b'\n') # we don't care about headers here
success = True
elif 'text' in request.form and request.form['text']:
with open(filename, mode='w', encoding='utf8') as handle:
handle.write(request.form['text'])
success = True
total_lines = len(request.form['text'].splitlines())
if not success and request.form.get('use_last_file', False):
success = True
filename, total_lines = get_temp_file()
elif success:
write_temp_file((filename, total_lines))
if not success:
return create_response('no file or text')
parameters = handle_parameters(request.form)
if parameters['has_header']:
total_lines -= 1
preview = []
valid_ip_addresses = None
valid_date_times = None
try:
with open(filename, encoding='utf8') as handle:
reader = csv.reader(handle, delimiter=parameters['delimiter'],
quotechar=parameters['quotechar'],
skipinitialspace=parameters['skipInitialSpace'],
escapechar=parameters['escapechar'],
)
for lineindex, line in enumerate(reader):
line = [col.replace(parameters['escapechar']*2, parameters['escapechar']) for col in line]
if parameters['skipInitialLines']:
if parameters['has_header'] and lineindex == 1:
for _ in range(parameters['skipInitialLines']):
line = next(reader)
elif not parameters['has_header'] and lineindex == 0:
for _ in range(parameters['skipInitialLines']):
line = next(reader)
if lineindex >= parameters['loadLinesMax'] + parameters['has_header']:
break
if valid_ip_addresses is None: # first data line
valid_ip_addresses = [0] * len(line)
valid_date_times = [0] * len(line)
for columnindex, value in enumerate(line):
if IPAddress.is_valid(value, sanitize=True):
valid_ip_addresses[columnindex] += 1
if DateTime.is_valid(value, sanitize=True):
valid_date_times[columnindex] += 1
preview.append(line)
except Exception as exc:
preview = [['Parse Error'], ['Is the number of columns consistent?']] + \
[[x] for x in traceback.format_exc().splitlines()]
column_types = ["IPAddress" if x/(total_lines if total_lines else 1) > 0.7 else None for x in valid_ip_addresses]
column_types = ["DateTime" if valid_date_times[i]/(total_lines if total_lines else 1) > 0.7 else x for i, x in enumerate(column_types)]
return create_response({"column_types": column_types,
"use_column": [bool(x) for x in column_types],
"preview": preview,
})
