def test_vuln_id_parser():
DEBUG = False
##DEBUG = True
if DEBUG: from pprint import pprint
print "Testing the vulnerability ID parsers..."
if DEBUG:
print "-" * 79
pprint(_test_case_extract_solution)
print "-" * 79
vulns = extract_vuln_ids(_test_case_extract)
if DEBUG:
pprint(vulns)
print "-" * 79
assert vulns == _test_case_extract_solution
all_vulns = []
for v in vulns.values():
all_vulns.extend(v)
all_vulns.sort()
if DEBUG:
pprint(all_vulns)
print "-" * 79
refs = convert_vuln_ids_to_references(all_vulns)
if DEBUG:
pprint(refs)
print "-" * 79
unrefs = convert_references_to_vuln_ids(refs)
if DEBUG:
pprint(unrefs)
print "-" * 79
assert unrefs == vulns
print "Testing reference URLs..."
import requests
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.62 Safari/537.36"}
for url in refs:
print "--> " + url
requests.get(url, headers=headers)
def test_vuln_id_parser():
DEBUG = False
##DEBUG = True
if DEBUG:
from pprint import pprint
print "Testing the vulnerability ID parsers..."
if DEBUG:
print "-" * 79
print "-- test case solution"
pprint(_test_case_extract_solution)
print "-" * 79
vulns = extract_vuln_ids(_test_case_extract)
if DEBUG:
print "-- extracted vuln ids"
pprint(vulns)
print "-" * 79
assert vulns == _test_case_extract_solution
all_vulns = []
for v in vulns.values():
all_vulns.extend(v)
all_vulns.sort()
if DEBUG:
print "-- only the ids"
pprint(all_vulns)
print "-" * 79
refs = convert_vuln_ids_to_references(all_vulns)
if DEBUG:
print "-- references"
pprint(refs)
print "-" * 79
unrefs = convert_references_to_vuln_ids(refs)
if DEBUG:
print "-- vuln ids back from references"
pprint(unrefs)
print "-" * 79
assert unrefs == vulns
urls = []
for url in _test_case_url.split("\n"):
url = url.strip()
if not url:
continue
urls.append(url)
parsed = set()
for vuln_ids in convert_references_to_vuln_ids(urls).itervalues():
parsed.update(vuln_ids)
if DEBUG:
print "-- test case"
pprint(urls)
print "-" * 79
print "-- extracted vuln ids"
pprint(sorted(parsed))
print "-" * 79
assert len(urls) == len(parsed), "%d vs %d" % (len(urls), len(parsed))
print "Testing reference URLs..."
import requests
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.62 Safari/537.36"
}
for url in refs:
print "--> " + url
requests.get(url, headers=headers, verify=False)
for url in urls:
if url not in refs:
print "--> " + url
requests.get(url, headers=headers, verify=False)
def parse_nikto_results(info, output_filename):
"""
Convert the output of a Nikto scan to the GoLismero data model.
:param info: Data object to link all results to (optional).
:type info: BaseURL
:param output_filename: Path to the output filename.
The format should always be CSV.
:type output_filename:
:returns: Results from the Nikto scan, and the vulnerability count.
:rtype: list(Data), int
"""
# Parse the scan results.
# On error log the exception and continue.
results = []
vuln_count = 0
hosts_seen = set()
urls_seen = {}
try:
with open(output_filename, "rU") as f:
csv_reader = reader(f)
for row in csv_reader:
try:
# Each row (except for the first) has always
# the same 7 columns, but some may be empty.
if len(row) < 7:
continue
host, ip, port, vuln_tag, method, path, text = row[:7]
# Report domain names and IP addresses.
if (
(info is None or host != info.hostname) and
host not in hosts_seen
):
hosts_seen.add(host)
if host in Config.audit_scope:
results.append( Domain(host) )
if ip not in hosts_seen:
hosts_seen.add(ip)
if ip in Config.audit_scope:
results.append( IP(ip) )
# Skip rows not informing of vulnerabilities.
if not vuln_tag:
continue
# Calculate the vulnerable URL.
if info is not None:
target = urljoin(info.url, path)
else:
if port == 443:
target = urljoin("https://%s/" % host, path)
else:
target = urljoin("http://%s/" % host, path)
# Skip if out of scope.
if target not in Config.audit_scope:
continue
# Report the URLs.
if (target, method) not in urls_seen:
url = URL(target, method)
urls_seen[ (target, method) ] = url
results.append(url)
else:
url = urls_seen[ (target, method) ]
# Get the reference URLs.
refs = extract_from_text(text)
refs.difference_update(urls_seen.itervalues())
# Report the vulnerabilities.
if vuln_tag == "OSVDB-0":
kwargs = {"level": "informational"}
else:
kwargs = extract_vuln_ids(
"%s: %s" % (vuln_tag, text))
kwargs["description"] = text if text else None
kwargs["references"] = refs
if "osvdb" in kwargs and "OSVDB-0" in kwargs["osvdb"]:
tmp = list(kwargs["osvdb"])
tmp.remove("OSVDB-0")
if tmp:
kwargs["osvdb"] = tuple(tmp)
else:
del kwargs["osvdb"]
if vuln_tag == "OSVDB-0":
vuln = UncategorizedVulnerability(url, **kwargs)
else:
vuln = VulnerableWebApp(url, **kwargs)
results.append(vuln)
vuln_count += 1
# On error, log the exception and continue.
except Exception, e:
Logger.log_error_verbose(str(e))
Logger.log_error_more_verbose(format_exc())
# On error, log the exception.
except Exception, e:
Logger.log_error_verbose(str(e))
Logger.log_error_more_verbose(format_exc())
def parse_nikto_results(info, output_filename):
"""
Convert the output of a Nikto scan to the GoLismero data model.
:param info: Data object to link all results to (optional).
:type info: BaseUrl
:param output_filename: Path to the output filename.
The format should always be CSV.
:type output_filename:
:returns: Results from the Nikto scan, and the vulnerability count.
:rtype: list(Data), int
"""
# Parse the scan results.
# On error log the exception and continue.
results = []
vuln_count = 0
hosts_seen = set()
urls_seen = {}
try:
with open(output_filename, "rU") as f:
csv_reader = reader(f)
for row in csv_reader:
try:
# Each row (except for the first) has always
# the same 7 columns, but some may be empty.
if len(row) < 7:
continue
host, ip, port, vuln_tag, method, path, text = row[:7]
# Report domain names and IP addresses.
if ((info is None or host != info.hostname)
and host not in hosts_seen):
hosts_seen.add(host)
if host in Config.audit_scope:
results.append(Domain(host))
if ip not in hosts_seen:
hosts_seen.add(ip)
if ip in Config.audit_scope:
results.append(IP(ip))
# Skip rows not informing of vulnerabilities.
if not vuln_tag:
continue
# Calculate the vulnerable URL.
if info is not None:
target = urljoin(info.url, path)
else:
if port == 443:
target = urljoin("https://%s/" % host, path)
else:
target = urljoin("http://%s/" % host, path)
# Skip if out of scope.
if target not in Config.audit_scope:
continue
# Report the URLs.
if (target, method) not in urls_seen:
url = Url(target, method)
urls_seen[(target, method)] = url
results.append(url)
else:
url = urls_seen[(target, method)]
# Get the reference URLs.
refs = extract_from_text(text)
refs.difference_update(urls_seen.itervalues())
# Report the vulnerabilities.
if vuln_tag == "OSVDB-0":
kwargs = {"level": "informational"}
else:
kwargs = extract_vuln_ids("%s: %s" %
(vuln_tag, text))
kwargs["description"] = text if text else None
kwargs["references"] = refs
if "osvdb" in kwargs and "OSVDB-0" in kwargs["osvdb"]:
tmp = list(kwargs["osvdb"])
tmp.remove("OSVDB-0")
if tmp:
kwargs["osvdb"] = tuple(tmp)
else:
del kwargs["osvdb"]
if vuln_tag == "OSVDB-0":
vuln = UncategorizedVulnerability(**kwargs)
vuln.add_resource(url)
else:
vuln = VulnerableWebApp(url, **kwargs)
results.append(vuln)
vuln_count += 1
# On error, log the exception and continue.
except Exception, e:
Logger.log_error_verbose(str(e))
Logger.log_error_more_verbose(format_exc())
# On error, log the exception.
except Exception, e:
Logger.log_error_verbose(str(e))
Logger.log_error_more_verbose(format_exc())
def test_vuln_id_parser():
DEBUG = False
##DEBUG = True
if DEBUG: from pprint import pprint
print "Testing the vulnerability ID parsers..."
if DEBUG:
print "-" * 79
print "-- test case solution"
pprint(_test_case_extract_solution)
print "-" * 79
vulns = extract_vuln_ids(_test_case_extract)
if DEBUG:
print "-- extracted vuln ids"
pprint(vulns)
print "-" * 79
assert vulns == _test_case_extract_solution
all_vulns = []
for v in vulns.values():
all_vulns.extend(v)
all_vulns.sort()
if DEBUG:
print "-- only the ids"
pprint(all_vulns)
print "-" * 79
refs = convert_vuln_ids_to_references(all_vulns)
if DEBUG:
print "-- references"
pprint(refs)
print "-" * 79
unrefs = convert_references_to_vuln_ids(refs)
if DEBUG:
print "-- vuln ids back from references"
pprint(unrefs)
print "-" * 79
assert unrefs == vulns
urls = []
for url in _test_case_url.split("\n"):
url = url.strip()
if not url:
continue
urls.append(url)
parsed = set()
for vuln_ids in convert_references_to_vuln_ids(urls).itervalues():
parsed.update(vuln_ids)
if DEBUG:
print "-- test case"
pprint(urls)
print "-" * 79
print "-- extracted vuln ids"
pprint(sorted(parsed))
print "-" * 79
assert len(urls) == len(parsed), "%d vs %d" % (len(urls), len(parsed))
print "Testing reference URLs..."
import requests
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.62 Safari/537.36"
}
for url in refs:
print "--> " + url
requests.get(url, headers=headers, verify=False)
for url in urls:
if url not in refs:
print "--> " + url
requests.get(url, headers=headers, verify=False)
