def test_vuln_id_parser(): DEBUG = False ##DEBUG = True if DEBUG: from pprint import pprint print "Testing the vulnerability ID parsers..." if DEBUG: print "-" * 79 pprint(_test_case_extract_solution) print "-" * 79 vulns = extract_vuln_ids(_test_case_extract) if DEBUG: pprint(vulns) print "-" * 79 assert vulns == _test_case_extract_solution all_vulns = [] for v in vulns.values(): all_vulns.extend(v) all_vulns.sort() if DEBUG: pprint(all_vulns) print "-" * 79 refs = convert_vuln_ids_to_references(all_vulns) if DEBUG: pprint(refs) print "-" * 79 unrefs = convert_references_to_vuln_ids(refs) if DEBUG: pprint(unrefs) print "-" * 79 assert unrefs == vulns print "Testing reference URLs..." import requests headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.62 Safari/537.36"} for url in refs: print "--> " + url requests.get(url, headers=headers)
def test_vuln_id_parser(): DEBUG = False ##DEBUG = True if DEBUG: from pprint import pprint print "Testing the vulnerability ID parsers..." if DEBUG: print "-" * 79 print "-- test case solution" pprint(_test_case_extract_solution) print "-" * 79 vulns = extract_vuln_ids(_test_case_extract) if DEBUG: print "-- extracted vuln ids" pprint(vulns) print "-" * 79 assert vulns == _test_case_extract_solution all_vulns = [] for v in vulns.values(): all_vulns.extend(v) all_vulns.sort() if DEBUG: print "-- only the ids" pprint(all_vulns) print "-" * 79 refs = convert_vuln_ids_to_references(all_vulns) if DEBUG: print "-- references" pprint(refs) print "-" * 79 unrefs = convert_references_to_vuln_ids(refs) if DEBUG: print "-- vuln ids back from references" pprint(unrefs) print "-" * 79 assert unrefs == vulns urls = [] for url in _test_case_url.split("\n"): url = url.strip() if not url: continue urls.append(url) parsed = set() for vuln_ids in convert_references_to_vuln_ids(urls).itervalues(): parsed.update(vuln_ids) if DEBUG: print "-- test case" pprint(urls) print "-" * 79 print "-- extracted vuln ids" pprint(sorted(parsed)) print "-" * 79 assert len(urls) == len(parsed), "%d vs %d" % (len(urls), len(parsed)) print "Testing reference URLs..." import requests headers = { "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.62 Safari/537.36" } for url in refs: print "--> " + url requests.get(url, headers=headers, verify=False) for url in urls: if url not in refs: print "--> " + url requests.get(url, headers=headers, verify=False)
def parse_nikto_results(info, output_filename): """ Convert the output of a Nikto scan to the GoLismero data model. :param info: Data object to link all results to (optional). :type info: BaseURL :param output_filename: Path to the output filename. The format should always be CSV. :type output_filename: :returns: Results from the Nikto scan, and the vulnerability count. :rtype: list(Data), int """ # Parse the scan results. # On error log the exception and continue. results = [] vuln_count = 0 hosts_seen = set() urls_seen = {} try: with open(output_filename, "rU") as f: csv_reader = reader(f) for row in csv_reader: try: # Each row (except for the first) has always # the same 7 columns, but some may be empty. if len(row) < 7: continue host, ip, port, vuln_tag, method, path, text = row[:7] # Report domain names and IP addresses. if ( (info is None or host != info.hostname) and host not in hosts_seen ): hosts_seen.add(host) if host in Config.audit_scope: results.append( Domain(host) ) if ip not in hosts_seen: hosts_seen.add(ip) if ip in Config.audit_scope: results.append( IP(ip) ) # Skip rows not informing of vulnerabilities. if not vuln_tag: continue # Calculate the vulnerable URL. if info is not None: target = urljoin(info.url, path) else: if port == 443: target = urljoin("https://%s/" % host, path) else: target = urljoin("http://%s/" % host, path) # Skip if out of scope. if target not in Config.audit_scope: continue # Report the URLs. if (target, method) not in urls_seen: url = URL(target, method) urls_seen[ (target, method) ] = url results.append(url) else: url = urls_seen[ (target, method) ] # Get the reference URLs. refs = extract_from_text(text) refs.difference_update(urls_seen.itervalues()) # Report the vulnerabilities. if vuln_tag == "OSVDB-0": kwargs = {"level": "informational"} else: kwargs = extract_vuln_ids( "%s: %s" % (vuln_tag, text)) kwargs["description"] = text if text else None kwargs["references"] = refs if "osvdb" in kwargs and "OSVDB-0" in kwargs["osvdb"]: tmp = list(kwargs["osvdb"]) tmp.remove("OSVDB-0") if tmp: kwargs["osvdb"] = tuple(tmp) else: del kwargs["osvdb"] if vuln_tag == "OSVDB-0": vuln = UncategorizedVulnerability(url, **kwargs) else: vuln = VulnerableWebApp(url, **kwargs) results.append(vuln) vuln_count += 1 # On error, log the exception and continue. except Exception, e: Logger.log_error_verbose(str(e)) Logger.log_error_more_verbose(format_exc()) # On error, log the exception. except Exception, e: Logger.log_error_verbose(str(e)) Logger.log_error_more_verbose(format_exc())
def parse_nikto_results(info, output_filename): """ Convert the output of a Nikto scan to the GoLismero data model. :param info: Data object to link all results to (optional). :type info: BaseUrl :param output_filename: Path to the output filename. The format should always be CSV. :type output_filename: :returns: Results from the Nikto scan, and the vulnerability count. :rtype: list(Data), int """ # Parse the scan results. # On error log the exception and continue. results = [] vuln_count = 0 hosts_seen = set() urls_seen = {} try: with open(output_filename, "rU") as f: csv_reader = reader(f) for row in csv_reader: try: # Each row (except for the first) has always # the same 7 columns, but some may be empty. if len(row) < 7: continue host, ip, port, vuln_tag, method, path, text = row[:7] # Report domain names and IP addresses. if ((info is None or host != info.hostname) and host not in hosts_seen): hosts_seen.add(host) if host in Config.audit_scope: results.append(Domain(host)) if ip not in hosts_seen: hosts_seen.add(ip) if ip in Config.audit_scope: results.append(IP(ip)) # Skip rows not informing of vulnerabilities. if not vuln_tag: continue # Calculate the vulnerable URL. if info is not None: target = urljoin(info.url, path) else: if port == 443: target = urljoin("https://%s/" % host, path) else: target = urljoin("http://%s/" % host, path) # Skip if out of scope. if target not in Config.audit_scope: continue # Report the URLs. if (target, method) not in urls_seen: url = Url(target, method) urls_seen[(target, method)] = url results.append(url) else: url = urls_seen[(target, method)] # Get the reference URLs. refs = extract_from_text(text) refs.difference_update(urls_seen.itervalues()) # Report the vulnerabilities. if vuln_tag == "OSVDB-0": kwargs = {"level": "informational"} else: kwargs = extract_vuln_ids("%s: %s" % (vuln_tag, text)) kwargs["description"] = text if text else None kwargs["references"] = refs if "osvdb" in kwargs and "OSVDB-0" in kwargs["osvdb"]: tmp = list(kwargs["osvdb"]) tmp.remove("OSVDB-0") if tmp: kwargs["osvdb"] = tuple(tmp) else: del kwargs["osvdb"] if vuln_tag == "OSVDB-0": vuln = UncategorizedVulnerability(**kwargs) vuln.add_resource(url) else: vuln = VulnerableWebApp(url, **kwargs) results.append(vuln) vuln_count += 1 # On error, log the exception and continue. except Exception, e: Logger.log_error_verbose(str(e)) Logger.log_error_more_verbose(format_exc()) # On error, log the exception. except Exception, e: Logger.log_error_verbose(str(e)) Logger.log_error_more_verbose(format_exc())