def _is_secure_renegotiation_supported(sec_rng, issue_id, asset, asset_port): if sec_rng is None: return False if sec_rng.attrib["supported"] != '1': return PatrowlEngineFinding( issue_id=issue_id, type="ssltest_secure_renegotiation", title="Secure renegotiation is not supported", description="Secure renegotiation is not supported on {}:{}".format( asset, asset_port), solution="Enable secure renegotiation on your server", severity="medium", confidence="firm", raw=sec_rng.attrib, target_addrs=[asset], meta_tags=["ssl", "tls"]) if sec_rng.attrib["supported"] == '1' and sec_rng.attrib["secure"] != '1': return PatrowlEngineFinding( issue_id=issue_id, type="ssltest_secure_renegotiation", title="Unsecure renegotiation is enabled", description="Unsecure renegotiation is enabled on {}:{}".format(asset, asset_port), solution="Disable unsecure renegotiation on your server", severity="high", confidence="firm", raw=sec_rng.attrib, target_addrs=[asset], meta_tags=["ssl", "tls"]) return False
def _get_heartbleed_vuln(items, issue_id, asset, asset_port): if items is None or not isinstance(items, list): return False is_vulnerable = False hb_links = ["http://heartbleed.com/"] hb_desc = "" for item in items: if item.get("vulnerable") == "1": hb_desc += "sslversion='{}' --> is VULNERABLE\n".format( item.get("sslversion")) is_vulnerable = True else: hb_desc += "sslversion='{}' --> is not vulnerable\n".format( item.get("sslversion")) if is_vulnerable: return PatrowlEngineFinding( issue_id=issue_id, type="ssltest_heartbleed", title="Heartbleed check on '{}:{}': VULNERABLE".format( asset, asset_port), description=hb_desc, solution="Update the version of the OpenSSL component used by the \ service listening on port '{}'".format(asset_port), severity="high", confidence="firm", raw=hb_desc, target_addrs=[asset], meta_tags=["heartbleed", "ssl", "tls"], meta_links=hb_links, meta_vuln_refs=[{ "CVE": ["CVE-2014-0160"] }]) else: return PatrowlEngineFinding( issue_id=issue_id, type="ssltest_heartbleed", title="Heartbleed check on '{}:{}': not vulnerable".format( asset, asset_port), description=hb_desc, solution="n/a", severity="info", confidence="firm", raw=hb_desc, target_addrs=[asset], meta_tags=["heartbleed", "ssl", "tls"], meta_links=hb_links)
def _get_ciphersuites(items, issue_id, asset, asset_port): if items is None or not isinstance(items, list): return False issue_desc = "Supported ciphersuites:\n" for item in items: add_info = "" if 'curve' in item.keys(): add_info += "Curve: ".format(item.get("curve")) if 'dhebits' in item.keys(): add_info += "DHEbits: ".format(item.get("dhebits")) if 'ecdhebits' in item.keys(): add_info += "ECDHEbits: ".format(item.get("ecdhebits")) issue_desc += "{:30} SSLVersion: {:8} Bits: {:4} Status: {:10} {}\n".format( item.get("cipher"), item.get("sslversion"), item.get("bits"), item.get("status"), add_info) return PatrowlEngineFinding( issue_id=issue_id, type="ssltest_supported_ciphersuites", title="Supported ciphersuites on '{}:{}'.".format(asset, asset_port), description=issue_desc, solution="n/a", severity="info", confidence="firm", raw=issue_desc, target_addrs=[asset], meta_tags=["ciphersuites", "ssl", "tls"])
def _spot_weak_ciphersuites(ciphers, issue_id, asset, asset_port): if ciphers is None: return False res = [] for cipher in ciphers: if cipher.attrib["strength"] in ("anonymous", "medium") and \ cipher.attrib["status"] in ("preferred", "accepted"): issue_id += 1 res.append(PatrowlEngineFinding( issue_id=issue_id, type="tls_supported_ciphersuites", title="Unsecure TLS ciphersuite detected : {}".format(cipher.attrib["cipher"]), description="Unsecure TLS ciphersuite {} was detected on {}:{}".format( cipher.attrib["cipher"], asset, asset_port), solution="Deactivate the ciphersuite {} on your TLS configuration".format(cipher.attrib["cipher"]), severity="medium", confidence="firm", raw=cipher.attrib, target_addrs=[asset], meta_tags=["ssl", "tls", "ciphersuites"])) if cipher.attrib["strength"] in ("null", "weak") and \ cipher.attrib["status"] in ("preferred", "accepted"): issue_id += 1 res.append(PatrowlEngineFinding( issue_id=issue_id, type="tls_supported_ciphersuites", title="Dangerous (weak) TLS ciphersuite detected : {}".format(cipher.attrib["cipher"]), description="Weak TLS ciphersuite {} was detected on {}:{}".format( cipher.attrib["cipher"], asset, asset_port), solution="Deactivate the ciphersuite {} on your TLS configuration".format(cipher.attrib["cipher"]), severity="medium", confidence="firm", raw=cipher.attrib, target_addrs=[asset], meta_tags=["ssl", "tls", "ciphersuites"])) return res
def _spot_weak_protocol(protocols, issue_id, asset, asset_port): if protocols is None: return False res = [] for protocol in protocols: if protocol.attrib["type"] == "ssl" and protocol.attrib["enabled"] == "1": issue_id += 1 res.append(PatrowlEngineFinding( issue_id=issue_id, type="tls_supported_protocols", title="Weak TLS protocol detected : SSLv{}".format(protocol.attrib["version"]), description="Weak TLS protocol SSLv{} was detected on {}:{}".format( protocol.attrib["version"], asset, asset_port), solution="Deactivate SSLv{} on your server".format(protocol.attrib["version"]), severity="high", confidence="firm", raw=protocol.attrib, target_addrs=[asset], meta_tags=["ssl", "tls"])) if protocol.attrib["type"] == "tls" and \ protocol.attrib["version"] in ("1.0", "1.1") and \ protocol.attrib["enabled"] == "1": issue_id += 1 res.append(PatrowlEngineFinding( issue_id=issue_id, type="tls_supported_protocols", title="Weak TLS protocol detected : TLSv{}".format(protocol.attrib["version"]), description="Weak TLS protocol TLSv{} was detected on {}:{}".format( protocol.attrib["version"], asset, asset_port), solution="Deactivate TLSv{} on your server".format(protocol.attrib["version"]), severity="medium", confidence="firm", raw=protocol.attrib, target_addrs=[asset], meta_tags=["ssl", "tls"])) return res
def _get_certificate_blob(cert_blob, issue_id, asset, asset_port): if cert_blob is None: return False cert_hash = hashlib.sha1(cert_blob.text).hexdigest().upper() return PatrowlEngineFinding( issue_id=issue_id, type="ssltest_certificate_pem", title="Certificate was retrieved from '{}:{}' with hash '{}'.".format( asset, asset_port, cert_hash[:6]), description="Following certificate was retrieved from the server:\n\ {}".format(cert_blob.text), solution="n/a", severity="info", confidence="firm", raw=cert_blob.text, target_addrs=[asset], meta_tags=["certificate", "ssl", "tls", "pem"])
def _is_fallback_supported(fallback, issue_id, asset, asset_port): if fallback is None: return False fallback_support = fallback.attrib["supported"] if fallback_support == '1': return False return PatrowlEngineFinding( issue_id=issue_id, type="ssltest_fallback_support", title="Downgrade attack prevention is not supported", description="Downgrade attack prevention is not supported on {}:{}".format( asset, asset_port), solution="Enable TLS_FALLBACK_SCSV option on your server", severity="low", confidence="firm", raw=fallback.attrib, target_addrs=[asset], meta_tags=["ssl", "tls"])
def _is_certificate_selfsigned(cert_tags, issue_id, asset, asset_port): if cert_tags is None: return False selfsigned_text = cert_tags.find("self-signed").text if len(selfsigned_text) == 0 or selfsigned_text == "false": return False return PatrowlEngineFinding( issue_id=issue_id, type="ssltest_certificate_selfsigned", title="Certificate from '{}:{}' is self-signed.".format( asset, asset_port), description="The SSL/TLS certificate retrieved from the server is \ self-signed.", solution="Renew the certificate on the service listening on '{}:{}' \ and sign it with a trusted CA.".format(asset, asset_port), severity="high", confidence="firm", raw=selfsigned_text, target_addrs=[asset], meta_tags=["certificate", "ssl", "tls", "self-signed"])
def _is_certificate_expired(cert_tags, issue_id, asset, asset_port): if cert_tags is None: return False expired_text = cert_tags.find("expired").text if len(expired_text) == 0 or expired_text == "false": return False return PatrowlEngineFinding( issue_id=issue_id, type="ssltest_certificate_expired", title="Certificate from '{}:{}' is expired.".format(asset, asset_port), description="The SSL/TLS certificate retrieved from the server is \ expired:\nNot valid before: {}\nNot valid after: {}".format( cert_tags.find("not-valid-before").text, cert_tags.find("not-valid-after").text), solution="Renew the certificate on the service listening on \ '{}:{}'.".format(asset, asset_port), severity="high", confidence="firm", raw=expired_text, target_addrs=[asset], meta_tags=["certificate", "ssl", "tls", "expired"])
def _search_twitter_thread(scan_id, asset_kw): issue_id = 0 findings = [] twitter = Twitter( auth=OAuth( engine.options["twitter_oauth_token"], engine.options["twitter_oauth_secret"], engine.options["twitter_consumer_key"], engine.options["twitter_consumer_secret"] ), retry=True ) # Set the Max count max_count = APP_SEARCH_TWITTER_MAX_COUNT_DEFAULT extra_kw = "" since = "" if "search_twitter_options" in engine.scans[scan_id]["options"].keys() and engine.scans[scan_id]["options"]["search_twitter_options"] is not None: if "max_count" in engine.scans[scan_id]["options"]["search_twitter_options"].keys() and engine.scans[scan_id]["options"]["search_twitter_options"]["max_count"] is not None and isinstance(engine.scans[scan_id]["options"]["search_twitter_options"]["max_count"], int): max_count = engine.scans[scan_id]["options"]["search_twitter_options"]["max_count"] if "extra_kw" in engine.scans[scan_id]["options"]["search_twitter_options"].keys() and engine.scans[scan_id]["options"]["search_twitter_options"]["extra_kw"] is not None and isinstance(engine.scans[scan_id]["options"]["search_twitter_options"]["extra_kw"], list): extra_kw = " OR ".join(engine.scans[scan_id]["options"]["search_twitter_options"]["extra_kw"]) if "since" in engine.scans[scan_id]["options"]["search_twitter_options"].keys() and engine.scans[scan_id]["options"]["search_twitter_options"]["since"] is not None and isinstance(engine.scans[scan_id]["options"]["search_twitter_options"]["since"], str): since = "since:{}".format(engine.scans[scan_id]["options"]["search_twitter_options"]["since"]) # WARNING a query should not exceed 500 chars, including filters and operators # print "query_string :", "\""+asset_kw+"\" "+extra_kw+" "+since+" -filter:retweets", "len:", len("\""+asset_kw+"\" "+extra_kw+" "+since+" -filter:retweets") results = twitter.search.tweets(q="\""+asset_kw+"\" "+extra_kw+" -filter:retweets", count=max_count) # print results if len(results["statuses"]) == 0: # no results metalink = "https://twitter.com/search"+results["search_metadata"]["refresh_url"] new_finding = PatrowlEngineFinding( issue_id=issue_id, type="twitter_leak", title="No matching tweets.", description="No matching tweet with following parameters:\n" + \ "Keyword (strict): {}\n".format(asset_kw) + \ "Extra key words: {}\n".format(extra_kw) + \ "URL: {}\n".format(metalink), solution="N/A", severity="info", confidence="firm", raw=results, target_addrs=[asset_kw], meta_links=[metalink]) findings.append(new_finding) else: for tweet in results["statuses"]: # print "id:", tweet["id"], "text:", tweet["text"] # print "user_id:", tweet["user"]["id"], "user_name:", tweet["user"]["name"], "user_nickname:", tweet["user"]["screen_name"] # print "tweet_url:", "https://twitter.com/i/web/status/"+tweet["id_str"] issue_id += 1 tw_hash = hashlib.sha1(str(tweet["text"]).encode('utf-8')).hexdigest()[:6] metalink = "https://twitter.com/search"+results["search_metadata"]["refresh_url"] new_finding = PatrowlEngineFinding( issue_id=issue_id, type="twitter_leak", title="Tweet matching search query (HASH: {}).".format(tw_hash), description="A tweet matching monitoring keywords has been found:\n" + \ "Query options:\nKeyword (strict): {}\n".format(asset_kw) + \ "Extra key words: {}\n".format(extra_kw) + \ "URL: {}\n".format(metalink), solution="Evaluate criticity. See internal procedures for incident reaction.", severity="high", confidence="firm", raw=tweet, target_addrs=[asset_kw], meta_links=[metalink]) findings.append(new_finding) # Write results under mutex scan_lock = threading.RLock() with scan_lock: engine.scans[scan_id]["findings"] = engine.scans[scan_id]["findings"] + findings
def _search_github_thread(scan_id, asset_kw): issue_id = 0 findings = [] asset_values = [a["value"] for a in engine.scans[scan_id]["assets"]] # qualifiers={} # if "github_qualifiers" in engine.scans[scan_id]["options"].keys() and engine.scans[scan_id]["options"]["github_qualifiers"] is not None: # for opt_qualifier in engine.scans[scan_id]["options"]["github_qualifiers"].keys(): # if opt_qualifier == "since_period": # num = re.search(r'\d+', engine.scans[scan_id]["options"]["github_qualifiers"]["since_period"]).group() # unit = re.search(r'[a-zA-Z]+', engine.scans[scan_id]["options"]["github_qualifiers"]["since_period"]).group() # if unit in ["weeks", "days", "hours", "minutes", "seconds"]: # since_date=date.today()-timedelta(**pa) # qualifiers.update({"created": ">="+str(since_date)}) # elif opt_qualifier == "from_date": # try: # from_date_str = engine.scans[scan_id]["options"]["github_qualifiers"]["from_date"] # from_date_check = datetime.strptime(engine.scans[scan_id]["options"]["github_qualifiers"]["from_date"], "%Y-%m-%d") # qualifiers.update({"created": ">="+str(from_date_str)}) # except Exception: # print "bad datetime format" # # elif opt_qualifier == "to_date": # try: # to_date_str = engine.scans[scan_id]["options"]["github_qualifiers"]["to_date"] # to_date_check = datetime.strptime(engine.scans[scan_id]["options"]["github_qualifiers"]["to_date"], "%Y-%m-%d") # qualifiers.update({"created": "<="+str(to_date_str)}) # except Exception: # print "bad datetime format" # g = Github(engine.options["github_username"], engine.options["github_password"]) # rate limit = 30 requests/min g = Github(engine.options["github_api_token"]) loops = 0 for git_code in g.search_code("\'"+asset_kw+"\'", sort="indexed", order="desc"): ititle = "File found in Github public repo (code): {}/{} (HASH: {})".format( git_code.name, git_code.repository.name, git_code.sha[:6]) iemail = "" if git_code.repository.owner.email is not None: git_code.repository.owner.email.encode("ascii", "ignore") idescription = "File found in Github public repo (code):\n\n" + \ "URL: {}\n\n".format(git_code.html_url) + \ "Repo: {}: {}\n\n".format(git_code.repository.name, git_code.repository.url) + \ "Owner:\nlogin:{}, name:{}, email:{}\n\n".format( git_code.repository.owner.login, git_code.repository.owner.name, iemail) + \ "Content ({} bits):{}".format(git_code.size, git_code.decoded_content) isolution = "Check if the snippet is legit or not. " + \ "If not, see internal procedures for incident reaction." issue_id += 1 new_finding = PatrowlEngineFinding( issue_id=issue_id, type="github_leak_code", title=ititle, description=idescription, solution=isolution, severity="high", confidence="firm", raw=git_code.raw_data, target_addrs=asset_values, meta_links=[git_code.html_url]) findings.append(new_finding) # Ratio limit trick: wait 3 seconds each 20 iters loops += 1 if loops % 20 == 0: time.sleep(3) # for git_commit in g.search_commits("\'"+asset_kw+"\'", sort="indexed", order="desc"): # print dir(git_commit) for git_issue in g.search_issues("\'"+asset_kw+"\'", sort="updated", order="desc"): ititle = "Matching issue found in Github public repo: {}... (HASH: {})".format( git_issue.title[:16], hashlib.sha1(str(git_issue.body).encode('utf-8')).hexdigest()[:6]) idescription = "Matching issue found in Github public repo:\n\n" + \ "URL: {}\n\n".format(git_issue.html_url) + \ "Repo: {}: {}\n\n".format(git_issue.repository.name, git_issue.repository.url) + \ "Owner:\nlogin:{}, name:{}, email:{}\n\n".format( git_issue.repository.owner.login, git_issue.repository.owner.name, git_issue.repository.owner.email) + \ "Content: {}".format(git_issue.body) isolution = "Check if the snippet is legit or not. " + \ "If not, see internal procedures for incident reaction." issue_id += 1 new_finding = PatrowlEngineFinding( issue_id=issue_id, type="github_leak_issue", title=ititle, description=idescription, solution=isolution, severity="high", confidence="firm", raw=git_issue.raw_data, target_addrs=asset_values, meta_links=[git_issue.html_url]) findings.append(new_finding) for git_repo in g.search_repositories("\'"+asset_kw+"\'", sort="updated", order="desc"): ititle = "Matching public Github repo: {} (HASH: {})".format( git_repo.name, hashlib.sha1(git_repo.description.encode('ascii', 'ignore')).hexdigest()[:6]) idescription = "Matching public Github repo:\n\n" + \ "URL: {}\n\n".format(git_repo.html_url) + \ "Repo: {}: {}\n\n".format(git_repo.name, git_repo.url) + \ "Owner:\nlogin:{}, name:{}, email:{}\n\n".format( git_repo.owner.login, git_repo.owner.name, git_repo.owner.email) + \ "Content: {}".format(git_repo.description.encode('ascii', 'ignore')) isolution = "Check if the snippet is legit or not. " + \ "If not, see internal procedures for incident reaction." issue_id += 1 new_finding = PatrowlEngineFinding( issue_id=issue_id, type="github_leak_repo", title=ititle, description=idescription, solution=isolution, severity="high", confidence="firm", raw=git_repo.raw_data, target_addrs=asset_values, meta_links=[git_repo.html_url]) findings.append(new_finding) for git_user in g.search_users(asset_kw, sort="joined", order="desc"): ititle = "Matching Github user: {} (HASH: {})".format( git_user.login, hashlib.sha1(str(git_user.login).encode('utf-8')).hexdigest()[:6]) ibio = "" if git_user.bio: ibio = git_user.bio.encode('ascii', 'ignore') idescription = "Matching Github user:\n\n" + \ "URL: {}\n\n".format(git_user.html_url) + \ "Info:\nlogin:{}, name:{}, email:{}\n\n".format( git_user.login, git_user.name.encode('ascii', 'ignore'), git_user.email) + \ "Bio: {}".format(ibio) isolution = "Check if the user is legit or not. " + \ "If not, see internal procedures for incident reaction." issue_id += 1 new_finding = PatrowlEngineFinding( issue_id=issue_id, type="github_leak_user", title=ititle, description=idescription, solution=isolution, severity="high", confidence="firm", raw=git_user.raw_data, target_addrs=asset_values, meta_links=[git_user.html_url]) findings.append(new_finding) # Write results under mutex scan_lock = threading.RLock() with scan_lock: engine.scans[scan_id]["findings"] = engine.scans[scan_id]["findings"] + findings
def _parse_results(scan_id): issues = [] issue_id = 1 nb_vulns = { "info": 0, "low": 0, "medium": 0, "high": 0, "critical": 0 } report_filename = "{}/results/{}.xml".format(APP_BASE_DIR, scan_id) if not os.path.isfile(report_filename): return False try: tree = ET.parse(report_filename) except Exception: # No Element found in XML file return False report = tree.getroot().find("report").find("report") # Map IP addresses to domains/fqdn/ all_assets = {} for host in report.findall("host"): host_ip = host.find("ip").text all_assets.update({host_ip: [host_ip]}) for detail in host.findall("detail"): if detail.find("name").text == "hostname": host_name = detail.find("value").text all_assets[host_ip].append(host_name) for result in report.find("results").findall("result"): issue_meta = {} issue_name = result.find("name").text issue_desc = result.find("description").text host_ip = result.find("host").text assets = all_assets[host_ip] host_port = result.find("port").text # Severity threat = result.find("threat").text severity = "info" if threat == "High": severity = "high" elif threat == "Medium": severity = "medium" elif threat == "Low": severity = "low" issue_cvss = float(result.find("severity").text) if result.find("nvt").find("cve") is not None and result.find("nvt").find("cve").text != "NOCVE": cvelist = str(result.find("nvt").find("cve").text) issue_meta.update({"CVE": cvelist.split(", ")}) if result.find("nvt").find("bid") is not None and result.find("nvt").find("bid").text != "NOBID": bid_list = str(result.find("nvt").find("bid").text) issue_meta.update({"BID": bid_list.split(", ")}) if result.find("nvt").find("xref") is not None and result.find("nvt").find("xref").text != "NOXREF": xref_list = str(result.find("nvt").find("xref").text) issue_meta.update({"XREF": xref_list.split(", ")}) issue = PatrowlEngineFinding( issue_id=issue_id, type="openvas_scan", title="{} ({})".format(issue_name, host_port), description=issue_desc, solution="n/a", severity=severity, confidence="firm", raw=ET.tostring(result, encoding='utf-8', method='xml'), target_addrs=assets, meta_tags=["openvas"], meta_risk={"cvss_base_score": issue_cvss}, meta_vuln_refs=issue_meta ) issues.append(issue._PatrowlEngineFinding__to_dict()) nb_vulns[severity] += 1 issue_id += 1 # report_id = engine.scans[scan_id]["report_id"] # for asset in engine.scans[scan_id]["findings"]: # if engine.scans[scan_id]["findings"][asset]["issues"]: # description = '' # cvss_max = float(0) # for eng in engine.scans[scan_id]["findings"][asset]["issues"]: # if float(eng[0]) > 0: # cvss_max = max(float(eng[0]), cvss_max) # description = description + "[%s] CVSS: %s - Associated CVE: %s" % (eng[2], eng[0], eng[1]) + "\n" # description = description + "For more detail go to 'https://%s/omp?cmd=get_report&report_id=%s'" % (engine.scanner["options"]["omp_host"]["value"], report_id) # # criticity = "high" # if cvss_max == 0: # criticity = "info" # elif cvss_max < 4.0: # criticity = "low" # elif cvss_max < 7.0: # criticity = "medium" # # nb_vulns[criticity] += 1 # # issues.append({ # "issue_id": len(issues)+1, # "severity": criticity, "confidence": "certain", # "target": {"addr": [asset], "protocol": "http"}, # "title": "'{}' identified in openvas".format(asset), # "solution": "n/a", # "metadata": {}, # "type": "openvas_report", # "timestamp": timestamp, # "description": description, # }) summary = { "nb_issues": len(issues), "nb_info": nb_vulns["info"], "nb_low": nb_vulns["low"], "nb_medium": nb_vulns["medium"], "nb_high": nb_vulns["high"], "nb_critical": 0, "engine_name": "openvas", "engine_version": engine.scanner["version"] } return issues, summary
def _scanowaspdc_thread(scan_id, asset_kw): issue_id = 0 findings = [] asset_values = [a["value"] for a in engine.scans[scan_id]["assets"]] # Create the scan's workdirs scan_wd = "{}/workdirs/scan_{}_{}".format(APP_BASE_DIR, scan_id, str(time.time())) if not os.path.exists(scan_wd): os.makedirs(scan_wd) for asset_value in asset_values: checked_files = [] # create the asset scan workdir scan_wd_asset = "{}/{}/src".format(scan_wd, hashlib.sha1(asset_value).hexdigest()[:6]) os.makedirs(scan_wd_asset) # print "scan_wd_asset:", scan_wd_asset # Check location and copy files to the workdir if not _check_location(scan_id, asset_value, scan_wd_asset): # Generate an error if it was not possible to get the source code summary_asset_finding = PatrowlEngineFinding( issue_id=issue_id, type="code_ext_jar_summary", title="OWASP-DC scan not performed for '{}' (Error)".format(asset_value), description="Scan error with source code available at this location: '{}'. Unknwon error.".format(asset_value), solution="n/a.", severity="info", confidence="firm", raw={}, target_addrs=[asset_value], meta_tags=["jar", "library", "owasp", "dependencies"]) issue_id+=1 findings.append(summary_asset_finding) continue time.sleep(2) # Start the scan cmd = 'libs/dependency-check/bin/dependency-check.sh --scan "{}" --format JSON --out "{}/oc_{}.json" --project "{}" --enableExperimental'.format( scan_wd_asset, scan_wd_asset, scan_id, scan_id) #print "cmd:", cmd p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) # Wait a little to ensure the report file is completely writen p.wait() time.sleep(2) report_filename = scan_wd_asset + "/oc_{}.json".format(scan_id) if not os.path.exists(report_filename): print("report file '{}' not found.".format(report_filename)) scan_results = json.load(open(report_filename)) for item in scan_results["dependencies"]: if "vulnerabilities" not in item.keys(): continue checked_files.append(item["filePath"]) for vuln in item["vulnerabilities"]: vuln_name = "" if vuln["name"].isdigit(): vuln_name = "NSP-{}".format(vuln["name"]) else: vuln_name = vuln["name"] item_title = "External library '{}' vulnerable ({})".format( item["fileName"], vuln_name) item_description = "Filepath: {}\nFilename: {}\n\n{}\n\nIdentifiers:\n{}".format( remove_prefix(item["filePath"], scan_wd_asset), item["fileName"], vuln["description"].encode('utf-8').strip(), "\n".join([vs["software"] for vs in vuln["vulnerableSoftware"]]) ) vuln_risks = {} if "cvssScore" in vuln.keys() and vuln["cvssScore"] != "": vuln_risks.update({"cvss_base_score": float(vuln["cvssScore"])}) vuln_links = [v["url"] for v in vuln["references"]] vuln_refs = {} if "cwe" in vuln.keys() and vuln["cwe"] != "": vuln_refs.update({"CWE": [vuln["cwe"].split(" ")[0]]}) if vuln["name"].startswith("CVE-"): vuln_refs.update({"CVE": [vuln["name"]]}) new_finding = PatrowlEngineFinding( issue_id=issue_id, type="code_ext_jar_missing_update", title=item_title, description=item_description, solution="Check the exploitability of the vulnerability in the application context. If the vulnerability is verified, consider updating the library.", severity=vuln["severity"].lower(), confidence="firm", raw=vuln, target_addrs=[asset_value], meta_links=vuln_links, meta_tags=["jar", "library", "update", "owasp", "dependencies"], meta_risk=vuln_risks, meta_vuln_refs=vuln_refs) issue_id+=1 findings.append(new_finding) # findings summary per asset (remove the workdir) checked_files_str = "\n".join([remove_prefix(ff, scan_wd_asset) for ff in sorted(checked_files)]) summary_asset_finding_hash = hashlib.sha1(checked_files_str).hexdigest()[:6] summary_asset_finding = PatrowlEngineFinding( issue_id=issue_id, type="code_ext_jar_summary", title="OWASP-DC scan summary for '{}' (#: {}, HASH: {})".format( asset_value, len(checked_files), summary_asset_finding_hash), description="Checked files:\n\n{}".format(checked_files_str), solution="n/a.", severity="info", confidence="firm", raw=[remove_prefix(ff, scan_wd_asset) for ff in checked_files], target_addrs=[asset_value], meta_tags=["jar", "library", "owasp", "dependencies"]) issue_id+=1 findings.append(summary_asset_finding) # Write results under mutex scan_lock = threading.RLock() with scan_lock: engine.scans[scan_id]["findings"] = engine.scans[scan_id]["findings"] + findings # Remove the workdir shutil.rmtree(scan_wd, ignore_errors=True)
def _scanjs_thread(scan_id, asset_kw): issue_id = 0 findings = [] asset_values = [a["value"] for a in engine.scans[scan_id]["assets"]] # Create the scan's workdirs scan_wd = "{}/workdirs/scan_{}_{}".format(APP_BASE_DIR, scan_id, str(time.time())) if not os.path.exists(scan_wd): os.makedirs(scan_wd) for asset_value in asset_values: checked_files = [] # create the asset scan workdir scan_wd_asset = "{}/{}".format(scan_wd, hashlib.sha1(asset_value).hexdigest()[:6]) os.makedirs(scan_wd_asset) # Check location and copy files to the workdir if not _check_location(scan_id, asset_value, scan_wd_asset): # Generate an error if it was not possible to get the source code summary_asset_finding = PatrowlEngineFinding( issue_id=issue_id, type="code_ext_js_summary", title="Retire.js scan not performed for '{}' (Error)".format(asset_value), description="Scan error with source code available at this location: '{}'. Unknwon error.".format(asset_value), solution="n/a.", severity="info", confidence="firm", raw={}, target_addrs=[asset_value], meta_tags=["js", "library", "retire.js"]) issue_id+=1 findings.append(summary_asset_finding) continue time.sleep(2) # Start the scan report_filename = "{}/oc_{}.json".format(scan_wd_asset, scan_id) cmd = 'retire -j --path="{}" --outputformat json --outputpath="{}" -v'.format( scan_wd_asset, report_filename) # p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) p = subprocess.Popen(cmd, shell=True, stdout=open("/dev/null", "w"), stderr=None) # Wait a little to ensure the report file is completely writen p.wait() time.sleep(2) if not os.path.exists(report_filename): print("report file '{}' not found.".format(report_filename)) engine.scans[scan_id]["status"] = "ERROR" os.killpg(os.getpgid(p.pid), signal.SIGTERM) # if psutil.pid_exists(p): # psutil.Process(p).terminate() return scan_results = json.load(open(report_filename)) for item in scan_results: checked_files.append(item["file"]) if len(item["results"]) == 0: continue for result in item["results"]: if "vulnerabilities" not in result.keys(): continue for vuln in result["vulnerabilities"]: vuln_summary = "n/a" if "summary" in vuln["identifiers"].keys(): vuln_summary = vuln["identifiers"]["summary"] # Title item_title = "'{}-{}' is vulnerable: '{}'".format( result["component"], result["version"], vuln_summary) # Description item_description = "An external JavaScript library has been found to be vulnerable:\n\nFilename: {}\nComponent: {}\nVersion: {}\nTitle: {}".format( item["file"], result["component"], result["version"], vuln_summary ) # Check CVE item_vuln_refs = {} if "CVE" in vuln["identifiers"].keys(): item_vuln_refs.update({"CVE": vuln["identifiers"]["CVE"]}) new_finding = PatrowlEngineFinding( issue_id=issue_id, type="code_js_missing_update", title=item_title, description=item_description, solution="Check the exploitability of the vulnerability in the application context. If the vulnerability is verified, consider updating the library.", severity=vuln["severity"], confidence="firm", raw=item, target_addrs=[asset_value], meta_links=vuln["info"], meta_tags=["js", "library", "update", "retire.js"], meta_vuln_refs=item_vuln_refs) issue_id+=1 findings.append(new_finding) # findings summary per asset (remove the workdir) checked_files_str = "\n".join([remove_prefix(ff, scan_wd_asset) for ff in sorted(checked_files)]) summary_asset_finding_hash = hashlib.sha1(checked_files_str).hexdigest()[:6] summary_asset_finding = PatrowlEngineFinding( issue_id=issue_id, type="code_js_summary", title="Retire.js scan summary for '{}' (#: {}, HASH: {})".format( asset_value, len(checked_files), summary_asset_finding_hash), description="Checked files:\n\n{}".format(checked_files_str), solution="n/a.", severity="info", confidence="firm", raw=checked_files, target_addrs=[asset_value], meta_tags=["js", "library", "retire.js"]) issue_id+=1 findings.append(summary_asset_finding) # Write results under mutex scan_lock = threading.RLock() with scan_lock: engine.scans[scan_id]["findings"] = engine.scans[scan_id]["findings"] + findings # Remove the workdir shutil.rmtree(scan_wd, ignore_errors=True)
def _parse_xml_results(scan_id, asset, asset_port): issue_id = 0 findings = [] filename = APP_BASE_DIR + "/results/" + scan_id + "/" + asset + "_" + asset_port + ".xml" # Check file try: findings_tree = ET.parse(filename) except Exception: print("No Element found in XML file: {}".format(filename)) return False xml_root = findings_tree.getroot() scan_results = findings_tree.find("ssltest") # Finding: Scan details issue_id += 1 new_finding = PatrowlEngineFinding( issue_id=issue_id, type="ssltest_scan_summary", title="SSLScan scan on '{}:{}'".format(asset, asset_port), description=ET.tostring(xml_root, encoding='utf-8', method='xml'), solution="n/a", severity="info", confidence="firm", raw=ET.tostring(xml_root, encoding='utf-8', method='xml'), target_addrs=[asset]) findings.append(new_finding) if scan_results: # Finding: Supported ciphersuites issue_id += 1 ciphersuites_issue = _get_ciphersuites( items=scan_results.findall("cipher"), issue_id=issue_id, asset=asset, asset_port=asset_port) if ciphersuites_issue: findings.append(ciphersuites_issue) # Finding: Certificate issue_id += 1 certificate_pem_issue = _get_certificate_blob( cert_blob=scan_results.find("certificate").find( "certificate-blob"), issue_id=issue_id, asset=asset, asset_port=asset_port) if certificate_pem_issue: findings.append(certificate_pem_issue) # Finding: Certificate is expired ? issue_id += 1 is_cert_expired_issue = _is_certificate_expired( cert_tags=scan_results.find(".//certificate/expired/.."), issue_id=issue_id, asset=asset, asset_port=asset_port) if is_cert_expired_issue: findings.append(is_cert_expired_issue) # Finding: Certificate is self-signed ? issue_id += 1 is_cert_selfsigned_issue = _is_certificate_selfsigned( cert_tags=scan_results.find(".//certificate/self-signed/.."), issue_id=issue_id, asset=asset, asset_port=asset_port) if is_cert_selfsigned_issue: findings.append(is_cert_selfsigned_issue) # Finding: Heartbleed issue_id += 1 hb_vuln = _get_heartbleed_vuln( items=scan_results.findall("heartbleed"), issue_id=issue_id, asset=asset, asset_port=asset_port) if hb_vuln: findings.append(hb_vuln) # Write results under mutex scan_lock = threading.RLock() with scan_lock: engine.scans[scan_id]["findings"] += findings return True
def _parse_xml_results(scan_id, asset, asset_port): issue_id = 0 findings = [] filename = APP_BASE_DIR+"/results/"+scan_id+"/"+asset+"_"+asset_port+".xml" # Check file try: findings_tree = ET.parse(filename) except Exception: print("No Element found in XML file: {}".format(filename)) return False xml_root = findings_tree.getroot() scan_results = findings_tree.find("ssltest") # Finding: Scan details issue_id += 1 new_finding = PatrowlEngineFinding( issue_id=issue_id, type="ssltest_scan_summary", title="SSLScan scan on '{}:{}'".format(asset, asset_port), description=ET.tostring(xml_root, encoding='utf-8', method='xml').decode('utf-8'), solution="n/a", severity="info", confidence="firm", raw=ET.tostring(xml_root, encoding='utf-8', method='xml').decode('utf-8'), target_addrs=[asset]) findings.append(new_finding) if scan_results is not None: # Finding: Supported ciphersuites issue_id += 1 ciphersuites_issue = _get_ciphersuites( items=scan_results.findall("cipher"), issue_id=issue_id, asset=asset, asset_port=asset_port) if ciphersuites_issue: findings.append(ciphersuites_issue) # Finding: Certificate if scan_results.find("certificate") is not None: issue_id += 1 certificate_pem_issue = _get_certificate_blob( cert_blob=scan_results.find("certificate").find("certificate-blob"), issue_id=issue_id, asset=asset, asset_port=asset_port) if certificate_pem_issue: findings.append(certificate_pem_issue) # Finding: Certificate is expired ? issue_id += 1 is_cert_expired_issue = _is_certificate_expired( cert_tags=scan_results.find(".//certificate/expired/.."), issue_id=issue_id, asset=asset, asset_port=asset_port) if is_cert_expired_issue: findings.append(is_cert_expired_issue) # Finding: Certificate is self-signed ? issue_id += 1 is_cert_selfsigned_issue = _is_certificate_selfsigned( cert_tags=scan_results.find(".//certificate/self-signed/.."), issue_id=issue_id, asset=asset, asset_port=asset_port) if is_cert_selfsigned_issue: findings.append(is_cert_selfsigned_issue) # Finding: Heartbleed issue_id += 1 hb_vuln = _get_heartbleed_vuln( items=scan_results.findall("heartbleed"), issue_id=issue_id, asset=asset, asset_port=asset_port) if hb_vuln: findings.append(hb_vuln) # Finding: Fallback supported ? issue_id += 1 is_fallback_supported_issue = _is_fallback_supported( fallback=scan_results.find("fallback"), issue_id=issue_id, asset=asset, asset_port=asset_port) if is_fallback_supported_issue: findings.append(is_fallback_supported_issue) # Finding: Secure renegotiation supported ? issue_id += 1 is_secure_renegotiation_issue = _is_secure_renegotiation_supported( sec_rng=scan_results.find("renegotiation"), issue_id=issue_id, asset=asset, asset_port=asset_port) if is_secure_renegotiation_issue: findings.append(is_secure_renegotiation_issue) # Finding: weak protocols # issue_id is handled inside the function wp_vuln = _spot_weak_protocol( protocols=scan_results.findall("protocol"), issue_id=issue_id, asset=asset, asset_port=asset_port) if wp_vuln: for weak_pr in wp_vuln: issue_id = weak_pr.__dict__["issue_id"] findings.append(weak_pr) # Finding: weak ciphersuites # issue_id is handled inside the function wc_vuln = _spot_weak_ciphersuites( ciphers=scan_results.findall("cipher"), issue_id=issue_id, asset=asset, asset_port=asset_port) if wc_vuln: for weak_cs in wc_vuln: issue_id = weak_cs.__dict__["issue_id"] findings.append(weak_cs) # Write results under mutex scan_lock = threading.RLock() with scan_lock: engine.scans[scan_id]["findings"] += findings return True