def _check_charset(url: str, res: Response) -> List[Result]: results: List[Result] = [] # if the body is empty, we really don't care about this if len(res.content) == 0: return results try: if "Content-Type" in res.headers: content_type = str(res.headers["Content-Type"]).lower() if "charset" not in content_type and "text/html" in content_type: # not charset specified results.append( Result.from_evidence( Evidence.from_response(res, {"content-type": content_type}), f"Charset Not Defined in '{res.headers['Content-Type']}' at {url}", Vulnerabilities.HTTP_HEADER_CONTENT_TYPE_NO_CHARSET, )) else: # content-type missing results.append( Result.from_evidence( Evidence.from_response(res), f"Content-Type Missing: {url} ({res.request.method} - {res.status_code})", Vulnerabilities.HTTP_HEADER_CONTENT_TYPE_MISSING, )) except Exception: output.debug_exception() return results
def check_json_user_enum(url: str) -> List[Result]: results = [] target = urljoin(url, "wp-json/wp/v2/users") res = network.http_get(target, False) body = res.text if res.status_code < 300 and "slug" in body: data = res.json() # log the enum finding results.append( Result.from_evidence( Evidence.from_response(res), f"WordPress WP-JSON User Enumeration at {target}", Vulnerabilities.APP_WORDPRESS_USER_ENUM_API, )) # log the individual users for user in data: results.append( Result.from_evidence( Evidence.from_response( res, { "user_id": user["id"], "user_slug": user["slug"], "user_name": user["name"], }, ), f"ID: {user['id']}\tUser Slug: '{user['slug']}'\t\tUser Name: '{user['name']}'", Vulnerabilities.APP_WORDPRESS_USER_FOUND, )) return results
def check_options(url: str) -> List[Result]: results: List[Result] = [] res = network.http_options(url) if "Allow" in res.headers: results.append( Result.from_evidence( Evidence.from_response(res), f"Allow HTTP Verbs (OPTIONS): {res.headers['Allow']}", Vln.HTTP_OPTIONS_ALLOW, ) ) if "Public" in res.headers: results.append( Result.from_evidence( Evidence.from_response(res), f"Public HTTP Verbs (OPTIONS): {res.headers['Public']}", Vln.HTTP_OPTIONS_PUBLIC, ) ) results += response_scanner.check_response(url, res) return results
def identify(url: str) -> Tuple[Union[str, None], List[Result]]: results = [] # find WordPress res, path = _identify_by_path(url, "") if path is None: res, path = _identify_by_path(url, "blog/") # check to see if we have a valid hit if path is not None: # we have a WordPress install, let's see if we can get a version body = res.text ver = "Unknown" # this works for modern versions m = re.search(r"login.min.css\?ver=\d+\.\d+\.?\d*", body) if m: ver = m.group(0).split("=")[1] else: # the current method doesn't work, fall back to an older method m = re.search(r"load-styles.php\?[\w,;=&%]+;ver=\d+\.\d+\.?\d*", body) if m: ver = m.group(0).split("=")[-1] # report that we found WordPress results.append( Result.from_evidence( Evidence.from_response(res, {"version": ver}), f"Found WordPress v{ver} at {path}", Vulnerabilities.APP_WORDPRESS_VERSION, ) ) # is this a current version? ver = cast(version.Version, version.parse(ver)) curr_version = version_checker.get_latest_version("wordpress", ver) if curr_version is not None and curr_version > ver: results.append( Result.from_evidence( Evidence.from_response( res, { "installed_version": str(ver), "current_verison": str(curr_version), }, ), f"WordPress Outdated: {ver} - Current: {curr_version}", Vulnerabilities.APP_WORDPRESS_OUTDATED, ) ) return path, results else: return None, []
def check_aspnet_handlers(url: str) -> List[Result]: results = [] file_name = secrets.token_hex(12) exts = ["ashx", "aspx", "asmx", "soap", "rem"] for ext in exts: target = urljoin(url, f"{file_name}.{ext}") vuln = False res = network.http_get(target, False) body = res.text if "Location" in res.headers and "aspxerrorpath" in res.headers["Location"]: vuln = True elif ( res.status_code >= 400 and "Remoting.RemotingException" in body or "HttpException" in body or "FileNotFoundException" in body ): vuln = True if vuln: results.append( Result.from_evidence( Evidence.from_response(res, {"handler": ext}), f"ASP.NET Handler Enumeration: {ext}", Vulnerabilities.SERVER_ASPNET_HANDLER_ENUM, ) ) return results
def check_path_disclosure(wp_url: str) -> List[Result]: # this is a list of files that are known to throw a fatal error when accessed directly # this is from a manual review of all plugins with at least 1M installs urls = [ "wp-content/plugins/hello.php", "wp-content/plugins/akismet/akismet.php", "wp-content/plugins/contact-form-7/includes/capabilities.php", "wp-content/plugins/wordpress-seo/admin/views/partial-alerts-errors.php", "wp-content/plugins/jetpack/load-jetpack.php", "wp-content/plugins/jetpack/uninstall.php", "wp-content/plugins/duplicate-post/duplicate-post-admin.php", "wp-content/plugins/wpforms-lite/includes/admin/class-welcome.php", "wp-content/plugins/wp-google-maps/base/includes/welcome.php", "wp-content/plugins/wp-super-cache/wp-cache.php", "wp-content/plugins/mailchimp-for-wp/integrations/wpforms/bootstrap.php", "wp-content/plugins/mailchimp-for-wp/integrations/bootstrap.php", "wp-content/plugins/regenerate-thumbnails/regenerate-thumbnails.php", "wp-content/plugins/advanced-custom-fields/includes/deprecated.php", "wp-content/plugins/redirection/redirection.php", "wp-content/plugins/wpforms-lite/includes/admin/importers/class-ninja-forms.php", "wp-content/plugins/ninja-forms/includes/deprecated.php", "wp-content/plugins/so-widgets-bundle/so-widgets-bundle.php", "wp-content/plugins/wp-fastest-cache/templates/preload.php", "wp-content/plugins/duplicate-page/duplicatepage.php", "wp-content/plugins/better-wp-security/better-wp-security.php", "wp-content/plugins/all-in-one-wp-security-and-firewall/other-includes/wp-security-unlock-request.php", "wp-content/plugins/related-posts/views/settings.php", "wp-content/plugins/wpcontentguard/views/settings.php", "wp-content/plugins/simple-social-icons/simple-social-icons.php", ] results: List[Result] = [] for url in urls: target = urljoin(wp_url, url) head = network.http_head(target, False) if head.status_code != 404: resp = network.http_get(target, False) if resp.status_code < 300 or resp.status_code >= 500: # we have some kind of response that could be useful if "<b>Fatal error</b>:" in resp.text: # we have an error pattern = r"<b>((\/|[A-Z]:\\).*.php)<\/b>" if re.search(pattern, resp.text): try: path = re.findall(pattern, resp.text)[0][0] results.append( Result.from_evidence( Evidence.from_response( resp, {"path": path}), f"WordPress File Path Disclosure: {target} ({path})", Vulnerabilities. APP_WORDPRESS_PATH_DISCLOSURE, )) except Exception: output.debug_exception() results += response_scanner.check_response(target, resp) return results
def _process(url: str, res: Response): nonlocal results if res.status_code == 200 and '<h1 class="p">PHP Version' in res.text: results.append( Result.from_evidence( Evidence.from_response(res), f"PHP Info Found: {url}", Vulnerabilities.SERVER_PHP_PHPINFO, ))
def _process(url: str, res: Response): nonlocal results if res.status_code == 200 and res.content.startswith( b"\0\0\0\1Bud1\0"): results.append( Result.from_evidence( Evidence.from_response(res), f".DS_Store File Found: {url}", Vulnerabilities.HTTP_DS_STORE_FILE, ))
def check_for_jira(session: Session) -> Tuple[List[Result], Union[str, None]]: # this checks for an instance of Jira relative to the session URL results: List[Result] = [] jira_url = None try: targets = [ f"{session.url}secure/Dashboard.jspa", f"{session.url}jira/secure/Dashboard.jspa", ] for target in targets: res = network.http_get(target, False) if ( res.status_code == 200 and 'name="application-name" content="JIRA"' in res.text ): # we have a Jira instance jira_url = target # try to get the version ver_str = "unknown" try: ver_pattern = ( r"<meta name=\"ajs-version-number\" content=\"([\d\.]+)\">" ) version = re.search(ver_pattern, res.text).group(1) build_pattern = ( r"<meta name=\"ajs-build-number\" content=\"(\d+)\">" ) build = re.search(build_pattern, res.text).group(1) ver_str = f"v{version}-{build}" except: output.debug_exception() results.append( Result.from_evidence( Evidence.from_response(res), f"Jira Installation Found ({ver_str}): {target}", Vulnerabilities.APP_JIRA_FOUND, ) ) results += response_scanner.check_response(target, res) break except Exception: output.debug_exception() return results, jira_url
def check_cve_2019_11043(session: Session, links: List[str]) -> List[Result]: min_qsl = 1500 max_qsl = 1950 qsl_step = 5 results = [] targets = [] if session.args.php_page is not None and len(session.args.php_page) > 0: php_page = str(session.args.php_page) if php_page.startswith("http://") or php_page.startswith("https://"): targets.append(urljoin(session.url, php_page)) elif php_page.startswith(session.url): targets.append(php_page) for link in links: if link.endswith(".php"): targets.append(link) elif link.endswith("/"): targets.append(f"{link}index.php") def _get_resp(url: str, q_count: int) -> Response: path_info = "/PHP\nindex.php" u = urlparse(url) orig_path = quote(u.path) new_path = quote(u.path + path_info) delta = len(new_path) - len(path_info) - len(orig_path) prime = q_count - delta / 2 req_url = urljoin(url, new_path + "?" + "Q" * int(prime)) return network.http_get(req_url, False) for target in targets: # start by making sure that we have a valid target if network.http_head(target, False).status_code < 400: # get our baseline status code res = _get_resp(target, 1500) base_status_code = res.status_code for qsl in range(min_qsl + qsl_step, max_qsl, qsl_step): res = _get_resp(target, qsl) if res.status_code != base_status_code: results.append( Result.from_evidence( Evidence.from_response(res, {"qsl": qsl}), f"Detected susceptibility to PHP Remote Code Execution (CVE-2019-11043) (QSL {qsl})", Vulnerabilities.SERVER_PHP_CVE_2019_11043, ) ) break return results
def _process(url: str, result: Tuple[bool, Response]): nonlocal results found, res = result if found and '<h1 class="p">PHP Version' in res.text: results.append( Result.from_evidence( Evidence.from_response(res), f"PHP Info Found: {url}", Vulnerabilities.SERVER_PHP_PHPINFO, ) )
def check_response(url: str, res: Response, body: Union[str, None] = None) -> List[Result]: global _data, _reports results = [] try: # make sure we actually have something if res is None: return [] if _data is None or len(_data) == 0: _get_data() if body is None: body = res.text for rule in _data: rule = cast(_MatchRule, rule) mtch = re.search(rule.pattern, body) if mtch: val = mtch.group(int(rule.match_group)) err_start = body.find(val) # get the error, plus 25 characters on each side err = body[err_start - 25:err_start + len(val) + 25] msg = (f"Found error message (confidence: {rule.confidence}) " f"on {url} ({res.request.method}): ...{err}...") if msg not in _reports: results.append( Result.from_evidence( Evidence.from_response(res), msg, Vulnerabilities.HTTP_ERROR_MESSAGE, )) _reports.append(msg) break else: output.debug(f"Ignored duplicate error message: {msg}") except Exception: output.debug_exception() return results
def _process(url: str, res: Response): nonlocal results, new_links if res.status_code == 200: # we found something! new_links.append(url) results.append( Result.from_evidence( Evidence.from_response(res), f"Found backup file: {url}", Vulnerabilities.HTTP_BACKUP_FILE, )) results += response_scanner.check_response(target, res)
def check_trace(url: str) -> List[Result]: results: List[Result] = [] res = network.http_custom("TRACE", url) body = res.text if res.status_code == 200 and "TRACE / HTTP/1.1" in body: results.append( Result.from_evidence( Evidence.from_response(res), "HTTP TRACE Enabled", Vln.HTTP_TRACE_ENABLED, )) results += response_scanner.check_response(url, res) return results
def get_results(soup: BeautifulSoup, url: str, res: Response) -> List[Result]: global _reports results: List[Result] = [] try: parsed = urlparse(url) domain = utils.get_domain(parsed.netloc) issues, r = _get_retirejs_results(soup, url, domain, res) results += r for js_url, issue in issues: comp = issue["component"] ver = issue["version"] if "vulnerabilities" in issue: for vuln in issue["vulnerabilities"]: info = ( f'Vulnerable JavaScript: {comp}-{ver} ({js_url}): Severity: {vuln["severity"]} - ' f'Info: {" ".join(vuln["info"])}' ) # make sure we haven't reported this issue before if info not in _reports: _reports.append(info) results.append( Result.from_evidence( Evidence.from_response( res, { "js_file": js_url, "js_lib": comp, "js_lib_ver": ver, "vuln_info": list(vuln["info"]), "vuln_sev": vuln["severity"], }, ), info, Vulnerabilities.JS_VULNERABLE_VERSION, ) ) except Exception: output.debug_exception() return results
def _get_retirejs_results( soup: BeautifulSoup, url: str, domain: str, res: Response ) -> Tuple[List[Tuple[str, Dict]], List[Result]]: global _data, _checked issues = [] results: List[Result] = [] if _data is None or len(_data) == 0: _get_data() if _data is not None: # get all the JS files files = [i.get("src") for i in soup.find_all("script") if i.get("src")] for file in files: # fix relative URLs if str(file).startswith("//"): file = f"https:{file}" if str(file).startswith("/") or (not str(file).startswith("http")): file = urljoin(url, file) if file not in _checked: findings = retirejs.scan_endpoint(file, _data) _checked.append(file) if domain not in file: # external JS file results.append( Result.from_evidence( Evidence.from_response(res, {"js_file": file}), f"External JavaScript File: {file}", Vulnerabilities.JS_EXTERNAL_FILE, ) ) for find in findings: issues.append((file, find)) return issues, results else: # this means we couldn't get the data, so bail return [], []
def check_telerik_rau_enabled(soup: BeautifulSoup, url: str) -> List[Result]: results: List[Result] = [] parsed = urlparse(url) domain = utils.get_domain(parsed.netloc) try: # get all the scripts files = [i.get("src") for i in soup.find_all("script") if i.get("src")] for file in files: if "Telerik.Web.UI.WebResource.axd" in file: # ok, they are using Telerik UI for ASP.NET AJAX # fix-up the URL if str(file).startswith("//"): file = f"https:{file}" if str(file).startswith("/") or (not str(file).startswith("http")): if parsed.scheme == "https": file = urljoin(f"https://{domain}", file) else: file = urljoin(f"http://{domain}", file) target = urlparse(file) target = target._replace(query="type=rau") if domain in target: res = network.http_get(urlunparse(target), False) # NOTE: Typo in "succesfully" is intentional - do not fix if "RadAsyncUpload handler is registered succesfully" in res.text: results.append( Result.from_evidence( Evidence.from_response(res, {"original_url": url}), f"Telerik UI for ASP.NET AJAX RadAsyncUpload Enabled " f"(Check for CVE-2019-18935) at {target}", Vulnerabilities.APP_TELERIK_UI_RAD_ASYNC_UPLOAD_ENABLED, ) ) break except Exception: output.debug_exception() return results
def check_propfind(url: str) -> List[Result]: results: List[Result] = [] res = network.http_custom("PROPFIND", url) body = res.text if res.status_code <= 400 and len(body) > 0: if "Content-Type" in res.headers and "text/xml" in res.headers["Content-Type"]: results.append( Result.from_evidence( Evidence.from_response(res), "Possible Info Disclosure: PROPFIND Enabled", Vln.HTTP_PROPFIND_ENABLED, ) ) results += response_scanner.check_response(url, res) return results
def check_jira_user_registration(jira_url: str) -> List[Result]: results: List[Result] = [] try: target = f"{jira_url.rsplit('/', 1)[0]}/Signup!default.jspa" res = network.http_get(target, False) if res.status_code == 200 and "<title>Sign up for Jira" in res.text: results.append( Result.from_evidence( Evidence.from_response(res), f"Jira User Registration Enabled: {target}", Vulnerabilities.APP_JIRA_USER_REG_ENABLED, )) results += response_scanner.check_response(target, res) except Exception: output.debug_exception() return results
def _check_url(url: str, targets: List[str]) -> Tuple[List[str], List[Result]]: files: List[str] = [] results: List[Result] = [] for target in targets: target_url = urljoin(url, target) res = network.http_get(target_url, False) results += response_scanner.check_response(target_url, res) if res.status_code < 300: files.append(target_url) results.append( Result.from_evidence( Evidence.from_response(res), f"File found: {target_url}", Vulnerabilities.SERVER_SPECIAL_FILE_EXPOSED, )) return files, results
def _get_retirejs_results( soup: BeautifulSoup, url: str, domain: str, res: Response) -> Tuple[List[Tuple[str, Dict]], List[Result]]: global _data, _checked issues = [] results: List[Result] = [] if _data is None or len(_data) == 0: _get_data() if _data is not None: # get all the JS files elements = [i for i in soup.find_all("script") if i.get("src")] for element in elements: file = element.get("src") # fix relative URLs if str(file).startswith("//"): file = f"https:{file}" if str(file).startswith("/") or (not str(file).startswith("http")): if urlparse(url).scheme == "https": file = urljoin(f"https://{domain}", file) else: file = urljoin(f"http://{domain}", file) if file not in _checked: findings = retirejs.scan_endpoint(file, _data) _checked.append(file) if domain not in file: # external JS file results.append( Result.from_evidence( Evidence.from_response(res, { "js_file": file, "element": str(element) }), f"External JavaScript File: {file}", Vulnerabilities.JS_EXTERNAL_FILE, )) # we have an external script; check for SRI if not element.get("integrity"): results.append( Result.from_evidence( Evidence.from_response(res, { "js_file": file, "element": str(element) }), f"External JavaScript Without SRI: {file}", Vulnerabilities.JS_EXTERNAL_NO_SRI, )) for find in findings: issues.append((file, find)) return issues, results else: # this means we couldn't get the data, so bail return [], []
def _check_cache_headers(url: str, res: Response) -> List[Result]: results = [] try: if "Cache-Control" in res.headers: # we have the header, check the content if "public" in str(res.headers["Cache-Control"]).lower(): results.append( Result.from_evidence( Evidence.from_response(res), f"Cache-Control: Public: {url}", Vulnerabilities.HTTP_HEADER_CACHE_CONTROL_PUBLIC, )) if "no-cache" not in str(res.headers["Cache-Control"]).lower(): results.append( Result.from_evidence( Evidence.from_response(res), f"Cache-Control: no-cache Not Found: {url}", Vulnerabilities. HTTP_HEADER_CACHE_CONTROL_NO_CACHE_MISSING, )) if "no-store" not in str(res.headers["Cache-Control"]).lower(): results.append( Result.from_evidence( Evidence.from_response(res), f"Cache-Control: no-store Not Found: {url}", Vulnerabilities. HTTP_HEADER_CACHE_CONTROL_NO_STORE_MISSING, )) if "private" not in str(res.headers["Cache-Control"]).lower(): results.append( Result.from_evidence( Evidence.from_response(res), f"Cache-Control: private Not Found: {url}", Vulnerabilities. HTTP_HEADER_CACHE_CONTROL_PRIVATE_MISSING, )) else: # header missing results.append( Result.from_evidence( Evidence.from_response(res), f"Cache-Control Header Not Found: {url}", Vulnerabilities.HTTP_HEADER_CACHE_CONTROL_MISSING, )) if "Expires" not in res.headers: results.append( Result.from_evidence( Evidence.from_response(res), f"Expires Header Not Found: {url}", Vulnerabilities.HTTP_HEADER_EXPIRES_MISSING, )) else: # TODO: parse the value and see if it's less than now pass if "Pragma" not in res.headers or "no-cache" not in str( res.headers["Pragma"]): results.append( Result.from_evidence( Evidence.from_response(res), f"Pragma: no-cache Not Found: {url}", Vulnerabilities.HTTP_HEADER_PRAGMA_NO_CACHE_MISSING, )) except Exception: output.debug_exception() return results
def _get_links(base_url: str, urls: List[str], queue, pool): global _links, _insecure, _tasks, _lock max_length = 1024 * 1024 * 3 # 3MB results: List[Result] = [] # fail-safe to make sure we don't go too crazy if len(_links) > 10000: # if we have more than 10,000 URLs in our list, just stop output.debug( "Spider: Link list contains > 10,000 items. Stopped gathering more links." ) return for url in urls: try: # list of pages found that will need to be processed to_process: List[str] = [] res = network.http_get(url, False) # get the length, so that we don't parse huge documents if "Content-Length" in res.headers: length = int(res.headers["Content-Length"]) else: length = len(res.content) if http_utils.is_text(res) and length < max_length: soup = BeautifulSoup(res.text, "html.parser") else: # no clue what this is soup = None results += response_scanner.check_response(url, res, soup) if soup is not None: for link in soup.find_all("a"): href = link.get("href") if str(href).startswith( "/") and not str(href).startswith("//"): href = urljoin(base_url, href) if href is not None: # check to see if this link is in scope if base_url in href and href not in _links: if "." in href.split("/")[-1]: file_ext = href.split("/")[-1].split(".")[-1] else: file_ext = None with _lock: _links.append(href) # filter out some of the obvious binary files if file_ext is None or file_ext not in [ "gzip", "jpg", "jpeg", "gif", "woff", "zip", "exe", "gz", "pdf", ]: if not _is_unsafe_link(href, link.string): to_process.append(href) else: output.debug( f"Skipping unsafe URL: {link.string} - {href}" ) else: output.debug( f'Skipping URL "{href}" due to file extension "{file_ext}"' ) else: if (base_url.startswith("https://") and str(href).startswith("http://") and str(href) not in _insecure): # link from secure to insecure with _lock: _insecure.append(str(href)) results.append( Result.from_evidence( Evidence.from_response( res, {"link": href}), f"Insecure Link: {url} links to {href}", Vulnerabilities.HTTP_INSECURE_LINK, )) # handle redirects if "Location" in res.headers: redirect = res.headers["Location"] # check for relative link if str(redirect).startswith("/"): redirect = urljoin(base_url, redirect) # make sure that we aren't redirected out of scope if base_url in redirect: to_process.append(redirect) if len(to_process) > 0: asy = pool.apply_async(_get_links, (base_url, to_process, queue, pool)) with _lock: _tasks.append(asy) except Exception: output.debug_exception() output.debug(f"GetLinks Task Completed - {len(results)} issues found.") queue.put(results)
def get_header_issues(res: Response, raw: str, url: str) -> List[Result]: results: List[Result] = [] try: headers = res.headers if "X-Powered-By" in headers: results.append( Result.from_evidence( Evidence.from_response(res), f'X-Powered-By Header Present: {headers["X-Powered-By"]} ({url})', Vln.HTTP_HEADER_X_POWERED_BY, )) # check to see if this is a php version results += php.check_version(headers["X-Powered-By"], raw, url) if "X-XSS-Protection" in headers: # header is present, check the value if "0" in headers["X-XSS-Protection"]: results.append( Result.from_evidence( Evidence.from_response(res), f"X-XSS-Protection Disabled Header Present ({url})", Vln.HTTP_HEADER_X_XSS_PROTECTION_DISABLED, )) else: results.append( Result.from_evidence( Evidence.from_response(res), f"X-XSS-Protection Header Not Present ({url})", Vln.HTTP_HEADER_X_XSS_PROTECTION_MISSING, )) if "X-Runtime" in headers: results.append( Result.from_evidence( Evidence.from_response(res), f"X-Runtime Header Present; likely indicates a RoR application ({url})", Vln.HTTP_HEADER_X_RUNTIME, )) if "X-Backend-Server" in headers: results.append( Result.from_evidence( Evidence.from_response(res), f'X-Backend-Server Header Present: {headers["X-Backend-Server"]} ({url})', Vln.HTTP_HEADER_X_BACKEND_SERVER, )) if "Via" in headers: results.append( Result.from_evidence( Evidence.from_response(res), f'Via Header Present: #{headers["Via"]} ({url})', Vln.HTTP_HEADER_VIA, )) if "X-Frame-Options" in headers: if "allow" in str(headers["X-Frame-Options"]).lower(): results.append( Result.from_evidence( Evidence.from_response(res), f'X-Frame-Options Header: {headers["X-Frame-Options"]} ({url})', Vln.HTTP_HEADER_X_FRAME_OPTIONS_ALLOW, )) else: results.append( Result.from_evidence( Evidence.from_response(res), f"X-Frame-Options Header Not Present ({url})", Vln.HTTP_HEADER_X_FRAME_OPTIONS_MISSING, )) if "X-Content-Type-Options" not in headers: results.append( Result.from_evidence( Evidence.from_response(res), f"X-Content-Type-Options Header Not Present ({url})", Vln.HTTP_HEADER_X_CONTENT_TYPE_OPTIONS_MISSING, )) if "Content-Security-Policy" not in headers: results.append( Result.from_evidence( Evidence.from_response(res), f"Content-Security-Policy Header Not Present ({url})", Vln.HTTP_HEADER_CONTENT_SECURITY_POLICY_MISSING, )) if "Referrer-Policy" not in headers: results.append( Result.from_evidence( Evidence.from_response(res), f"Referrer-Policy Header Not Present ({url})", Vln.HTTP_HEADER_REFERRER_POLICY_MISSING, )) if "Feature-Policy" not in headers: results.append( Result.from_evidence( Evidence.from_response(res), f"Feature-Policy Header Not Present ({url})", Vln.HTTP_HEADER_FEATURE_POLICY_MISSING, )) if "Access-Control-Allow-Origin" in headers: if headers["Access-Control-Allow-Origin"] == "*": results.append( Result.from_evidence( Evidence.from_response(res), f"Access-Control-Allow-Origin: Unrestricted ({url})", Vln.HTTP_HEADER_CORS_ACAO_UNRESTRICTED, )) if "Strict-Transport-Security" not in headers: results.append( Result.from_evidence( Evidence.from_response(res), f"Strict-Transport-Security Header Not Present ({url})", Vln.HTTP_HEADER_HSTS_MISSING, )) if "Server" in headers: results += get_server_banner_issues(headers["Server"], raw, url, headers) except Exception: output.debug_exception() return results
def get_header_issues(res: Response, raw: str, url: str) -> List[Result]: results: List[Result] = [] try: headers = res.headers if "X-Powered-By" in headers: results.append( Result.from_evidence( Evidence.from_response(res), f'X-Powered-By Header Present: {headers["X-Powered-By"]} ({url})', Vln.HTTP_HEADER_X_POWERED_BY, ) ) # check to see if this is a php version results += php.check_version(headers["X-Powered-By"], raw, url) if "X-XSS-Protection" in headers: # header is present, check the value if "0" in headers["X-XSS-Protection"]: results.append( Result.from_evidence( Evidence.from_response(res), f"X-XSS-Protection Disabled Header Present ({url})", Vln.HTTP_HEADER_X_XSS_PROTECTION_DISABLED, ) ) else: results.append( Result.from_evidence( Evidence.from_response(res), f"X-XSS-Protection Header Not Present ({url})", Vln.HTTP_HEADER_X_XSS_PROTECTION_MISSING, ) ) if "X-Runtime" in headers: results.append( Result.from_evidence( Evidence.from_response(res), f"X-Runtime Header Present; likely indicates a RoR application ({url})", Vln.HTTP_HEADER_X_RUNTIME, ) ) if "X-Backend-Server" in headers: results.append( Result.from_evidence( Evidence.from_response(res), f'X-Backend-Server Header Present: {headers["X-Backend-Server"]} ({url})', Vln.HTTP_HEADER_X_BACKEND_SERVER, ) ) if "Via" in headers: results.append( Result.from_evidence( Evidence.from_response(res), f'Via Header Present: #{headers["Via"]} ({url})', Vln.HTTP_HEADER_VIA, ) ) if "X-Frame-Options" in headers: if "allow" in str(headers["X-Frame-Options"]).lower(): results.append( Result.from_evidence( Evidence.from_response(res), f'X-Frame-Options Header: {headers["X-Frame-Options"]} ({url})', Vln.HTTP_HEADER_X_FRAME_OPTIONS_ALLOW, ) ) else: results.append( Result.from_evidence( Evidence.from_response(res), f"X-Frame-Options Header Not Present ({url})", Vln.HTTP_HEADER_X_FRAME_OPTIONS_MISSING, ) ) if "X-Content-Type-Options" not in headers: results.append( Result.from_evidence( Evidence.from_response(res), f"X-Content-Type-Options Header Not Present ({url})", Vln.HTTP_HEADER_X_CONTENT_TYPE_OPTIONS_MISSING, ) ) if "Content-Security-Policy" not in headers: results.append( Result.from_evidence( Evidence.from_response(res), f"Content-Security-Policy Header Not Present ({url})", Vln.HTTP_HEADER_CONTENT_SECURITY_POLICY_MISSING, ) ) if "Referrer-Policy" not in headers: results.append( Result.from_evidence( Evidence.from_response(res), f"Referrer-Policy Header Not Present ({url})", Vln.HTTP_HEADER_REFERRER_POLICY_MISSING, ) ) if "Feature-Policy" not in headers: results.append( Result.from_evidence( Evidence.from_response(res), f"Feature-Policy Header Not Present ({url})", Vln.HTTP_HEADER_FEATURE_POLICY_MISSING, ) ) if "Access-Control-Allow-Origin" in headers: if headers["Access-Control-Allow-Origin"] == "*": results.append( Result.from_evidence( Evidence.from_response(res), f"Access-Control-Allow-Origin: Unrestricted ({url})", Vln.HTTP_HEADER_CORS_ACAO_UNRESTRICTED, ) ) if "Strict-Transport-Security" not in headers: results.append( Result.from_evidence( Evidence.from_response(res), f"Strict-Transport-Security Header Not Present ({url})", Vln.HTTP_HEADER_HSTS_MISSING, ) ) if "Server" in headers: results += get_server_banner_issues(headers["Server"], raw, url, headers) # check to see if any headers are duplicated. # we have to access a private member, as it's the only access to the raw headers if res.raw._original_response is not None: raw_headers = str(res.raw._original_response.headers).splitlines(False) raw_headers_checked: List[str] = [] for raw_header in raw_headers: header_name = raw_header.split(":")[0] if header_name not in raw_headers_checked: raw_headers_checked.append(header_name) for dup in raw_headers: dup_name = dup.split(":")[0] if dup_name == header_name and dup != raw_header: # we have a second header, with a different value results.append( Result.from_evidence( Evidence.from_response(res), f"Header {header_name} set multiple times with different values at {url}", Vln.HTTP_HEADER_DUPLICATE, ) ) break except Exception: output.debug_exception() return results
def _get_cookie_issues(cookies: List[str], url: str, res: Response) -> List[Result]: global _checked_cookies # setup the checked list if Vulnerabilities.COOKIE_MISSING_SECURE_FLAG not in _checked_cookies: _checked_cookies[Vulnerabilities.COOKIE_MISSING_SECURE_FLAG] = [] if Vulnerabilities.COOKIE_INVALID_SECURE_FLAG not in _checked_cookies: _checked_cookies[Vulnerabilities.COOKIE_INVALID_SECURE_FLAG] = [] if Vulnerabilities.COOKIE_MISSING_HTTPONLY_FLAG not in _checked_cookies: _checked_cookies[Vulnerabilities.COOKIE_MISSING_HTTPONLY_FLAG] = [] if Vulnerabilities.COOKIE_MISSING_SAMESITE_FLAG not in _checked_cookies: _checked_cookies[Vulnerabilities.COOKIE_MISSING_SAMESITE_FLAG] = [] if Vulnerabilities.COOKIE_WITH_SAMESITE_NONE_FLAG not in _checked_cookies: _checked_cookies[Vulnerabilities.COOKIE_WITH_SAMESITE_NONE_FLAG] = [] if Vulnerabilities.COOKIE_INVALID_SAMESITE_NONE_FLAG not in _checked_cookies: _checked_cookies[ Vulnerabilities.COOKIE_INVALID_SAMESITE_NONE_FLAG] = [] results: List[Result] = [] try: parsed = urlparse(url) for cookie in cookies: comp = cookie.split(";") # get the name name = comp[0].split("=")[0] # normalize the components comp = list(map(str.strip, comp)) comp = list(map(str.lower, comp)) # check Secure flag if "secure" not in comp and parsed.scheme == "https": if (name not in _checked_cookies[ Vulnerabilities.COOKIE_MISSING_SECURE_FLAG]): results.append( Result.from_evidence( Evidence.from_response(res, {"cookie": name}), f"Cookie Missing Secure Flag: {cookie}", Vulnerabilities.COOKIE_MISSING_SECURE_FLAG, )) _checked_cookies[Vulnerabilities. COOKIE_MISSING_SECURE_FLAG].append(name) elif "secure" in comp and parsed.scheme == "http": # secure flag over HTTP is invalid if (name not in _checked_cookies[ Vulnerabilities.COOKIE_INVALID_SECURE_FLAG]): results.append( Result.from_evidence( Evidence.from_response(res, {"cookie": name}), f"Cookie Secure Flag Invalid (over HTTP): {cookie}", Vulnerabilities.COOKIE_INVALID_SECURE_FLAG, )) _checked_cookies[Vulnerabilities. COOKIE_INVALID_SECURE_FLAG].append(name) # check HttpOnly flag if "httponly" not in comp: if (name not in _checked_cookies[ Vulnerabilities.COOKIE_MISSING_HTTPONLY_FLAG]): results.append( Result.from_evidence( Evidence.from_response(res, {"cookie": name}), f"Cookie Missing HttpOnly Flag: {cookie}", Vulnerabilities.COOKIE_MISSING_HTTPONLY_FLAG, )) _checked_cookies[Vulnerabilities. COOKIE_MISSING_HTTPONLY_FLAG].append(name) # check SameSite flag if ("samesite=lax" not in comp and "samesite=strict" not in comp and "samesite=none" not in comp): if (name not in _checked_cookies[ Vulnerabilities.COOKIE_MISSING_SAMESITE_FLAG]): results.append( Result.from_evidence( Evidence.from_response(res, {"cookie": name}), f"Cookie Missing SameSite Flag: {cookie}", Vulnerabilities.COOKIE_MISSING_SAMESITE_FLAG, )) _checked_cookies[Vulnerabilities. COOKIE_MISSING_SAMESITE_FLAG].append(name) # check SameSite=None flag if "samesite=none" in comp: if "secure" in comp: if (name not in _checked_cookies[ Vulnerabilities.COOKIE_WITH_SAMESITE_NONE_FLAG]): results.append( Result.from_evidence( Evidence.from_response(res, {"cookie": name}), f"Cookie With SameSite=None Flag: {cookie}", Vulnerabilities.COOKIE_WITH_SAMESITE_NONE_FLAG, )) _checked_cookies[ Vulnerabilities. COOKIE_WITH_SAMESITE_NONE_FLAG].append(name) else: if (name not in _checked_cookies[ Vulnerabilities.COOKIE_INVALID_SAMESITE_NONE_FLAG] ): results.append( Result.from_evidence( Evidence.from_response(res, {"cookie": name}), f"Cookie SameSite=None Flag Invalid (without Secure flag): {cookie}", Vulnerabilities. COOKIE_INVALID_SAMESITE_NONE_FLAG, )) _checked_cookies[ Vulnerabilities. COOKIE_INVALID_SAMESITE_NONE_FLAG].append(name) except Exception: output.debug_exception() return results
def _file_search(session: Session, orig_links: List[str]) -> List[str]: new_files: List[str] = [] file_good, file_res, path_good, path_res = network.check_404_response(session.url) # these are here for data typing results: Union[List[Result], None] links: Union[List[str], None] if not file_good: reporter.display( "Web server does not respond properly to file 404 errors.", Issue( Vulnerabilities.SERVER_INVALID_404_FILE, session.url, Evidence.from_response(file_res), ), ) if not path_good: reporter.display( "Web server does not respond properly to path 404 errors.", Issue( Vulnerabilities.SERVER_INVALID_404_PATH, session.url, Evidence.from_response(path_res), ), ) if not (file_good or path_good): output.norm( "Site does not respond properly to non-existent file/path requests; skipping some checks." ) if file_good: links, results = special_files.check_special_files(session.url) if results: reporter.display_results(results, "\t") new_files += links if session.args.files: output.empty() output.norm("Searching for common files (this will take a few minutes)...") with Spinner(): try: links, results = file_search.find_files(session.url) except Exception as error: output.debug_exception() output.error(f"Error running scan: {str(error)}") results = None links = None if results is not None and results: reporter.display_results(results, "\t") if links is not None and links: new_files += links for l in links: if l not in orig_links: output.norm(f"\tNew file found: {l}") output.empty() # check for common backup files all_links = orig_links + new_files with Spinner(): backups, res = file_search.find_backups(all_links) if res: reporter.display_results(res, "\t") if backups: new_files += backups if path_good: links, results = special_files.check_special_paths(session.url) if results: reporter.display_results(results, "\t") new_files += links if session.args.dir: output.empty() output.norm( "Searching for common directories (this will take a few minutes)..." ) with Spinner(): try: links, results = file_search.find_directories( session.url, session.args.dirlistredir, session.args.dirrecursive, ) except Exception as error: output.debug_exception() output.error(f"Error running scan: {str(error)}") results = None links = None if results is not None and results: reporter.display_results(results, "\t") if links is not None and links: new_files += links for l in links: if l not in orig_links: output.norm(f"\tNew directory found: {l}") output.empty() # check for .DS_Store files if file_good: res = file_search.find_ds_store(new_files) if res: reporter.display_results(res, "\t") return new_files
def _get_cookie_issues(cookies: List[str], url: str, res: Response) -> List[Result]: global _checked_cookies # setup the checked list if Vln.COOKIE_MISSING_SECURE_FLAG not in _checked_cookies: _checked_cookies[Vln.COOKIE_MISSING_SECURE_FLAG] = [] if Vln.COOKIE_INVALID_SECURE_FLAG not in _checked_cookies: _checked_cookies[Vln.COOKIE_INVALID_SECURE_FLAG] = [] if Vln.COOKIE_MISSING_HTTPONLY_FLAG not in _checked_cookies: _checked_cookies[Vln.COOKIE_MISSING_HTTPONLY_FLAG] = [] if Vln.COOKIE_MISSING_SAMESITE_FLAG not in _checked_cookies: _checked_cookies[Vln.COOKIE_MISSING_SAMESITE_FLAG] = [] if Vln.COOKIE_WITH_SAMESITE_NONE_FLAG not in _checked_cookies: _checked_cookies[Vln.COOKIE_WITH_SAMESITE_NONE_FLAG] = [] if Vln.COOKIE_INVALID_SAMESITE_NONE_FLAG not in _checked_cookies: _checked_cookies[Vln.COOKIE_INVALID_SAMESITE_NONE_FLAG] = [] if Vln.COOKIE_BIGIP_IP_DISCLOSURE not in _checked_cookies: _checked_cookies[Vln.COOKIE_BIGIP_IP_DISCLOSURE] = [] results: List[Result] = [] try: parsed = urlparse(url) for cookie in cookies: comp = cookie.split(";") # get the name name = comp[0].split("=")[0] # get the value value = comp[0].split("=")[1] # normalize the components comp = list(map(str.strip, comp)) # trim the string to clear the spaces comp = list( map(str.lower, comp) # make it all lowercase, to simplify checks ) # check for BigIP IP Disclosure if "BIGip" in name: if name not in _checked_cookies[Vln.COOKIE_BIGIP_IP_DISCLOSURE]: _checked_cookies[Vln.COOKIE_BIGIP_IP_DISCLOSURE].append(name) decoded = _decode_big_ip_cookie(value) if decoded is not None: results.append( Result.from_evidence( Evidence.from_response(res, {"cookie": name}), f"Big-IP Internal IP Address Disclosure: {name}: {decoded}", Vln.COOKIE_BIGIP_IP_DISCLOSURE, ) ) # check Secure flag if "secure" not in comp and parsed.scheme == "https": if name not in _checked_cookies[Vln.COOKIE_MISSING_SECURE_FLAG]: results.append( Result.from_evidence( Evidence.from_response(res, {"cookie": name}), f"Cookie Missing Secure Flag: {cookie}", Vln.COOKIE_MISSING_SECURE_FLAG, ) ) _checked_cookies[Vln.COOKIE_MISSING_SECURE_FLAG].append(name) elif "secure" in comp and parsed.scheme == "http": # secure flag over HTTP is invalid if name not in _checked_cookies[Vln.COOKIE_INVALID_SECURE_FLAG]: results.append( Result.from_evidence( Evidence.from_response(res, {"cookie": name}), f"Cookie Secure Flag Invalid (over HTTP): {cookie}", Vln.COOKIE_INVALID_SECURE_FLAG, ) ) _checked_cookies[Vln.COOKIE_INVALID_SECURE_FLAG].append(name) # check HttpOnly flag if "httponly" not in comp: if name not in _checked_cookies[Vln.COOKIE_MISSING_HTTPONLY_FLAG]: results.append( Result.from_evidence( Evidence.from_response(res, {"cookie": name}), f"Cookie Missing HttpOnly Flag: {cookie}", Vln.COOKIE_MISSING_HTTPONLY_FLAG, ) ) _checked_cookies[Vln.COOKIE_MISSING_HTTPONLY_FLAG].append(name) # check SameSite flag if ( "samesite=lax" not in comp and "samesite=strict" not in comp and "samesite=none" not in comp ): if name not in _checked_cookies[Vln.COOKIE_MISSING_SAMESITE_FLAG]: results.append( Result.from_evidence( Evidence.from_response(res, {"cookie": name}), f"Cookie Missing SameSite Flag: {cookie}", Vln.COOKIE_MISSING_SAMESITE_FLAG, ) ) _checked_cookies[Vln.COOKIE_MISSING_SAMESITE_FLAG].append(name) # check SameSite=None flag if "samesite=none" in comp: if "secure" in comp: if name not in _checked_cookies[Vln.COOKIE_WITH_SAMESITE_NONE_FLAG]: results.append( Result.from_evidence( Evidence.from_response(res, {"cookie": name}), f"Cookie With SameSite=None Flag: {cookie}", Vln.COOKIE_WITH_SAMESITE_NONE_FLAG, ) ) _checked_cookies[Vln.COOKIE_WITH_SAMESITE_NONE_FLAG].append( name ) else: if ( name not in _checked_cookies[Vln.COOKIE_INVALID_SAMESITE_NONE_FLAG] ): results.append( Result.from_evidence( Evidence.from_response(res, {"cookie": name}), f"Cookie SameSite=None Flag Invalid (without Secure flag): {cookie}", Vln.COOKIE_INVALID_SAMESITE_NONE_FLAG, ) ) _checked_cookies[Vln.COOKIE_INVALID_SAMESITE_NONE_FLAG].append( name ) except Exception: output.debug_exception() return results
def _check_cache_headers(url: str, res: Response) -> List[Result]: results = [] try: if "Cache-Control" in res.headers: # we have the header, check the content if "public" in str(res.headers["Cache-Control"]).lower(): results.append( Result.from_evidence( Evidence.from_response(res), f"Cache-Control: Public: {url}", Vulnerabilities.HTTP_HEADER_CACHE_CONTROL_PUBLIC, ) ) if "no-cache" not in str(res.headers["Cache-Control"]).lower(): results.append( Result.from_evidence( Evidence.from_response(res), f"Cache-Control: no-cache Not Found: {url}", Vulnerabilities.HTTP_HEADER_CACHE_CONTROL_NO_CACHE_MISSING, ) ) if "no-store" not in str(res.headers["Cache-Control"]).lower(): results.append( Result.from_evidence( Evidence.from_response(res), f"Cache-Control: no-store Not Found: {url}", Vulnerabilities.HTTP_HEADER_CACHE_CONTROL_NO_STORE_MISSING, ) ) if "private" not in str(res.headers["Cache-Control"]).lower(): results.append( Result.from_evidence( Evidence.from_response(res), f"Cache-Control: private Not Found: {url}", Vulnerabilities.HTTP_HEADER_CACHE_CONTROL_PRIVATE_MISSING, ) ) else: # header missing results.append( Result.from_evidence( Evidence.from_response(res), f"Cache-Control Header Not Found: {url}", Vulnerabilities.HTTP_HEADER_CACHE_CONTROL_MISSING, ) ) if "Expires" not in res.headers: results.append( Result.from_evidence( Evidence.from_response(res), f"Expires Header Not Found: {url}", Vulnerabilities.HTTP_HEADER_EXPIRES_MISSING, ) ) else: # parse the date, and check to see if it's in the past try: # using fuzzy=true here could lead to some false positives due to it doing whatever it can to produce # a valid date - but it is the most forgiving option we have to ensure odd servers don't cause issues dt = parse(res.headers["Expires"], fuzzy=True) if dt > datetime.now(tz.UTC): # Expires is in the future - it's an issue results.append( Result.from_evidence( Evidence.from_response(res), f"Expires Header - Future Dated ({res.headers['Expires']}): {url}", Vulnerabilities.HTTP_HEADER_EXPIRES_FUTURE, ) ) except Exception: output.debug_exception() if "Pragma" not in res.headers or "no-cache" not in str(res.headers["Pragma"]): results.append( Result.from_evidence( Evidence.from_response(res), f"Pragma: no-cache Not Found: {url}", Vulnerabilities.HTTP_HEADER_PRAGMA_NO_CACHE_MISSING, ) ) except Exception: output.debug_exception() return results