def spider(url) -> Tuple[List[str], List[Result]]: global _links, _insecure, _tasks, _lock results: List[Result] = [] # create processing pool pool = Pool() mgr = Manager() queue = mgr.Queue() asy = pool.apply_async(_get_links, (url, [url], queue, pool)) with _lock: _tasks.append(asy) while True: if all(t is None or t.ready() for t in _tasks): break else: count_none = 0 count_ready = 0 count_not_ready = 0 for t in _tasks: if t is None: count_none += 1 elif t.ready(): count_ready += 1 else: count_not_ready += 1 output.debug( f"Spider Task Status: None: {count_none}, Ready: {count_ready}, Not Ready: {count_not_ready}" ) time.sleep(3) pool.close() for t in _tasks: try: t.get() except Exception: output.debug_exception() while not queue.empty(): res = queue.get() if len(res) > 0: for re in res: if re not in results: results.append(re) # copy data and reset links = _links[:] _links = [] _insecure = [] _tasks = [] return links, results
def http_build_raw_response(res: Response) -> str: if res.raw.version == 11: res_line = f"HTTP/1.1 {res.raw.status} {res.raw.reason}" else: res_line = f"HTTP/1.0 {res.raw.status} {res.raw.reason}" res_string = res_line + "\r\n" if res.raw._original_response is not None: res_string += "\r\n".join( str(res.raw._original_response.headers).splitlines(False) ) else: res_string += "\r\n".join(f"{k}: {v}" for k, v in res.headers.items()) try: if response_body_is_text(res): txt = res.text if txt != "": res_string += "\r\n\r\n" res_string += txt elif len(res.content) > 0: # the body is binary - no real value in keeping it res_string += "\r\n\r\n<BINARY DATA EXCLUDED>" except Exception: output.debug_exception() return res_string
def _get_version_info() -> str: try: data, code = network.http_json("https://pypi.org/pypi/yawast/json") except Exception: output.debug_exception() return "Supported Version: (Unable to get version information)" if code != 200: ret = "Supported Version: (PyPi returned an error code while fetching current version)" else: if "info" in data and "version" in data["info"]: ver = cast(version.Version, version.parse(get_version())) curr_version = cast(version.Version, version.parse(data["info"]["version"])) ret = f"Supported Version: {curr_version} - " if ver == curr_version: ret += "You are on the latest version." elif ver > curr_version or "dev" in get_version(): ret += "You are on a pre-release version. Take care." else: ret += "Please update to the current version." else: ret = "Supported Version: (PyPi returned invalid data while fetching current version)" return ret
def check_cve_2019_5418(url: str) -> List[Result]: global _checked # this only applies to controllers, so skip the check unless the link ends with '/' if not url.endswith("/") or url in _checked: return [] results: List[Result] = [] _checked.append(url) try: res = network.http_get( url, False, {"Accept": "../../../../../../../../../e*c/p*sswd{{"} ) body = res.text req = network.http_build_raw_request(res.request) results += response_scanner.check_response(url, res) pattern = r"root:[a-zA-Z0-9]+:0:0:.+$" mtch = re.search(pattern, body) if mtch: results.append( Result( f"Rails CVE-2019-5418: File Content Disclosure: {url} - {mtch.group(0)}", Vulnerabilities.SERVER_RAILS_CVE_2019_5418, url, [body, req], ) ) except Exception: output.debug_exception() return results
def _analyze(domain: str, new=False) -> Dict[str, Any]: new_path = "host={target}&publish=off&startNew=on&all=done&ignoreMismatch=on".format( target=domain ) status_path = "host={target}&publish=off&all=done&ignoreMismatch=on".format( target=domain ) if new: path = new_path else: path = status_path try: body, code = network.http_json(API_SERVER + "/api/v3/analyze?" + path) except Exception: output.debug_exception() raise # check for error messages if body.get("errors") is not None: raise ValueError( "SSL Labs returned the following error(s): {errors}".format( errors=str(body["errors"]) ) ) # next up, check to see what error code we have if code != 200: # if we got anything but 200, it's a problem raise ValueError("SSL Labs returned error code: {code}".format(code=code)) return body
def start(session: Session): print(f"Scanning: {session.url}") # make sure it resolves try: socket.gethostbyname(session.domain) except socket.gaierror as error: print( f"Fatal Error: Unable to resolve {session.domain} ({str(error)})") return try: cutils.check_redirect(session) except ValueError as error: print(f"Unable to continue: {str(error)}") return # check to see if we are looking at an HTTPS server if session.url_parsed.scheme == "https": if (session.args.internalssl or utils.is_ip(session.domain) or utils.get_port(session.url) != 443): # use internal scanner ssl_internal.scan(session) else: try: ssl_labs.scan(session) except Exception as error: output.debug_exception() output.error(f"Error running scan with SSL Labs: {str(error)}") if session.args.tdessessioncount: ssl_sweet32.scan(session)
def _is_unsafe_link(href: str, description: str) -> bool: """ Check for strings that indicate an unsafe link :param href: :param description: :return: """ unsafe_fragments = [ "logoff", "log off", "log_off", "logout", "log out", "log_out", "delete", "destroy", ] ret = False try: description = str( description).lower() if description is not None else "" href = str(href).lower() for frag in unsafe_fragments: if frag in href or frag in description: return True except Exception: output.debug_exception() return ret
def _check_charset(url: str, res: Response) -> List[Result]: results: List[Result] = [] # if the body is empty, we really don't care about this if len(res.content) == 0: return results try: if "Content-Type" in res.headers: content_type = str(res.headers["Content-Type"]).lower() if "charset" not in content_type and "text/html" in content_type: # not charset specified results.append( Result.from_evidence( Evidence.from_response(res, {"content-type": content_type}), f"Charset Not Defined in '{res.headers['Content-Type']}' at {url}", Vulnerabilities.HTTP_HEADER_CONTENT_TYPE_NO_CHARSET, )) else: # content-type missing results.append( Result.from_evidence( Evidence.from_response(res), f"Content-Type Missing: {url} ({res.request.method} - {res.status_code})", Vulnerabilities.HTTP_HEADER_CONTENT_TYPE_MISSING, )) except Exception: output.debug_exception() return results
def network_info(ip): global _failure, _cache # first, check the cache if _cache.get(ip) is not None: return _cache[ip] # now, make sure we haven't turn this off due to errors if _failure: return "Network Information disabled due to prior failure" try: info, code = network.http_json("https://api.iptoasn.com/v1/as/ip/%s" % ip) if code == 200: ret = "%s - %s" % (info["as_country_code"], info["as_description"]) _cache[ip] = ret return ret else: _failure = True return "IP To ASN Service returned code: %s" % code except (ValueError, KeyError) as error: output.debug_exception() _failure = True return "IP To ASN Service error: %s" % str(error)
def http_build_raw_response(res: Response) -> str: if res.raw.version == 11: res_line = f"HTTP/1.1 {res.raw.status} {res.raw.reason}" else: res_line = f"HTTP/1.0 {res.raw.status} {res.raw.reason}" res_string = res_line + "\r\n" if res.raw._original_response is not None: res_string += "\r\n".join( str(res.raw._original_response.headers).splitlines(False)) else: res_string += "\r\n".join(f"{k}: {v}" for k, v in res.headers.items()) try: txt = res.text if txt != "": res_string += "\r\n\r\n" res_string += txt except Exception: output.debug_exception() return res_string
def _is_port_open(url: str, ip: str, rec, queue): sock = socket.socket() port = rec["port"] # set a timeout - this has a huge speed impact sock.settimeout(0.75) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) connected = False try: connected = sock.connect_ex((ip, port)) is 0 sock.close() except Exception: # this shouldn't happen, but just in case output.debug_exception() if connected: queue.put( Result( f"Open Port: IP: {ip} - Port: {port} (Assigned Service: {rec['name']} - {rec['desc']})", Vulnerabilities.NETWORK_OPEN_PORT, url, { "ip": ip, "port": port }, ))
def check_path_disclosure(wp_url: str) -> List[Result]: # this is a list of files that are known to throw a fatal error when accessed directly # this is from a manual review of all plugins with at least 1M installs urls = [ "wp-content/plugins/hello.php", "wp-content/plugins/akismet/akismet.php", "wp-content/plugins/contact-form-7/includes/capabilities.php", "wp-content/plugins/wordpress-seo/admin/views/partial-alerts-errors.php", "wp-content/plugins/jetpack/load-jetpack.php", "wp-content/plugins/jetpack/uninstall.php", "wp-content/plugins/duplicate-post/duplicate-post-admin.php", "wp-content/plugins/wpforms-lite/includes/admin/class-welcome.php", "wp-content/plugins/wp-google-maps/base/includes/welcome.php", "wp-content/plugins/wp-super-cache/wp-cache.php", "wp-content/plugins/mailchimp-for-wp/integrations/wpforms/bootstrap.php", "wp-content/plugins/mailchimp-for-wp/integrations/bootstrap.php", "wp-content/plugins/regenerate-thumbnails/regenerate-thumbnails.php", "wp-content/plugins/advanced-custom-fields/includes/deprecated.php", "wp-content/plugins/redirection/redirection.php", "wp-content/plugins/wpforms-lite/includes/admin/importers/class-ninja-forms.php", "wp-content/plugins/ninja-forms/includes/deprecated.php", "wp-content/plugins/so-widgets-bundle/so-widgets-bundle.php", "wp-content/plugins/wp-fastest-cache/templates/preload.php", "wp-content/plugins/duplicate-page/duplicatepage.php", "wp-content/plugins/better-wp-security/better-wp-security.php", "wp-content/plugins/all-in-one-wp-security-and-firewall/other-includes/wp-security-unlock-request.php", "wp-content/plugins/related-posts/views/settings.php", "wp-content/plugins/wpcontentguard/views/settings.php", "wp-content/plugins/simple-social-icons/simple-social-icons.php", ] results: List[Result] = [] for url in urls: target = urljoin(wp_url, url) head = network.http_head(target, False) if head.status_code != 404: resp = network.http_get(target, False) if resp.status_code < 300 or resp.status_code >= 500: # we have some kind of response that could be useful if "<b>Fatal error</b>:" in resp.text: # we have an error pattern = r"<b>((\/|[A-Z]:\\).*.php)<\/b>" if re.search(pattern, resp.text): try: path = re.findall(pattern, resp.text)[0][0] results.append( Result.from_evidence( Evidence.from_response( resp, {"path": path}), f"WordPress File Path Disclosure: {target} ({path})", Vulnerabilities. APP_WORDPRESS_PATH_DISCLOSURE, )) except Exception: output.debug_exception() results += response_scanner.check_response(target, resp) return results
def get_ips(domain: str): ips = [] try: answers_v4 = resolver.query(domain, "A") for data in answers_v4: ips.append(str(data)) except (resolver.NoAnswer, resolver.NXDOMAIN, exception.Timeout): pass except (resolver.NoNameservers, resolver.NotAbsolute, resolver.NoRootSOA): output.debug_exception() pass try: answers_v6 = resolver.query(domain, "AAAA") for data in answers_v6: ips.append(str(data)) except (resolver.NoAnswer, resolver.NXDOMAIN, exception.Timeout): pass except (resolver.NoNameservers, resolver.NotAbsolute, resolver.NoRootSOA): output.debug_exception() pass return ips
def check_cve_2019_0232(links: List[str]) -> List[Result]: results: List[Result] = [] try: targets: List[str] = [] for link in links: if "cgi-bin" in link: if "?" in link: targets.append(f"{link}&dir") else: targets.append(f"{link}?dir") for target in targets: res = network.http_get(target, False) body = res.text if "<DIR>" in body: # we have a hit results.append( Result( f"Apache Tomcat RCE (CVE-2019-0232): {target}", Vulnerabilities.SERVER_TOMCAT_CVE_2019_0232, target, [ network.http_build_raw_request(res.request), network.http_build_raw_response(res), ], )) results += response_scanner.check_response(target, res) except Exception: output.debug_exception() return results
def _get_ip_info(ip: str) -> Tuple[str, str]: try: server_ip = socket.gethostbyname(ip) ni = network_info.network_info(str(server_ip)) except Exception: server_ip = "(Unavailable)" ni = "(Unavailable)" output.debug_exception() return server_ip, ni
def get_host(ip): name = "N/A" try: rev_name = reversename.from_address(str(ip)) name = str(resolver.query(rev_name, "PTR", lifetime=3)[0])[:-1] except (resolver.NoAnswer, resolver.NXDOMAIN, exception.Timeout): pass except (resolver.NoNameservers, resolver.NotAbsolute, resolver.NoRootSOA): output.debug_exception() return name
def _check_version_verb(url: str) -> List[Result]: results: List[Result] = [] try: res = network.http_custom("XYZ", url) if res.status_code > 400: results += get_version(url, res, "Invalid HTTP Verb") except Exception: output.debug_exception() return results
def check_for_jira(session: Session) -> Tuple[List[Result], Union[str, None]]: # this checks for an instance of Jira relative to the session URL results: List[Result] = [] jira_url = None try: targets = [ f"{session.url}secure/Dashboard.jspa", f"{session.url}jira/secure/Dashboard.jspa", ] for target in targets: res = network.http_get(target, False) if ( res.status_code == 200 and 'name="application-name" content="JIRA"' in res.text ): # we have a Jira instance jira_url = target # try to get the version ver_str = "unknown" try: ver_pattern = ( r"<meta name=\"ajs-version-number\" content=\"([\d\.]+)\">" ) version = re.search(ver_pattern, res.text).group(1) build_pattern = ( r"<meta name=\"ajs-build-number\" content=\"(\d+)\">" ) build = re.search(build_pattern, res.text).group(1) ver_str = f"v{version}-{build}" except: output.debug_exception() results.append( Result.from_evidence( Evidence.from_response(res), f"Jira Installation Found ({ver_str}): {target}", Vulnerabilities.APP_JIRA_FOUND, ) ) results += response_scanner.check_response(target, res) break except Exception: output.debug_exception() return results, jira_url
def _check_version_post(url: str) -> List[Result]: results: List[Result] = [] try: res = network.http_custom("POST", url) if res.status_code > 400: results += get_version(url, res, "POST to root") except Exception: output.debug_exception() return results
def start(args, url): print(f"Scanning: {url}") # parse the URL, we'll need this parsed = urlparse(url) # get rid of any port number & credentials that may exist domain = utils.get_domain(parsed.netloc) # make sure it resolves try: socket.gethostbyname(domain) except socket.gaierror as error: print(f"Fatal Error: Unable to resolve {domain} ({str(error)})") return if parsed.scheme == "http": try: # check for TLS redirect tls_redirect = network.check_ssl_redirect(url) if tls_redirect != url: print(f"Server redirects to TLS: Scanning: {tls_redirect}") url = tls_redirect parsed = urlparse(url) except Exception as error: output.debug_exception() output.error(f"Failed to connect to {url} ({str(error)})") return www_redirect = network.check_www_redirect(url) if www_redirect is not None and www_redirect != url: print(f"Server performs WWW redirect: Scanning: {www_redirect}") url = www_redirect parsed = urlparse(url) # check to see if we are looking at an HTTPS server if parsed.scheme == "https": if args.internalssl or utils.is_ip(domain) or utils.get_port(url) != 443: # use internal scanner ssl_internal.scan(args, url, domain) else: try: ssl_labs.scan(args, url, domain) except Exception as error: output.debug_exception() output.error(f"Error running scan with SSL Labs: {str(error)}") if args.tdessessioncount: ssl_sweet32.scan(args, url, domain)
def _get_data() -> None: global _data data_url = "https://raw.githubusercontent.com/augustd/burp-suite-error-message-checks/master/src/main/resources/burp/match-rules.tab" try: raw = network.http_get(data_url).text for line in raw.splitlines(): _data.append(_MatchRule(line)) except Exception as error: output.debug(f"Failed to get version data: {error}") output.debug_exception()
def _get_caa_records(domain: str, resv: Resolver) -> List[str]: records: List[str] = [] try: answers = resv.query(domain, "CAA", lifetime=3) for data in answers: records.append(data.to_text()) except (resolver.NoAnswer, resolver.NXDOMAIN, exception.Timeout): pass except (resolver.NoNameservers, resolver.NotAbsolute, resolver.NoRootSOA): output.debug_exception() return records
def get_mx(domain): records = [] try: answers = resolver.query(domain, "MX") for data in answers: records.append([str(data.exchange), str(data.preference)]) except (resolver.NoAnswer, resolver.NXDOMAIN, exception.Timeout): pass except (resolver.NoNameservers, resolver.NotAbsolute, resolver.NoRootSOA): output.debug_exception() return records
def _check_connection(url: str) -> str: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) result = "Connection Failed" try: headers = {"User-Agent": SERVICE_UA} res = requests.get(url, headers=headers, verify=False) result = res.text.strip() except Exception: output.debug_exception() return result
def get_ns(domain): records = [] try: answers = resolver.query(domain, "NS") for data in answers: records.append(str(data)) except (resolver.NoAnswer, resolver.NXDOMAIN, exception.Timeout): pass except (resolver.NoNameservers, resolver.NotAbsolute, resolver.NoRootSOA): output.debug_exception() return records
def _get_cname(domain: str, resv: Resolver) -> Union[str, None]: name = None try: answers = resv.query(domain, "CNAME", lifetime=3) for data in answers: name = str(data.target) except (resolver.NoAnswer, resolver.NXDOMAIN, exception.Timeout): pass except (resolver.NoNameservers, resolver.NotAbsolute, resolver.NoRootSOA): output.debug_exception() return name
def check_response(url: str, res: Response, body: Union[str, None] = None) -> List[Result]: global _data, _reports results = [] try: # make sure we actually have something if res is None: return [] if _data is None or len(_data) == 0: _get_data() if body is None: body = res.text for rule in _data: rule = cast(_MatchRule, rule) mtch = re.search(rule.pattern, body) if mtch: val = mtch.group(int(rule.match_group)) err_start = body.find(val) # get the error, plus 25 characters on each side err = body[err_start - 25:err_start + len(val) + 25] msg = (f"Found error message (confidence: {rule.confidence}) " f"on {url} ({res.request.method}): ...{err}...") if msg not in _reports: results.append( Result.from_evidence( Evidence.from_response(res), msg, Vulnerabilities.HTTP_ERROR_MESSAGE, )) _reports.append(msg) break else: output.debug(f"Ignored duplicate error message: {msg}") except Exception: output.debug_exception() return results
def _get_info(self) -> str: # prime the call to cpu_percent, as the first call doesn't return useful data self.process.cpu_percent() # force a collection; not ideal, but seems to help gc.collect(2) # use oneshot() to cache the data, so we minimize hits with self.process.oneshot(): pct = self.process.cpu_percent() times = self.process.cpu_times() mem = self.process.memory_info() mem_res = "{0:cM}".format(Size(mem.rss)) mem_virt = "{0:cM}".format(Size(mem.vms)) if mem.rss > self.peak_mem_res: self.peak_mem_res = mem.rss output.debug(f"New high-memory threshold: {self.peak_mem_res}") thr = self.process.num_threads() vm = psutil.virtual_memory() mem_total = "{0:cM}".format(Size(vm.total)) mem_avail_bytes = vm.available mem_avail = "{0:cM}".format(Size(vm.available)) if mem_avail_bytes < self.WARNING_THRESHOLD and not self.low_mem_warning: self.low_mem_warning = True output.error(f"Low RAM Available: {mem_avail}") cons = -1 try: cons = len(self.process.connections(kind="inet")) except Exception: # we don't care if this fails output.debug_exception() cpu_freq = psutil.cpu_freq() info = (f"Process Stats: CPU: {pct}% - Sys: {times.system} - " f"User: {times.user} - Res: {mem_res} - Virt: {mem_virt} - " f"Available: {mem_avail}/{mem_total} - Threads: {thr} - " f"Connections: {cons} - CPU Freq: " f"{int(cpu_freq.current)}MHz/{int(cpu_freq.max)}MHz - " f"GC Objects: {len(gc.get_objects())}") return info
def get_info_message() -> List[str]: path = "/api/v3/info" messages: List[str] = [] try: body, code = network.http_json(API_SERVER + path) if len(body["messages"]) > 0: for msg in body["messages"]: messages.append(msg) except Exception: output.debug_exception() raise return messages
def _get_version_data() -> None: global _versions data: Union[Dict[str, Dict[str, Dict[str, str]]], None] = None data_url = "https://raw.githubusercontent.com/adamcaudill/current_versions/master/current_versions.json" try: data, _ = network.http_json(data_url) except Exception as error: output.debug(f"Failed to get version data: {error}") output.debug_exception() if data is not None and "software" in data: _versions = data["software"] else: _versions = None