def __init__( self, msg: str, vuln: Vulnerabilities, url: str, evidence: Union[str, List[str], Dict[str, Any], None] = None, ): self.message = msg self.vulnerability = vuln self.url = url if evidence is not None: if type(evidence) is dict or type(evidence) is Evidence: self.evidence = evidence elif type(evidence) is str: # if the evidence is a string, lets tack on the message as an extra element self.evidence = {"e": str(evidence), "message": msg} else: self.evidence = {"e": evidence} else: # fall back to the message if we don't have evidence - better than nothing self.evidence = {"message": msg} self.id = uuid.uuid4().hex output.debug( f"Result Created: {self.id} - {self.vulnerability.name} - {self.url}" )
def http_custom( verb: str, url: str, additional_headers: Union[None, Dict] = None, timeout: Optional[int] = 30, ) -> Response: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) global _requester headers = {"User-Agent": YAWAST_UA} if additional_headers is not None: headers = {**headers, **additional_headers} res = _requester.request(verb, url, headers=headers, verify=False, timeout=timeout) output.debug( f"{res.request.method}: {url} - completed ({res.status_code}) in " f"{int(res.elapsed.total_seconds() * 1000)}ms.") return res
def _check_url(urls: List[str], queue, follow_redirections, recursive) -> None: files: List[str] = [] results: List[Result] = [] for url in urls: try: # get the HEAD first, we only really care about actual files res = network.http_head(url, False) if res.status_code < 300: # run a scan on the full result, so we can ensure that we get any issues results += response_scanner.check_response( url, network.http_get(url, False)) files.append(url) if recursive: fl, re = find_directories(url, follow_redirections, recursive) files.extend(fl) results.extend(re) elif res.status_code < 400 and follow_redirections: if "Location" in res.headers: _check_url([res.headers["Location"]], queue, follow_redirections, recursive) except Exception as error: output.debug(f"Error checking URL ({url}): {str(error)}") queue.put((files, results))
def check_asp_net_debug(url: str) -> List[Result]: results: List[Result] = [] res = network.http_custom( "DEBUG", url, additional_headers={"Command": "stop-debug", "Accept": "*/*"} ) if res.status_code == 200 and "OK" in res.text: # we've got a hit, but could be a false positive # try this again, with a different verb xres = network.http_custom( "XDEBUG", url, additional_headers={"Command": "stop-debug", "Accept": "*/*"} ) # if we get a 200 when using an invalid verb, it's a false positive # if we get something else, then the DEBUG actually did something if xres.status_code != 200: results.append( Result( "ASP.NET Debugging Enabled", Vulnerabilities.SERVER_ASPNET_DEBUG_ENABLED, url, [ network.http_build_raw_request(res.request), network.http_build_raw_response(res), ], ) ) else: output.debug("Server responds to invalid HTTP verbs with status 200") results += response_scanner.check_response(url, res) return results
def get_latest_version( package: str, base_version: Union[str, version.Version]) -> Union[version.Version, None]: global _versions if _versions is not None: # make sure that we have data loaded if len(_versions) == 0: _get_version_data() if package in _versions: # check the type of base_version, and parse as needed if type(base_version) is version.Version: base_version = ".".join(str(base_version).split(".")[0:2]) if base_version in _versions[package]: return version.parse(_versions[package][base_version]) else: return version.parse(_versions[package]["latest"]) else: return None else: # if it's none, that means that we've attempted to get the version data, and it failed output.debug( f"_versions is None; skipping version check for {package}:{base_version}" ) return None
def _shutdown(): global _start_time, _monitor, _has_shutdown if _has_shutdown: return _has_shutdown = True output.debug("Shutting down...") elapsed = datetime.now() - _start_time mem_res = "{0:cM}".format(Size(_monitor.peak_mem_res)) reporter.register_info("peak_memory", _monitor.peak_mem_res) output.empty() if _monitor.peak_mem_res > 0: output.norm( f"Completed (Elapsed: {str(elapsed)} - Peak Memory: {mem_res})") else: # if we don't have memory info - likely not running in a terminal, don't print junk output.norm(f"Completed (Elapsed: {str(elapsed)})") if reporter.get_output_file() != "": with Spinner() as spinner: reporter.save_output(spinner)
def start_scan(domain: str) -> Tuple[str, Dict[str, Any]]: resp = _analyze(domain, True) status = resp["status"] output.debug(f"Started SSL Labs scan: {resp}") return status, resp
def spider(url) -> Tuple[List[str], List[Result]]: global _links, _insecure, _tasks, _lock results: List[Result] = [] # create processing pool pool = Pool() mgr = Manager() queue = mgr.Queue() asy = pool.apply_async(_get_links, (url, [url], queue, pool)) with _lock: _tasks.append(asy) while True: if all(t is None or t.ready() for t in _tasks): break else: count_none = 0 count_ready = 0 count_not_ready = 0 for t in _tasks: if t is None: count_none += 1 elif t.ready(): count_ready += 1 else: count_not_ready += 1 output.debug( f"Spider Task Status: None: {count_none}, Ready: {count_ready}, Not Ready: {count_not_ready}" ) time.sleep(3) pool.close() for t in _tasks: try: t.get() except Exception: output.debug_exception() while not queue.empty(): res = queue.get() if len(res) > 0: for re in res: if re not in results: results.append(re) # copy data and reset links = _links[:] _links = [] _insecure = [] _tasks = [] return links, results
def http_put( url: str, data: str, allow_redirects=True, additional_headers: Union[None, Dict] = None, timeout: Optional[int] = 30, ) -> Response: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) headers = {"User-Agent": YAWAST_UA} if additional_headers is not None: headers = {**headers, **additional_headers} res = _requester.put( url, data=data, headers=headers, allow_redirects=allow_redirects, timeout=timeout, ) output.debug( f"{res.request.method}: {url} - completed ({res.status_code}) in " f"{int(res.elapsed.total_seconds() * 1000)}ms " f"(Body: {len(res.content)})" ) return res
def _get_mem(self): mem = self.process.memory_info() if mem.rss > self.peak_mem_res: self.peak_mem_res = mem.rss output.debug(f"New high-memory threshold: {self.peak_mem_res}") return mem
def check_scan(domain: str) -> Tuple[str, Dict[str, Any]]: resp = _analyze(domain) status = resp["status"] if status != "READY": output.debug(f"SSL Labs status: {resp}") return status, resp
def main(): global _start_time, _monitor signal.signal(signal.SIGINT, signal_handler) warnings.simplefilter("ignore") try: if str(sys.stdout.encoding).lower() != "utf-8": print( f"Output encoding is {sys.stdout.encoding}: changing to UTF-8") sys.stdout.reconfigure(encoding="utf-8") except Exception as error: print(f"Unable to set UTF-8 encoding: {str(error)}") parser = command_line.build_parser() args, urls = parser.parse_known_args() # setup the output system output.setup(args.debug, args.nocolors, args.nowrap) output.debug("Starting application...") proxy = args.proxy if "proxy" in args else None cookie = args.cookie if "cookie" in args else None header = args.header if "header" in args else None network.init(proxy, cookie, header) # if we made it this far, it means that the parsing worked. # version doesn't require any URLs, so it gets special handing if args.command != "version": urls = command_line.process_urls(urls) else: urls = [] # we are good to keep going print_header() if args.output is not None: reporter.init(args.output) _set_basic_info() print(f"Saving output to '{reporter.get_output_file()}'") print() try: with _KeyMonitor(): with _ProcessMonitor() as pm: _monitor = pm args.func(args, urls) except KeyboardInterrupt: output.empty() output.error("Scan cancelled by user.") finally: _shutdown()
def http_options(url: str, timeout: Optional[int] = 30) -> Response: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) headers = {"User-Agent": YAWAST_UA} res = _requester.options(url, headers=headers, timeout=timeout) output.debug( f"{res.request.method}: {url} - completed ({res.status_code}) in " f"{int(res.elapsed.total_seconds() * 1000)}ms." ) return res
def http_get( url: str, allow_redirects: Optional[bool] = True, additional_headers: Union[None, Dict] = None, timeout: Optional[int] = 30, ) -> Response: max_size = 5 * 1024 * 1024 # 5MB chunk_size = 10 * 1024 # 10KB - this is the default used by requests urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) headers = {"User-Agent": YAWAST_UA} if additional_headers is not None: headers = {**headers, **additional_headers} res = _requester.get( url, headers=headers, allow_redirects=allow_redirects, timeout=timeout, stream=True, ) # if we have a content-length use that first, as it'll be a faster check if ( "content-length" in res.headers and int(res.headers["content-length"]) > max_size ): raise ValueError(f"File '{url}' exceeds the maximum size of {max_size} bytes.") length = 0 content = bytes() for chunk in res.iter_content(chunk_size): length += len(chunk) content += chunk if length > max_size: raise ValueError( f"File '{url}' exceeds the maximum size of {max_size} bytes." ) # hack: set the Response's content directly, as it doesn't keep it in memory if you stream the data res._content = content output.debug( f"{res.request.method}: {url} - completed ({res.status_code}) in " f"{int(res.elapsed.total_seconds() * 1000)}ms " f"(Body: {len(res.content)})" ) return res
def _get_data() -> None: global _data data_url = "https://raw.githubusercontent.com/augustd/burp-suite-error-message-checks/master/src/main/resources/burp/match-rules.tab" try: raw = network.http_get(data_url).text for line in raw.splitlines(): _data.append(_MatchRule(line)) except Exception as error: output.debug(f"Failed to get version data: {error}") output.debug_exception()
def _get_info(self) -> str: # prime the call to cpu_percent, as the first call doesn't return useful data self.process.cpu_percent() # force a collection; not ideal, but seems to help gc.collect(2) # use oneshot() to cache the data, so we minimize hits with self.process.oneshot(): pct = self.process.cpu_percent() times = self.process.cpu_times() mem = self.process.memory_info() mem_res = "{0:cM}".format(Size(mem.rss)) mem_virt = "{0:cM}".format(Size(mem.vms)) if mem.rss > self.peak_mem_res: self.peak_mem_res = mem.rss output.debug(f"New high-memory threshold: {self.peak_mem_res}") thr = self.process.num_threads() vm = psutil.virtual_memory() mem_total = "{0:cM}".format(Size(vm.total)) mem_avail_bytes = vm.available mem_avail = "{0:cM}".format(Size(vm.available)) if mem_avail_bytes < self.WARNING_THRESHOLD and not self.low_mem_warning: self.low_mem_warning = True output.error(f"Low RAM Available: {mem_avail}") cons = -1 try: cons = len(self.process.connections(kind="inet")) except Exception: # we don't care if this fails output.debug_exception() cpu_freq = psutil.cpu_freq() info = (f"Process Stats: CPU: {pct}% - Sys: {times.system} - " f"User: {times.user} - Res: {mem_res} - Virt: {mem_virt} - " f"Available: {mem_avail}/{mem_total} - Threads: {thr} - " f"Connections: {cons} - CPU Freq: " f"{int(cpu_freq.current)}MHz/{int(cpu_freq.max)}MHz - " f"GC Objects: {len(gc.get_objects())}") return info
def check_response(url: str, res: Response, body: Union[str, None] = None) -> List[Result]: global _data, _reports results = [] try: # make sure we actually have something if res is None: return [] if _data is None or len(_data) == 0: _get_data() if body is None: body = res.text for rule in _data: rule = cast(_MatchRule, rule) mtch = re.search(rule.pattern, body) if mtch: val = mtch.group(int(rule.match_group)) err_start = body.find(val) # get the error, plus 25 characters on each side err = body[err_start - 25:err_start + len(val) + 25] msg = (f"Found error message (confidence: {rule.confidence}) " f"on {url} ({res.request.method}): ...{err}...") if msg not in _reports: results.append( Result.from_evidence( Evidence.from_response(res), msg, Vulnerabilities.HTTP_ERROR_MESSAGE, )) _reports.append(msg) break else: output.debug(f"Ignored duplicate error message: {msg}") except Exception: output.debug_exception() return results
def _get_version_data() -> None: global _versions data: Union[Dict[str, Dict[str, Dict[str, str]]], None] = None data_url = "https://raw.githubusercontent.com/adamcaudill/current_versions/master/current_versions.json" try: data, _ = network.http_json(data_url) except Exception as error: output.debug(f"Failed to get version data: {error}") output.debug_exception() if data is not None and "software" in data: _versions = data["software"] else: _versions = None
def monitor_task(self): if sys.stdout.isatty(): while self.busy: try: info = self._get_info() output.debug(info) time.sleep(5) except Exception: output.debug_exception() self.busy = False else: # if this isn't a TTY, no point in doing any of this self.busy = False
def register(issue: Issue) -> None: # make sure the Dict for _domain exists - this shouldn't normally be an issue, but is for unit tests if _domain not in _issues: _issues[_domain] = {} # add the evidence to the evidence list, and swap the value in the object for its hash. # the point of this is to minimize cases where we are holding the same (large) string # multiple times in memory. should reduce memory by up to 20% if _domain not in _evidence: _evidence[_domain] = {} if "request" in issue.evidence and issue.evidence["request"] is not None: req = str(issue.evidence["request"]).encode("utf-8") req_id = hashlib.blake2b(req, digest_size=16).hexdigest() if req_id not in _evidence[_domain]: _evidence[_domain][req_id] = issue.evidence["request"] issue.evidence["request"] = req_id if "response" in issue.evidence and issue.evidence["response"] is not None: res = str(issue.evidence["response"]).encode("utf-8") res_id = hashlib.blake2b(res, digest_size=16).hexdigest() if res_id not in _evidence[_domain]: _evidence[_domain][res_id] = issue.evidence["response"] issue.evidence["response"] = res_id # if we haven't handled this issue yet, create a List for it if not is_registered(issue.vulnerability): _issues[_domain][issue.vulnerability] = [] # we need to check to see if we already have this issue, for this URL, so we don't create dups # TODO: This isn't exactly efficient - refactor findings = _issues[_domain][issue.vulnerability] findings = cast(List[Issue], findings) for finding in findings: if finding.url == issue.url and finding.evidence == issue.evidence: # just bail out output.debug( f"Duplicate Issue: {issue.id} (duplicate of {finding.id})") return _issues[_domain][issue.vulnerability].append(issue)
def _get_data() -> None: global _data data: Union[Dict[Any, Any], None] = None data_url = "https://raw.githubusercontent.com/RetireJS/retire.js/master/repository/jsrepository.json" try: raw = network.http_get(data_url).content raw_js = raw.decode("utf-8").replace("§§version§§", "[0-9][0-9.a-z_\\\\-]+") data = json.loads(raw_js) except Exception as error: output.debug(f"Failed to get version data: {error}") output.debug_exception() _data = data
def _shutdown(): global _start_time, _monitor, _has_shutdown if _has_shutdown: return _has_shutdown = True output.debug("Shutting down...") elapsed = datetime.now() - _start_time mem_res = "{0:cM}".format(Size(_monitor.peak_mem_res)) output.empty() output.norm( f"Completed (Elapsed: {str(elapsed)} - Peak Memory: {mem_res})") if reporter.get_output_file() != "": with Spinner(): reporter.save_output()
def _convert_keys(dct: Dict) -> Dict: ret = {} for k, v in dct.items(): if type(k) is Vulnerabilities: k = k.name if type(v) is dict: v = _convert_keys(v) try: _ = json.dumps(v) except Exception as error: output.debug(f"Error serializing data: {str(error)}") # convert to string - this may be wrong, but at least it won't fail v = str(v) ret[k] = v return ret
def wait_task(self): if sys.stdout.isatty(): while self.busy: try: with utils.INPUT_LOCK: key = getchar() if key != "": output.debug(f"Received from keyboard: {key}") if key == "d": output.toggle_debug() time.sleep(0.1) except Exception: output.debug_exception() self.busy = False else: # if this isn't a TTY, no point in doing any of this self.busy = False
def http_head(url: str, allow_redirects: Optional[bool] = True, timeout: Optional[int] = 30) -> Response: global _requester urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) headers = {"User-Agent": YAWAST_UA} res = _requester.head( url, headers=headers, verify=False, allow_redirects=allow_redirects, timeout=timeout, ) output.debug( f"{res.request.method}: {url} - completed ({res.status_code}) in " f"{int(res.elapsed.total_seconds() * 1000)}ms.") return res
def is_printable_str(b: bytes) -> bool: decoders = ["utf_8", "latin_1", "cp1251"] printable = False good_decoder = None with ExecutionTimer() as timer: for decoder in decoders: s = b.decode(decoder, "backslashreplace") if not any( repr(ch).startswith("'\\x") or repr(ch).startswith("'\\u") for ch in s): printable = True good_decoder = decoder break if good_decoder is not None: output.debug(f"Decoded string as {good_decoder} in {timer.to_ms()}ms") return printable
def find_srv_records(domain, path=None): records = [] res = resolver.Resolver() res.nameservers.insert(0, "8.8.8.8") res.nameservers.insert(0, "1.1.1.1") res.search = [] # read the data in from the data directory if path is None: file_path = pkg_resources.resource_filename("yawast", "resources/srv.txt") else: file_path = path with open(file_path) as file: for line in file: host = line.strip() + "." + domain + "." try: answers = res.query(host, "SRV", lifetime=3, raise_on_no_answer=False) for data in answers: target = data.target.to_text() port = str(data.port) records.append([host, target, port]) except (resolver.NoAnswer, resolver.NXDOMAIN, exception.Timeout) as error: output.debug(f"SRV: {host} received error: {str(error)}") except (resolver.NoNameservers, resolver.NotAbsolute, resolver.NoRootSOA): output.debug_exception() pass return records
def main(): global _start_time, _monitor signal.signal(signal.SIGINT, signal_handler) parser = command_line.build_parser() args, urls = parser.parse_known_args() # setup the output system output.setup(args.debug, args.nocolors) output.debug("Starting application...") network.init(args.proxy, args.cookie) # if we made it this far, it means that the parsing worked. urls = command_line.process_urls(urls) # we are good to keep going print_header() if args.output is not None: reporter.init(args.output) _set_basic_info() print(f"Saving output to '{reporter.get_output_file()}'") print() try: with _KeyMonitor(): with _ProcessMonitor() as pm: _monitor = pm args.func(args, urls) except KeyboardInterrupt: output.empty() output.error("Scan cancelled by user.") finally: _shutdown()
def monitor_task(self): if sys.stdout.isatty(): while self.busy: try: # only print the data out every 10 seconds if datetime.now().second / 10 == 0: info = self._get_info() output.debug(info) else: # call get_mem so that we record peak more accurately self._get_mem() time.sleep(1) except Exception: output.debug_exception() self.busy = False pass else: # if this isn't a TTY, no point in doing any of this self.busy = False
def register(issue: Issue) -> None: global _issues, _domain # make sure the Dict for _domain exists - this shouldn't normally be an issue, but is for unit tests if _domain not in _issues: _issues[_domain] = {} # if we haven't handled this issue yet, create a List for it if not is_registered(issue.vulnerability): _issues[_domain][issue.vulnerability] = [] # we need to check to see if we already have this issue, for this URL, so we don't create dups # TODO: This isn't exactly efficient - refactor findings = _issues[_domain][issue.vulnerability] findings = cast(List[Issue], findings) for finding in findings: if finding.url == issue.url and finding.evidence == issue.evidence: # just bail out output.debug( f"Duplicate Issue: {issue.id} (duplicate of {finding.id})") return _issues[_domain][issue.vulnerability].append(issue)