def __init__(
self,
msg: str,
vuln: Vulnerabilities,
url: str,
evidence: Union[str, List[str], Dict[str, Any], None] = None,
):
self.message = msg
self.vulnerability = vuln
self.url = url
if evidence is not None:
if type(evidence) is dict or type(evidence) is Evidence:
self.evidence = evidence
elif type(evidence) is str:
# if the evidence is a string, lets tack on the message as an extra element
self.evidence = {"e": str(evidence), "message": msg}
else:
self.evidence = {"e": evidence}
else:
# fall back to the message if we don't have evidence - better than nothing
self.evidence = {"message": msg}
self.id = uuid.uuid4().hex
output.debug(
f"Result Created: {self.id} - {self.vulnerability.name} - {self.url}"
)
def http_custom(
verb: str,
url: str,
additional_headers: Union[None, Dict] = None,
timeout: Optional[int] = 30,
) -> Response:
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
global _requester
headers = {"User-Agent": YAWAST_UA}
if additional_headers is not None:
headers = {**headers, **additional_headers}
res = _requester.request(verb,
url,
headers=headers,
verify=False,
timeout=timeout)
output.debug(
f"{res.request.method}: {url} - completed ({res.status_code}) in "
f"{int(res.elapsed.total_seconds() * 1000)}ms.")
return res
def _check_url(urls: List[str], queue, follow_redirections, recursive) -> None:
files: List[str] = []
results: List[Result] = []
for url in urls:
try:
# get the HEAD first, we only really care about actual files
res = network.http_head(url, False)
if res.status_code < 300:
# run a scan on the full result, so we can ensure that we get any issues
results += response_scanner.check_response(
url, network.http_get(url, False))
files.append(url)
if recursive:
fl, re = find_directories(url, follow_redirections,
recursive)
files.extend(fl)
results.extend(re)
elif res.status_code < 400 and follow_redirections:
if "Location" in res.headers:
_check_url([res.headers["Location"]], queue,
follow_redirections, recursive)
except Exception as error:
output.debug(f"Error checking URL ({url}): {str(error)}")
queue.put((files, results))
def check_asp_net_debug(url: str) -> List[Result]:
results: List[Result] = []
res = network.http_custom(
"DEBUG", url, additional_headers={"Command": "stop-debug", "Accept": "*/*"}
)
if res.status_code == 200 and "OK" in res.text:
# we've got a hit, but could be a false positive
# try this again, with a different verb
xres = network.http_custom(
"XDEBUG", url, additional_headers={"Command": "stop-debug", "Accept": "*/*"}
)
# if we get a 200 when using an invalid verb, it's a false positive
# if we get something else, then the DEBUG actually did something
if xres.status_code != 200:
results.append(
Result(
"ASP.NET Debugging Enabled",
Vulnerabilities.SERVER_ASPNET_DEBUG_ENABLED,
url,
[
network.http_build_raw_request(res.request),
network.http_build_raw_response(res),
],
)
)
else:
output.debug("Server responds to invalid HTTP verbs with status 200")
results += response_scanner.check_response(url, res)
return results
def get_latest_version(
package: str,
base_version: Union[str,
version.Version]) -> Union[version.Version, None]:
global _versions
if _versions is not None:
# make sure that we have data loaded
if len(_versions) == 0:
_get_version_data()
if package in _versions:
# check the type of base_version, and parse as needed
if type(base_version) is version.Version:
base_version = ".".join(str(base_version).split(".")[0:2])
if base_version in _versions[package]:
return version.parse(_versions[package][base_version])
else:
return version.parse(_versions[package]["latest"])
else:
return None
else:
# if it's none, that means that we've attempted to get the version data, and it failed
output.debug(
f"_versions is None; skipping version check for {package}:{base_version}"
)
return None
def _shutdown():
global _start_time, _monitor, _has_shutdown
if _has_shutdown:
return
_has_shutdown = True
output.debug("Shutting down...")
elapsed = datetime.now() - _start_time
mem_res = "{0:cM}".format(Size(_monitor.peak_mem_res))
reporter.register_info("peak_memory", _monitor.peak_mem_res)
output.empty()
if _monitor.peak_mem_res > 0:
output.norm(
f"Completed (Elapsed: {str(elapsed)} - Peak Memory: {mem_res})")
else:
# if we don't have memory info - likely not running in a terminal, don't print junk
output.norm(f"Completed (Elapsed: {str(elapsed)})")
if reporter.get_output_file() != "":
with Spinner() as spinner:
reporter.save_output(spinner)
def start_scan(domain: str) -> Tuple[str, Dict[str, Any]]:
resp = _analyze(domain, True)
status = resp["status"]
output.debug(f"Started SSL Labs scan: {resp}")
return status, resp
def spider(url) -> Tuple[List[str], List[Result]]:
global _links, _insecure, _tasks, _lock
results: List[Result] = []
# create processing pool
pool = Pool()
mgr = Manager()
queue = mgr.Queue()
asy = pool.apply_async(_get_links, (url, [url], queue, pool))
with _lock:
_tasks.append(asy)
while True:
if all(t is None or t.ready() for t in _tasks):
break
else:
count_none = 0
count_ready = 0
count_not_ready = 0
for t in _tasks:
if t is None:
count_none += 1
elif t.ready():
count_ready += 1
else:
count_not_ready += 1
output.debug(
f"Spider Task Status: None: {count_none}, Ready: {count_ready}, Not Ready: {count_not_ready}"
)
time.sleep(3)
pool.close()
for t in _tasks:
try:
t.get()
except Exception:
output.debug_exception()
while not queue.empty():
res = queue.get()
if len(res) > 0:
for re in res:
if re not in results:
results.append(re)
# copy data and reset
links = _links[:]
_links = []
_insecure = []
_tasks = []
return links, results
def http_put(
url: str,
data: str,
allow_redirects=True,
additional_headers: Union[None, Dict] = None,
timeout: Optional[int] = 30,
) -> Response:
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
headers = {"User-Agent": YAWAST_UA}
if additional_headers is not None:
headers = {**headers, **additional_headers}
res = _requester.put(
url,
data=data,
headers=headers,
allow_redirects=allow_redirects,
timeout=timeout,
)
output.debug(
f"{res.request.method}: {url} - completed ({res.status_code}) in "
f"{int(res.elapsed.total_seconds() * 1000)}ms "
f"(Body: {len(res.content)})"
)
return res
def _get_mem(self):
mem = self.process.memory_info()
if mem.rss > self.peak_mem_res:
self.peak_mem_res = mem.rss
output.debug(f"New high-memory threshold: {self.peak_mem_res}")
return mem
def check_scan(domain: str) -> Tuple[str, Dict[str, Any]]:
resp = _analyze(domain)
status = resp["status"]
if status != "READY":
output.debug(f"SSL Labs status: {resp}")
return status, resp
def main():
global _start_time, _monitor
signal.signal(signal.SIGINT, signal_handler)
warnings.simplefilter("ignore")
try:
if str(sys.stdout.encoding).lower() != "utf-8":
print(
f"Output encoding is {sys.stdout.encoding}: changing to UTF-8")
sys.stdout.reconfigure(encoding="utf-8")
except Exception as error:
print(f"Unable to set UTF-8 encoding: {str(error)}")
parser = command_line.build_parser()
args, urls = parser.parse_known_args()
# setup the output system
output.setup(args.debug, args.nocolors, args.nowrap)
output.debug("Starting application...")
proxy = args.proxy if "proxy" in args else None
cookie = args.cookie if "cookie" in args else None
header = args.header if "header" in args else None
network.init(proxy, cookie, header)
# if we made it this far, it means that the parsing worked.
# version doesn't require any URLs, so it gets special handing
if args.command != "version":
urls = command_line.process_urls(urls)
else:
urls = []
# we are good to keep going
print_header()
if args.output is not None:
reporter.init(args.output)
_set_basic_info()
print(f"Saving output to '{reporter.get_output_file()}'")
print()
try:
with _KeyMonitor():
with _ProcessMonitor() as pm:
_monitor = pm
args.func(args, urls)
except KeyboardInterrupt:
output.empty()
output.error("Scan cancelled by user.")
finally:
_shutdown()
def http_options(url: str, timeout: Optional[int] = 30) -> Response:
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
headers = {"User-Agent": YAWAST_UA}
res = _requester.options(url, headers=headers, timeout=timeout)
output.debug(
f"{res.request.method}: {url} - completed ({res.status_code}) in "
f"{int(res.elapsed.total_seconds() * 1000)}ms."
)
return res
def http_get(
url: str,
allow_redirects: Optional[bool] = True,
additional_headers: Union[None, Dict] = None,
timeout: Optional[int] = 30,
) -> Response:
max_size = 5 * 1024 * 1024 # 5MB
chunk_size = 10 * 1024 # 10KB - this is the default used by requests
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
headers = {"User-Agent": YAWAST_UA}
if additional_headers is not None:
headers = {**headers, **additional_headers}
res = _requester.get(
url,
headers=headers,
allow_redirects=allow_redirects,
timeout=timeout,
stream=True,
)
# if we have a content-length use that first, as it'll be a faster check
if (
"content-length" in res.headers
and int(res.headers["content-length"]) > max_size
):
raise ValueError(f"File '{url}' exceeds the maximum size of {max_size} bytes.")
length = 0
content = bytes()
for chunk in res.iter_content(chunk_size):
length += len(chunk)
content += chunk
if length > max_size:
raise ValueError(
f"File '{url}' exceeds the maximum size of {max_size} bytes."
)
# hack: set the Response's content directly, as it doesn't keep it in memory if you stream the data
res._content = content
output.debug(
f"{res.request.method}: {url} - completed ({res.status_code}) in "
f"{int(res.elapsed.total_seconds() * 1000)}ms "
f"(Body: {len(res.content)})"
)
return res
def _get_data() -> None:
global _data
data_url = "https://raw.githubusercontent.com/augustd/burp-suite-error-message-checks/master/src/main/resources/burp/match-rules.tab"
try:
raw = network.http_get(data_url).text
for line in raw.splitlines():
_data.append(_MatchRule(line))
except Exception as error:
output.debug(f"Failed to get version data: {error}")
output.debug_exception()
def _get_info(self) -> str:
# prime the call to cpu_percent, as the first call doesn't return useful data
self.process.cpu_percent()
# force a collection; not ideal, but seems to help
gc.collect(2)
# use oneshot() to cache the data, so we minimize hits
with self.process.oneshot():
pct = self.process.cpu_percent()
times = self.process.cpu_times()
mem = self.process.memory_info()
mem_res = "{0:cM}".format(Size(mem.rss))
mem_virt = "{0:cM}".format(Size(mem.vms))
if mem.rss > self.peak_mem_res:
self.peak_mem_res = mem.rss
output.debug(f"New high-memory threshold: {self.peak_mem_res}")
thr = self.process.num_threads()
vm = psutil.virtual_memory()
mem_total = "{0:cM}".format(Size(vm.total))
mem_avail_bytes = vm.available
mem_avail = "{0:cM}".format(Size(vm.available))
if mem_avail_bytes < self.WARNING_THRESHOLD and not self.low_mem_warning:
self.low_mem_warning = True
output.error(f"Low RAM Available: {mem_avail}")
cons = -1
try:
cons = len(self.process.connections(kind="inet"))
except Exception:
# we don't care if this fails
output.debug_exception()
cpu_freq = psutil.cpu_freq()
info = (f"Process Stats: CPU: {pct}% - Sys: {times.system} - "
f"User: {times.user} - Res: {mem_res} - Virt: {mem_virt} - "
f"Available: {mem_avail}/{mem_total} - Threads: {thr} - "
f"Connections: {cons} - CPU Freq: "
f"{int(cpu_freq.current)}MHz/{int(cpu_freq.max)}MHz - "
f"GC Objects: {len(gc.get_objects())}")
return info
def check_response(url: str,
res: Response,
body: Union[str, None] = None) -> List[Result]:
global _data, _reports
results = []
try:
# make sure we actually have something
if res is None:
return []
if _data is None or len(_data) == 0:
_get_data()
if body is None:
body = res.text
for rule in _data:
rule = cast(_MatchRule, rule)
mtch = re.search(rule.pattern, body)
if mtch:
val = mtch.group(int(rule.match_group))
err_start = body.find(val)
# get the error, plus 25 characters on each side
err = body[err_start - 25:err_start + len(val) + 25]
msg = (f"Found error message (confidence: {rule.confidence}) "
f"on {url} ({res.request.method}): ...{err}...")
if msg not in _reports:
results.append(
Result.from_evidence(
Evidence.from_response(res),
msg,
Vulnerabilities.HTTP_ERROR_MESSAGE,
))
_reports.append(msg)
break
else:
output.debug(f"Ignored duplicate error message: {msg}")
except Exception:
output.debug_exception()
return results
def _get_version_data() -> None:
global _versions
data: Union[Dict[str, Dict[str, Dict[str, str]]], None] = None
data_url = "https://raw.githubusercontent.com/adamcaudill/current_versions/master/current_versions.json"
try:
data, _ = network.http_json(data_url)
except Exception as error:
output.debug(f"Failed to get version data: {error}")
output.debug_exception()
if data is not None and "software" in data:
_versions = data["software"]
else:
_versions = None
def monitor_task(self):
if sys.stdout.isatty():
while self.busy:
try:
info = self._get_info()
output.debug(info)
time.sleep(5)
except Exception:
output.debug_exception()
self.busy = False
else:
# if this isn't a TTY, no point in doing any of this
self.busy = False
def register(issue: Issue) -> None:
# make sure the Dict for _domain exists - this shouldn't normally be an issue, but is for unit tests
if _domain not in _issues:
_issues[_domain] = {}
# add the evidence to the evidence list, and swap the value in the object for its hash.
# the point of this is to minimize cases where we are holding the same (large) string
# multiple times in memory. should reduce memory by up to 20%
if _domain not in _evidence:
_evidence[_domain] = {}
if "request" in issue.evidence and issue.evidence["request"] is not None:
req = str(issue.evidence["request"]).encode("utf-8")
req_id = hashlib.blake2b(req, digest_size=16).hexdigest()
if req_id not in _evidence[_domain]:
_evidence[_domain][req_id] = issue.evidence["request"]
issue.evidence["request"] = req_id
if "response" in issue.evidence and issue.evidence["response"] is not None:
res = str(issue.evidence["response"]).encode("utf-8")
res_id = hashlib.blake2b(res, digest_size=16).hexdigest()
if res_id not in _evidence[_domain]:
_evidence[_domain][res_id] = issue.evidence["response"]
issue.evidence["response"] = res_id
# if we haven't handled this issue yet, create a List for it
if not is_registered(issue.vulnerability):
_issues[_domain][issue.vulnerability] = []
# we need to check to see if we already have this issue, for this URL, so we don't create dups
# TODO: This isn't exactly efficient - refactor
findings = _issues[_domain][issue.vulnerability]
findings = cast(List[Issue], findings)
for finding in findings:
if finding.url == issue.url and finding.evidence == issue.evidence:
# just bail out
output.debug(
f"Duplicate Issue: {issue.id} (duplicate of {finding.id})")
return
_issues[_domain][issue.vulnerability].append(issue)
def _get_data() -> None:
global _data
data: Union[Dict[Any, Any], None] = None
data_url = "https://raw.githubusercontent.com/RetireJS/retire.js/master/repository/jsrepository.json"
try:
raw = network.http_get(data_url).content
raw_js = raw.decode("utf-8").replace("§§version§§", "[0-9][0-9.a-z_\\\\-]+")
data = json.loads(raw_js)
except Exception as error:
output.debug(f"Failed to get version data: {error}")
output.debug_exception()
_data = data
def _shutdown():
global _start_time, _monitor, _has_shutdown
if _has_shutdown:
return
_has_shutdown = True
output.debug("Shutting down...")
elapsed = datetime.now() - _start_time
mem_res = "{0:cM}".format(Size(_monitor.peak_mem_res))
output.empty()
output.norm(
f"Completed (Elapsed: {str(elapsed)} - Peak Memory: {mem_res})")
if reporter.get_output_file() != "":
with Spinner():
reporter.save_output()
def _convert_keys(dct: Dict) -> Dict:
ret = {}
for k, v in dct.items():
if type(k) is Vulnerabilities:
k = k.name
if type(v) is dict:
v = _convert_keys(v)
try:
_ = json.dumps(v)
except Exception as error:
output.debug(f"Error serializing data: {str(error)}")
# convert to string - this may be wrong, but at least it won't fail
v = str(v)
ret[k] = v
return ret
def wait_task(self):
if sys.stdout.isatty():
while self.busy:
try:
with utils.INPUT_LOCK:
key = getchar()
if key != "":
output.debug(f"Received from keyboard: {key}")
if key == "d":
output.toggle_debug()
time.sleep(0.1)
except Exception:
output.debug_exception()
self.busy = False
else:
# if this isn't a TTY, no point in doing any of this
self.busy = False
def http_head(url: str,
allow_redirects: Optional[bool] = True,
timeout: Optional[int] = 30) -> Response:
global _requester
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
headers = {"User-Agent": YAWAST_UA}
res = _requester.head(
url,
headers=headers,
verify=False,
allow_redirects=allow_redirects,
timeout=timeout,
)
output.debug(
f"{res.request.method}: {url} - completed ({res.status_code}) in "
f"{int(res.elapsed.total_seconds() * 1000)}ms.")
return res
def is_printable_str(b: bytes) -> bool:
decoders = ["utf_8", "latin_1", "cp1251"]
printable = False
good_decoder = None
with ExecutionTimer() as timer:
for decoder in decoders:
s = b.decode(decoder, "backslashreplace")
if not any(
repr(ch).startswith("'\\x") or repr(ch).startswith("'\\u")
for ch in s):
printable = True
good_decoder = decoder
break
if good_decoder is not None:
output.debug(f"Decoded string as {good_decoder} in {timer.to_ms()}ms")
return printable
def find_srv_records(domain, path=None):
records = []
res = resolver.Resolver()
res.nameservers.insert(0, "8.8.8.8")
res.nameservers.insert(0, "1.1.1.1")
res.search = []
# read the data in from the data directory
if path is None:
file_path = pkg_resources.resource_filename("yawast",
"resources/srv.txt")
else:
file_path = path
with open(file_path) as file:
for line in file:
host = line.strip() + "." + domain + "."
try:
answers = res.query(host,
"SRV",
lifetime=3,
raise_on_no_answer=False)
for data in answers:
target = data.target.to_text()
port = str(data.port)
records.append([host, target, port])
except (resolver.NoAnswer, resolver.NXDOMAIN,
exception.Timeout) as error:
output.debug(f"SRV: {host} received error: {str(error)}")
except (resolver.NoNameservers, resolver.NotAbsolute,
resolver.NoRootSOA):
output.debug_exception()
pass
return records
def main():
global _start_time, _monitor
signal.signal(signal.SIGINT, signal_handler)
parser = command_line.build_parser()
args, urls = parser.parse_known_args()
# setup the output system
output.setup(args.debug, args.nocolors)
output.debug("Starting application...")
network.init(args.proxy, args.cookie)
# if we made it this far, it means that the parsing worked.
urls = command_line.process_urls(urls)
# we are good to keep going
print_header()
if args.output is not None:
reporter.init(args.output)
_set_basic_info()
print(f"Saving output to '{reporter.get_output_file()}'")
print()
try:
with _KeyMonitor():
with _ProcessMonitor() as pm:
_monitor = pm
args.func(args, urls)
except KeyboardInterrupt:
output.empty()
output.error("Scan cancelled by user.")
finally:
_shutdown()
def monitor_task(self):
if sys.stdout.isatty():
while self.busy:
try:
# only print the data out every 10 seconds
if datetime.now().second / 10 == 0:
info = self._get_info()
output.debug(info)
else:
# call get_mem so that we record peak more accurately
self._get_mem()
time.sleep(1)
except Exception:
output.debug_exception()
self.busy = False
pass
else:
# if this isn't a TTY, no point in doing any of this
self.busy = False
def register(issue: Issue) -> None:
global _issues, _domain
# make sure the Dict for _domain exists - this shouldn't normally be an issue, but is for unit tests
if _domain not in _issues:
_issues[_domain] = {}
# if we haven't handled this issue yet, create a List for it
if not is_registered(issue.vulnerability):
_issues[_domain][issue.vulnerability] = []
# we need to check to see if we already have this issue, for this URL, so we don't create dups
# TODO: This isn't exactly efficient - refactor
findings = _issues[_domain][issue.vulnerability]
findings = cast(List[Issue], findings)
for finding in findings:
if finding.url == issue.url and finding.evidence == issue.evidence:
# just bail out
output.debug(
f"Duplicate Issue: {issue.id} (duplicate of {finding.id})")
return
_issues[_domain][issue.vulnerability].append(issue)
