def spider(url) -> Tuple[List[str], List[Result]]:
global _links, _insecure, _tasks, _lock
results: List[Result] = []
# create processing pool
pool = Pool()
mgr = Manager()
queue = mgr.Queue()
asy = pool.apply_async(_get_links, (url, [url], queue, pool))
with _lock:
_tasks.append(asy)
while True:
if all(t is None or t.ready() for t in _tasks):
break
else:
count_none = 0
count_ready = 0
count_not_ready = 0
for t in _tasks:
if t is None:
count_none += 1
elif t.ready():
count_ready += 1
else:
count_not_ready += 1
output.debug(
f"Spider Task Status: None: {count_none}, Ready: {count_ready}, Not Ready: {count_not_ready}"
)
time.sleep(3)
pool.close()
for t in _tasks:
try:
t.get()
except Exception:
output.debug_exception()
while not queue.empty():
res = queue.get()
if len(res) > 0:
for re in res:
if re not in results:
results.append(re)
# copy data and reset
links = _links[:]
_links = []
_insecure = []
_tasks = []
return links, results
def http_build_raw_response(res: Response) -> str:
if res.raw.version == 11:
res_line = f"HTTP/1.1 {res.raw.status} {res.raw.reason}"
else:
res_line = f"HTTP/1.0 {res.raw.status} {res.raw.reason}"
res_string = res_line + "\r\n"
if res.raw._original_response is not None:
res_string += "\r\n".join(
str(res.raw._original_response.headers).splitlines(False)
)
else:
res_string += "\r\n".join(f"{k}: {v}" for k, v in res.headers.items())
try:
if response_body_is_text(res):
txt = res.text
if txt != "":
res_string += "\r\n\r\n"
res_string += txt
elif len(res.content) > 0:
# the body is binary - no real value in keeping it
res_string += "\r\n\r\n<BINARY DATA EXCLUDED>"
except Exception:
output.debug_exception()
return res_string
def _get_version_info() -> str:
try:
data, code = network.http_json("https://pypi.org/pypi/yawast/json")
except Exception:
output.debug_exception()
return "Supported Version: (Unable to get version information)"
if code != 200:
ret = "Supported Version: (PyPi returned an error code while fetching current version)"
else:
if "info" in data and "version" in data["info"]:
ver = cast(version.Version, version.parse(get_version()))
curr_version = cast(version.Version,
version.parse(data["info"]["version"]))
ret = f"Supported Version: {curr_version} - "
if ver == curr_version:
ret += "You are on the latest version."
elif ver > curr_version or "dev" in get_version():
ret += "You are on a pre-release version. Take care."
else:
ret += "Please update to the current version."
else:
ret = "Supported Version: (PyPi returned invalid data while fetching current version)"
return ret
def check_cve_2019_5418(url: str) -> List[Result]:
global _checked
# this only applies to controllers, so skip the check unless the link ends with '/'
if not url.endswith("/") or url in _checked:
return []
results: List[Result] = []
_checked.append(url)
try:
res = network.http_get(
url, False, {"Accept": "../../../../../../../../../e*c/p*sswd{{"}
)
body = res.text
req = network.http_build_raw_request(res.request)
results += response_scanner.check_response(url, res)
pattern = r"root:[a-zA-Z0-9]+:0:0:.+$"
mtch = re.search(pattern, body)
if mtch:
results.append(
Result(
f"Rails CVE-2019-5418: File Content Disclosure: {url} - {mtch.group(0)}",
Vulnerabilities.SERVER_RAILS_CVE_2019_5418,
url,
[body, req],
)
)
except Exception:
output.debug_exception()
return results
def _analyze(domain: str, new=False) -> Dict[str, Any]:
new_path = "host={target}&publish=off&startNew=on&all=done&ignoreMismatch=on".format(
target=domain
)
status_path = "host={target}&publish=off&all=done&ignoreMismatch=on".format(
target=domain
)
if new:
path = new_path
else:
path = status_path
try:
body, code = network.http_json(API_SERVER + "/api/v3/analyze?" + path)
except Exception:
output.debug_exception()
raise
# check for error messages
if body.get("errors") is not None:
raise ValueError(
"SSL Labs returned the following error(s): {errors}".format(
errors=str(body["errors"])
)
)
# next up, check to see what error code we have
if code != 200:
# if we got anything but 200, it's a problem
raise ValueError("SSL Labs returned error code: {code}".format(code=code))
return body
def start(session: Session):
print(f"Scanning: {session.url}")
# make sure it resolves
try:
socket.gethostbyname(session.domain)
except socket.gaierror as error:
print(
f"Fatal Error: Unable to resolve {session.domain} ({str(error)})")
return
try:
cutils.check_redirect(session)
except ValueError as error:
print(f"Unable to continue: {str(error)}")
return
# check to see if we are looking at an HTTPS server
if session.url_parsed.scheme == "https":
if (session.args.internalssl or utils.is_ip(session.domain)
or utils.get_port(session.url) != 443):
# use internal scanner
ssl_internal.scan(session)
else:
try:
ssl_labs.scan(session)
except Exception as error:
output.debug_exception()
output.error(f"Error running scan with SSL Labs: {str(error)}")
if session.args.tdessessioncount:
ssl_sweet32.scan(session)
def _is_unsafe_link(href: str, description: str) -> bool:
"""
Check for strings that indicate an unsafe link
:param href:
:param description:
:return:
"""
unsafe_fragments = [
"logoff",
"log off",
"log_off",
"logout",
"log out",
"log_out",
"delete",
"destroy",
]
ret = False
try:
description = str(
description).lower() if description is not None else ""
href = str(href).lower()
for frag in unsafe_fragments:
if frag in href or frag in description:
return True
except Exception:
output.debug_exception()
return ret
def _check_charset(url: str, res: Response) -> List[Result]:
results: List[Result] = []
# if the body is empty, we really don't care about this
if len(res.content) == 0:
return results
try:
if "Content-Type" in res.headers:
content_type = str(res.headers["Content-Type"]).lower()
if "charset" not in content_type and "text/html" in content_type:
# not charset specified
results.append(
Result.from_evidence(
Evidence.from_response(res,
{"content-type": content_type}),
f"Charset Not Defined in '{res.headers['Content-Type']}' at {url}",
Vulnerabilities.HTTP_HEADER_CONTENT_TYPE_NO_CHARSET,
))
else:
# content-type missing
results.append(
Result.from_evidence(
Evidence.from_response(res),
f"Content-Type Missing: {url} ({res.request.method} - {res.status_code})",
Vulnerabilities.HTTP_HEADER_CONTENT_TYPE_MISSING,
))
except Exception:
output.debug_exception()
return results
def network_info(ip):
global _failure, _cache
# first, check the cache
if _cache.get(ip) is not None:
return _cache[ip]
# now, make sure we haven't turn this off due to errors
if _failure:
return "Network Information disabled due to prior failure"
try:
info, code = network.http_json("https://api.iptoasn.com/v1/as/ip/%s" % ip)
if code == 200:
ret = "%s - %s" % (info["as_country_code"], info["as_description"])
_cache[ip] = ret
return ret
else:
_failure = True
return "IP To ASN Service returned code: %s" % code
except (ValueError, KeyError) as error:
output.debug_exception()
_failure = True
return "IP To ASN Service error: %s" % str(error)
def http_build_raw_response(res: Response) -> str:
if res.raw.version == 11:
res_line = f"HTTP/1.1 {res.raw.status} {res.raw.reason}"
else:
res_line = f"HTTP/1.0 {res.raw.status} {res.raw.reason}"
res_string = res_line + "\r\n"
if res.raw._original_response is not None:
res_string += "\r\n".join(
str(res.raw._original_response.headers).splitlines(False))
else:
res_string += "\r\n".join(f"{k}: {v}" for k, v in res.headers.items())
try:
txt = res.text
if txt != "":
res_string += "\r\n\r\n"
res_string += txt
except Exception:
output.debug_exception()
return res_string
def _is_port_open(url: str, ip: str, rec, queue):
sock = socket.socket()
port = rec["port"]
# set a timeout - this has a huge speed impact
sock.settimeout(0.75)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
connected = False
try:
connected = sock.connect_ex((ip, port)) is 0
sock.close()
except Exception:
# this shouldn't happen, but just in case
output.debug_exception()
if connected:
queue.put(
Result(
f"Open Port: IP: {ip} - Port: {port} (Assigned Service: {rec['name']} - {rec['desc']})",
Vulnerabilities.NETWORK_OPEN_PORT,
url,
{
"ip": ip,
"port": port
},
))
def check_path_disclosure(wp_url: str) -> List[Result]:
# this is a list of files that are known to throw a fatal error when accessed directly
# this is from a manual review of all plugins with at least 1M installs
urls = [
"wp-content/plugins/hello.php",
"wp-content/plugins/akismet/akismet.php",
"wp-content/plugins/contact-form-7/includes/capabilities.php",
"wp-content/plugins/wordpress-seo/admin/views/partial-alerts-errors.php",
"wp-content/plugins/jetpack/load-jetpack.php",
"wp-content/plugins/jetpack/uninstall.php",
"wp-content/plugins/duplicate-post/duplicate-post-admin.php",
"wp-content/plugins/wpforms-lite/includes/admin/class-welcome.php",
"wp-content/plugins/wp-google-maps/base/includes/welcome.php",
"wp-content/plugins/wp-super-cache/wp-cache.php",
"wp-content/plugins/mailchimp-for-wp/integrations/wpforms/bootstrap.php",
"wp-content/plugins/mailchimp-for-wp/integrations/bootstrap.php",
"wp-content/plugins/regenerate-thumbnails/regenerate-thumbnails.php",
"wp-content/plugins/advanced-custom-fields/includes/deprecated.php",
"wp-content/plugins/redirection/redirection.php",
"wp-content/plugins/wpforms-lite/includes/admin/importers/class-ninja-forms.php",
"wp-content/plugins/ninja-forms/includes/deprecated.php",
"wp-content/plugins/so-widgets-bundle/so-widgets-bundle.php",
"wp-content/plugins/wp-fastest-cache/templates/preload.php",
"wp-content/plugins/duplicate-page/duplicatepage.php",
"wp-content/plugins/better-wp-security/better-wp-security.php",
"wp-content/plugins/all-in-one-wp-security-and-firewall/other-includes/wp-security-unlock-request.php",
"wp-content/plugins/related-posts/views/settings.php",
"wp-content/plugins/wpcontentguard/views/settings.php",
"wp-content/plugins/simple-social-icons/simple-social-icons.php",
]
results: List[Result] = []
for url in urls:
target = urljoin(wp_url, url)
head = network.http_head(target, False)
if head.status_code != 404:
resp = network.http_get(target, False)
if resp.status_code < 300 or resp.status_code >= 500:
# we have some kind of response that could be useful
if "<b>Fatal error</b>:" in resp.text:
# we have an error
pattern = r"<b>((\/|[A-Z]:\\).*.php)<\/b>"
if re.search(pattern, resp.text):
try:
path = re.findall(pattern, resp.text)[0][0]
results.append(
Result.from_evidence(
Evidence.from_response(
resp, {"path": path}),
f"WordPress File Path Disclosure: {target} ({path})",
Vulnerabilities.
APP_WORDPRESS_PATH_DISCLOSURE,
))
except Exception:
output.debug_exception()
results += response_scanner.check_response(target, resp)
return results
def get_ips(domain: str):
ips = []
try:
answers_v4 = resolver.query(domain, "A")
for data in answers_v4:
ips.append(str(data))
except (resolver.NoAnswer, resolver.NXDOMAIN, exception.Timeout):
pass
except (resolver.NoNameservers, resolver.NotAbsolute, resolver.NoRootSOA):
output.debug_exception()
pass
try:
answers_v6 = resolver.query(domain, "AAAA")
for data in answers_v6:
ips.append(str(data))
except (resolver.NoAnswer, resolver.NXDOMAIN, exception.Timeout):
pass
except (resolver.NoNameservers, resolver.NotAbsolute, resolver.NoRootSOA):
output.debug_exception()
pass
return ips
def check_cve_2019_0232(links: List[str]) -> List[Result]:
results: List[Result] = []
try:
targets: List[str] = []
for link in links:
if "cgi-bin" in link:
if "?" in link:
targets.append(f"{link}&dir")
else:
targets.append(f"{link}?dir")
for target in targets:
res = network.http_get(target, False)
body = res.text
if "<DIR>" in body:
# we have a hit
results.append(
Result(
f"Apache Tomcat RCE (CVE-2019-0232): {target}",
Vulnerabilities.SERVER_TOMCAT_CVE_2019_0232,
target,
[
network.http_build_raw_request(res.request),
network.http_build_raw_response(res),
],
))
results += response_scanner.check_response(target, res)
except Exception:
output.debug_exception()
return results
def _get_ip_info(ip: str) -> Tuple[str, str]:
try:
server_ip = socket.gethostbyname(ip)
ni = network_info.network_info(str(server_ip))
except Exception:
server_ip = "(Unavailable)"
ni = "(Unavailable)"
output.debug_exception()
return server_ip, ni
def get_host(ip):
name = "N/A"
try:
rev_name = reversename.from_address(str(ip))
name = str(resolver.query(rev_name, "PTR", lifetime=3)[0])[:-1]
except (resolver.NoAnswer, resolver.NXDOMAIN, exception.Timeout):
pass
except (resolver.NoNameservers, resolver.NotAbsolute, resolver.NoRootSOA):
output.debug_exception()
return name
def _check_version_verb(url: str) -> List[Result]:
results: List[Result] = []
try:
res = network.http_custom("XYZ", url)
if res.status_code > 400:
results += get_version(url, res, "Invalid HTTP Verb")
except Exception:
output.debug_exception()
return results
def check_for_jira(session: Session) -> Tuple[List[Result], Union[str, None]]:
# this checks for an instance of Jira relative to the session URL
results: List[Result] = []
jira_url = None
try:
targets = [
f"{session.url}secure/Dashboard.jspa",
f"{session.url}jira/secure/Dashboard.jspa",
]
for target in targets:
res = network.http_get(target, False)
if (
res.status_code == 200
and 'name="application-name" content="JIRA"' in res.text
):
# we have a Jira instance
jira_url = target
# try to get the version
ver_str = "unknown"
try:
ver_pattern = (
r"<meta name=\"ajs-version-number\" content=\"([\d\.]+)\">"
)
version = re.search(ver_pattern, res.text).group(1)
build_pattern = (
r"<meta name=\"ajs-build-number\" content=\"(\d+)\">"
)
build = re.search(build_pattern, res.text).group(1)
ver_str = f"v{version}-{build}"
except:
output.debug_exception()
results.append(
Result.from_evidence(
Evidence.from_response(res),
f"Jira Installation Found ({ver_str}): {target}",
Vulnerabilities.APP_JIRA_FOUND,
)
)
results += response_scanner.check_response(target, res)
break
except Exception:
output.debug_exception()
return results, jira_url
def _check_version_post(url: str) -> List[Result]:
results: List[Result] = []
try:
res = network.http_custom("POST", url)
if res.status_code > 400:
results += get_version(url, res, "POST to root")
except Exception:
output.debug_exception()
return results
def start(args, url):
print(f"Scanning: {url}")
# parse the URL, we'll need this
parsed = urlparse(url)
# get rid of any port number & credentials that may exist
domain = utils.get_domain(parsed.netloc)
# make sure it resolves
try:
socket.gethostbyname(domain)
except socket.gaierror as error:
print(f"Fatal Error: Unable to resolve {domain} ({str(error)})")
return
if parsed.scheme == "http":
try:
# check for TLS redirect
tls_redirect = network.check_ssl_redirect(url)
if tls_redirect != url:
print(f"Server redirects to TLS: Scanning: {tls_redirect}")
url = tls_redirect
parsed = urlparse(url)
except Exception as error:
output.debug_exception()
output.error(f"Failed to connect to {url} ({str(error)})")
return
www_redirect = network.check_www_redirect(url)
if www_redirect is not None and www_redirect != url:
print(f"Server performs WWW redirect: Scanning: {www_redirect}")
url = www_redirect
parsed = urlparse(url)
# check to see if we are looking at an HTTPS server
if parsed.scheme == "https":
if args.internalssl or utils.is_ip(domain) or utils.get_port(url) != 443:
# use internal scanner
ssl_internal.scan(args, url, domain)
else:
try:
ssl_labs.scan(args, url, domain)
except Exception as error:
output.debug_exception()
output.error(f"Error running scan with SSL Labs: {str(error)}")
if args.tdessessioncount:
ssl_sweet32.scan(args, url, domain)
def _get_data() -> None:
global _data
data_url = "https://raw.githubusercontent.com/augustd/burp-suite-error-message-checks/master/src/main/resources/burp/match-rules.tab"
try:
raw = network.http_get(data_url).text
for line in raw.splitlines():
_data.append(_MatchRule(line))
except Exception as error:
output.debug(f"Failed to get version data: {error}")
output.debug_exception()
def _get_caa_records(domain: str, resv: Resolver) -> List[str]:
records: List[str] = []
try:
answers = resv.query(domain, "CAA", lifetime=3)
for data in answers:
records.append(data.to_text())
except (resolver.NoAnswer, resolver.NXDOMAIN, exception.Timeout):
pass
except (resolver.NoNameservers, resolver.NotAbsolute, resolver.NoRootSOA):
output.debug_exception()
return records
def get_mx(domain):
records = []
try:
answers = resolver.query(domain, "MX")
for data in answers:
records.append([str(data.exchange), str(data.preference)])
except (resolver.NoAnswer, resolver.NXDOMAIN, exception.Timeout):
pass
except (resolver.NoNameservers, resolver.NotAbsolute, resolver.NoRootSOA):
output.debug_exception()
return records
def _check_connection(url: str) -> str:
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
result = "Connection Failed"
try:
headers = {"User-Agent": SERVICE_UA}
res = requests.get(url, headers=headers, verify=False)
result = res.text.strip()
except Exception:
output.debug_exception()
return result
def get_ns(domain):
records = []
try:
answers = resolver.query(domain, "NS")
for data in answers:
records.append(str(data))
except (resolver.NoAnswer, resolver.NXDOMAIN, exception.Timeout):
pass
except (resolver.NoNameservers, resolver.NotAbsolute, resolver.NoRootSOA):
output.debug_exception()
return records
def _get_cname(domain: str, resv: Resolver) -> Union[str, None]:
name = None
try:
answers = resv.query(domain, "CNAME", lifetime=3)
for data in answers:
name = str(data.target)
except (resolver.NoAnswer, resolver.NXDOMAIN, exception.Timeout):
pass
except (resolver.NoNameservers, resolver.NotAbsolute, resolver.NoRootSOA):
output.debug_exception()
return name
def check_response(url: str,
res: Response,
body: Union[str, None] = None) -> List[Result]:
global _data, _reports
results = []
try:
# make sure we actually have something
if res is None:
return []
if _data is None or len(_data) == 0:
_get_data()
if body is None:
body = res.text
for rule in _data:
rule = cast(_MatchRule, rule)
mtch = re.search(rule.pattern, body)
if mtch:
val = mtch.group(int(rule.match_group))
err_start = body.find(val)
# get the error, plus 25 characters on each side
err = body[err_start - 25:err_start + len(val) + 25]
msg = (f"Found error message (confidence: {rule.confidence}) "
f"on {url} ({res.request.method}): ...{err}...")
if msg not in _reports:
results.append(
Result.from_evidence(
Evidence.from_response(res),
msg,
Vulnerabilities.HTTP_ERROR_MESSAGE,
))
_reports.append(msg)
break
else:
output.debug(f"Ignored duplicate error message: {msg}")
except Exception:
output.debug_exception()
return results
def _get_info(self) -> str:
# prime the call to cpu_percent, as the first call doesn't return useful data
self.process.cpu_percent()
# force a collection; not ideal, but seems to help
gc.collect(2)
# use oneshot() to cache the data, so we minimize hits
with self.process.oneshot():
pct = self.process.cpu_percent()
times = self.process.cpu_times()
mem = self.process.memory_info()
mem_res = "{0:cM}".format(Size(mem.rss))
mem_virt = "{0:cM}".format(Size(mem.vms))
if mem.rss > self.peak_mem_res:
self.peak_mem_res = mem.rss
output.debug(f"New high-memory threshold: {self.peak_mem_res}")
thr = self.process.num_threads()
vm = psutil.virtual_memory()
mem_total = "{0:cM}".format(Size(vm.total))
mem_avail_bytes = vm.available
mem_avail = "{0:cM}".format(Size(vm.available))
if mem_avail_bytes < self.WARNING_THRESHOLD and not self.low_mem_warning:
self.low_mem_warning = True
output.error(f"Low RAM Available: {mem_avail}")
cons = -1
try:
cons = len(self.process.connections(kind="inet"))
except Exception:
# we don't care if this fails
output.debug_exception()
cpu_freq = psutil.cpu_freq()
info = (f"Process Stats: CPU: {pct}% - Sys: {times.system} - "
f"User: {times.user} - Res: {mem_res} - Virt: {mem_virt} - "
f"Available: {mem_avail}/{mem_total} - Threads: {thr} - "
f"Connections: {cons} - CPU Freq: "
f"{int(cpu_freq.current)}MHz/{int(cpu_freq.max)}MHz - "
f"GC Objects: {len(gc.get_objects())}")
return info
def get_info_message() -> List[str]:
path = "/api/v3/info"
messages: List[str] = []
try:
body, code = network.http_json(API_SERVER + path)
if len(body["messages"]) > 0:
for msg in body["messages"]:
messages.append(msg)
except Exception:
output.debug_exception()
raise
return messages
def _get_version_data() -> None:
global _versions
data: Union[Dict[str, Dict[str, Dict[str, str]]], None] = None
data_url = "https://raw.githubusercontent.com/adamcaudill/current_versions/master/current_versions.json"
try:
data, _ = network.http_json(data_url)
except Exception as error:
output.debug(f"Failed to get version data: {error}")
output.debug_exception()
if data is not None and "software" in data:
_versions = data["software"]
else:
_versions = None