async def _async_try_login_post(
self, username: str, password: str,
auth_url: str) -> Tuple[bool, dict, List[str]]:
# Fetch the login page and try to extract the login form
try:
page = await self.async_get(web.Request(auth_url),
follow_redirects=True)
form = {}
disconnect_urls = []
login_form, username_field_idx, password_field_idx = page.find_login_form(
)
if login_form:
post_params = login_form.post_params
get_params = login_form.get_params
if login_form.method == "POST":
post_params[username_field_idx][1] = username
post_params[password_field_idx][1] = password
form["login_field"] = post_params[username_field_idx][0]
form["password_field"] = post_params[password_field_idx][0]
else:
get_params[username_field_idx][1] = username
get_params[password_field_idx][1] = password
form["login_field"] = get_params[username_field_idx][0]
form["password_field"] = get_params[password_field_idx][0]
login_request = web.Request(path=login_form.url,
method=login_form.method,
post_params=post_params,
get_params=get_params,
referer=login_form.referer,
link_depth=login_form.link_depth)
login_response = await self.async_send(login_request,
follow_redirects=True)
# ensure logged in
if login_response.soup.find_all(
text=re.compile(DISCONNECT_REGEX)):
self.is_logged_in = True
logging.success(_("Login success"))
disconnect_urls = self._extract_disconnect_urls(
login_response)
else:
logging.warning(
_("Login failed") + " : " +
_("Credentials might be invalid"))
else:
logging.warning(
_("Login failed") + " : " + _("No login form detected"))
return self.is_logged_in, form, disconnect_urls
except ConnectionError:
logging.error(_("[!] Connection error with URL"), auth_url)
except httpx.RequestError as error:
logging.error(
_("[!] {} with url {}").format(error.__class__.__name__,
auth_url))
async def update(self):
"""Update the Wappalizer database from the web and load the patterns."""
try:
await self._load_wapp_database(self.WAPP_CATEGORIES_URL,
self.WAPP_TECHNOLOGIES_BASE_URL,
self.WAPP_GROUPS_URL)
except IOError:
logging.error(_("Error downloading wapp database."))
async def update(self):
"""Update the HashThePlanet database from the web."""
try:
await self._download_htp_database(
self.HTP_DATABASE_URL,
os.path.join(self.user_config_dir, self.HTP_DATABASE)
)
except IOError:
logging.error(_("Error downloading htp database."))
def _is_valid_dns(self, dns_endpoint: str) -> bool:
if dns_endpoint is None:
return False
try:
self._dns_host = socket.gethostbyname(dns_endpoint)
except OSError:
logging.error(
_("Error: {} is not a valid domain name").format(dns_endpoint))
return False
return True
async def attack(self, request: Request):
page = request.path
for mutated_request, parameter, _payload, _flags in self.mutator.mutate(
request):
log_verbose(f"[¨] {mutated_request.url}")
try:
response = await self.crawler.async_send(mutated_request)
except ReadTimeout:
self.network_errors += 1
await self.add_anom_medium(request_id=request.path_id,
category=Messages.RES_CONSUMPTION,
request=mutated_request,
parameter=parameter,
info="Timeout (" + parameter + ")",
wstg=RESOURCE_CONSUMPTION_WSTG_CODE)
log_orange("---")
log_orange(Messages.MSG_TIMEOUT, page)
log_orange(Messages.MSG_EVIL_REQUEST)
log_orange(mutated_request.http_repr())
log_orange("---")
except HTTPStatusError:
self.network_errors += 1
logging.error(
_("Error: The server did not understand this request"))
except RequestError:
self.network_errors += 1
else:
if "wapiti" in response.headers:
await self.add_vuln_low(
request_id=request.path_id,
category=NAME,
request=mutated_request,
parameter=parameter,
info=_(
"{0} via injection in the parameter {1}").format(
self.MSG_VULN, parameter),
wstg=WSTG_CODE)
if parameter == "QUERY_STRING":
injection_msg = Messages.MSG_QS_INJECT
else:
injection_msg = Messages.MSG_PARAM_INJECT
log_red("---")
log_red(injection_msg, self.MSG_VULN, page, parameter)
log_red(Messages.MSG_EVIL_REQUEST)
log_red(mutated_request.http_repr())
log_red("---")
async def async_try_login(self, auth_url: str,
auth_type: str) -> Tuple[bool, dict, List[str]]:
"""
Try to authenticate with the provided url and credentials.
Returns if the the authentication has been successful, the used form variables and the disconnect urls.
"""
if len(self._auth_credentials) != 2:
logging.error(
_("Login failed") + " : " + _("Invalid credentials format"))
return False, {}, []
username, password = self._auth_credentials
if auth_type == "post":
return await self._async_try_login_post(username, password,
auth_url)
return await self._async_try_login_basic_digest_ntlm(auth_url)
async def update(self):
"""Update the Nikto database from the web and load the patterns."""
try:
request = Request(self.NIKTO_DB_URL)
response = await self.crawler.async_send(request)
csv.register_dialect("nikto", quoting=csv.QUOTE_ALL, doublequote=False, escapechar="\\")
reader = csv.reader(response.content.split("\n"), "nikto")
self.nikto_db = [line for line in reader if line != [] and line[0].isdigit()]
with open(
os.path.join(self.user_config_dir, self.NIKTO_DB),
"w",
errors="ignore",
encoding='utf-8'
) as nikto_db_file:
writer = csv.writer(nikto_db_file)
writer.writerows(self.nikto_db)
except IOError:
logging.error(_("Error downloading nikto database."))
async def finish(self):
endpoint_url = f"{self.internal_endpoint}get_ssrf.php?session_id={self._session_id}"
logging.info(_("[*] Asking endpoint URL {} for results, please wait...").format(endpoint_url))
await sleep(2)
# A la fin des attaques on questionne le endpoint pour savoir s'il a été contacté
endpoint_request = Request(endpoint_url)
try:
response = await self.crawler.async_send(endpoint_request)
except RequestError:
self.network_errors += 1
logging.error(_("[!] Unable to request endpoint URL '{}'").format(self.internal_endpoint))
else:
data = response.json
if isinstance(data, dict):
for request_id in data:
original_request = await self.persister.get_path_by_id(request_id)
if original_request is None:
raise ValueError("Could not find the original request with that ID")
page = original_request.path
for hex_param in data[request_id]:
parameter = unhexlify(hex_param).decode("utf-8")
for infos in data[request_id][hex_param]:
request_url = infos["url"]
# Date in ISO format
request_date = infos["date"]
request_ip = infos["ip"]
request_method = infos["method"]
# request_size = infos["size"]
if parameter == "QUERY_STRING":
vuln_message = Messages.MSG_QS_INJECT.format(self.MSG_VULN, page)
else:
vuln_message = _(
"{0} via injection in the parameter {1}.\n"
"The target performed an outgoing HTTP {2} request at {3} with IP {4}.\n"
"Full request can be seen at {5}"
).format(
self.MSG_VULN,
parameter,
request_method,
request_date,
request_ip,
request_url
)
mutator = Mutator(
methods="G" if original_request.method == "GET" else "PF",
payloads=[("http://external.url/page", Flags())],
qs_inject=self.must_attack_query_string,
parameters=[parameter],
skip=self.options.get("skipped_parameters")
)
mutated_request, __, __, __ = next(mutator.mutate(original_request))
await self.add_vuln_critical(
request_id=original_request.path_id,
category=NAME,
request=mutated_request,
info=vuln_message,
parameter=parameter,
wstg=WSTG_CODE
)
log_red("---")
log_red(
Messages.MSG_QS_INJECT if parameter == "QUERY_STRING"
else Messages.MSG_PARAM_INJECT,
self.MSG_VULN,
page,
parameter
)
log_red(Messages.MSG_EVIL_REQUEST)
log_red(mutated_request.http_repr())
log_red("---")
async def attack(self, request: Request):
self.finished = True
request_to_root = Request(request.url)
categories_file_path = os.path.join(self.user_config_dir,
self.WAPP_CATEGORIES)
groups_file_path = os.path.join(self.user_config_dir, self.WAPP_GROUPS)
technologies_file_path = os.path.join(self.user_config_dir,
self.WAPP_TECHNOLOGIES)
await self._verify_wapp_database(categories_file_path,
technologies_file_path,
groups_file_path)
try:
application_data = ApplicationData(categories_file_path,
groups_file_path,
technologies_file_path)
except FileNotFoundError as exception:
logging.error(exception)
logging.error(
_("Try using --store-session option, or update apps.json using --update option."
))
return
except ApplicationDataException as exception:
logging.error(exception)
return
detected_applications = await self._detect_applications(
request.url, application_data)
if len(detected_applications) > 0:
log_blue("---")
for application_name in sorted(detected_applications,
key=lambda x: x.lower()):
versions = detected_applications[application_name]["versions"]
categories = detected_applications[application_name]["categories"]
groups = detected_applications[application_name]["groups"]
log_blue(MSG_TECHNO_VERSIONED, application_name, versions)
log_blue(MSG_CATEGORIES, categories)
log_blue(MSG_GROUPS, groups)
log_blue("")
await self.add_addition(
category=TECHNO_DETECTED,
request=request_to_root,
info=json.dumps(detected_applications[application_name]),
wstg=TECHNO_DETECTED_WSTG_CODE)
if versions:
if "Web servers" in categories:
await self.add_vuln_info(
category=WEB_SERVER_VERSIONED,
request=request_to_root,
info=json.dumps(
detected_applications[application_name]),
wstg=WEB_SERVER_WSTG_CODE)
else:
await self.add_vuln_info(
category=WEB_APP_VERSIONED,
request=request_to_root,
info=json.dumps(
detected_applications[application_name]),
wstg=WEB_APP_WSTG_CODE)
async def finish(self):
endpoint_url = f"{self.internal_endpoint}get_xxe.php?session_id={self._session_id}"
logging.info(
_("[*] Asking endpoint URL {} for results, please wait...").format(
endpoint_url))
await sleep(2)
# A la fin des attaques on questionne le endpoint pour savoir s'il a été contacté
endpoint_request = Request(endpoint_url)
try:
response = await self.crawler.async_send(endpoint_request)
except RequestError:
self.network_errors += 1
logging.error(
_("[!] Unable to request endpoint URL '{}'").format(
self.internal_endpoint))
return
data = response.json
if not isinstance(data, dict):
return
for request_id in data:
original_request = await self.persister.get_path_by_id(request_id)
if original_request is None:
continue
# raise ValueError("Could not find the original request with ID {}".format(request_id))
page = original_request.path
for hex_param in data[request_id]:
parameter = unhexlify(hex_param).decode("utf-8")
for infos in data[request_id][hex_param]:
request_url = infos["url"]
# Date in ISO format
request_date = infos["date"]
request_ip = infos["ip"]
request_size = infos["size"]
payload_name = infos["payload"]
if parameter == "QUERY_STRING":
vuln_message = Messages.MSG_QS_INJECT.format(
self.MSG_VULN, page)
elif parameter == "raw body":
vuln_message = _(
"Out-Of-Band {0} by sending raw XML in request body"
).format(self.MSG_VULN)
else:
vuln_message = _(
"Out-Of-Band {0} via injection in the parameter {1}"
).format(self.MSG_VULN, parameter)
more_infos = _(
"The target sent {0} bytes of data to the endpoint at {1} with IP {2}.\n"
"Received data can be seen at {3}.").format(
request_size, request_date, request_ip,
request_url)
vuln_message += "\n" + more_infos
# placeholder if shit happens
payload = (
"<xml>"
"See https://phonexicum.github.io/infosec/xxe.html#attack-vectors"
"</xml>")
for payload, _flags in self.payloads:
if f"{payload_name}.dtd" in payload:
payload = payload.replace(
"[PATH_ID]", str(original_request.path_id))
payload = payload.replace("[PARAM_AS_HEX]",
"72617720626f6479")
break
if parameter == "raw body":
mutated_request = Request(original_request.path,
method="POST",
enctype="text/xml",
post_params=payload)
elif parameter == "QUERY_STRING":
mutated_request = Request(
f"{original_request.path}?{quote(payload)}",
method="GET")
elif parameter in original_request.get_keys or parameter in original_request.post_keys:
mutator = Mutator(
methods="G"
if original_request.method == "GET" else "P",
payloads=[(payload, Flags())],
qs_inject=self.must_attack_query_string,
parameters=[parameter],
skip=self.options.get("skipped_parameters"))
mutated_request, __, __, __ = next(
mutator.mutate(original_request))
else:
mutator = FileMutator(
payloads=[(payload, Flags())],
parameters=[parameter],
skip=self.options.get("skipped_parameters"))
mutated_request, __, __, __ = next(
mutator.mutate(original_request))
await self.add_vuln_high(
request_id=original_request.path_id,
category=NAME,
request=mutated_request,
info=vuln_message,
parameter=parameter,
wstg=WSTG_CODE)
log_red("---")
log_red(vuln_message)
log_red(Messages.MSG_EVIL_REQUEST)
log_red(mutated_request.http_repr())
log_red("---")
async def attack(self, request: Request):
page = request.path
saw_internal_error = False
current_parameter = None
vulnerable_parameter = False
for mutated_request, parameter, _payload, _flags in self.mutator.mutate(
request):
if current_parameter != parameter:
# Forget what we know about current parameter
current_parameter = parameter
vulnerable_parameter = False
elif vulnerable_parameter:
# If parameter is vulnerable, just skip till next parameter
continue
log_verbose(f"[¨] {mutated_request}")
try:
response = await self.crawler.async_send(mutated_request)
except ReadTimeout:
# The request with time based payload did timeout, what about a regular request?
if await self.does_timeout(request):
self.network_errors += 1
logging.error(
"[!] Too much lag from website, can't reliably test time-based blind SQL"
)
break
if parameter == "QUERY_STRING":
vuln_message = Messages.MSG_QS_INJECT.format(
self.MSG_VULN, page)
log_message = Messages.MSG_QS_INJECT
else:
vuln_message = _(
"{0} via injection in the parameter {1}").format(
self.MSG_VULN, parameter)
log_message = Messages.MSG_PARAM_INJECT
await self.add_vuln_critical(request_id=request.path_id,
category=NAME,
request=mutated_request,
info=vuln_message,
parameter=parameter,
wstg=WSTG_CODE)
log_red("---")
log_red(log_message, self.MSG_VULN, page, parameter)
log_red(Messages.MSG_EVIL_REQUEST)
log_red(mutated_request.http_repr())
log_red("---")
# We reached maximum exploitation for this parameter, don't send more payloads
vulnerable_parameter = True
continue
except RequestError:
self.network_errors += 1
continue
else:
if response.is_server_error and not saw_internal_error:
saw_internal_error = True
if parameter == "QUERY_STRING":
anom_msg = Messages.MSG_QS_500
else:
anom_msg = Messages.MSG_PARAM_500.format(parameter)
await self.add_anom_high(request_id=request.path_id,
category=Messages.ERROR_500,
request=mutated_request,
info=anom_msg,
parameter=parameter,
wstg=INTERNAL_ERROR_WSTG_CODE)
log_orange("---")
log_orange(Messages.MSG_500, page)
log_orange(Messages.MSG_EVIL_REQUEST)
log_orange(mutated_request.http_repr())
log_orange("---")
async def async_explore(self,
to_explore: deque,
excluded_urls: list = None):
"""Explore a single TLD or the whole Web starting with a URL
@param to_explore: A list of Request of URLs (str) to scan the scan with.
@type to_explore: list
@param excluded_urls: A list of URLs to skip. Request objects or strings which may contain wildcards.
@type excluded_urls: list
@rtype: generator
"""
if isinstance(excluded_urls, list):
while True:
try:
bad_request = excluded_urls.pop()
except IndexError:
break
else:
if isinstance(bad_request, str):
self._regexes.append(wildcard_translate(bad_request))
elif isinstance(bad_request, web.Request):
self._processed_requests.append(bad_request)
self._crawler.stream = True
if self._max_depth < 0:
return
task_to_request = {}
while True:
while to_explore:
# Concurrent tasks are limited through the use of the semaphore BUT we don't want the to_explore
# queue to be empty everytime (as we may need to extract remaining URLs) and overload the event loop
# with pending tasks.
if len(task_to_request) > self._max_tasks:
break
if self._stopped.is_set():
break
request = to_explore.popleft()
if not isinstance(request, web.Request):
# We treat start_urls as if they are all valid URLs (ie in scope)
request = web.Request(request, link_depth=0)
if request in self._processed_requests:
continue
resource_url = request.url
if request.link_depth > self._max_depth:
continue
dir_name = request.dir_name
if self._max_files_per_dir and self._file_counts[
dir_name] >= self._max_files_per_dir:
continue
# Won't enter if qs_limit is 0 (aka insane mode)
if self._qs_limit:
if request.parameters_count:
try:
if self._pattern_counts[request.pattern] >= 220 / (
math.exp(request.parameters_count *
self._qs_limit)**2):
continue
except OverflowError:
# Oh boy... that's not good to try to attack a form with more than 600 input fields
# but I guess insane mode can do it as it is insane
continue
if self.is_forbidden(resource_url):
continue
task = asyncio.create_task(self.async_analyze(request))
task_to_request[task] = request
if task_to_request:
done, __ = await asyncio.wait(
task_to_request,
timeout=0.25,
return_when=asyncio.FIRST_COMPLETED)
else:
done = []
# process any completed task
for task in done:
request = task_to_request[task]
try:
success, resources = await task
except Exception as exception:
logging.error(
f"{request} generated an exception: {exception.__class__.__name__}"
)
else:
if success:
yield request
accepted_urls = 0
for unfiltered_request in resources:
if BAD_URL_REGEX.search(unfiltered_request.file_path):
# Malformed link due to HTML issues
continue
if not self._crawler.is_in_scope(unfiltered_request):
continue
if unfiltered_request.hostname not in self._hostnames:
unfiltered_request.link_depth = 0
if unfiltered_request not in self._processed_requests and unfiltered_request not in to_explore:
to_explore.append(unfiltered_request)
accepted_urls += 1
# TODO: fix this, it doesn't looks valid
# if self._max_per_depth and accepted_urls >= self._max_per_depth:
# break
# remove the now completed task
del task_to_request[task]
if not task_to_request and (self._stopped.is_set()
or not to_explore):
break
self._crawler.stream = False
async def async_analyze(self, request) -> Tuple[bool, List]:
async with self._sem:
self._processed_requests.append(request) # thread safe
log_verbose(f"[+] {request}")
dir_name = request.dir_name
# Currently not exploited. Would be interesting though but then it should be implemented separately
# Maybe in another task as we don't want to spend to much time in this function
# async with self._shared_lock:
# # lock to prevent launching duplicates requests that would otherwise waste time
# if dir_name not in self._custom_404_codes:
# invalid_page = "zqxj{0}.html".format("".join([choice(ascii_letters) for __ in range(10)]))
# invalid_resource = web.Request(dir_name + invalid_page)
# try:
# page = await self._crawler.async_get(invalid_resource)
# self._custom_404_codes[dir_name] = page.status
# except httpx.RequestError:
# pass
self._hostnames.add(request.hostname)
resource_url = request.url
try:
page = await self._crawler.async_send(request)
except (TypeError, UnicodeDecodeError) as exception:
logging.debug(f"{exception} with url {resource_url}") # debug
return False, []
# TODO: what to do of connection errors ? sleep a while before retrying ?
except ConnectionError:
logging.error(_("[!] Connection error with URL"), resource_url)
return False, []
except httpx.RequestError as error:
logging.error(
_("[!] {} with url {}").format(error.__class__.__name__,
resource_url))
return False, []
if self._max_files_per_dir:
async with self._shared_lock:
self._file_counts[dir_name] += 1
if self._qs_limit and request.parameters_count:
async with self._shared_lock:
self._pattern_counts[request.pattern] += 1
if request.link_depth == self._max_depth:
# We are at the edge of the depth so next links will have depth + 1 so to need to parse the page.
return True, []
# Above this line we need the content of the page. As we are in stream mode we must force reading the body.
await page.read()
# Sur les ressources statiques le content-length est généralement indiqué
if self._max_page_size > 0:
if page.raw_size > self._max_page_size:
await page.clean()
return False, []
await asyncio.sleep(0)
resources = self.extract_links(page, request)
# TODO: there's more situations where we would not want to attack the resource... must check this
if page.is_directory_redirection:
return False, resources
return True, resources
