async def _async_try_login_post( self, username: str, password: str, auth_url: str) -> Tuple[bool, dict, List[str]]: # Fetch the login page and try to extract the login form try: page = await self.async_get(web.Request(auth_url), follow_redirects=True) form = {} disconnect_urls = [] login_form, username_field_idx, password_field_idx = page.find_login_form( ) if login_form: post_params = login_form.post_params get_params = login_form.get_params if login_form.method == "POST": post_params[username_field_idx][1] = username post_params[password_field_idx][1] = password form["login_field"] = post_params[username_field_idx][0] form["password_field"] = post_params[password_field_idx][0] else: get_params[username_field_idx][1] = username get_params[password_field_idx][1] = password form["login_field"] = get_params[username_field_idx][0] form["password_field"] = get_params[password_field_idx][0] login_request = web.Request(path=login_form.url, method=login_form.method, post_params=post_params, get_params=get_params, referer=login_form.referer, link_depth=login_form.link_depth) login_response = await self.async_send(login_request, follow_redirects=True) # ensure logged in if login_response.soup.find_all( text=re.compile(DISCONNECT_REGEX)): self.is_logged_in = True logging.success(_("Login success")) disconnect_urls = self._extract_disconnect_urls( login_response) else: logging.warning( _("Login failed") + " : " + _("Credentials might be invalid")) else: logging.warning( _("Login failed") + " : " + _("No login form detected")) return self.is_logged_in, form, disconnect_urls except ConnectionError: logging.error(_("[!] Connection error with URL"), auth_url) except httpx.RequestError as error: logging.error( _("[!] {} with url {}").format(error.__class__.__name__, auth_url))
async def update(self): """Update the Wappalizer database from the web and load the patterns.""" try: await self._load_wapp_database(self.WAPP_CATEGORIES_URL, self.WAPP_TECHNOLOGIES_BASE_URL, self.WAPP_GROUPS_URL) except IOError: logging.error(_("Error downloading wapp database."))
async def update(self): """Update the HashThePlanet database from the web.""" try: await self._download_htp_database( self.HTP_DATABASE_URL, os.path.join(self.user_config_dir, self.HTP_DATABASE) ) except IOError: logging.error(_("Error downloading htp database."))
def _is_valid_dns(self, dns_endpoint: str) -> bool: if dns_endpoint is None: return False try: self._dns_host = socket.gethostbyname(dns_endpoint) except OSError: logging.error( _("Error: {} is not a valid domain name").format(dns_endpoint)) return False return True
async def attack(self, request: Request): page = request.path for mutated_request, parameter, _payload, _flags in self.mutator.mutate( request): log_verbose(f"[¨] {mutated_request.url}") try: response = await self.crawler.async_send(mutated_request) except ReadTimeout: self.network_errors += 1 await self.add_anom_medium(request_id=request.path_id, category=Messages.RES_CONSUMPTION, request=mutated_request, parameter=parameter, info="Timeout (" + parameter + ")", wstg=RESOURCE_CONSUMPTION_WSTG_CODE) log_orange("---") log_orange(Messages.MSG_TIMEOUT, page) log_orange(Messages.MSG_EVIL_REQUEST) log_orange(mutated_request.http_repr()) log_orange("---") except HTTPStatusError: self.network_errors += 1 logging.error( _("Error: The server did not understand this request")) except RequestError: self.network_errors += 1 else: if "wapiti" in response.headers: await self.add_vuln_low( request_id=request.path_id, category=NAME, request=mutated_request, parameter=parameter, info=_( "{0} via injection in the parameter {1}").format( self.MSG_VULN, parameter), wstg=WSTG_CODE) if parameter == "QUERY_STRING": injection_msg = Messages.MSG_QS_INJECT else: injection_msg = Messages.MSG_PARAM_INJECT log_red("---") log_red(injection_msg, self.MSG_VULN, page, parameter) log_red(Messages.MSG_EVIL_REQUEST) log_red(mutated_request.http_repr()) log_red("---")
async def async_try_login(self, auth_url: str, auth_type: str) -> Tuple[bool, dict, List[str]]: """ Try to authenticate with the provided url and credentials. Returns if the the authentication has been successful, the used form variables and the disconnect urls. """ if len(self._auth_credentials) != 2: logging.error( _("Login failed") + " : " + _("Invalid credentials format")) return False, {}, [] username, password = self._auth_credentials if auth_type == "post": return await self._async_try_login_post(username, password, auth_url) return await self._async_try_login_basic_digest_ntlm(auth_url)
async def update(self): """Update the Nikto database from the web and load the patterns.""" try: request = Request(self.NIKTO_DB_URL) response = await self.crawler.async_send(request) csv.register_dialect("nikto", quoting=csv.QUOTE_ALL, doublequote=False, escapechar="\\") reader = csv.reader(response.content.split("\n"), "nikto") self.nikto_db = [line for line in reader if line != [] and line[0].isdigit()] with open( os.path.join(self.user_config_dir, self.NIKTO_DB), "w", errors="ignore", encoding='utf-8' ) as nikto_db_file: writer = csv.writer(nikto_db_file) writer.writerows(self.nikto_db) except IOError: logging.error(_("Error downloading nikto database."))
async def finish(self): endpoint_url = f"{self.internal_endpoint}get_ssrf.php?session_id={self._session_id}" logging.info(_("[*] Asking endpoint URL {} for results, please wait...").format(endpoint_url)) await sleep(2) # A la fin des attaques on questionne le endpoint pour savoir s'il a été contacté endpoint_request = Request(endpoint_url) try: response = await self.crawler.async_send(endpoint_request) except RequestError: self.network_errors += 1 logging.error(_("[!] Unable to request endpoint URL '{}'").format(self.internal_endpoint)) else: data = response.json if isinstance(data, dict): for request_id in data: original_request = await self.persister.get_path_by_id(request_id) if original_request is None: raise ValueError("Could not find the original request with that ID") page = original_request.path for hex_param in data[request_id]: parameter = unhexlify(hex_param).decode("utf-8") for infos in data[request_id][hex_param]: request_url = infos["url"] # Date in ISO format request_date = infos["date"] request_ip = infos["ip"] request_method = infos["method"] # request_size = infos["size"] if parameter == "QUERY_STRING": vuln_message = Messages.MSG_QS_INJECT.format(self.MSG_VULN, page) else: vuln_message = _( "{0} via injection in the parameter {1}.\n" "The target performed an outgoing HTTP {2} request at {3} with IP {4}.\n" "Full request can be seen at {5}" ).format( self.MSG_VULN, parameter, request_method, request_date, request_ip, request_url ) mutator = Mutator( methods="G" if original_request.method == "GET" else "PF", payloads=[("http://external.url/page", Flags())], qs_inject=self.must_attack_query_string, parameters=[parameter], skip=self.options.get("skipped_parameters") ) mutated_request, __, __, __ = next(mutator.mutate(original_request)) await self.add_vuln_critical( request_id=original_request.path_id, category=NAME, request=mutated_request, info=vuln_message, parameter=parameter, wstg=WSTG_CODE ) log_red("---") log_red( Messages.MSG_QS_INJECT if parameter == "QUERY_STRING" else Messages.MSG_PARAM_INJECT, self.MSG_VULN, page, parameter ) log_red(Messages.MSG_EVIL_REQUEST) log_red(mutated_request.http_repr()) log_red("---")
async def attack(self, request: Request): self.finished = True request_to_root = Request(request.url) categories_file_path = os.path.join(self.user_config_dir, self.WAPP_CATEGORIES) groups_file_path = os.path.join(self.user_config_dir, self.WAPP_GROUPS) technologies_file_path = os.path.join(self.user_config_dir, self.WAPP_TECHNOLOGIES) await self._verify_wapp_database(categories_file_path, technologies_file_path, groups_file_path) try: application_data = ApplicationData(categories_file_path, groups_file_path, technologies_file_path) except FileNotFoundError as exception: logging.error(exception) logging.error( _("Try using --store-session option, or update apps.json using --update option." )) return except ApplicationDataException as exception: logging.error(exception) return detected_applications = await self._detect_applications( request.url, application_data) if len(detected_applications) > 0: log_blue("---") for application_name in sorted(detected_applications, key=lambda x: x.lower()): versions = detected_applications[application_name]["versions"] categories = detected_applications[application_name]["categories"] groups = detected_applications[application_name]["groups"] log_blue(MSG_TECHNO_VERSIONED, application_name, versions) log_blue(MSG_CATEGORIES, categories) log_blue(MSG_GROUPS, groups) log_blue("") await self.add_addition( category=TECHNO_DETECTED, request=request_to_root, info=json.dumps(detected_applications[application_name]), wstg=TECHNO_DETECTED_WSTG_CODE) if versions: if "Web servers" in categories: await self.add_vuln_info( category=WEB_SERVER_VERSIONED, request=request_to_root, info=json.dumps( detected_applications[application_name]), wstg=WEB_SERVER_WSTG_CODE) else: await self.add_vuln_info( category=WEB_APP_VERSIONED, request=request_to_root, info=json.dumps( detected_applications[application_name]), wstg=WEB_APP_WSTG_CODE)
async def finish(self): endpoint_url = f"{self.internal_endpoint}get_xxe.php?session_id={self._session_id}" logging.info( _("[*] Asking endpoint URL {} for results, please wait...").format( endpoint_url)) await sleep(2) # A la fin des attaques on questionne le endpoint pour savoir s'il a été contacté endpoint_request = Request(endpoint_url) try: response = await self.crawler.async_send(endpoint_request) except RequestError: self.network_errors += 1 logging.error( _("[!] Unable to request endpoint URL '{}'").format( self.internal_endpoint)) return data = response.json if not isinstance(data, dict): return for request_id in data: original_request = await self.persister.get_path_by_id(request_id) if original_request is None: continue # raise ValueError("Could not find the original request with ID {}".format(request_id)) page = original_request.path for hex_param in data[request_id]: parameter = unhexlify(hex_param).decode("utf-8") for infos in data[request_id][hex_param]: request_url = infos["url"] # Date in ISO format request_date = infos["date"] request_ip = infos["ip"] request_size = infos["size"] payload_name = infos["payload"] if parameter == "QUERY_STRING": vuln_message = Messages.MSG_QS_INJECT.format( self.MSG_VULN, page) elif parameter == "raw body": vuln_message = _( "Out-Of-Band {0} by sending raw XML in request body" ).format(self.MSG_VULN) else: vuln_message = _( "Out-Of-Band {0} via injection in the parameter {1}" ).format(self.MSG_VULN, parameter) more_infos = _( "The target sent {0} bytes of data to the endpoint at {1} with IP {2}.\n" "Received data can be seen at {3}.").format( request_size, request_date, request_ip, request_url) vuln_message += "\n" + more_infos # placeholder if shit happens payload = ( "<xml>" "See https://phonexicum.github.io/infosec/xxe.html#attack-vectors" "</xml>") for payload, _flags in self.payloads: if f"{payload_name}.dtd" in payload: payload = payload.replace( "[PATH_ID]", str(original_request.path_id)) payload = payload.replace("[PARAM_AS_HEX]", "72617720626f6479") break if parameter == "raw body": mutated_request = Request(original_request.path, method="POST", enctype="text/xml", post_params=payload) elif parameter == "QUERY_STRING": mutated_request = Request( f"{original_request.path}?{quote(payload)}", method="GET") elif parameter in original_request.get_keys or parameter in original_request.post_keys: mutator = Mutator( methods="G" if original_request.method == "GET" else "P", payloads=[(payload, Flags())], qs_inject=self.must_attack_query_string, parameters=[parameter], skip=self.options.get("skipped_parameters")) mutated_request, __, __, __ = next( mutator.mutate(original_request)) else: mutator = FileMutator( payloads=[(payload, Flags())], parameters=[parameter], skip=self.options.get("skipped_parameters")) mutated_request, __, __, __ = next( mutator.mutate(original_request)) await self.add_vuln_high( request_id=original_request.path_id, category=NAME, request=mutated_request, info=vuln_message, parameter=parameter, wstg=WSTG_CODE) log_red("---") log_red(vuln_message) log_red(Messages.MSG_EVIL_REQUEST) log_red(mutated_request.http_repr()) log_red("---")
async def attack(self, request: Request): page = request.path saw_internal_error = False current_parameter = None vulnerable_parameter = False for mutated_request, parameter, _payload, _flags in self.mutator.mutate( request): if current_parameter != parameter: # Forget what we know about current parameter current_parameter = parameter vulnerable_parameter = False elif vulnerable_parameter: # If parameter is vulnerable, just skip till next parameter continue log_verbose(f"[¨] {mutated_request}") try: response = await self.crawler.async_send(mutated_request) except ReadTimeout: # The request with time based payload did timeout, what about a regular request? if await self.does_timeout(request): self.network_errors += 1 logging.error( "[!] Too much lag from website, can't reliably test time-based blind SQL" ) break if parameter == "QUERY_STRING": vuln_message = Messages.MSG_QS_INJECT.format( self.MSG_VULN, page) log_message = Messages.MSG_QS_INJECT else: vuln_message = _( "{0} via injection in the parameter {1}").format( self.MSG_VULN, parameter) log_message = Messages.MSG_PARAM_INJECT await self.add_vuln_critical(request_id=request.path_id, category=NAME, request=mutated_request, info=vuln_message, parameter=parameter, wstg=WSTG_CODE) log_red("---") log_red(log_message, self.MSG_VULN, page, parameter) log_red(Messages.MSG_EVIL_REQUEST) log_red(mutated_request.http_repr()) log_red("---") # We reached maximum exploitation for this parameter, don't send more payloads vulnerable_parameter = True continue except RequestError: self.network_errors += 1 continue else: if response.is_server_error and not saw_internal_error: saw_internal_error = True if parameter == "QUERY_STRING": anom_msg = Messages.MSG_QS_500 else: anom_msg = Messages.MSG_PARAM_500.format(parameter) await self.add_anom_high(request_id=request.path_id, category=Messages.ERROR_500, request=mutated_request, info=anom_msg, parameter=parameter, wstg=INTERNAL_ERROR_WSTG_CODE) log_orange("---") log_orange(Messages.MSG_500, page) log_orange(Messages.MSG_EVIL_REQUEST) log_orange(mutated_request.http_repr()) log_orange("---")
async def async_explore(self, to_explore: deque, excluded_urls: list = None): """Explore a single TLD or the whole Web starting with a URL @param to_explore: A list of Request of URLs (str) to scan the scan with. @type to_explore: list @param excluded_urls: A list of URLs to skip. Request objects or strings which may contain wildcards. @type excluded_urls: list @rtype: generator """ if isinstance(excluded_urls, list): while True: try: bad_request = excluded_urls.pop() except IndexError: break else: if isinstance(bad_request, str): self._regexes.append(wildcard_translate(bad_request)) elif isinstance(bad_request, web.Request): self._processed_requests.append(bad_request) self._crawler.stream = True if self._max_depth < 0: return task_to_request = {} while True: while to_explore: # Concurrent tasks are limited through the use of the semaphore BUT we don't want the to_explore # queue to be empty everytime (as we may need to extract remaining URLs) and overload the event loop # with pending tasks. if len(task_to_request) > self._max_tasks: break if self._stopped.is_set(): break request = to_explore.popleft() if not isinstance(request, web.Request): # We treat start_urls as if they are all valid URLs (ie in scope) request = web.Request(request, link_depth=0) if request in self._processed_requests: continue resource_url = request.url if request.link_depth > self._max_depth: continue dir_name = request.dir_name if self._max_files_per_dir and self._file_counts[ dir_name] >= self._max_files_per_dir: continue # Won't enter if qs_limit is 0 (aka insane mode) if self._qs_limit: if request.parameters_count: try: if self._pattern_counts[request.pattern] >= 220 / ( math.exp(request.parameters_count * self._qs_limit)**2): continue except OverflowError: # Oh boy... that's not good to try to attack a form with more than 600 input fields # but I guess insane mode can do it as it is insane continue if self.is_forbidden(resource_url): continue task = asyncio.create_task(self.async_analyze(request)) task_to_request[task] = request if task_to_request: done, __ = await asyncio.wait( task_to_request, timeout=0.25, return_when=asyncio.FIRST_COMPLETED) else: done = [] # process any completed task for task in done: request = task_to_request[task] try: success, resources = await task except Exception as exception: logging.error( f"{request} generated an exception: {exception.__class__.__name__}" ) else: if success: yield request accepted_urls = 0 for unfiltered_request in resources: if BAD_URL_REGEX.search(unfiltered_request.file_path): # Malformed link due to HTML issues continue if not self._crawler.is_in_scope(unfiltered_request): continue if unfiltered_request.hostname not in self._hostnames: unfiltered_request.link_depth = 0 if unfiltered_request not in self._processed_requests and unfiltered_request not in to_explore: to_explore.append(unfiltered_request) accepted_urls += 1 # TODO: fix this, it doesn't looks valid # if self._max_per_depth and accepted_urls >= self._max_per_depth: # break # remove the now completed task del task_to_request[task] if not task_to_request and (self._stopped.is_set() or not to_explore): break self._crawler.stream = False
async def async_analyze(self, request) -> Tuple[bool, List]: async with self._sem: self._processed_requests.append(request) # thread safe log_verbose(f"[+] {request}") dir_name = request.dir_name # Currently not exploited. Would be interesting though but then it should be implemented separately # Maybe in another task as we don't want to spend to much time in this function # async with self._shared_lock: # # lock to prevent launching duplicates requests that would otherwise waste time # if dir_name not in self._custom_404_codes: # invalid_page = "zqxj{0}.html".format("".join([choice(ascii_letters) for __ in range(10)])) # invalid_resource = web.Request(dir_name + invalid_page) # try: # page = await self._crawler.async_get(invalid_resource) # self._custom_404_codes[dir_name] = page.status # except httpx.RequestError: # pass self._hostnames.add(request.hostname) resource_url = request.url try: page = await self._crawler.async_send(request) except (TypeError, UnicodeDecodeError) as exception: logging.debug(f"{exception} with url {resource_url}") # debug return False, [] # TODO: what to do of connection errors ? sleep a while before retrying ? except ConnectionError: logging.error(_("[!] Connection error with URL"), resource_url) return False, [] except httpx.RequestError as error: logging.error( _("[!] {} with url {}").format(error.__class__.__name__, resource_url)) return False, [] if self._max_files_per_dir: async with self._shared_lock: self._file_counts[dir_name] += 1 if self._qs_limit and request.parameters_count: async with self._shared_lock: self._pattern_counts[request.pattern] += 1 if request.link_depth == self._max_depth: # We are at the edge of the depth so next links will have depth + 1 so to need to parse the page. return True, [] # Above this line we need the content of the page. As we are in stream mode we must force reading the body. await page.read() # Sur les ressources statiques le content-length est généralement indiqué if self._max_page_size > 0: if page.raw_size > self._max_page_size: await page.clean() return False, [] await asyncio.sleep(0) resources = self.extract_links(page, request) # TODO: there's more situations where we would not want to attack the resource... must check this if page.is_directory_redirection: return False, resources return True, resources