def check_http_traffic(request): settings.TOTAL_OF_REQUESTS = settings.TOTAL_OF_REQUESTS + 1 # Delay in seconds between each HTTP request time.sleep(int(settings.DELAY)) class do_connection(_http_client.HTTPConnection): def send(self, req): headers = req.decode() http_method = headers[:4].strip() if menu.options.traffic_file: logs.log_traffic("-" * 37 + "\n" + info_msg + "\n" + "-" * 37) request_http_headers = str(headers).split("\r\n") for header in request_http_headers: if settings.VERBOSITY_LEVEL >= 2: if http_method == "GET" and len( header) > 1 or http_method == "POST": print(settings.print_traffic(header)) if menu.options.traffic_file: logs.log_traffic("\n" + header) if menu.options.traffic_file: if settings.VERBOSITY_LEVEL > 1: logs.log_traffic("\n\n" + "#" * 77 + "\n\n") else: logs.log_traffic("\n\n") _http_client.HTTPConnection.send(self, req) class connection_handler(_urllib.request.HTTPHandler, _urllib.request.HTTPSHandler): if settings.SCHEME == 'https': def https_open(self, req): try: return self.do_open(do_connection, req) except Exception as err_msg: try: error_msg = str(err_msg.args[0]).split("] ")[1] + "." except IndexError: error_msg = str(err_msg.args[0]) + "." if settings.INIT_TEST == True: if settings.VERBOSITY_LEVEL < 2: print(settings.FAIL_STATUS) else: if settings.VERBOSITY_LEVEL < 1: print("") print(settings.print_critical_msg(error_msg)) raise SystemExit() else: def http_open(self, req): try: return self.do_open(do_connection, req) except Exception as err_msg: try: error_msg = str(err_msg.args[0]).split("] ")[1] + "." except IndexError: error_msg = str(err_msg.args[0]) + "." if settings.INIT_TEST == True: if settings.VERBOSITY_LEVEL < 2: print(settings.FAIL_STATUS) else: if settings.VERBOSITY_LEVEL < 1: print("") print(settings.print_critical_msg(error_msg)) raise SystemExit() if settings.REVERSE_TCP == False and settings.BIND_TCP == False: opener = _urllib.request.build_opener(connection_handler()) response = False current_attempt = 0 unauthorized = False while not response and current_attempt <= settings.MAX_RETRIES and not unauthorized: if settings.VERBOSITY_LEVEL >= 2: req_msg = "HTTP request:" print(settings.print_request_msg(req_msg)) try: opener.open(request) response = True if settings.VERBOSITY_LEVEL < 2: if current_attempt != 0: info_msg = "Checking connection to the target URL. " sys.stdout.write(settings.print_info_msg(info_msg)) sys.stdout.flush() if settings.INIT_TEST == True and not settings.UNAUTHORIZED: print(settings.SUCCESS_STATUS) if not settings.CHECK_INTERNET: settings.INIT_TEST = False except _urllib.error.HTTPError as err_msg: if settings.UNAUTHORIZED_ERROR in str(err_msg): if settings.VERBOSITY_LEVEL < 2 and not settings.UNAUTHORIZED: print(settings.FAIL_STATUS) settings.UNAUTHORIZED = unauthorized = True except _urllib.error.URLError as err_msg: if current_attempt == 0: if settings.VERBOSITY_LEVEL < 2: print(settings.FAIL_STATUS) try: error_msg = str(err_msg.args[0]).split("] ")[1] + ". " except IndexError: error_msg = "" error_msg += "Please wait while retring the request(s)." print(settings.print_critical_msg(error_msg)) warn_msg = "In case the provided target URL is valid, try to rerun with" warn_msg += " the switch '--random-agent' and/or proxy switch." print(settings.print_warning_msg(warn_msg)) if settings.VERBOSITY_LEVEL >= 2 or current_attempt == 1: info_msg = "Please wait while retring the request(s)." print(settings.print_info_msg(info_msg)) current_attempt = current_attempt + 1 time.sleep(3) except _http_client.BadStatusLine as err_msg: if settings.VERBOSITY_LEVEL < 2: print(settings.FAIL_STATUS) if len(err_msg.line) > 2: print(err_msg.line, err_msg.message) raise SystemExit() except ValueError as err: if settings.VERBOSITY_LEVEL < 2: print(settings.FAIL_STATUS) err_msg = "Invalid target URL has been given." print(settings.print_critical_msg(err_msg)) raise SystemExit() except AttributeError: raise SystemExit() try: response = _urllib.request.urlopen(request) code = response.getcode() # Check the HTTP response headers. response_headers = response.info() page = response.read() try: # Fix for Python 2.7 page = page.encode(settings.DEFAULT_ENCODING) except AttributeError: pass if response_headers.get('Content-Encoding') == 'gzip': page = gzip.GzipFile("", "rb", 9, io.BytesIO(page)).read() request.add_header('Accept-Encoding', 'deflate') if len(settings.ENCODING) != 0: page = page.decode(settings.ENCODING) else: if type(page) != str: page = page.decode(settings.DEFAULT_ENCODING) response_headers[settings.URI_HTTP_HEADER] = response.geturl() response_headers = str(response_headers).strip("\n") if settings.VERBOSITY_LEVEL > 2: print_http_response(response_headers, code, page) # Checks regarding a potential CAPTCHA protection mechanism. checks.captcha_check(page) # Checks regarding a potential browser verification protection mechanism. checks.browser_verification(page) # Checks regarding recognition of generic "your ip has been blocked" messages. checks.blocked_ip(page) # This is useful when handling exotic HTTP errors (i.e requests for authentication). except _urllib.error.HTTPError as err: if settings.VERBOSITY_LEVEL > 2: print_http_response(err.info(), err.code, err.read()) error_msg = "Got " + str(err).replace(": ", " (") # Check for 4xx and/or 5xx HTTP error codes. if str(err.code).startswith('4') or \ str(err.code).startswith('5'): if settings.VERBOSITY_LEVEL > 1: if len(str(err).split(": ")[1]) == 0: error_msg = error_msg + "Non-standard HTTP status code" warn_msg = error_msg print(settings.print_warning_msg(warn_msg + ").")) pass else: error_msg = str(err).replace(": ", " (") if len(str(err).split(": ")[1]) == 0: err_msg = error_msg + "Non-standard HTTP status code" else: err_msg = error_msg print(settings.print_critical_msg(err_msg + ").")) raise SystemExit() # The handlers raise this exception when they run into a problem. except (socket.error, _http_client.HTTPException, _urllib.error.URLError) as err: if settings.VERBOSITY_LEVEL > 2: print_http_response(response_headers=err.info(), code=err.code, page=err.read()) err_msg = "Unable to connect to the target URL" try: err_msg += " (" + str(err.args[0]).split("] ")[1] + ")." except IndexError: err_msg += "." print(settings.print_critical_msg(err_msg)) raise SystemExit() except _http_client.IncompleteRead as err: print(settings.print_critical_msg(str(err))) raise SystemExit() except UnicodeDecodeError as err: print(settings.print_critical_msg(str(err))) raise SystemExit() except LookupError as err: print(settings.print_critical_msg(str(err))) raise SystemExit() # Raise exception regarding existing connection was forcibly closed by the remote host. except SocketError as err: if err.errno == errno.ECONNRESET: error_msg = "Connection reset by peer." print(settings.print_critical_msg(error_msg)) elif err.errno == errno.ECONNREFUSED: error_msg = "Connection refused." print(settings.print_critical_msg(error_msg)) raise SystemExit()
def check_http_traffic(request): settings.TOTAL_OF_REQUESTS = settings.TOTAL_OF_REQUESTS + 1 # Delay in seconds between each HTTP request time.sleep(int(settings.DELAY)) if settings.SCHEME == 'https': http_client = _http_client.HTTPSConnection else: http_client = _http_client.HTTPConnection class connection(http_client): def send(self, req): headers = req.decode() request_http_headers = str(headers).split("\r\n") unique_request_http_headers = [] [ unique_request_http_headers.append(item) for item in request_http_headers if item not in unique_request_http_headers ] request_http_headers = unique_request_http_headers for header in request_http_headers: if settings.VERBOSITY_LEVEL >= 2: print(settings.print_traffic(header)) if menu.options.traffic_file: logs.log_traffic("\n" + header) http_client.send(self, req) class connection_handler(_urllib.request.HTTPSHandler, _urllib.request.HTTPHandler, object): def http_open(self, req): try: self.do_open(connection, req) return super(connection_handler, self).http_open(req) except (_urllib.error.HTTPError, _urllib.error.URLError) as err_msg: try: error_msg = str(err_msg.args[0]).split("] ")[1] + "." except IndexError: error_msg = str(err_msg.args[0]) + "." error_msg = "Connection to the target URL " + error_msg except _http_client.InvalidURL as err_msg: settings.VALID_URL = False error_msg = err_msg if current_attempt == 0 and settings.VERBOSITY_LEVEL < 2: print(settings.SINGLE_WHITESPACE) print(settings.print_critical_msg(error_msg)) if not settings.VALID_URL: raise SystemExit() def https_open(self, req): try: self.do_open(connection, req) return super(connection_handler, self).https_open(req) except (_urllib.error.HTTPError, _urllib.error.URLError) as err_msg: try: error_msg = str(err_msg.args[0]).split("] ")[1] + "." except IndexError: error_msg = str(err_msg.args[0]) + "." error_msg = "Connection to the target URL " + error_msg except _http_client.InvalidURL as err_msg: settings.VALID_URL = False error_msg = err_msg if current_attempt == 0 and settings.VERBOSITY_LEVEL < 2: print(settings.SINGLE_WHITESPACE) print(settings.print_critical_msg(error_msg)) if not settings.VALID_URL: raise SystemExit() opener = _urllib.request.build_opener(connection_handler()) if len(settings.HTTP_METHOD) != 0: request.get_method = lambda: settings.HTTP_METHOD _ = False current_attempt = 0 unauthorized = False while not _ and current_attempt <= settings.MAX_RETRIES and unauthorized is False: if settings.VERBOSITY_LEVEL >= 2 or menu.options.traffic_file: if settings.VERBOSITY_LEVEL >= 2: req_msg = "HTTP request [" + settings.print_request_num( settings.TOTAL_OF_REQUESTS) + "]:" print(settings.print_request_msg(req_msg)) if menu.options.traffic_file: req_msg = "HTTP request [#" + str( settings.TOTAL_OF_REQUESTS) + "]:" logs.log_traffic(req_msg) try: response = opener.open(request, timeout=settings.TIMEOUT) page = checks.page_encoding(response, action="encode") _ = True if settings.VERBOSITY_LEVEL < 2: if current_attempt != 0: info_msg = "Testing connection to the target URL." sys.stdout.write(settings.print_info_msg(info_msg)) sys.stdout.flush() if settings.INIT_TEST == True and not settings.UNAUTHORIZED: print(settings.SINGLE_WHITESPACE) if not settings.CHECK_INTERNET: settings.INIT_TEST = False except _urllib.error.HTTPError as err_msg: if settings.UNAUTHORIZED_ERROR in str(err_msg): settings.UNAUTHORIZED = unauthorized = True if [ True for err_code in settings.HTTP_ERROR_CODES if err_code in str(err_msg) ]: break except (_urllib.error.URLError, _http_client.BadStatusLine, _http_client.IncompleteRead) as err_msg: if current_attempt == 0: if settings.VERBOSITY_LEVEL < 2 and "has closed the connection" in str( err_msg): print(settings.SINGLE_WHITESPACE) if "IncompleteRead" in str(err_msg): warn_msg = "There was an incomplete read error while retrieving data " warn_msg += "from the target URL " else: warn_msg = "The provided target URL seems not reachable. " warn_msg += "In case that it is, please try to re-run using " if not menu.options.random_agent: warn_msg += "'--random-agent' switch and/or " warn_msg += "'--proxy' option." print(settings.print_warning_msg(warn_msg)) info_msg = settings.APPLICATION.capitalize( ) + " is going to retry the request(s)." print(settings.print_info_msg(info_msg)) current_attempt = current_attempt + 1 time.sleep(3) except ValueError as err: if settings.VERBOSITY_LEVEL < 2: print(settings.SINGLE_WHITESPACE) err_msg = "Invalid target URL has been given." print(settings.print_critical_msg(err_msg)) raise SystemExit() except AttributeError: raise SystemExit() try: response = _urllib.request.urlopen(request, timeout=settings.TIMEOUT) code = response.getcode() response_headers = response.info() page = checks.page_encoding(response, action="encode") response_headers[settings.URI_HTTP_HEADER] = response.geturl() response_headers = str(response_headers).strip("\n") if settings.VERBOSITY_LEVEL > 2 or menu.options.traffic_file: print_http_response(response_headers, code, page) # Checks regarding a potential CAPTCHA protection mechanism. checks.captcha_check(page) # Checks regarding a potential browser verification protection mechanism. checks.browser_verification(page) # Checks regarding recognition of generic "your ip has been blocked" messages. checks.blocked_ip(page) # Checks for not declared cookie(s), while server wants to set its own. if menu.options.cookie == None and not menu.options.drop_set_cookie: checks.not_declared_cookies(response) # This is useful when handling exotic HTTP errors (i.e requests for authentication). except _urllib.error.HTTPError as err: if settings.VERBOSITY_LEVEL != 0: print_http_response(err.info(), err.code, err.read()) if not settings.PERFORM_CRACKING and \ not settings.IS_JSON and \ not settings.IS_XML and \ not str(err.code) == settings.INTERNAL_SERVER_ERROR and \ not str(err.code) == settings.BAD_REQUEST: print(settings.SINGLE_WHITESPACE) # error_msg = "Got " + str(err).replace(": "," (") # Check for 3xx, 4xx, 5xx HTTP error codes. if str(err.code).startswith(('3', '4', '5')): if settings.VERBOSITY_LEVEL >= 2: if len(str(err).split(": ")[1]) == 0: error_msg = "Non-standard HTTP status code" pass else: error_msg = str(err).replace(": ", " (") if len(str(err).split(": ")[1]) == 0: err_msg = error_msg + "Non-standard HTTP status code" else: err_msg = error_msg print(settings.print_critical_msg(err_msg + ").")) raise SystemExit() # The handlers raise this exception when they run into a problem. except (_http_client.HTTPException, _urllib.error.URLError, _http_client.IncompleteRead) as err: if any(_ in str(err) for _ in ("timed out", "IncompleteRead", "Interrupted system call")): pass else: err_msg = "Unable to connect to the target URL" try: err_msg += " (Reason: " + str( err.args[0]).split("] ")[-1].lower() + ")." except IndexError: err_msg += "." if menu.options.bulkfile: raise else: print(settings.print_critical_msg(err_msg)) raise SystemExit() # Raise exception regarding existing connection was forcibly closed by the remote host. except SocketError as err: if err.errno == errno.ECONNRESET: error_msg = "Connection reset by peer." print(settings.print_critical_msg(error_msg)) elif err.errno == errno.ECONNREFUSED: error_msg = "Connection refused." print(settings.print_critical_msg(error_msg)) raise SystemExit()
def check_http_traffic(request): settings.TOTAL_OF_REQUESTS = settings.TOTAL_OF_REQUESTS + 1 # Delay in seconds between each HTTP request time.sleep(int(settings.DELAY)) if settings.SCHEME == 'https': http_client = _http_client.HTTPSConnection else: http_client = _http_client.HTTPConnection class connection(http_client): def send(self, req): headers = req.decode() request_http_headers = str(headers).split("\r\n") unique_request_http_headers = [] [ unique_request_http_headers.append(item) for item in request_http_headers if item not in unique_request_http_headers ] request_http_headers = unique_request_http_headers for header in request_http_headers: if settings.VERBOSITY_LEVEL >= 2: print(settings.print_traffic(header)) if menu.options.traffic_file: logs.log_traffic("\n" + header) http_client.send(self, req) class connection_handler(_urllib.request.HTTPSHandler, _urllib.request.HTTPHandler, object): def http_open(self, req): try: self.do_open(connection, req) return super(connection_handler, self).http_open(req) except (_urllib.error.HTTPError, _urllib.error.URLError) as err_msg: try: error_msg = str(err_msg.args[0]).split("] ")[1] + "." except IndexError: error_msg = str(err_msg.args[0]) + "." error_msg = "Connection to the target URL " + error_msg if current_attempt == 0 and settings.VERBOSITY_LEVEL <= 1: print("") print(settings.print_critical_msg(error_msg)) def https_open(self, req): try: self.do_open(connection, req) return super(connection_handler, self).https_open(req) except (_urllib.error.HTTPError, _urllib.error.URLError) as err_msg: try: error_msg = str(err_msg.args[0]).split("] ")[1] + "." except IndexError: error_msg = str(err_msg.args[0]) + "." error_msg = "Connection to the target URL " + error_msg if current_attempt == 0 and settings.VERBOSITY_LEVEL < 1: print("") print(settings.print_critical_msg(error_msg)) if settings.REVERSE_TCP == False and settings.BIND_TCP == False: opener = _urllib.request.build_opener(connection_handler()) response = False current_attempt = 0 unauthorized = False while not response and current_attempt <= settings.MAX_RETRIES and unauthorized is False: if settings.VERBOSITY_LEVEL >= 2 or menu.options.traffic_file: if settings.VERBOSITY_LEVEL >= 2: req_msg = "HTTP request [" + settings.print_request_num( settings.TOTAL_OF_REQUESTS) + "]:" print(settings.print_request_msg(req_msg)) if menu.options.traffic_file: req_msg = "HTTP request [#" + str( settings.TOTAL_OF_REQUESTS) + "]:" logs.log_traffic(req_msg) try: opener.open(request, timeout=settings.TIMEOUT) response = True if settings.VERBOSITY_LEVEL < 2: if current_attempt != 0: info_msg = "Testing connection to the target URL." sys.stdout.write(settings.print_info_msg(info_msg)) sys.stdout.flush() if settings.INIT_TEST == True and not settings.UNAUTHORIZED: print(settings.SUCCESS_STATUS) if not settings.CHECK_INTERNET: settings.INIT_TEST = False except _urllib.error.HTTPError as err_msg: if settings.UNAUTHORIZED_ERROR in str(err_msg): if settings.VERBOSITY_LEVEL < 2 and not settings.UNAUTHORIZED: print(settings.FAIL_STATUS) settings.UNAUTHORIZED = unauthorized = True http_errors = [settings.BAD_REQUEST, settings.FORBIDDEN_ERROR, settings.NOT_FOUND_ERROR,\ settings.NOT_ACCEPTABLE_ERROR, settings.INTERNAL_SERVER_ERROR] if [ True for err_code in http_errors if err_code in str(err_msg) ]: break except _urllib.error.URLError as err_msg: if current_attempt == 0: warn_msg = "The provided target URL seems not reachable. " warn_msg += "In case that it is, please try to re-run using " if not menu.options.random_agent: warn_msg += "'--random-agent' switch and/or " warn_msg += "'--proxy' option." print(settings.print_warning_msg(warn_msg)) if settings.VERBOSITY_LEVEL >= 1: debug_msg = settings.APPLICATION + " is going to retry the request(s)." print(settings.print_debug_msg(debug_msg)) current_attempt = current_attempt + 1 time.sleep(3) except _http_client.BadStatusLine as err_msg: if settings.VERBOSITY_LEVEL < 2: print(settings.FAIL_STATUS) if len(err_msg.line) > 2: print(err_msg.line, err_msg.message) raise SystemExit() except ValueError as err: if settings.VERBOSITY_LEVEL < 2: print(settings.FAIL_STATUS) err_msg = "Invalid target URL has been given." print(settings.print_critical_msg(err_msg)) raise SystemExit() except AttributeError: raise SystemExit() try: response = _urllib.request.urlopen(request, timeout=settings.TIMEOUT) code = response.getcode() # Check the HTTP response headers. response_headers = response.info() page = response.read() try: # Fix for Python 2.7 page = page.encode(settings.DEFAULT_ENCODING) except (UnicodeDecodeError, AttributeError) as err: pass if response_headers.get('Content-Encoding') == 'gzip': page = gzip.GzipFile("", "rb", 9, io.BytesIO(page)).read() request.add_header('Accept-Encoding', 'deflate') if len(settings.ENCODING) != 0: page = page.decode(settings.ENCODING) else: if type(page) != str: page = page.decode(settings.DEFAULT_ENCODING) response_headers[settings.URI_HTTP_HEADER] = response.geturl() response_headers = str(response_headers).strip("\n") if settings.VERBOSITY_LEVEL > 2 or menu.options.traffic_file: print_http_response(response_headers, code, page) # Checks regarding a potential CAPTCHA protection mechanism. checks.captcha_check(page) # Checks regarding a potential browser verification protection mechanism. checks.browser_verification(page) # Checks regarding recognition of generic "your ip has been blocked" messages. checks.blocked_ip(page) # This is useful when handling exotic HTTP errors (i.e requests for authentication). except _urllib.error.HTTPError as err: if settings.VERBOSITY_LEVEL > 2: print_http_response(err.info(), err.code, err.read()) error_msg = "Got " + str(err).replace(": ", " (") # Check for 4xx and/or 5xx HTTP error codes. if str(err.code).startswith('4') or \ str(err.code).startswith('5'): if settings.VERBOSITY_LEVEL > 1: if len(str(err).split(": ")[1]) == 0: error_msg = error_msg + "Non-standard HTTP status code" warn_msg = error_msg print(settings.print_warning_msg(warn_msg + ").")) pass else: error_msg = str(err).replace(": ", " (") if len(str(err).split(": ")[1]) == 0: err_msg = error_msg + "Non-standard HTTP status code" else: err_msg = error_msg print(settings.print_critical_msg(err_msg + ").")) raise SystemExit() except (UnicodeDecodeError, LookupError) as err: pass # The handlers raise this exception when they run into a problem. except (_http_client.HTTPException, _urllib.error.URLError, _http_client.IncompleteRead) as err: # if settings.VERBOSITY_LEVEL > 2: # print_http_response(response_headers=err.info(), code=err.code, page=err.read()) err_msg = "Unable to connect to the target URL" try: err_msg += " (" + str(err.args[0]).split("] ")[1] + ")." except IndexError: err_msg += "." print(settings.print_critical_msg(err_msg)) raise SystemExit() # Raise exception regarding existing connection was forcibly closed by the remote host. except SocketError as err: if err.errno == errno.ECONNRESET: error_msg = "Connection reset by peer." print(settings.print_critical_msg(error_msg)) elif err.errno == errno.ECONNREFUSED: error_msg = "Connection refused." print(settings.print_critical_msg(error_msg)) raise SystemExit()