def test_normalize_path_with_querystring_no_filename(self): url_0 = URL('https://w3af.org/?id=1') normalized_path_0 = FourOhFourResponse.normalize_path(url_0) url_1 = URL('https://w3af.org/?id=3') normalized_path_1 = FourOhFourResponse.normalize_path(url_1) self.assertEqual(normalized_path_0, normalized_path_1)
def test_normalize_path_filenames(self): url_0 = URL('https://w3af.org/assets/uploads/2015/09/index.php') normalized_path_0 = FourOhFourResponse.normalize_path(url_0) url_1 = URL('https://w3af.org/assets/uploads/2015/09/nidex.php') normalized_path_1 = FourOhFourResponse.normalize_path(url_1) url_2 = URL('https://w3af.org/assets/uploads/2015/09/hppvresion.php') normalized_path_2 = FourOhFourResponse.normalize_path(url_2) self.assertEqual(normalized_path_0, normalized_path_1) self.assertEqual(normalized_path_1, normalized_path_2)
def test_msgpack_transformations(self): url = URL('http://w3af.com') headers = Headers([('Content-Type', 'text/html')]) body = '' http_response = HTTPResponse(200, body, headers, url, url) clean_response = FourOhFourResponse.from_http_response(http_response) clean_response.diff = '' clean_response.diff_with_id = 1 clean_response_serialized = clean_response.dumps() clean_response_from_msgpack = FourOhFourResponse.loads( clean_response_serialized) self.assertEqual(clean_response, clean_response_from_msgpack)
def test_dict_transformations(self): url = URL('http://w3af.com') headers = Headers([('Content-Type', 'text/html')]) body = '' http_response = HTTPResponse(200, body, headers, url, url) clean_response = FourOhFourResponse.from_http_response(http_response) clean_response.diff = '' clean_response.diff_with_id = 1 clean_response_dict = clean_response.to_dict() clean_response_from_dict = FourOhFourResponse.from_dict( clean_response_dict) self.assertEqual(clean_response, clean_response_from_dict)
def get_response_cache_key(http_response, clean_response=None, headers=None): """ Note: query.body has been cleaned by get_clean_body() :param http_response: The HTTP response we want to get a cache key for :param clean_response: The FourOhFourResponse associated with the HTTPResponse passed as parameter (optional, will be calculated if not provided) :param headers: A string containing the HTTP response headers that have to be used to calculate the hash :return: Hash of the HTTP response body """ headers = '' or headers # # Only some HTTP responses benefit from the XML-bones signature # if _should_use_xml_bones(http_response): body = get_xml_bones(http_response.get_body()) normalized_path = FourOhFourResponse.normalize_path( http_response.get_uri()) else: # # Get a clean_response if it was not provided # if clean_response is None: clean_response = FourOhFourResponse.from_http_response( http_response) body = clean_response.body normalized_path = clean_response.normalized_path # # Calculate the hash using all the captured information # key = ''.join([ str(http_response.get_code()), smart_str_ignore(normalized_path), str(headers), smart_str_ignore(body) ]) return quick_hash(key)
def _is_404_complex(self, http_response): # 404_body stored in the DB was cleaned when creating the # FourOhFourResponse class. # # Clean the body received as parameter in order to have a fair # comparison query = FourOhFourResponse.from_http_response(http_response) return self._is_404_complex_impl(http_response, query)
def send_request_generate_404(uri_opener, http_response, debugging_id, exclude=None): exclude = [] if exclude is None else exclude # # Make sure to generate a URL which is not in exclude! # url_404 = None for seed in xrange(25): url_404 = get_url_for_404_request(http_response, seed=seed) if url_404.url_string not in exclude: break response_404 = send_404(uri_opener, url_404, debugging_id=debugging_id) return FourOhFourResponse.from_http_response(response_404)
def _get_404_response(self, http_response, query, debugging_id): """ :return: A FourOhFourResponse instance. * First try to get the response from the 404 DB * If the data is not there then send an HTTP request with a randomly generated path or name to force a 404, save the data to the DB and then return it. """ serialized_known_404 = self._404_responses.get(query.normalized_path, None) if serialized_known_404 is not None: return FourOhFourResponse.loads(serialized_known_404) known_404 = send_request_generate_404(self._uri_opener, http_response, debugging_id) self._404_responses[query.normalized_path] = known_404.dumps() return known_404
def test_normalize_path_paths(self): url = URL('https://w3af.org/a/b/c/') normalized_path = FourOhFourResponse.normalize_path(url) self.assertEqual(normalized_path, 'https://w3af.org/a/b/path/')
def get_call_key(self, http_response): return FourOhFourResponse.normalize_path(http_response.get_uri())
def _is_404_complex(self, http_response): """ Verifies if the response is a 404 by comparing it with other responses which are known to be 404s, potentially sends HTTP requests to the server. :param http_response: The HTTP response :return: True if the HTTP response is a 404 """ response_did = http_response.get_debugging_id() debugging_id = response_did if response_did is not None else rand_alnum( 8) # 404_body stored in the DB was cleaned when creating the # FourOhFourResponse class. # # Clean the body received as parameter in order to have a fair # comparison query = FourOhFourResponse(http_response) # # Compare query with a known 404 from the DB (or a generated one # if there is none with the same path in the DB) # known_404 = self._get_404_response(http_response, query, debugging_id) # Trivial performance improvement that prevents running fuzzy_equal if query.code in NOT_404_RESPONSE_CODES and known_404.code == 404: msg = ('"%s" (id:%s, code:%s, len:%s, did:%s) is NOT a 404' ' [known 404 with ID %s uses 404 code]') args = (http_response.get_url(), http_response.id, http_response.get_code(), len(http_response.get_body()), debugging_id, known_404.id) om.out.debug(msg % args) return False # Since the fuzzy_equal function is CPU-intensive we want to # avoid calling it for cases where we know it won't match, for # example in comparing an image and an html if query.doc_type != known_404.doc_type: msg = ('"%s" (id:%s, code:%s, len:%s, did:%s) is NOT a 404' ' [document type mismatch with known 404 with ID %s]') args = (http_response.get_url(), http_response.id, http_response.get_code(), len(http_response.get_body()), debugging_id, known_404.id) om.out.debug(msg % args) return False # This is the simplest case. If they are 100% equal, no matter how # large or complex the responses are, then query is a 404 if known_404.body == query.body: msg = ('"%s" (id:%s, code:%s, len:%s, did:%s) is a 404' ' [string equals with 404 DB entry with ID %s]') args = (http_response.get_url(), http_response.id, http_response.get_code(), len(http_response.get_body()), debugging_id, known_404.id) om.out.debug(msg % args) return True is_fuzzy_equal = fuzzy_equal(known_404.body, query.body, IS_EQUAL_RATIO) if not is_fuzzy_equal: msg = ('"%s" (id:%s, code:%s, len:%s, did:%s) is NOT a 404' ' [similarity_ratio < %s with known 404 with ID %s]') args = (http_response.get_url(), http_response.id, http_response.get_code(), len(http_response.get_body()), debugging_id, IS_EQUAL_RATIO, known_404.id) om.out.debug(msg % args) return False if len(query.body) < MAX_FUZZY_LENGTH: # The response bodies are fuzzy-equal, and the length is less than # MAX_FUZZY_LENGTH. This is good, it means that they are equal and # long headers / footers in HTTP response bodies are not # interfering with fuzzy-equals. # # Some sites have really large headers and footers which they # include for all pages, including 404s. When that happens one page # might look like: # # {header-4000bytes} # Hello world # {footer-4000bytes} # # The header might contain large CSS and the footer might include # JQuery or some other large JS. Then, the 404 might look like: # # {header-4000bytes} # Not found # {footer-4000bytes} # # A user with a browser might only see the text, and clearly # identify one as a valid page and another as a 404, but the # fuzzy_equal() function will return True, indicating that they # are equal because 99% of the bytes are the same. msg = ('"%s" (id:%s, code:%s, len:%s, did:%s) is a 404' ' [similarity_ratio > %s with 404 DB entry with ID %s]') args = (http_response.get_url(), http_response.id, http_response.get_code(), len(http_response.get_body()), debugging_id, IS_EQUAL_RATIO, known_404.id) om.out.debug(msg % args) return True else: # See the large comment above on why we need to check for # MAX_FUZZY_LENGTH. # # The way to handle this case is to send an extra HTTP # request that will act as a tie-breaker. return self._handle_large_http_responses(http_response, query, known_404, debugging_id)
def get_call_key(self, args): http_response = args[1] return FourOhFourResponse.normalize_path(http_response.get_uri())
def get_call_key(self, args): http_response = args[1] return FourOhFourResponse.normalize_path(http_response.get_uri())