def test_normalize_path_with_querystring_no_filename(self):
url_0 = URL('https://w3af.org/?id=1')
normalized_path_0 = FourOhFourResponse.normalize_path(url_0)
url_1 = URL('https://w3af.org/?id=3')
normalized_path_1 = FourOhFourResponse.normalize_path(url_1)
self.assertEqual(normalized_path_0, normalized_path_1)
def test_normalize_path_filenames(self):
url_0 = URL('https://w3af.org/assets/uploads/2015/09/index.php')
normalized_path_0 = FourOhFourResponse.normalize_path(url_0)
url_1 = URL('https://w3af.org/assets/uploads/2015/09/nidex.php')
normalized_path_1 = FourOhFourResponse.normalize_path(url_1)
url_2 = URL('https://w3af.org/assets/uploads/2015/09/hppvresion.php')
normalized_path_2 = FourOhFourResponse.normalize_path(url_2)
self.assertEqual(normalized_path_0, normalized_path_1)
self.assertEqual(normalized_path_1, normalized_path_2)
def test_msgpack_transformations(self):
url = URL('http://w3af.com')
headers = Headers([('Content-Type', 'text/html')])
body = ''
http_response = HTTPResponse(200, body, headers, url, url)
clean_response = FourOhFourResponse.from_http_response(http_response)
clean_response.diff = ''
clean_response.diff_with_id = 1
clean_response_serialized = clean_response.dumps()
clean_response_from_msgpack = FourOhFourResponse.loads(
clean_response_serialized)
self.assertEqual(clean_response, clean_response_from_msgpack)
def test_dict_transformations(self):
url = URL('http://w3af.com')
headers = Headers([('Content-Type', 'text/html')])
body = ''
http_response = HTTPResponse(200, body, headers, url, url)
clean_response = FourOhFourResponse.from_http_response(http_response)
clean_response.diff = ''
clean_response.diff_with_id = 1
clean_response_dict = clean_response.to_dict()
clean_response_from_dict = FourOhFourResponse.from_dict(
clean_response_dict)
self.assertEqual(clean_response, clean_response_from_dict)
def get_response_cache_key(http_response, clean_response=None, headers=None):
"""
Note: query.body has been cleaned by get_clean_body()
:param http_response: The HTTP response we want to get a cache key for
:param clean_response: The FourOhFourResponse associated with the HTTPResponse
passed as parameter (optional, will be calculated if not
provided)
:param headers: A string containing the HTTP response headers that have to be
used to calculate the hash
:return: Hash of the HTTP response body
"""
headers = '' or headers
#
# Only some HTTP responses benefit from the XML-bones signature
#
if _should_use_xml_bones(http_response):
body = get_xml_bones(http_response.get_body())
normalized_path = FourOhFourResponse.normalize_path(
http_response.get_uri())
else:
#
# Get a clean_response if it was not provided
#
if clean_response is None:
clean_response = FourOhFourResponse.from_http_response(
http_response)
body = clean_response.body
normalized_path = clean_response.normalized_path
#
# Calculate the hash using all the captured information
#
key = ''.join([
str(http_response.get_code()),
smart_str_ignore(normalized_path),
str(headers),
smart_str_ignore(body)
])
return quick_hash(key)
def _is_404_complex(self, http_response):
# 404_body stored in the DB was cleaned when creating the
# FourOhFourResponse class.
#
# Clean the body received as parameter in order to have a fair
# comparison
query = FourOhFourResponse.from_http_response(http_response)
return self._is_404_complex_impl(http_response, query)
def send_request_generate_404(uri_opener,
http_response,
debugging_id,
exclude=None):
exclude = [] if exclude is None else exclude
#
# Make sure to generate a URL which is not in exclude!
#
url_404 = None
for seed in xrange(25):
url_404 = get_url_for_404_request(http_response, seed=seed)
if url_404.url_string not in exclude:
break
response_404 = send_404(uri_opener, url_404, debugging_id=debugging_id)
return FourOhFourResponse.from_http_response(response_404)
def _get_404_response(self, http_response, query, debugging_id):
"""
:return: A FourOhFourResponse instance.
* First try to get the response from the 404 DB
* If the data is not there then send an HTTP request
with a randomly generated path or name to force a 404,
save the data to the DB and then return it.
"""
serialized_known_404 = self._404_responses.get(query.normalized_path,
None)
if serialized_known_404 is not None:
return FourOhFourResponse.loads(serialized_known_404)
known_404 = send_request_generate_404(self._uri_opener, http_response,
debugging_id)
self._404_responses[query.normalized_path] = known_404.dumps()
return known_404
def test_normalize_path_paths(self):
url = URL('https://w3af.org/a/b/c/')
normalized_path = FourOhFourResponse.normalize_path(url)
self.assertEqual(normalized_path, 'https://w3af.org/a/b/path/')
def get_call_key(self, http_response):
return FourOhFourResponse.normalize_path(http_response.get_uri())
def _is_404_complex(self, http_response):
"""
Verifies if the response is a 404 by comparing it with other responses
which are known to be 404s, potentially sends HTTP requests to the
server.
:param http_response: The HTTP response
:return: True if the HTTP response is a 404
"""
response_did = http_response.get_debugging_id()
debugging_id = response_did if response_did is not None else rand_alnum(
8)
# 404_body stored in the DB was cleaned when creating the
# FourOhFourResponse class.
#
# Clean the body received as parameter in order to have a fair
# comparison
query = FourOhFourResponse(http_response)
#
# Compare query with a known 404 from the DB (or a generated one
# if there is none with the same path in the DB)
#
known_404 = self._get_404_response(http_response, query, debugging_id)
# Trivial performance improvement that prevents running fuzzy_equal
if query.code in NOT_404_RESPONSE_CODES and known_404.code == 404:
msg = ('"%s" (id:%s, code:%s, len:%s, did:%s) is NOT a 404'
' [known 404 with ID %s uses 404 code]')
args = (http_response.get_url(), http_response.id,
http_response.get_code(), len(http_response.get_body()),
debugging_id, known_404.id)
om.out.debug(msg % args)
return False
# Since the fuzzy_equal function is CPU-intensive we want to
# avoid calling it for cases where we know it won't match, for
# example in comparing an image and an html
if query.doc_type != known_404.doc_type:
msg = ('"%s" (id:%s, code:%s, len:%s, did:%s) is NOT a 404'
' [document type mismatch with known 404 with ID %s]')
args = (http_response.get_url(), http_response.id,
http_response.get_code(), len(http_response.get_body()),
debugging_id, known_404.id)
om.out.debug(msg % args)
return False
# This is the simplest case. If they are 100% equal, no matter how
# large or complex the responses are, then query is a 404
if known_404.body == query.body:
msg = ('"%s" (id:%s, code:%s, len:%s, did:%s) is a 404'
' [string equals with 404 DB entry with ID %s]')
args = (http_response.get_url(), http_response.id,
http_response.get_code(), len(http_response.get_body()),
debugging_id, known_404.id)
om.out.debug(msg % args)
return True
is_fuzzy_equal = fuzzy_equal(known_404.body, query.body,
IS_EQUAL_RATIO)
if not is_fuzzy_equal:
msg = ('"%s" (id:%s, code:%s, len:%s, did:%s) is NOT a 404'
' [similarity_ratio < %s with known 404 with ID %s]')
args = (http_response.get_url(), http_response.id,
http_response.get_code(), len(http_response.get_body()),
debugging_id, IS_EQUAL_RATIO, known_404.id)
om.out.debug(msg % args)
return False
if len(query.body) < MAX_FUZZY_LENGTH:
# The response bodies are fuzzy-equal, and the length is less than
# MAX_FUZZY_LENGTH. This is good, it means that they are equal and
# long headers / footers in HTTP response bodies are not
# interfering with fuzzy-equals.
#
# Some sites have really large headers and footers which they
# include for all pages, including 404s. When that happens one page
# might look like:
#
# {header-4000bytes}
# Hello world
# {footer-4000bytes}
#
# The header might contain large CSS and the footer might include
# JQuery or some other large JS. Then, the 404 might look like:
#
# {header-4000bytes}
# Not found
# {footer-4000bytes}
#
# A user with a browser might only see the text, and clearly
# identify one as a valid page and another as a 404, but the
# fuzzy_equal() function will return True, indicating that they
# are equal because 99% of the bytes are the same.
msg = ('"%s" (id:%s, code:%s, len:%s, did:%s) is a 404'
' [similarity_ratio > %s with 404 DB entry with ID %s]')
args = (http_response.get_url(), http_response.id,
http_response.get_code(), len(http_response.get_body()),
debugging_id, IS_EQUAL_RATIO, known_404.id)
om.out.debug(msg % args)
return True
else:
# See the large comment above on why we need to check for
# MAX_FUZZY_LENGTH.
#
# The way to handle this case is to send an extra HTTP
# request that will act as a tie-breaker.
return self._handle_large_http_responses(http_response, query,
known_404, debugging_id)
def get_call_key(self, args):
http_response = args[1]
return FourOhFourResponse.normalize_path(http_response.get_uri())
def get_call_key(self, args):
http_response = args[1]
return FourOhFourResponse.normalize_path(http_response.get_uri())