class TestXUrllibTimeout(unittest.TestCase):
def setUp(self):
self.uri_opener = ExtendedUrllib()
def tearDown(self):
self.uri_opener.end()
def test_timeout(self):
upper_daemon = UpperDaemon(TimeoutTCPHandler)
upper_daemon.start()
upper_daemon.wait_for_start()
port = upper_daemon.get_port()
url = URL('http://127.0.0.1:%s/' % port)
self.uri_opener.settings.set_configured_timeout(0.5)
self.uri_opener.clear_timeout()
# We can mock this because it's being tested at TestXUrllibDelayOnError
self.uri_opener._pause_on_http_error = Mock()
start = time.time()
try:
self.uri_opener.GET(url)
except HTTPRequestException, hre:
self.assertEqual(hre.message, 'HTTP timeout error')
except Exception, e:
msg = 'Not expecting: "%s"'
self.assertTrue(False, msg % e.__class__.__name__)
class TestXUrllibTimeout(unittest.TestCase):
def setUp(self):
self.uri_opener = ExtendedUrllib()
def tearDown(self):
self.uri_opener.end()
def test_timeout(self):
upper_daemon = UpperDaemon(TimeoutTCPHandler)
upper_daemon.start()
upper_daemon.wait_for_start()
port = upper_daemon.get_port()
url = URL('http://127.0.0.1:%s/' % port)
self.uri_opener.settings.set_configured_timeout(0.5)
self.uri_opener.clear_timeout()
# We can mock this because it's being tested at TestXUrllibDelayOnError
self.uri_opener._pause_on_http_error = Mock()
start = time.time()
try:
self.uri_opener.GET(url)
except HTTPRequestException, hre:
self.assertEqual(hre.message, 'HTTP timeout error')
except Exception, e:
msg = 'Not expecting: "%s"'
self.assertTrue(False, msg % e.__class__.__name__)
class TestXUrllibDelayOnError(unittest.TestCase):
def setUp(self):
self.uri_opener = ExtendedUrllib()
def tearDown(self):
self.uri_opener.end()
def test_increasing_delay_on_errors(self):
expected_log = {
0: False,
70: False,
40: False,
10: False,
80: False,
50: False,
20: False,
90: False,
60: False,
30: False,
100: False
}
self.assertEqual(self.uri_opener._sleep_log, expected_log)
return_empty_daemon = UpperDaemon(EmptyTCPHandler)
return_empty_daemon.start()
return_empty_daemon.wait_for_start()
port = return_empty_daemon.get_port()
# No retries means that the test is easier to read/understand
self.uri_opener.settings.set_max_http_retries(0)
# We want to keep going, don't test the _should_stop_scan here.
self.uri_opener._should_stop_scan = lambda x: False
self.uri_opener._rate_limit = lambda: True
url = URL('http://127.0.0.1:%s/' % port)
http_exception_count = 0
loops = 100
# Now check the delays
with patch('w3af.core.data.url.extended_urllib.time.sleep') as sleepm:
for i in xrange(loops):
try:
self.uri_opener.GET(url, cache=False)
except HTTPRequestException:
http_exception_count += 1
except Exception as e:
msg = 'Not expecting: "%s"'
self.assertTrue(False, msg % e.__class__.__name__)
else:
self.assertTrue(False, 'Expecting HTTPRequestException')
self.assertEqual(loops - 1, i)
# Note that the timeouts are increasing based on the error rate and
# SOCKET_ERROR_DELAY
expected_calls = [
call(1.5),
call(3.0),
call(4.5),
call(6.0),
call(7.5),
call(9.0),
call(10.5),
call(12.0),
call(13.5)
]
expected_log = {
0: False,
70: True,
40: True,
10: True,
80: True,
50: True,
20: True,
90: True,
60: True,
30: True,
100: False
}
self.assertEqual(expected_calls, sleepm.call_args_list)
self.assertEqual(http_exception_count, 100)
self.assertEqual(self.uri_opener._sleep_log, expected_log)
# This one should also clear the log
try:
self.uri_opener.GET(url, cache=False)
except HTTPRequestException:
pass
else:
self.assertTrue(False, 'Expected HTTPRequestException')
# The log was cleared, all values should be False
self.assertTrue(
all([not v for v in self.uri_opener._sleep_log.values()]))
def test_error_handling_disable_per_request(self):
upper_daemon = UpperDaemon(TimeoutTCPHandler)
upper_daemon.start()
upper_daemon.wait_for_start()
port = upper_daemon.get_port()
self.uri_opener.settings.set_configured_timeout(1)
self.uri_opener.clear_timeout()
self.uri_opener._retry = Mock()
url = URL('http://127.0.0.1:%s/' % port)
try:
self.uri_opener.GET(url, error_handling=False)
except HTTPRequestException:
self.assertEqual(self.uri_opener._retry.call_count, 0)
else:
self.assertTrue(False, 'Exception not raised')
self.uri_opener.settings.set_default_values()
def test_exception_is_raised_always_after_stop(self):
return_empty_daemon = UpperDaemon(EmptyTCPHandler)
return_empty_daemon.start()
return_empty_daemon.wait_for_start()
port = return_empty_daemon.get_port()
# No retries means that the test is easier to read/understand
self.uri_opener.settings.set_max_http_retries(0)
# Don't rate limit
self.uri_opener._rate_limit = lambda: True
url = URL('http://127.0.0.1:%s/' % port)
http_exception_count = 0
loops = 100
# Loop until we reach a must stop exception
for i in xrange(loops):
try:
self.uri_opener.GET(url, cache=False)
except HTTPRequestException:
http_exception_count += 1
except ScanMustStopByKnownReasonExc, smse:
break
except Exception as e:
msg = 'Not expecting: "%s"'
self.assertTrue(False, msg % e.__class__.__name__)
else:
class TestXUrllibTimeout(unittest.TestCase):
def setUp(self):
self.uri_opener = ExtendedUrllib()
def tearDown(self):
self.uri_opener.end()
def test_timeout(self):
upper_daemon = UpperDaemon(TimeoutTCPHandler)
upper_daemon.start()
upper_daemon.wait_for_start()
port = upper_daemon.get_port()
url = URL('http://127.0.0.1:%s/' % port)
self.uri_opener.settings.set_configured_timeout(0.5)
self.uri_opener.clear_timeout()
# We can mock this because it's being tested at TestXUrllibDelayOnError
self.uri_opener._pause_on_http_error = Mock()
start = time.time()
try:
self.uri_opener.GET(url)
except HTTPRequestException as hre:
self.assertEqual(hre.message, 'HTTP timeout error')
except Exception as e:
msg = 'Not expecting: "%s"'
self.assertTrue(False, msg % e.__class__.__name__)
else:
self.assertTrue(False, 'Expected HTTPRequestException.')
end = time.time()
self.uri_opener.settings.set_default_values()
self.assertLess(end-start, 1.5)
def test_timeout_ssl(self):
ssl_daemon = RawSSLDaemon(TimeoutTCPHandler)
ssl_daemon.start()
ssl_daemon.wait_for_start()
port = ssl_daemon.get_port()
url = URL('https://127.0.0.1:%s/' % port)
self.uri_opener.settings.set_max_http_retries(0)
self.uri_opener.settings.set_configured_timeout(1)
self.uri_opener.clear_timeout()
start = time.time()
self.assertRaises(HTTPRequestException, self.uri_opener.GET, url)
end = time.time()
self.uri_opener.settings.set_default_values()
# We set the upper limit to 4 because the uri opener needs to timeout
# all the connections (one for each SSL protocol) and then, because of
# some very relaxed handshake it needs to timeout a SSL protocol 3
# connection which passes handshake phase but then fails to send/get
# the headers
self.assertLess(end-start, 80)
def test_timeout_many(self):
upper_daemon = UpperDaemon(TimeoutTCPHandler)
upper_daemon.start()
upper_daemon.wait_for_start()
port = upper_daemon.get_port()
self.uri_opener.settings.set_configured_timeout(0.5)
self.uri_opener.clear_timeout()
# We can mock this because it's being tested at TestXUrllibDelayOnError
self.uri_opener._pause_on_http_error = Mock()
url = URL('http://127.0.0.1:%s/' % port)
http_request_e = 0
scan_stop_e = 0
for _ in xrange(MAX_ERROR_COUNT):
try:
self.uri_opener.GET(url)
except HTTPRequestException as hre:
http_request_e += 1
self.assertEqual(hre.message, 'HTTP timeout error')
except ScanMustStopException:
scan_stop_e += 1
self.assertTrue(True)
break
except Exception as e:
msg = 'Not expecting: "%s"'
self.assertTrue(False, msg % e.__class__.__name__)
else:
self.assertTrue(False, 'Expecting timeout')
else:
self.assertTrue(False, 'Expected ScanMustStopException')
self.uri_opener.settings.set_default_values()
self.assertEqual(http_request_e, 4)
self.assertEqual(scan_stop_e, 1)
def test_timeout_auto_adjust(self):
upper_daemon = UpperDaemon(Ok200SmallDelayHandler)
upper_daemon.start()
upper_daemon.wait_for_start()
port = upper_daemon.get_port()
# Enable timeout auto-adjust
self.uri_opener.settings.set_configured_timeout(0)
self.uri_opener.clear_timeout()
# We can mock this because it's being tested at TestXUrllibDelayOnError
self.uri_opener._pause_on_http_error = Mock()
# Mock to verify the calls
self.uri_opener.set_timeout = Mock()
# Make sure we start from the desired timeout value
self.assertEqual(self.uri_opener.get_timeout('127.0.0.1'),
DEFAULT_TIMEOUT)
url = URL('http://127.0.0.1:%s/' % port)
sent_requests = 0
self.uri_opener.GET(url)
time.sleep(TIMEOUT_UPDATE_ELAPSED_MIN + 1)
for _ in xrange(TIMEOUT_ADJUST_LIMIT * 3):
try:
self.uri_opener.GET(url)
except Exception:
raise
else:
sent_requests += 1
if self.uri_opener.set_timeout.call_count:
break
self.assertEqual(self.uri_opener.set_timeout.call_count, 1)
# pylint: disable=E1136
rtt = self.uri_opener.get_average_rtt()[0]
adjusted_tout = self.uri_opener.set_timeout.call_args[0][0]
expected_tout = TIMEOUT_MULT_CONST * rtt
delta = rtt * 0.2
# pylint: enable=E1136
self.assertGreaterEqual(adjusted_tout, expected_tout - delta)
self.assertLessEqual(adjusted_tout, expected_tout + delta)
self.assertLess(adjusted_tout, DEFAULT_TIMEOUT)
self.assertEqual(sent_requests, TIMEOUT_ADJUST_LIMIT)
def test_timeout_parameter_overrides_global_timeout(self):
upper_daemon = UpperDaemon(Ok200SmallDelayWithLongTriggeredTimeoutHandler)
upper_daemon.start()
upper_daemon.wait_for_start()
port = upper_daemon.get_port()
# Enable timeout auto-adjust
self.uri_opener.settings.set_configured_timeout(0)
self.uri_opener.clear_timeout()
# Make sure we start from the desired timeout value
self.assertEqual(self.uri_opener.get_timeout('127.0.0.1'),
DEFAULT_TIMEOUT)
url = URL('http://127.0.0.1:%s/' % port)
self.uri_opener.GET(url)
time.sleep(TIMEOUT_UPDATE_ELAPSED_MIN + 1)
for _ in xrange(TIMEOUT_ADJUST_LIMIT * 3):
self.uri_opener.GET(url)
# These make sure that the HTTP connection pool is full, this is
# required because we want to check if the timeout applies to
# existing connections, not new ones
for _ in xrange(ConnectionManager.MAX_CONNECTIONS):
self.uri_opener.GET(url)
# Make sure we reached the desired timeout after our HTTP
# requests to the test server
self.assertEqual(self.uri_opener.get_timeout('127.0.0.1'),
MIN_TIMEOUT)
timeout_url = URL('http://127.0.0.1:%s/timeout' % port)
# And now the real test, this one makes sure that the timeout
# parameter sent to GET overrides the configured value
response = self.uri_opener.GET(timeout_url, timeout=8.0)
self.assertEqual(response.get_code(), 200)
self.assertEqual(self.uri_opener.get_timeout('127.0.0.1'),
MIN_TIMEOUT)
# When timeout is not specified and the server returns in more
# than the expected time, an exception is raised
self.assertRaises(Exception, self.uri_opener.GET, timeout_url)