class TestXUrllibTimeout(unittest.TestCase): def setUp(self): self.uri_opener = ExtendedUrllib() def tearDown(self): self.uri_opener.end() def test_timeout(self): upper_daemon = UpperDaemon(TimeoutTCPHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() url = URL('http://127.0.0.1:%s/' % port) self.uri_opener.settings.set_configured_timeout(0.5) self.uri_opener.clear_timeout() # We can mock this because it's being tested at TestXUrllibDelayOnError self.uri_opener._pause_on_http_error = Mock() start = time.time() try: self.uri_opener.GET(url) except HTTPRequestException, hre: self.assertEqual(hre.message, 'HTTP timeout error') except Exception, e: msg = 'Not expecting: "%s"' self.assertTrue(False, msg % e.__class__.__name__)
class TestXUrllibTimeout(unittest.TestCase): def setUp(self): self.uri_opener = ExtendedUrllib() def tearDown(self): self.uri_opener.end() def test_timeout(self): upper_daemon = UpperDaemon(TimeoutTCPHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() url = URL('http://127.0.0.1:%s/' % port) self.uri_opener.settings.set_configured_timeout(0.5) self.uri_opener.clear_timeout() # We can mock this because it's being tested at TestXUrllibDelayOnError self.uri_opener._pause_on_http_error = Mock() start = time.time() try: self.uri_opener.GET(url) except HTTPRequestException, hre: self.assertEqual(hre.message, 'HTTP timeout error') except Exception, e: msg = 'Not expecting: "%s"' self.assertTrue(False, msg % e.__class__.__name__)
class TestXUrllibDelayOnError(unittest.TestCase): def setUp(self): self.uri_opener = ExtendedUrllib() def tearDown(self): self.uri_opener.end() def test_increasing_delay_on_errors(self): expected_log = { 0: False, 70: False, 40: False, 10: False, 80: False, 50: False, 20: False, 90: False, 60: False, 30: False, 100: False } self.assertEqual(self.uri_opener._sleep_log, expected_log) return_empty_daemon = UpperDaemon(EmptyTCPHandler) return_empty_daemon.start() return_empty_daemon.wait_for_start() port = return_empty_daemon.get_port() # No retries means that the test is easier to read/understand self.uri_opener.settings.set_max_http_retries(0) # We want to keep going, don't test the _should_stop_scan here. self.uri_opener._should_stop_scan = lambda x: False self.uri_opener._rate_limit = lambda: True url = URL('http://127.0.0.1:%s/' % port) http_exception_count = 0 loops = 100 # Now check the delays with patch('w3af.core.data.url.extended_urllib.time.sleep') as sleepm: for i in xrange(loops): try: self.uri_opener.GET(url, cache=False) except HTTPRequestException: http_exception_count += 1 except Exception as e: msg = 'Not expecting: "%s"' self.assertTrue(False, msg % e.__class__.__name__) else: self.assertTrue(False, 'Expecting HTTPRequestException') self.assertEqual(loops - 1, i) # Note that the timeouts are increasing based on the error rate and # SOCKET_ERROR_DELAY expected_calls = [ call(1.5), call(3.0), call(4.5), call(6.0), call(7.5), call(9.0), call(10.5), call(12.0), call(13.5) ] expected_log = { 0: False, 70: True, 40: True, 10: True, 80: True, 50: True, 20: True, 90: True, 60: True, 30: True, 100: False } self.assertEqual(expected_calls, sleepm.call_args_list) self.assertEqual(http_exception_count, 100) self.assertEqual(self.uri_opener._sleep_log, expected_log) # This one should also clear the log try: self.uri_opener.GET(url, cache=False) except HTTPRequestException: pass else: self.assertTrue(False, 'Expected HTTPRequestException') # The log was cleared, all values should be False self.assertTrue( all([not v for v in self.uri_opener._sleep_log.values()])) def test_error_handling_disable_per_request(self): upper_daemon = UpperDaemon(TimeoutTCPHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() self.uri_opener.settings.set_configured_timeout(1) self.uri_opener.clear_timeout() self.uri_opener._retry = Mock() url = URL('http://127.0.0.1:%s/' % port) try: self.uri_opener.GET(url, error_handling=False) except HTTPRequestException: self.assertEqual(self.uri_opener._retry.call_count, 0) else: self.assertTrue(False, 'Exception not raised') self.uri_opener.settings.set_default_values() def test_exception_is_raised_always_after_stop(self): return_empty_daemon = UpperDaemon(EmptyTCPHandler) return_empty_daemon.start() return_empty_daemon.wait_for_start() port = return_empty_daemon.get_port() # No retries means that the test is easier to read/understand self.uri_opener.settings.set_max_http_retries(0) # Don't rate limit self.uri_opener._rate_limit = lambda: True url = URL('http://127.0.0.1:%s/' % port) http_exception_count = 0 loops = 100 # Loop until we reach a must stop exception for i in xrange(loops): try: self.uri_opener.GET(url, cache=False) except HTTPRequestException: http_exception_count += 1 except ScanMustStopByKnownReasonExc, smse: break except Exception as e: msg = 'Not expecting: "%s"' self.assertTrue(False, msg % e.__class__.__name__) else:
class TestXUrllibTimeout(unittest.TestCase): def setUp(self): self.uri_opener = ExtendedUrllib() def tearDown(self): self.uri_opener.end() def test_timeout(self): upper_daemon = UpperDaemon(TimeoutTCPHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() url = URL('http://127.0.0.1:%s/' % port) self.uri_opener.settings.set_configured_timeout(0.5) self.uri_opener.clear_timeout() # We can mock this because it's being tested at TestXUrllibDelayOnError self.uri_opener._pause_on_http_error = Mock() start = time.time() try: self.uri_opener.GET(url) except HTTPRequestException as hre: self.assertEqual(hre.message, 'HTTP timeout error') except Exception as e: msg = 'Not expecting: "%s"' self.assertTrue(False, msg % e.__class__.__name__) else: self.assertTrue(False, 'Expected HTTPRequestException.') end = time.time() self.uri_opener.settings.set_default_values() self.assertLess(end-start, 1.5) def test_timeout_ssl(self): ssl_daemon = RawSSLDaemon(TimeoutTCPHandler) ssl_daemon.start() ssl_daemon.wait_for_start() port = ssl_daemon.get_port() url = URL('https://127.0.0.1:%s/' % port) self.uri_opener.settings.set_max_http_retries(0) self.uri_opener.settings.set_configured_timeout(1) self.uri_opener.clear_timeout() start = time.time() self.assertRaises(HTTPRequestException, self.uri_opener.GET, url) end = time.time() self.uri_opener.settings.set_default_values() # We set the upper limit to 4 because the uri opener needs to timeout # all the connections (one for each SSL protocol) and then, because of # some very relaxed handshake it needs to timeout a SSL protocol 3 # connection which passes handshake phase but then fails to send/get # the headers self.assertLess(end-start, 80) def test_timeout_many(self): upper_daemon = UpperDaemon(TimeoutTCPHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() self.uri_opener.settings.set_configured_timeout(0.5) self.uri_opener.clear_timeout() # We can mock this because it's being tested at TestXUrllibDelayOnError self.uri_opener._pause_on_http_error = Mock() url = URL('http://127.0.0.1:%s/' % port) http_request_e = 0 scan_stop_e = 0 for _ in xrange(MAX_ERROR_COUNT): try: self.uri_opener.GET(url) except HTTPRequestException as hre: http_request_e += 1 self.assertEqual(hre.message, 'HTTP timeout error') except ScanMustStopException: scan_stop_e += 1 self.assertTrue(True) break except Exception as e: msg = 'Not expecting: "%s"' self.assertTrue(False, msg % e.__class__.__name__) else: self.assertTrue(False, 'Expecting timeout') else: self.assertTrue(False, 'Expected ScanMustStopException') self.uri_opener.settings.set_default_values() self.assertEqual(http_request_e, 4) self.assertEqual(scan_stop_e, 1) def test_timeout_auto_adjust(self): upper_daemon = UpperDaemon(Ok200SmallDelayHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() # Enable timeout auto-adjust self.uri_opener.settings.set_configured_timeout(0) self.uri_opener.clear_timeout() # We can mock this because it's being tested at TestXUrllibDelayOnError self.uri_opener._pause_on_http_error = Mock() # Mock to verify the calls self.uri_opener.set_timeout = Mock() # Make sure we start from the desired timeout value self.assertEqual(self.uri_opener.get_timeout('127.0.0.1'), DEFAULT_TIMEOUT) url = URL('http://127.0.0.1:%s/' % port) sent_requests = 0 self.uri_opener.GET(url) time.sleep(TIMEOUT_UPDATE_ELAPSED_MIN + 1) for _ in xrange(TIMEOUT_ADJUST_LIMIT * 3): try: self.uri_opener.GET(url) except Exception: raise else: sent_requests += 1 if self.uri_opener.set_timeout.call_count: break self.assertEqual(self.uri_opener.set_timeout.call_count, 1) # pylint: disable=E1136 rtt = self.uri_opener.get_average_rtt()[0] adjusted_tout = self.uri_opener.set_timeout.call_args[0][0] expected_tout = TIMEOUT_MULT_CONST * rtt delta = rtt * 0.2 # pylint: enable=E1136 self.assertGreaterEqual(adjusted_tout, expected_tout - delta) self.assertLessEqual(adjusted_tout, expected_tout + delta) self.assertLess(adjusted_tout, DEFAULT_TIMEOUT) self.assertEqual(sent_requests, TIMEOUT_ADJUST_LIMIT) def test_timeout_parameter_overrides_global_timeout(self): upper_daemon = UpperDaemon(Ok200SmallDelayWithLongTriggeredTimeoutHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() # Enable timeout auto-adjust self.uri_opener.settings.set_configured_timeout(0) self.uri_opener.clear_timeout() # Make sure we start from the desired timeout value self.assertEqual(self.uri_opener.get_timeout('127.0.0.1'), DEFAULT_TIMEOUT) url = URL('http://127.0.0.1:%s/' % port) self.uri_opener.GET(url) time.sleep(TIMEOUT_UPDATE_ELAPSED_MIN + 1) for _ in xrange(TIMEOUT_ADJUST_LIMIT * 3): self.uri_opener.GET(url) # These make sure that the HTTP connection pool is full, this is # required because we want to check if the timeout applies to # existing connections, not new ones for _ in xrange(ConnectionManager.MAX_CONNECTIONS): self.uri_opener.GET(url) # Make sure we reached the desired timeout after our HTTP # requests to the test server self.assertEqual(self.uri_opener.get_timeout('127.0.0.1'), MIN_TIMEOUT) timeout_url = URL('http://127.0.0.1:%s/timeout' % port) # And now the real test, this one makes sure that the timeout # parameter sent to GET overrides the configured value response = self.uri_opener.GET(timeout_url, timeout=8.0) self.assertEqual(response.get_code(), 200) self.assertEqual(self.uri_opener.get_timeout('127.0.0.1'), MIN_TIMEOUT) # When timeout is not specified and the server returns in more # than the expected time, an exception is raised self.assertRaises(Exception, self.uri_opener.GET, timeout_url)