def test_timeout(self): upper_daemon = UpperDaemon(TimeoutTCPHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() url = URL('http://127.0.0.1:%s/' % port) self.uri_opener.settings.set_configured_timeout(0.5) self.uri_opener.clear_timeout() # We can mock this because it's being tested at TestXUrllibDelayOnError self.uri_opener._pause_on_http_error = Mock() start = time.time() try: self.uri_opener.GET(url) except HTTPRequestException as hre: self.assertEqual(hre.message, 'HTTP timeout error') except Exception as e: msg = 'Not expecting: "%s"' self.assertTrue(False, msg % e.__class__.__name__) else: self.assertTrue(False, 'Expected HTTPRequestException.') end = time.time() self.uri_opener.settings.set_default_values() self.assertLess(end-start, 1.5)
def test_url_port_not_http_many(self): upper_daemon = UpperDaemon(EmptyTCPHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() self.uri_opener.settings.set_max_http_retries(0) url = URL('http://127.0.0.1:%s/' % port) http_request_e = 0 scan_must_stop_e = 0 for _ in xrange(MAX_ERROR_COUNT): try: self.uri_opener.GET(url) except HTTPRequestException: http_request_e += 1 except ScanMustStopException as smse: scan_must_stop_e += 1 break except Exception as e: msg = 'Not expecting "%s".' self.assertTrue(False, msg % e.__class__.__name__) self.assertEqual(scan_must_stop_e, 1) self.assertEqual(http_request_e, 9)
def test_timeout_many(self): upper_daemon = UpperDaemon(TimeoutTCPHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() self.uri_opener.settings.set_timeout(1) url = URL('http://127.0.0.1:%s/' % port) for _ in xrange(MAX_ERROR_COUNT): try: self.uri_opener.GET(url) except ScanMustStopByUnknownReasonExc: self.assertTrue(False, 'Not expecting this exception type.') except ScanMustStopOnUrlError: self.assertTrue(True) except ScanMustStopException: self.assertTrue(True) break else: self.assertTrue(False) self.uri_opener.settings.set_default_values()
def test_exception_is_raised_always_after_stop(self): return_empty_daemon = UpperDaemon(EmptyTCPHandler) return_empty_daemon.start() return_empty_daemon.wait_for_start() port = return_empty_daemon.get_port() # No retries means that the test is easier to read/understand self.uri_opener.settings.set_max_http_retries(0) # Don't rate limit self.uri_opener._rate_limit = lambda: True url = URL('http://127.0.0.1:%s/' % port) http_exception_count = 0 loops = 100 # Loop until we reach a must stop exception for i in xrange(loops): try: self.uri_opener.GET(url, cache=False) except HTTPRequestException: http_exception_count += 1 except ScanMustStopByKnownReasonExc, smse: break except Exception, e: msg = 'Not expecting: "%s"' self.assertTrue(False, msg % e.__class__.__name__)
def test_http_timeout_with_plugin(self): """ This is very related with the tests at: w3af/core/data/url/tests/test_xurllib.py Very similar test is TestXUrllib.test_timeout :see: https://github.com/andresriancho/w3af/issues/7112 """ upper_daemon = UpperDaemon(TimeoutTCPHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() url = URL('http://127.0.0.1:%s/' % port) freq = FuzzableRequest(url) plugin_inst = self.w3af.plugins.get_plugin_inst('audit', 'sqli') plugin_inst._uri_opener.settings.set_configured_timeout(1) plugin_inst._uri_opener.clear_timeout() # We expect the server to timeout and the response to be a 204 resp = plugin_inst.get_original_response(freq) self.assertEqual(resp.get_url(), url) self.assertEqual(resp.get_code(), 204) plugin_inst._uri_opener.settings.set_default_values()
def test_timeout_many(self): upper_daemon = UpperDaemon(TimeoutTCPHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() self.uri_opener.settings.set_configured_timeout(0.5) self.uri_opener.clear_timeout() # We can mock this because it's being tested at TestXUrllibDelayOnError self.uri_opener._pause_on_http_error = Mock() url = URL('http://127.0.0.1:%s/' % port) http_request_e = 0 scan_stop_e = 0 for _ in xrange(MAX_ERROR_COUNT): try: self.uri_opener.GET(url) except HTTPRequestException, hre: http_request_e += 1 self.assertEqual(hre.message, 'HTTP timeout error') except ScanMustStopException: scan_stop_e += 1 self.assertTrue(True) break
def test_timeout_many(self): upper_daemon = UpperDaemon(TimeoutTCPHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() self.uri_opener.settings.set_timeout(1) url = URL('http://127.0.0.1:%s/' % port) http_request_e = 0 scan_stop_e = 0 for _ in xrange(MAX_ERROR_COUNT): try: self.uri_opener.GET(url) except HTTPRequestException: http_request_e += 1 self.assertTrue(True) except ScanMustStopException: scan_stop_e += 1 self.assertTrue(True) break except Exception, e: msg = 'Not expecting: "%s"' self.assertTrue(False, msg % e.__class__.__name__)
def test_url_port_not_http(self): upper_daemon = UpperDaemon(EmptyTCPHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() url = URL('http://127.0.0.1:%s/' % port) self.assertRaises(ScanMustStopOnUrlError, self.uri_opener.GET, url)
def test_ssl_fail_when_requesting_http(self): http_daemon = UpperDaemon(Ok200Handler) http_daemon.start() http_daemon.wait_for_start() port = http_daemon.get_port() # Note that here I'm using httpS <<---- "S" and that I've started an # HTTP server. We should get an exception url = URL('https://127.0.0.1:%s/' % port) self.assertRaises(HTTPRequestException, self.uri_opener.GET, url)
def test_url_port_not_http(self): upper_daemon = UpperDaemon(EmptyTCPHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() url = URL('http://127.0.0.1:%s/' % port) try: self.uri_opener.GET(url) except HTTPRequestException, hre: self.assertEqual(hre.value, "Bad HTTP response status line: ''")
def test_timeout_auto_adjust(self): upper_daemon = UpperDaemon(Ok200SmallDelayHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() # Enable timeout auto-adjust self.uri_opener.settings.set_configured_timeout(0) self.uri_opener.clear_timeout() # We can mock this because it's being tested at TestXUrllibDelayOnError self.uri_opener._pause_on_http_error = Mock() # Mock to verify the calls self.uri_opener.set_timeout = Mock() # Make sure we start from the desired timeout value self.assertEqual(self.uri_opener.get_timeout('127.0.0.1'), DEFAULT_TIMEOUT) url = URL('http://127.0.0.1:%s/' % port) sent_requests = 0 self.uri_opener.GET(url) time.sleep(TIMEOUT_UPDATE_ELAPSED_MIN + 1) for _ in xrange(TIMEOUT_ADJUST_LIMIT * 3): try: self.uri_opener.GET(url) except Exception: raise else: sent_requests += 1 if self.uri_opener.set_timeout.call_count: break self.assertEqual(self.uri_opener.set_timeout.call_count, 1) # pylint: disable=E1136 rtt = self.uri_opener.get_average_rtt()[0] adjusted_tout = self.uri_opener.set_timeout.call_args[0][0] expected_tout = TIMEOUT_MULT_CONST * rtt delta = rtt * 0.2 # pylint: enable=E1136 self.assertGreaterEqual(adjusted_tout, expected_tout - delta) self.assertLessEqual(adjusted_tout, expected_tout + delta) self.assertLess(adjusted_tout, DEFAULT_TIMEOUT) self.assertEqual(sent_requests, TIMEOUT_ADJUST_LIMIT)
def test_timeout(self): upper_daemon = UpperDaemon(TimeoutTCPHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() url = URL('http://127.0.0.1:%s/' % port) self.uri_opener.settings.set_timeout(1) self.assertRaises(ScanMustStopOnUrlError, self.uri_opener.GET, url) self.uri_opener.settings.set_default_values()
def test_timeout_parameter_overrides_global_timeout(self): upper_daemon = UpperDaemon(Ok200SmallDelayWithLongTriggeredTimeoutHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() # Enable timeout auto-adjust self.uri_opener.settings.set_configured_timeout(0) self.uri_opener.clear_timeout() # Make sure we start from the desired timeout value self.assertEqual(self.uri_opener.get_timeout('127.0.0.1'), DEFAULT_TIMEOUT) url = URL('http://127.0.0.1:%s/' % port) self.uri_opener.GET(url) time.sleep(TIMEOUT_UPDATE_ELAPSED_MIN + 1) for _ in xrange(TIMEOUT_ADJUST_LIMIT * 3): self.uri_opener.GET(url) # These make sure that the HTTP connection pool is full, this is # required because we want to check if the timeout applies to # existing connections, not new ones for _ in xrange(ConnectionManager.MAX_CONNECTIONS): self.uri_opener.GET(url) # Make sure we reached the desired timeout after our HTTP # requests to the test server self.assertEqual(self.uri_opener.get_timeout('127.0.0.1'), MIN_TIMEOUT) timeout_url = URL('http://127.0.0.1:%s/timeout' % port) # And now the real test, this one makes sure that the timeout # parameter sent to GET overrides the configured value response = self.uri_opener.GET(timeout_url, timeout=8.0) self.assertEqual(response.get_code(), 200) self.assertEqual(self.uri_opener.get_timeout('127.0.0.1'), MIN_TIMEOUT) # When timeout is not specified and the server returns in more # than the expected time, an exception is raised self.assertRaises(Exception, self.uri_opener.GET, timeout_url)
def test_timeout_auto_adjust(self): upper_daemon = UpperDaemon(Ok200SmallDelayHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() # Enable timeout auto-adjust self.uri_opener.settings.set_configured_timeout(0) self.uri_opener.clear_timeout() # We can mock this because it's being tested at TestXUrllibDelayOnError self.uri_opener._pause_on_http_error = Mock() # Mock to verify the calls self.uri_opener.set_timeout = Mock() # Make sure we start from the desired timeout value self.assertEqual(self.uri_opener.get_timeout('127.0.0.1'), DEFAULT_TIMEOUT) url = URL('http://127.0.0.1:%s/' % port) sent_requests = 0 for _ in xrange(TIMEOUT_ADJUST_LIMIT * 2): try: self.uri_opener.GET(url) except Exception: raise else: sent_requests += 1 if self.uri_opener.set_timeout.call_count: break self.assertEqual(self.uri_opener.set_timeout.call_count, 1) rtt = self.uri_opener.get_average_rtt()[0] adjusted_tout = self.uri_opener.set_timeout.call_args[0][0] expected_tout = TIMEOUT_MULT_CONST * rtt delta = rtt * 0.2 self.assertGreaterEqual(adjusted_tout, expected_tout - delta) self.assertLessEqual(adjusted_tout, expected_tout + delta) self.assertLess(adjusted_tout, DEFAULT_TIMEOUT) self.assertEqual(sent_requests, TIMEOUT_ADJUST_LIMIT + 1)
def test_timeout(self): upper_daemon = UpperDaemon(TimeoutTCPHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() url = URL('http://127.0.0.1:%s/' % port) self.uri_opener.settings.set_configured_timeout(0.5) self.uri_opener.clear_timeout() # We can mock this because it's being tested at TestXUrllibDelayOnError self.uri_opener._pause_on_http_error = Mock() start = time.time() try: self.uri_opener.GET(url) except HTTPRequestException, hre: self.assertEqual(hre.message, 'HTTP timeout error')
def test_ignore_errors(self): upper_daemon = UpperDaemon(TimeoutTCPHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() self.uri_opener.settings.set_timeout(1) self.uri_opener._retry = Mock() url = URL('http://127.0.0.1:%s/' % port) try: self.uri_opener.GET(url, ignore_errors=True) except ScanMustStopOnUrlError: self.assertEqual(self.uri_opener._retry.call_count, 0) else: self.assertTrue(False, 'Exception not raised') self.uri_opener.settings.set_default_values()
def test_error_handling_disable_per_request(self): upper_daemon = UpperDaemon(TimeoutTCPHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() self.uri_opener.settings.set_configured_timeout(1) self.uri_opener.clear_timeout() self.uri_opener._retry = Mock() url = URL('http://127.0.0.1:%s/' % port) try: self.uri_opener.GET(url, error_handling=False) except HTTPRequestException: self.assertEqual(self.uri_opener._retry.call_count, 0) else: self.assertTrue(False, 'Exception not raised') self.uri_opener.settings.set_default_values()
class TestUpperDaemon(unittest.TestCase): """ This is a unittest for the UpperDaemon which lives in upper_daemon.py @author: Andres Riancho <andres . riancho | gmail . com> """ def setUp(self): self.upper_daemon = UpperDaemon() self.upper_daemon.start() self.upper_daemon.wait_for_start() def test_basic(self): sent = 'abc' sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(('127.0.0.1', self.upper_daemon.get_port())) sock.sendall(sent) received = sock.recv(3) self.assertEqual(received, sent.upper())
def test_timeout_many(self): upper_daemon = UpperDaemon(TimeoutTCPHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() self.uri_opener.settings.set_configured_timeout(0.5) self.uri_opener.clear_timeout() # We can mock this because it's being tested at TestXUrllibDelayOnError self.uri_opener._pause_on_http_error = Mock() url = URL('http://127.0.0.1:%s/' % port) http_request_e = 0 scan_stop_e = 0 for _ in xrange(MAX_ERROR_COUNT): try: self.uri_opener.GET(url) except HTTPRequestException as hre: http_request_e += 1 self.assertEqual(hre.message, 'HTTP timeout error') except ScanMustStopException: scan_stop_e += 1 self.assertTrue(True) break except Exception as e: msg = 'Not expecting: "%s"' self.assertTrue(False, msg % e.__class__.__name__) else: self.assertTrue(False, 'Expecting timeout') else: self.assertTrue(False, 'Expected ScanMustStopException') self.uri_opener.settings.set_default_values() self.assertEqual(http_request_e, 4) self.assertEqual(scan_stop_e, 1)
def test_url_port_not_http_many(self): upper_daemon = UpperDaemon(EmptyTCPHandler) upper_daemon.start() upper_daemon.wait_for_start() port = upper_daemon.get_port() self.uri_opener.settings.set_max_http_retries(0) url = URL('http://127.0.0.1:%s/' % port) http_request_e = 0 scan_must_stop_e = 0 for _ in xrange(MAX_ERROR_COUNT): try: self.uri_opener.GET(url) except HTTPRequestException: http_request_e += 1 except ScanMustStopException, smse: scan_must_stop_e += 1 break except Exception, e: msg = 'Not expecting "%s".' self.assertTrue(False, msg % e.__class__.__name__)
def test_increasing_delay_on_errors(self): expected_log = {0: False, 70: False, 40: False, 10: False, 80: False, 50: False, 20: False, 90: False, 60: False, 30: False, 100: False} self.assertEqual(self.uri_opener._sleep_log, expected_log) return_empty_daemon = UpperDaemon(EmptyTCPHandler) return_empty_daemon.start() return_empty_daemon.wait_for_start() port = return_empty_daemon.get_port() # No retries means that the test is easier to read/understand self.uri_opener.settings.set_max_http_retries(0) # We want to keep going, don't test the _should_stop_scan here. self.uri_opener._should_stop_scan = lambda x: False url = URL('http://127.0.0.1:%s/' % port) http_exception_count = 0 loops = 100 # Not check the delays with patch('w3af.core.data.url.extended_urllib.time.sleep') as sleepm: for i in xrange(loops): try: self.uri_opener.GET(url, cache=False) except HTTPRequestException: http_exception_count += 1 except Exception, e: msg = 'Not expecting: "%s"' self.assertTrue(False, msg % e.__class__.__name__) else: self.assertTrue(False, 'Expecting HTTPRequestException') self.assertEqual(loops - 1, i) # Note that the timeouts are increasing based on the error rate and # SOCKET_ERROR_DELAY expected_calls = [call(1.5), call(3.0), call(4.5), call(6.0), call(7.5), call(9.0), call(10.5), call(12.0), call(13.5)] expected_log = {0: False, 70: True, 40: True, 10: True, 80: True, 50: True, 20: True, 90: True, 60: True, 30: True, 100: False} self.assertEqual(expected_calls, sleepm.call_args_list) self.assertEqual(http_exception_count, 100) self.assertEqual(self.uri_opener._sleep_log, expected_log) # This one should also clear the log try: self.uri_opener.GET(url, cache=False) except HTTPRequestException: pass else: self.assertTrue(False, 'Expected HTTPRequestException') # The log was cleared, all values should be False self.assertTrue(all([not v for v in self.uri_opener._sleep_log.values()]))