def test_timeout_elapsed(self, current_time): current_time.return_value = TIMEOUT_EPOCH timeout = Timeout(total=3) self.assertRaises(TimeoutStateError, timeout.get_connect_duration) timeout.start_connect() self.assertRaises(TimeoutStateError, timeout.start_connect) current_time.return_value = TIMEOUT_EPOCH + 2 self.assertEqual(timeout.get_connect_duration(), 2) current_time.return_value = TIMEOUT_EPOCH + 37 self.assertEqual(timeout.get_connect_duration(), 37)
def test_timeout_elapsed(self, current_time): current_time.return_value = TIMEOUT_EPOCH timeout = Timeout(total=3) with pytest.raises(TimeoutStateError): timeout.get_connect_duration() timeout.start_connect() with pytest.raises(TimeoutStateError): timeout.start_connect() current_time.return_value = TIMEOUT_EPOCH + 2 assert timeout.get_connect_duration() == 2 current_time.return_value = TIMEOUT_EPOCH + 37 assert timeout.get_connect_duration() == 37
def test_ssl_read_timeout(self): timed_out = Event() def socket_handler(listener): sock = listener.accept()[0] ssl_sock = ssl.wrap_socket(sock, server_side=True, keyfile=DEFAULT_CERTS['keyfile'], certfile=DEFAULT_CERTS['certfile'], ca_certs=DEFAULT_CA) buf = b'' while not buf.endswith(b'\r\n\r\n'): buf += ssl_sock.recv(65536) # Send incomplete message (note Content-Length) ssl_sock.send(('HTTP/1.1 200 OK\r\n' 'Content-Type: text/plain\r\n' 'Content-Length: 10\r\n' '\r\n' 'Hi-').encode('utf-8')) timed_out.wait() sock.close() ssl_sock.close() self._start_server(socket_handler) pool = HTTPSConnectionPool(self.host, self.port) response = pool.urlopen('GET', '/', retries=0, preload_content=False, timeout=Timeout(connect=1, read=0.001)) try: self.assertRaises(ReadTimeoutError, response.read) finally: timed_out.set()
def test_timeout(self): # Requests should time out when expected block_event = Event() ready_event = self.start_basic_handler(block_send=block_event, num=3) # Pool-global timeout short_timeout = Timeout(read=SHORT_TIMEOUT) with HTTPConnectionPool(self.host, self.port, timeout=short_timeout, retries=False) as pool: wait_for_socket(ready_event) block_event.clear() with pytest.raises(ReadTimeoutError): pool.request("GET", "/") block_event.set() # Release request # Request-specific timeouts should raise errors with HTTPConnectionPool(self.host, self.port, timeout=short_timeout, retries=False) as pool: wait_for_socket(ready_event) now = time.time() with pytest.raises(ReadTimeoutError): pool.request("GET", "/", timeout=LONG_TIMEOUT) delta = time.time() - now message = "timeout was pool-level SHORT_TIMEOUT rather than request-level LONG_TIMEOUT" assert delta >= LONG_TIMEOUT, message block_event.set() # Release request # Timeout passed directly to request should raise a request timeout wait_for_socket(ready_event) with pytest.raises(ReadTimeoutError): pool.request("GET", "/", timeout=SHORT_TIMEOUT) block_event.set() # Release request
def post(self, url, data, timeout=None): """Request an URL. Args: url (str): The web location we want to retrieve. data (dict[str, str]): A dict of key/value pairs. Note: On py2.7 value is unicode. timeout (Optional[int|float]): If this value is specified, use it as the read timeout from the server (instead of the one specified during creation of the connection pool). Returns: A JSON object. """ urlopen_kwargs = {} if timeout is not None: urlopen_kwargs['timeout'] = Timeout(read=timeout, connect=self._connect_timeout) if InputFile.is_inputfile(data): data = InputFile(data) result = self._request_wrapper('POST', url, body=data.to_form(), headers=data.headers, **urlopen_kwargs) else: data = json.dumps(data) result = self._request_wrapper( 'POST', url, body=data.encode(), headers={'Content-Type': 'application/json'}, **urlopen_kwargs) return self._parse(result)
def request(self, req, res): options = {} headers = req['headers'] or {} req_url = req['url'] if req_url.endswith('?'): req_url = req['url'][:-1] #req_url = req['url'] #if not '://' in req_url: # req_url = 'http://%s' % req_url #url_parts = urlsplit(req_url) #if url_parts.scheme not in ('http', 'https'): # emsg = 'Unknown scheme [%s]' % url_parts.scheme # # second argument saved as `err.transport_error` # # and used then to generate short code of error # # for statistics # raise error.InvalidUrlError( # emsg, error.InvalidUrlError(emsg), # ) pool = self.get_pool(req, use_cache=(not req['close_connection'])) self.op_started = time.time() if req['resolve']: if req['proxy']: raise error.IowebConfigError( 'Request option `resolve` could not be used along option `proxy`' ) for host, ip in req['resolve'].items(): pool.resolving_cache[host] = ip if req['content_encoding']: if not any(x.lower() == 'accept-encoding' for x in headers): headers['Accept-Encoding'] = req['content_encoding'] if req['data']: if isinstance(req['data'], dict): if req['multipart']: body, ctype = encode_multipart_formdata(req['data']) else: body = urlencode(req['data']) ctype = 'application/x-www-form-urlencoded' options['body'] = body headers['Content-Type'] = ctype elif isinstance(req['data'], bytes): options['body'] = req['data'] elif isinstance(req['data'], str): options['body'] = req['data'].encode('utf-8') else: raise error.IowebConfigError( 'Invalid type of request data option: %s' % type(req['data'])) headers['Content-Length'] = len(options['body']) with self.handle_network_error(req): retry_opts = { # total - set to None to remove this constraint # and fall back on other counts. 'connect': False, 'read': False, } if req['follow_redirect']: retry_opts.update({ 'total': req['max_redirects'], 'redirect': req['max_redirects'], 'raise_on_redirect': True, }) else: retry_opts.update({ 'total': False, 'redirect': False, 'raise_on_redirect': False, }) self.urllib3_response = pool.urlopen( req.method(), req_url, headers=headers, retries=Retry(**retry_opts), timeout=Timeout( connect=req['connect_timeout'], read=req['timeout'], ), preload_content=False, decode_content=req['decode_content'], **options)
def pool(self): timeout = Timeout(connect=self._conn_timeout, read=self._read_timeout) return PoolManager(num_pools=self._num_pools, timeout=timeout)
if not APP_NAME: raise NoAppNameError('APP_NAME is mandatory parameter!') WORK_DIR = os.path.dirname(os.path.abspath(__file__)) LOG_PATH = os.path.join(WORK_DIR, 'log') LOG = os.path.join(LOG_PATH, 'log.txt') LOGSTASH_HOST = os.getenv('LOGSTASH_HOST') HTTP_CONNECT_TIMEOUT = os.getenv('HTTP_CONNECT_TIMEOUT') HTTP_READ_TIMEOUT = os.getenv('HTTP_READ_TIMEOUT') HTTP_RETRY_COUNT = int(os.getenv('HTTP_RETRY_COUNT') or 4) HTTP_RETRY_BACKOFF = float(os.getenv('HTTP_RETRY_BACKOFF') or 15) if HTTP_CONNECT_TIMEOUT or HTTP_READ_TIMEOUT: HTTP_TIMEOUT = Timeout(connect=int(HTTP_CONNECT_TIMEOUT or 0) or _Default, read=int(HTTP_READ_TIMEOUT or 0) or _Default) else: http_timeout = int(os.getenv('HTTP_TIMEOUT') or 60) HTTP_TIMEOUT = Timeout(connect=http_timeout, read=http_timeout) TIME_ZONE = 'UTC' LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'formatters': { 'default': { 'class': 'watcher.logger.DSMConsoleFormatter', 'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s' } },
out += " #" + str(count) print("Grabbed deck: " + out) ## configure readline histfile = os.path.join(os.path.expanduser("~"), ".tu_deck_grabber_history") if os.path.exists(histfile): readline.read_history_file(histfile) readline.set_history_length(1000) readline.read_init_file() atexit.register(readline.write_history_file, histfile) with PoolManager( 1, timeout=Timeout(connect=15.0, read=20.0, total=30.0), retries=Retry(total=3), cert_reqs='CERT_REQUIRED', ca_certs=certifi.where(), ) as http: while True: try: line = input("{} ~> ".format(login)) except EOFError: print() line = None if not line: break line = line.strip().lower() if line == 'exit': break
def test_invalid_timeouts(self, kwargs, message): with pytest.raises(ValueError) as e: Timeout(**kwargs) assert message in str(e.value)
def test_timeout(self): # Requests should time out when expected block_event = Event() ready_event = self.start_basic_handler(block_send=block_event, num=6) # Pool-global timeout timeout = Timeout(read=SHORT_TIMEOUT) pool = HTTPConnectionPool(self.host, self.port, timeout=timeout, retries=False) self.addCleanup(pool.close) wait_for_socket(ready_event) conn = pool._get_conn() self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', '/') pool._put_conn(conn) block_event.set() # Release request wait_for_socket(ready_event) block_event.clear() self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/') block_event.set() # Release request # Request-specific timeouts should raise errors pool = HTTPConnectionPool(self.host, self.port, timeout=LONG_TIMEOUT, retries=False) self.addCleanup(pool.close) conn = pool._get_conn() wait_for_socket(ready_event) now = time.time() self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', '/', timeout=timeout) delta = time.time() - now block_event.set() # Release request message = "timeout was pool-level LONG_TIMEOUT rather than request-level SHORT_TIMEOUT" self.assertTrue(delta < LONG_TIMEOUT, message) pool._put_conn(conn) wait_for_socket(ready_event) now = time.time() self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/', timeout=timeout) delta = time.time() - now message = "timeout was pool-level LONG_TIMEOUT rather than request-level SHORT_TIMEOUT" self.assertTrue(delta < LONG_TIMEOUT, message) block_event.set() # Release request # Timeout int/float passed directly to request and _make_request should # raise a request timeout wait_for_socket(ready_event) self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/', timeout=SHORT_TIMEOUT) block_event.set() # Release request wait_for_socket(ready_event) conn = pool._new_conn() # FIXME: This assert flakes sometimes. Not sure why. self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', '/', timeout=SHORT_TIMEOUT) block_event.set() # Release request
cookiejar = cookies.RequestsCookieJar() cookiejar.update(cookies_) assert response.text == '\n'.join('{}: {}'.format(*item) for item in cookiejar.items()) # Timeouts should go last, because '/slow' hangs the HTTP server @pytest.mark.parametrize('timeout', [0.1, (None, 0.1)]) def test_get_timeout(http_server, timeout): with pytest.raises(requests.Timeout): requests.get(http_server.base_url + '/slow', timeout=timeout) @pytest.mark.skipif('urllib3' not in sys.modules, reason='urllib3 not available') @pytest.mark.parametrize('timeout', [Timeout(read=0.1), Timeout(total=0.1)]) def test_get_timeout_urllib3(http_server, timeout): with pytest.raises(requests.Timeout): requests.get(http_server.base_url + '/slow', timeout=timeout) @pytest.mark.parametrize( 'timeout', [0.1, (0.1, None), Timeout(connect=0.1), Timeout(total=0.1)]) def test_get_connect_timeout(http_server, timeout): with pytest.raises(requests.Timeout): # Use RFC-5737 TEST-NET-1 address since it should always be unreachable requests.get('http://192.0.2.1', timeout=timeout) requests.get(http_server.base_url + '/slow', timeout=timeout)
def test_timeout(self): # Requests should time out when expected block_event = Event() ready_event = self.start_basic_handler(block_send=block_event, num=6) # Pool-global timeout timeout = Timeout(read=SHORT_TIMEOUT) with HTTPConnectionPool(self.host, self.port, timeout=timeout, retries=False) as pool: wait_for_socket(ready_event) conn = pool._get_conn() with pytest.raises(ReadTimeoutError): pool._make_request(conn, "GET", "/") pool._put_conn(conn) block_event.set() # Release request wait_for_socket(ready_event) block_event.clear() with pytest.raises(ReadTimeoutError): pool.request("GET", "/") block_event.set() # Release request # Request-specific timeouts should raise errors with HTTPConnectionPool(self.host, self.port, timeout=LONG_TIMEOUT, retries=False) as pool: conn = pool._get_conn() wait_for_socket(ready_event) now = time.time() with pytest.raises(ReadTimeoutError): pool._make_request(conn, "GET", "/", timeout=timeout) delta = time.time() - now block_event.set() # Release request message = "timeout was pool-level LONG_TIMEOUT rather than request-level SHORT_TIMEOUT" assert delta < LONG_TIMEOUT, message pool._put_conn(conn) wait_for_socket(ready_event) now = time.time() with pytest.raises(ReadTimeoutError): pool.request("GET", "/", timeout=timeout) delta = time.time() - now message = "timeout was pool-level LONG_TIMEOUT rather than request-level SHORT_TIMEOUT" assert delta < LONG_TIMEOUT, message block_event.set() # Release request # Timeout int/float passed directly to request and _make_request should # raise a request timeout wait_for_socket(ready_event) with pytest.raises(ReadTimeoutError): pool.request("GET", "/", timeout=SHORT_TIMEOUT) block_event.set() # Release request wait_for_socket(ready_event) conn = pool._new_conn() # FIXME: This assert flakes sometimes. Not sure why. with pytest.raises(ReadTimeoutError): pool._make_request(conn, "GET", "/", timeout=SHORT_TIMEOUT) block_event.set() # Release request
def test_create_connection_timeout(self): timeout = Timeout(connect=SHORT_TIMEOUT, total=LONG_TIMEOUT) pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=timeout, retries=False) self.addCleanup(pool.close) conn = pool._new_conn() self.assertRaises(ConnectTimeoutError, conn.connect)
def test_connection_closed_on_read_timeout_preload_false(self): timed_out = Event() def socket_handler(listener): sock = listener.accept()[0] # Consume request buf = b'' while not buf.endswith(b'\r\n\r\n'): buf = sock.recv(65535) # Send partial chunked response and then hang. sock.send(('HTTP/1.1 200 OK\r\n' 'Content-Type: text/plain\r\n' 'Transfer-Encoding: chunked\r\n' '\r\n' '8\r\n' '12345678\r\n').encode('utf-8')) timed_out.wait(5) # Expect a new request, but keep hold of the old socket to avoid # leaking it. Because we don't want to hang this thread, we # actually use select.select to confirm that a new request is # coming in: this lets us time the thread out. rlist, _, _ = select.select([listener], [], [], 1) assert rlist new_sock = listener.accept()[0] # Consume request buf = b'' while not buf.endswith(b'\r\n\r\n'): buf = new_sock.recv(65535) # Send complete chunked response. new_sock.send(('HTTP/1.1 200 OK\r\n' 'Content-Type: text/plain\r\n' 'Transfer-Encoding: chunked\r\n' '\r\n' '8\r\n' '12345678\r\n' '0\r\n\r\n').encode('utf-8')) new_sock.close() sock.close() self._start_server(socket_handler) with HTTPConnectionPool(self.host, self.port) as pool: # First request should fail. response = pool.urlopen('GET', '/', retries=0, preload_content=False, timeout=Timeout(connect=1, read=0.001)) try: self.assertRaises(ReadTimeoutError, response.read) finally: timed_out.set() # Second should succeed. response = pool.urlopen('GET', '/', retries=0, preload_content=False, timeout=Timeout(connect=1, read=0.1)) self.assertEqual(len(response.read()), 8)
self.host = host if pools > 1: self.ports = np.array(ports, dtype=np.uint16) self._port = self._roll else: self.ports = ports[0] self._port = lambda: self.ports def _roll(self): with self._lock: self.ports = np.roll(self.ports, 1) return self.ports[0] def json(self, path='/', url=None, method='GET'): if not url: url = '{}:{:d}{}'.format(self.host, self._port(), path) try: r = self.request(method, url) if r.status == 200: return json.loads(r.data.decode()) return [] except Exception: return [] timeout = Timeout(connect=2.0, read=4.0) api_client = Autobahn([ 3232, ], timeout=timeout)
def test_release_conn_param_is_respected_after_timeout_retry(self): """For successful ```urlopen(release_conn=False)```, the connection isn't released, even after a retry. This test allows a retry: one request fails, the next request succeeds. This is a regression test for issue #651 [1], where the connection would be released if the initial request failed, even if a retry succeeded. [1] <https://github.com/shazow/urllib3/issues/651> """ def socket_handler(listener): sock = listener.accept()[0] consume_socket(sock) # Close the connection, without sending any response (not even the # HTTP status line). This will trigger a `Timeout` on the client, # inside `urlopen()`. sock.close() # Expect a new request. Because we don't want to hang this thread, # we actually use select.select to confirm that a new request is # coming in: this lets us time the thread out. rlist, _, _ = select.select([listener], [], [], 5) assert rlist sock = listener.accept()[0] consume_socket(sock) # Send complete chunked response. sock.send(('HTTP/1.1 200 OK\r\n' 'Content-Type: text/plain\r\n' 'Transfer-Encoding: chunked\r\n' '\r\n' '8\r\n' '12345678\r\n' '0\r\n\r\n').encode('utf-8')) sock.close() self._start_server(socket_handler) with HTTPConnectionPool(self.host, self.port, maxsize=1) as pool: # First request should fail, but the timeout and `retries=1` should # save it. response = pool.urlopen('GET', '/', retries=1, release_conn=False, preload_content=False, timeout=Timeout(connect=1, read=0.001)) # The connection should still be on the response object, and none # should be in the pool. We opened two though. self.assertEqual(pool.num_connections, 2) self.assertEqual(pool.pool.qsize(), 0) self.assertTrue(response.connection is not None) # Consume the data. This should put the connection back. response.read() self.assertEqual(pool.pool.qsize(), 1) self.assertTrue(response.connection is None)
def test_invalid_timeouts(self, kwargs: Dict[str, Union[int, bool]], message: str) -> None: with pytest.raises(ValueError, match=message): Timeout(**kwargs)
def request(self): req = self._request if req.proxy: if req.proxy_userpwd: headers = make_headers(proxy_basic_auth=req.proxy_userpwd) else: headers = None proxy_url = '%s://%s' % (req.proxy_type, req.proxy) if req.proxy_type == 'socks5': pool = SOCKSProxyManager( proxy_url, cert_reqs='CERT_REQUIRED', ca_certs=certifi.where()) # , proxy_headers=headers) else: pool = ProxyManager(proxy_url, proxy_headers=headers, cert_reqs='CERT_REQUIRED', ca_certs=certifi.where()) else: pool = self.pool with self.wrap_transport_error(): # Retries can be disabled by passing False: # http://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#module-urllib3.util.retry # Do not use False because of warning: # Converted retries value: False -> Retry(total=False, # connect=None, read=None, redirect=0, status=None) retry = Retry( total=False, connect=False, read=False, redirect=0, status=None, ) # The read timeout is not total response time timeout # It is the timeout on read of next data chunk from the server # Total response timeout is handled by Grab timeout = Timeout(connect=req.connect_timeout, read=req.timeout) #req_headers = dict((make_unicode(x), make_unicode(y)) # for (x, y) in req.headers.items()) if six.PY3: req_url = make_unicode(req.url) req_method = make_unicode(req.method) else: req_url = make_str(req.url) req_method = req.method req.op_started = time.time() try: res = pool.urlopen(req_method, req_url, body=req.data, timeout=timeout, retries=retry, headers=req.headers, preload_content=False) except UnicodeError as ex: raise error.GrabConnectionError('GrabInvalidUrl', ex) #except exceptions.ReadTimeoutError as ex: # raise error.GrabTimeoutError('ReadTimeoutError', ex) #except exceptions.ConnectTimeoutError as ex: # raise error.GrabConnectionError('ConnectTimeoutError', ex) #except exceptions.ProtocolError as ex: # # TODO: # # the code # # raise error.GrabConnectionError(ex.args[1][0], ex.args[1][1]) # # fails # # with error TypeError: 'OSError' object is not subscriptable # raise error.GrabConnectionError('ProtocolError', ex) #except exceptions.SSLError as ex: # raise error.GrabConnectionError('SSLError', ex) # WTF? self.request_head = b'' self.request_body = b'' self.request_log = b'' self._response = res
def test_timeout(self): """ Requests should time out when expected """ url = '/sleep?seconds=0.003' timeout = Timeout(read=0.001) # Pool-global timeout pool = HTTPConnectionPool(self.host, self.port, timeout=timeout, retries=False) conn = pool._get_conn() self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', url) pool._put_conn(conn) time.sleep(0.02) # Wait for server to start receiving again. :( self.assertRaises(ReadTimeoutError, pool.request, 'GET', url) # Request-specific timeouts should raise errors pool = HTTPConnectionPool(self.host, self.port, timeout=0.1, retries=False) conn = pool._get_conn() self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', url, timeout=timeout) pool._put_conn(conn) time.sleep(0.02) # Wait for server to start receiving again. :( self.assertRaises(ReadTimeoutError, pool.request, 'GET', url, timeout=timeout) # Timeout int/float passed directly to request and _make_request should # raise a request timeout self.assertRaises(ReadTimeoutError, pool.request, 'GET', url, timeout=0.001) conn = pool._new_conn() self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', url, timeout=0.001) pool._put_conn(conn) # Timeout int/float passed directly to _make_request should not raise a # request timeout if it's a high value pool.request('GET', url, timeout=1)
def test_timeout_str(self): timeout = Timeout(connect=1, read=2, total=3) self.assertEqual(str(timeout), "Timeout(connect=1, read=2, total=3)") timeout = Timeout(connect=1, read=None, total=3) self.assertEqual(str(timeout), "Timeout(connect=1, read=None, total=3)")
def request(self, req, res): options = {} headers = req['headers'] or {} pool = self.get_pool(req) self.op_started = time.time() if req['resolve']: if req['proxy']: raise error.IowebConfigError( 'Request option `resolve` could not be used along option `proxy`' ) for host, ip in req['resolve'].items(): pool.resolving_cache[host] = ip if req['content_encoding']: if not any(x.lower() == 'accept-encoding' for x in headers): headers['Accept-Encoding'] = req['content_encoding'] if req['data']: if isinstance(req['data'], dict): if req['multipart']: body, ctype = encode_multipart_formata(req['data']) else: body = urlencode(req['data']) ctype = 'application/x-www-form-urlencoded' options['body'] = body headers['Content-Type'] = ctype elif isinstance(req['data'], bytes): options['body'] = req['data'] elif isinstance(req['data'], str): options['body'] = req['data'].encode('utf-8') else: raise IowebConfigError( 'Invalid type of request data option: %s' % type(req['data'])) headers['Content-Length'] = len(options['body']) with self.handle_network_error(): if req['follow_redirect']: retry_opts = { 'redirect': req['max_redirects'], 'raise_on_redirect': True, } else: retry_opts = { 'redirect': False, 'raise_on_redirect': False, } self.urllib3_response = pool.urlopen( req.method(), req['url'], headers=headers, # total - set to None to remove this constraint # and fall back on other counts. retries=Retry( total=None, connect=False, read=False, **retry_opts, ), timeout=Timeout( connect=req['connect_timeout'], read=req['timeout'], ), preload_content=False, decode_content=req['decode_content'], **options)
def test_timeout(self): # Requests should time out when expected block_event = Event() ready_event = self.start_basic_handler(block_send=block_event, num=6) # Pool-global timeout timeout = Timeout(read=SHORT_TIMEOUT) pool = HTTPConnectionPool(self.host, self.port, timeout=timeout, retries=False) wait_for_socket(ready_event) conn = pool._get_conn() with self.assertRaises(ReadTimeoutError) as cmgr: pool._make_request(conn, 'GET', '/') self.assertEqual(cmgr.exception.args[0].split()[-1], 'timeout=%s)' % timeout.read_timeout) pool._put_conn(conn) block_event.set() # Release request wait_for_socket(ready_event) block_event.clear() with self.assertRaises(ReadTimeoutError) as cmgr: pool.request('GET', '/') self.assertEqual(cmgr.exception.args[0].split()[-1], 'timeout=%s)' % timeout.read_timeout) block_event.set() # Release request # Request-specific timeouts should raise errors pool = HTTPConnectionPool(self.host, self.port, timeout=LONG_TIMEOUT, retries=False) conn = pool._get_conn() wait_for_socket(ready_event) now = time.time() with self.assertRaises(ReadTimeoutError) as cmgr: pool._make_request(conn, 'GET', '/', timeout=timeout) self.assertEqual(cmgr.exception.args[0].split()[-1], 'timeout=%s)' % timeout.read_timeout) delta = time.time() - now block_event.set() # Release request self.assertTrue( delta < LONG_TIMEOUT, "timeout was pool-level LONG_TIMEOUT rather than request-level SHORT_TIMEOUT" ) pool._put_conn(conn) wait_for_socket(ready_event) now = time.time() with self.assertRaises(ReadTimeoutError) as cmgr: pool.request('GET', '/', timeout=timeout) self.assertEqual(cmgr.exception.args[0].split()[-1], 'timeout=%s)' % timeout.read_timeout) delta = time.time() - now self.assertTrue( delta < LONG_TIMEOUT, "timeout was pool-level LONG_TIMEOUT rather than request-level SHORT_TIMEOUT" ) block_event.set() # Release request # Timeout int/float passed directly to request and _make_request should # raise a request timeout wait_for_socket(ready_event) with self.assertRaises(ReadTimeoutError) as cmgr: pool.request('GET', '/', timeout=SHORT_TIMEOUT) self.assertEqual(cmgr.exception.args[0].split()[-1], 'timeout=%s)' % SHORT_TIMEOUT) block_event.set() # Release request wait_for_socket(ready_event) conn = pool._new_conn() # FIXME: This assert flakes sometimes. Not sure why. with self.assertRaises(ReadTimeoutError) as cmgr: pool._make_request(conn, 'GET', '/', timeout=SHORT_TIMEOUT) block_event.set() # Release request self.assertEqual(cmgr.exception.args[0].split()[-1], 'timeout=%s)' % SHORT_TIMEOUT)
def test_invalid_timeouts(self, kwargs, message): with pytest.raises(ValueError, match=message): Timeout(**kwargs)
try: from urllib3 import disable_warnings, PoolManager from urllib3.util.timeout import Timeout except ImportError: ver = version_info[0] if version_info[0] >= 3 else "" raise ("\n * Package urllib3 not installed. Please install the package urllib3 before continue.\n" + " Example: \n" + " # apt-get install python%s-pip ; easy_install%s urllib3\n" % (ver, ver)) from urllib3 import disable_warnings, PoolManager from urllib3.util.timeout import Timeout disable_warnings() timeout = Timeout(connect=3.0, read=6.0) pool = PoolManager(timeout=timeout, cert_reqs='CERT_NONE') user_agents = ["Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:38.0) Gecko/20100101 Firefox/38.0", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36", "Mozilla/5.0 (Windows NT 5.1; rv:40.0) Gecko/20100101 Firefox/40.0", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)", "Mozilla/5.0 (compatible; MSIE 6.0; Windows NT 5.1)", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20100101 Firefox/31.0", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36", "Opera/9.80 (Windows NT 6.2; Win64; x64) Presto/2.12.388 Version/12.17", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0",
def check_vul(url): """ Test if a GET to a URL is successful :param url: The URL to test :return: A dict with the exploit type as the keys, and the HTTP status code as the value """ if gl_args.mode == 'auto-scan' or gl_args.mode == 'file-scan': timeout = Timeout(connect=1.0, read=3.0) pool = PoolManager(timeout=timeout, retries=1, cert_reqs='CERT_NONE') else: timeout = Timeout(connect=3.0, read=6.0) pool = PoolManager(timeout=timeout, cert_reqs='CERT_NONE') url_check = parse_url(url) if '443' in str(url_check.port) and url_check.scheme != 'https': url = "https://" + str(url_check.host) + ":" + str(url_check.port) print(GREEN + "\n ** Checking Host: %s **\n" % url) headers = { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Connection": "keep-alive", "User-Agent": user_agents[randint(0, len(user_agents) - 1)] } paths = { "jmx-console": "/jmx-console/HtmlAdaptor?action=inspectMBean&name=jboss.system:type=ServerInfo", "web-console": "/web-console/ServerInfo.jsp", "JMXInvokerServlet": "/invoker/JMXInvokerServlet", "admin-console": "/admin-console/" } for i in paths.keys(): if gl_interrupted: break try: print(GREEN + " * Checking %s: \t" % i + ENDC), r = pool.request('HEAD', url + str(paths[i]), redirect=False, headers=headers) paths[i] = r.status if paths[i] in (301, 302, 303, 307, 308): url_redirect = r.get_redirect_location() print(GREEN + "[ REDIRECT ]\n * The server sent a redirect to: %s\n" % url_redirect) elif paths[i] == 200 or paths[i] == 500: if i == "admin-console": print(RED + "[ EXPOSED ]" + ENDC) else: print(RED + "[ VULNERABLE ]" + ENDC) else: print(GREEN + "[ OK ]") except: print(RED + "\n * An error occurred while connecting to the host %s\n" % url + ENDC) paths[i] = 505 return paths
def test_timeout_str(self): timeout = Timeout(connect=1, read=2, total=3) assert str(timeout) == "Timeout(connect=1, read=2, total=3)" timeout = Timeout(connect=1, read=None, total=3) assert str(timeout) == "Timeout(connect=1, read=None, total=3)"
def post(self, url, data, timeout=None): """Request an URL. Args: url (:obj:`str`): The web location we want to retrieve. data (dict[str, str|int]): A dict of key/value pairs. Note: On py2.7 value is unicode. timeout (:obj:`int` | :obj:`float`): If this value is specified, use it as the read timeout from the server (instead of the one specified during creation of the connection pool). Returns: A JSON object. """ urlopen_kwargs = {} if timeout is not None: urlopen_kwargs['timeout'] = Timeout(read=timeout, connect=self._connect_timeout) # Are we uploading files? files = False for key, val in data.copy().items(): if isinstance(val, InputFile): # Convert the InputFile to urllib3 field format data[key] = val.field_tuple files = True elif isinstance(val, (float, int)): # Urllib3 doesn't like floats it seems data[key] = str(val) elif key == 'media': # One media or multiple if isinstance(val, InputMedia): # Attach and set val to attached name data[key] = val.to_json() if isinstance(val.media, InputFile): data[val.media.attach] = val.media.field_tuple else: # Attach and set val to attached name for all media = [] for m in val: media.append(m.to_dict()) if isinstance(m.media, InputFile): data[m.media.attach] = m.media.field_tuple data[key] = json.dumps(media) files = True # Use multipart upload if we're uploading files, otherwise use JSON if files: result = self._request_wrapper('POST', url, fields=data, **urlopen_kwargs) else: result = self._request_wrapper( 'POST', url, body=json.dumps(data).encode('utf-8'), headers={'Content-Type': 'application/json'}, **urlopen_kwargs) return self._parse(result)