def test_timeout_elapsed(self, current_time):
current_time.return_value = TIMEOUT_EPOCH
timeout = Timeout(total=3)
self.assertRaises(TimeoutStateError, timeout.get_connect_duration)
timeout.start_connect()
self.assertRaises(TimeoutStateError, timeout.start_connect)
current_time.return_value = TIMEOUT_EPOCH + 2
self.assertEqual(timeout.get_connect_duration(), 2)
current_time.return_value = TIMEOUT_EPOCH + 37
self.assertEqual(timeout.get_connect_duration(), 37)
def test_timeout_elapsed(self, current_time):
current_time.return_value = TIMEOUT_EPOCH
timeout = Timeout(total=3)
with pytest.raises(TimeoutStateError):
timeout.get_connect_duration()
timeout.start_connect()
with pytest.raises(TimeoutStateError):
timeout.start_connect()
current_time.return_value = TIMEOUT_EPOCH + 2
assert timeout.get_connect_duration() == 2
current_time.return_value = TIMEOUT_EPOCH + 37
assert timeout.get_connect_duration() == 37
def test_ssl_read_timeout(self):
timed_out = Event()
def socket_handler(listener):
sock = listener.accept()[0]
ssl_sock = ssl.wrap_socket(sock,
server_side=True,
keyfile=DEFAULT_CERTS['keyfile'],
certfile=DEFAULT_CERTS['certfile'],
ca_certs=DEFAULT_CA)
buf = b''
while not buf.endswith(b'\r\n\r\n'):
buf += ssl_sock.recv(65536)
# Send incomplete message (note Content-Length)
ssl_sock.send(('HTTP/1.1 200 OK\r\n'
'Content-Type: text/plain\r\n'
'Content-Length: 10\r\n'
'\r\n'
'Hi-').encode('utf-8'))
timed_out.wait()
sock.close()
ssl_sock.close()
self._start_server(socket_handler)
pool = HTTPSConnectionPool(self.host, self.port)
response = pool.urlopen('GET',
'/',
retries=0,
preload_content=False,
timeout=Timeout(connect=1, read=0.001))
try:
self.assertRaises(ReadTimeoutError, response.read)
finally:
timed_out.set()
def test_timeout(self):
# Requests should time out when expected
block_event = Event()
ready_event = self.start_basic_handler(block_send=block_event, num=3)
# Pool-global timeout
short_timeout = Timeout(read=SHORT_TIMEOUT)
with HTTPConnectionPool(self.host,
self.port,
timeout=short_timeout,
retries=False) as pool:
wait_for_socket(ready_event)
block_event.clear()
with pytest.raises(ReadTimeoutError):
pool.request("GET", "/")
block_event.set() # Release request
# Request-specific timeouts should raise errors
with HTTPConnectionPool(self.host,
self.port,
timeout=short_timeout,
retries=False) as pool:
wait_for_socket(ready_event)
now = time.time()
with pytest.raises(ReadTimeoutError):
pool.request("GET", "/", timeout=LONG_TIMEOUT)
delta = time.time() - now
message = "timeout was pool-level SHORT_TIMEOUT rather than request-level LONG_TIMEOUT"
assert delta >= LONG_TIMEOUT, message
block_event.set() # Release request
# Timeout passed directly to request should raise a request timeout
wait_for_socket(ready_event)
with pytest.raises(ReadTimeoutError):
pool.request("GET", "/", timeout=SHORT_TIMEOUT)
block_event.set() # Release request
def post(self, url, data, timeout=None):
"""Request an URL.
Args:
url (str): The web location we want to retrieve.
data (dict[str, str]): A dict of key/value pairs. Note: On py2.7 value is unicode.
timeout (Optional[int|float]): If this value is specified, use it as the read timeout
from the server (instead of the one specified during creation of the connection
pool).
Returns:
A JSON object.
"""
urlopen_kwargs = {}
if timeout is not None:
urlopen_kwargs['timeout'] = Timeout(read=timeout,
connect=self._connect_timeout)
if InputFile.is_inputfile(data):
data = InputFile(data)
result = self._request_wrapper('POST',
url,
body=data.to_form(),
headers=data.headers,
**urlopen_kwargs)
else:
data = json.dumps(data)
result = self._request_wrapper(
'POST',
url,
body=data.encode(),
headers={'Content-Type': 'application/json'},
**urlopen_kwargs)
return self._parse(result)
def request(self, req, res):
options = {}
headers = req['headers'] or {}
req_url = req['url']
if req_url.endswith('?'):
req_url = req['url'][:-1]
#req_url = req['url']
#if not '://' in req_url:
# req_url = 'http://%s' % req_url
#url_parts = urlsplit(req_url)
#if url_parts.scheme not in ('http', 'https'):
# emsg = 'Unknown scheme [%s]' % url_parts.scheme
# # second argument saved as `err.transport_error`
# # and used then to generate short code of error
# # for statistics
# raise error.InvalidUrlError(
# emsg, error.InvalidUrlError(emsg),
# )
pool = self.get_pool(req, use_cache=(not req['close_connection']))
self.op_started = time.time()
if req['resolve']:
if req['proxy']:
raise error.IowebConfigError(
'Request option `resolve` could not be used along option `proxy`'
)
for host, ip in req['resolve'].items():
pool.resolving_cache[host] = ip
if req['content_encoding']:
if not any(x.lower() == 'accept-encoding' for x in headers):
headers['Accept-Encoding'] = req['content_encoding']
if req['data']:
if isinstance(req['data'], dict):
if req['multipart']:
body, ctype = encode_multipart_formdata(req['data'])
else:
body = urlencode(req['data'])
ctype = 'application/x-www-form-urlencoded'
options['body'] = body
headers['Content-Type'] = ctype
elif isinstance(req['data'], bytes):
options['body'] = req['data']
elif isinstance(req['data'], str):
options['body'] = req['data'].encode('utf-8')
else:
raise error.IowebConfigError(
'Invalid type of request data option: %s' %
type(req['data']))
headers['Content-Length'] = len(options['body'])
with self.handle_network_error(req):
retry_opts = {
# total - set to None to remove this constraint
# and fall back on other counts.
'connect': False,
'read': False,
}
if req['follow_redirect']:
retry_opts.update({
'total': req['max_redirects'],
'redirect': req['max_redirects'],
'raise_on_redirect': True,
})
else:
retry_opts.update({
'total': False,
'redirect': False,
'raise_on_redirect': False,
})
self.urllib3_response = pool.urlopen(
req.method(),
req_url,
headers=headers,
retries=Retry(**retry_opts),
timeout=Timeout(
connect=req['connect_timeout'],
read=req['timeout'],
),
preload_content=False,
decode_content=req['decode_content'],
**options)
def pool(self):
timeout = Timeout(connect=self._conn_timeout, read=self._read_timeout)
return PoolManager(num_pools=self._num_pools, timeout=timeout)
if not APP_NAME:
raise NoAppNameError('APP_NAME is mandatory parameter!')
WORK_DIR = os.path.dirname(os.path.abspath(__file__))
LOG_PATH = os.path.join(WORK_DIR, 'log')
LOG = os.path.join(LOG_PATH, 'log.txt')
LOGSTASH_HOST = os.getenv('LOGSTASH_HOST')
HTTP_CONNECT_TIMEOUT = os.getenv('HTTP_CONNECT_TIMEOUT')
HTTP_READ_TIMEOUT = os.getenv('HTTP_READ_TIMEOUT')
HTTP_RETRY_COUNT = int(os.getenv('HTTP_RETRY_COUNT') or 4)
HTTP_RETRY_BACKOFF = float(os.getenv('HTTP_RETRY_BACKOFF') or 15)
if HTTP_CONNECT_TIMEOUT or HTTP_READ_TIMEOUT:
HTTP_TIMEOUT = Timeout(connect=int(HTTP_CONNECT_TIMEOUT or 0) or _Default,
read=int(HTTP_READ_TIMEOUT or 0) or _Default)
else:
http_timeout = int(os.getenv('HTTP_TIMEOUT') or 60)
HTTP_TIMEOUT = Timeout(connect=http_timeout, read=http_timeout)
TIME_ZONE = 'UTC'
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'default': {
'class': 'watcher.logger.DSMConsoleFormatter',
'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
}
},
out += " #" + str(count)
print("Grabbed deck: " + out)
## configure readline
histfile = os.path.join(os.path.expanduser("~"), ".tu_deck_grabber_history")
if os.path.exists(histfile):
readline.read_history_file(histfile)
readline.set_history_length(1000)
readline.read_init_file()
atexit.register(readline.write_history_file, histfile)
with PoolManager(
1,
timeout=Timeout(connect=15.0, read=20.0, total=30.0),
retries=Retry(total=3),
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where(),
) as http:
while True:
try:
line = input("{} ~> ".format(login))
except EOFError:
print()
line = None
if not line:
break
line = line.strip().lower()
if line == 'exit':
break
def test_invalid_timeouts(self, kwargs, message):
with pytest.raises(ValueError) as e:
Timeout(**kwargs)
assert message in str(e.value)
def test_timeout(self):
# Requests should time out when expected
block_event = Event()
ready_event = self.start_basic_handler(block_send=block_event, num=6)
# Pool-global timeout
timeout = Timeout(read=SHORT_TIMEOUT)
pool = HTTPConnectionPool(self.host,
self.port,
timeout=timeout,
retries=False)
self.addCleanup(pool.close)
wait_for_socket(ready_event)
conn = pool._get_conn()
self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET',
'/')
pool._put_conn(conn)
block_event.set() # Release request
wait_for_socket(ready_event)
block_event.clear()
self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/')
block_event.set() # Release request
# Request-specific timeouts should raise errors
pool = HTTPConnectionPool(self.host,
self.port,
timeout=LONG_TIMEOUT,
retries=False)
self.addCleanup(pool.close)
conn = pool._get_conn()
wait_for_socket(ready_event)
now = time.time()
self.assertRaises(ReadTimeoutError,
pool._make_request,
conn,
'GET',
'/',
timeout=timeout)
delta = time.time() - now
block_event.set() # Release request
message = "timeout was pool-level LONG_TIMEOUT rather than request-level SHORT_TIMEOUT"
self.assertTrue(delta < LONG_TIMEOUT, message)
pool._put_conn(conn)
wait_for_socket(ready_event)
now = time.time()
self.assertRaises(ReadTimeoutError,
pool.request,
'GET',
'/',
timeout=timeout)
delta = time.time() - now
message = "timeout was pool-level LONG_TIMEOUT rather than request-level SHORT_TIMEOUT"
self.assertTrue(delta < LONG_TIMEOUT, message)
block_event.set() # Release request
# Timeout int/float passed directly to request and _make_request should
# raise a request timeout
wait_for_socket(ready_event)
self.assertRaises(ReadTimeoutError,
pool.request,
'GET',
'/',
timeout=SHORT_TIMEOUT)
block_event.set() # Release request
wait_for_socket(ready_event)
conn = pool._new_conn()
# FIXME: This assert flakes sometimes. Not sure why.
self.assertRaises(ReadTimeoutError,
pool._make_request,
conn,
'GET',
'/',
timeout=SHORT_TIMEOUT)
block_event.set() # Release request
cookiejar = cookies.RequestsCookieJar()
cookiejar.update(cookies_)
assert response.text == '\n'.join('{}: {}'.format(*item)
for item in cookiejar.items())
# Timeouts should go last, because '/slow' hangs the HTTP server
@pytest.mark.parametrize('timeout', [0.1, (None, 0.1)])
def test_get_timeout(http_server, timeout):
with pytest.raises(requests.Timeout):
requests.get(http_server.base_url + '/slow', timeout=timeout)
@pytest.mark.skipif('urllib3' not in sys.modules,
reason='urllib3 not available')
@pytest.mark.parametrize('timeout', [Timeout(read=0.1), Timeout(total=0.1)])
def test_get_timeout_urllib3(http_server, timeout):
with pytest.raises(requests.Timeout):
requests.get(http_server.base_url + '/slow', timeout=timeout)
@pytest.mark.parametrize(
'timeout',
[0.1,
(0.1, None), Timeout(connect=0.1),
Timeout(total=0.1)])
def test_get_connect_timeout(http_server, timeout):
with pytest.raises(requests.Timeout):
# Use RFC-5737 TEST-NET-1 address since it should always be unreachable
requests.get('http://192.0.2.1', timeout=timeout)
requests.get(http_server.base_url + '/slow', timeout=timeout)
def test_timeout(self):
# Requests should time out when expected
block_event = Event()
ready_event = self.start_basic_handler(block_send=block_event, num=6)
# Pool-global timeout
timeout = Timeout(read=SHORT_TIMEOUT)
with HTTPConnectionPool(self.host,
self.port,
timeout=timeout,
retries=False) as pool:
wait_for_socket(ready_event)
conn = pool._get_conn()
with pytest.raises(ReadTimeoutError):
pool._make_request(conn, "GET", "/")
pool._put_conn(conn)
block_event.set() # Release request
wait_for_socket(ready_event)
block_event.clear()
with pytest.raises(ReadTimeoutError):
pool.request("GET", "/")
block_event.set() # Release request
# Request-specific timeouts should raise errors
with HTTPConnectionPool(self.host,
self.port,
timeout=LONG_TIMEOUT,
retries=False) as pool:
conn = pool._get_conn()
wait_for_socket(ready_event)
now = time.time()
with pytest.raises(ReadTimeoutError):
pool._make_request(conn, "GET", "/", timeout=timeout)
delta = time.time() - now
block_event.set() # Release request
message = "timeout was pool-level LONG_TIMEOUT rather than request-level SHORT_TIMEOUT"
assert delta < LONG_TIMEOUT, message
pool._put_conn(conn)
wait_for_socket(ready_event)
now = time.time()
with pytest.raises(ReadTimeoutError):
pool.request("GET", "/", timeout=timeout)
delta = time.time() - now
message = "timeout was pool-level LONG_TIMEOUT rather than request-level SHORT_TIMEOUT"
assert delta < LONG_TIMEOUT, message
block_event.set() # Release request
# Timeout int/float passed directly to request and _make_request should
# raise a request timeout
wait_for_socket(ready_event)
with pytest.raises(ReadTimeoutError):
pool.request("GET", "/", timeout=SHORT_TIMEOUT)
block_event.set() # Release request
wait_for_socket(ready_event)
conn = pool._new_conn()
# FIXME: This assert flakes sometimes. Not sure why.
with pytest.raises(ReadTimeoutError):
pool._make_request(conn, "GET", "/", timeout=SHORT_TIMEOUT)
block_event.set() # Release request
def test_create_connection_timeout(self):
timeout = Timeout(connect=SHORT_TIMEOUT, total=LONG_TIMEOUT)
pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=timeout, retries=False)
self.addCleanup(pool.close)
conn = pool._new_conn()
self.assertRaises(ConnectTimeoutError, conn.connect)
def test_connection_closed_on_read_timeout_preload_false(self):
timed_out = Event()
def socket_handler(listener):
sock = listener.accept()[0]
# Consume request
buf = b''
while not buf.endswith(b'\r\n\r\n'):
buf = sock.recv(65535)
# Send partial chunked response and then hang.
sock.send(('HTTP/1.1 200 OK\r\n'
'Content-Type: text/plain\r\n'
'Transfer-Encoding: chunked\r\n'
'\r\n'
'8\r\n'
'12345678\r\n').encode('utf-8'))
timed_out.wait(5)
# Expect a new request, but keep hold of the old socket to avoid
# leaking it. Because we don't want to hang this thread, we
# actually use select.select to confirm that a new request is
# coming in: this lets us time the thread out.
rlist, _, _ = select.select([listener], [], [], 1)
assert rlist
new_sock = listener.accept()[0]
# Consume request
buf = b''
while not buf.endswith(b'\r\n\r\n'):
buf = new_sock.recv(65535)
# Send complete chunked response.
new_sock.send(('HTTP/1.1 200 OK\r\n'
'Content-Type: text/plain\r\n'
'Transfer-Encoding: chunked\r\n'
'\r\n'
'8\r\n'
'12345678\r\n'
'0\r\n\r\n').encode('utf-8'))
new_sock.close()
sock.close()
self._start_server(socket_handler)
with HTTPConnectionPool(self.host, self.port) as pool:
# First request should fail.
response = pool.urlopen('GET',
'/',
retries=0,
preload_content=False,
timeout=Timeout(connect=1, read=0.001))
try:
self.assertRaises(ReadTimeoutError, response.read)
finally:
timed_out.set()
# Second should succeed.
response = pool.urlopen('GET',
'/',
retries=0,
preload_content=False,
timeout=Timeout(connect=1, read=0.1))
self.assertEqual(len(response.read()), 8)
self.host = host
if pools > 1:
self.ports = np.array(ports, dtype=np.uint16)
self._port = self._roll
else:
self.ports = ports[0]
self._port = lambda: self.ports
def _roll(self):
with self._lock:
self.ports = np.roll(self.ports, 1)
return self.ports[0]
def json(self, path='/', url=None, method='GET'):
if not url:
url = '{}:{:d}{}'.format(self.host, self._port(), path)
try:
r = self.request(method, url)
if r.status == 200:
return json.loads(r.data.decode())
return []
except Exception:
return []
timeout = Timeout(connect=2.0, read=4.0)
api_client = Autobahn([
3232,
], timeout=timeout)
def test_release_conn_param_is_respected_after_timeout_retry(self):
"""For successful ```urlopen(release_conn=False)```,
the connection isn't released, even after a retry.
This test allows a retry: one request fails, the next request succeeds.
This is a regression test for issue #651 [1], where the connection
would be released if the initial request failed, even if a retry
succeeded.
[1] <https://github.com/shazow/urllib3/issues/651>
"""
def socket_handler(listener):
sock = listener.accept()[0]
consume_socket(sock)
# Close the connection, without sending any response (not even the
# HTTP status line). This will trigger a `Timeout` on the client,
# inside `urlopen()`.
sock.close()
# Expect a new request. Because we don't want to hang this thread,
# we actually use select.select to confirm that a new request is
# coming in: this lets us time the thread out.
rlist, _, _ = select.select([listener], [], [], 5)
assert rlist
sock = listener.accept()[0]
consume_socket(sock)
# Send complete chunked response.
sock.send(('HTTP/1.1 200 OK\r\n'
'Content-Type: text/plain\r\n'
'Transfer-Encoding: chunked\r\n'
'\r\n'
'8\r\n'
'12345678\r\n'
'0\r\n\r\n').encode('utf-8'))
sock.close()
self._start_server(socket_handler)
with HTTPConnectionPool(self.host, self.port, maxsize=1) as pool:
# First request should fail, but the timeout and `retries=1` should
# save it.
response = pool.urlopen('GET',
'/',
retries=1,
release_conn=False,
preload_content=False,
timeout=Timeout(connect=1, read=0.001))
# The connection should still be on the response object, and none
# should be in the pool. We opened two though.
self.assertEqual(pool.num_connections, 2)
self.assertEqual(pool.pool.qsize(), 0)
self.assertTrue(response.connection is not None)
# Consume the data. This should put the connection back.
response.read()
self.assertEqual(pool.pool.qsize(), 1)
self.assertTrue(response.connection is None)
def test_invalid_timeouts(self, kwargs: Dict[str, Union[int, bool]],
message: str) -> None:
with pytest.raises(ValueError, match=message):
Timeout(**kwargs)
def request(self):
req = self._request
if req.proxy:
if req.proxy_userpwd:
headers = make_headers(proxy_basic_auth=req.proxy_userpwd)
else:
headers = None
proxy_url = '%s://%s' % (req.proxy_type, req.proxy)
if req.proxy_type == 'socks5':
pool = SOCKSProxyManager(
proxy_url,
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where()) # , proxy_headers=headers)
else:
pool = ProxyManager(proxy_url,
proxy_headers=headers,
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where())
else:
pool = self.pool
with self.wrap_transport_error():
# Retries can be disabled by passing False:
# http://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#module-urllib3.util.retry
# Do not use False because of warning:
# Converted retries value: False -> Retry(total=False,
# connect=None, read=None, redirect=0, status=None)
retry = Retry(
total=False,
connect=False,
read=False,
redirect=0,
status=None,
)
# The read timeout is not total response time timeout
# It is the timeout on read of next data chunk from the server
# Total response timeout is handled by Grab
timeout = Timeout(connect=req.connect_timeout, read=req.timeout)
#req_headers = dict((make_unicode(x), make_unicode(y))
# for (x, y) in req.headers.items())
if six.PY3:
req_url = make_unicode(req.url)
req_method = make_unicode(req.method)
else:
req_url = make_str(req.url)
req_method = req.method
req.op_started = time.time()
try:
res = pool.urlopen(req_method,
req_url,
body=req.data,
timeout=timeout,
retries=retry,
headers=req.headers,
preload_content=False)
except UnicodeError as ex:
raise error.GrabConnectionError('GrabInvalidUrl', ex)
#except exceptions.ReadTimeoutError as ex:
# raise error.GrabTimeoutError('ReadTimeoutError', ex)
#except exceptions.ConnectTimeoutError as ex:
# raise error.GrabConnectionError('ConnectTimeoutError', ex)
#except exceptions.ProtocolError as ex:
# # TODO:
# # the code
# # raise error.GrabConnectionError(ex.args[1][0], ex.args[1][1])
# # fails
# # with error TypeError: 'OSError' object is not subscriptable
# raise error.GrabConnectionError('ProtocolError', ex)
#except exceptions.SSLError as ex:
# raise error.GrabConnectionError('SSLError', ex)
# WTF?
self.request_head = b''
self.request_body = b''
self.request_log = b''
self._response = res
def test_timeout(self):
""" Requests should time out when expected """
url = '/sleep?seconds=0.003'
timeout = Timeout(read=0.001)
# Pool-global timeout
pool = HTTPConnectionPool(self.host,
self.port,
timeout=timeout,
retries=False)
conn = pool._get_conn()
self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET',
url)
pool._put_conn(conn)
time.sleep(0.02) # Wait for server to start receiving again. :(
self.assertRaises(ReadTimeoutError, pool.request, 'GET', url)
# Request-specific timeouts should raise errors
pool = HTTPConnectionPool(self.host,
self.port,
timeout=0.1,
retries=False)
conn = pool._get_conn()
self.assertRaises(ReadTimeoutError,
pool._make_request,
conn,
'GET',
url,
timeout=timeout)
pool._put_conn(conn)
time.sleep(0.02) # Wait for server to start receiving again. :(
self.assertRaises(ReadTimeoutError,
pool.request,
'GET',
url,
timeout=timeout)
# Timeout int/float passed directly to request and _make_request should
# raise a request timeout
self.assertRaises(ReadTimeoutError,
pool.request,
'GET',
url,
timeout=0.001)
conn = pool._new_conn()
self.assertRaises(ReadTimeoutError,
pool._make_request,
conn,
'GET',
url,
timeout=0.001)
pool._put_conn(conn)
# Timeout int/float passed directly to _make_request should not raise a
# request timeout if it's a high value
pool.request('GET', url, timeout=1)
def test_timeout_str(self):
timeout = Timeout(connect=1, read=2, total=3)
self.assertEqual(str(timeout), "Timeout(connect=1, read=2, total=3)")
timeout = Timeout(connect=1, read=None, total=3)
self.assertEqual(str(timeout),
"Timeout(connect=1, read=None, total=3)")
def request(self, req, res):
options = {}
headers = req['headers'] or {}
pool = self.get_pool(req)
self.op_started = time.time()
if req['resolve']:
if req['proxy']:
raise error.IowebConfigError(
'Request option `resolve` could not be used along option `proxy`'
)
for host, ip in req['resolve'].items():
pool.resolving_cache[host] = ip
if req['content_encoding']:
if not any(x.lower() == 'accept-encoding' for x in headers):
headers['Accept-Encoding'] = req['content_encoding']
if req['data']:
if isinstance(req['data'], dict):
if req['multipart']:
body, ctype = encode_multipart_formata(req['data'])
else:
body = urlencode(req['data'])
ctype = 'application/x-www-form-urlencoded'
options['body'] = body
headers['Content-Type'] = ctype
elif isinstance(req['data'], bytes):
options['body'] = req['data']
elif isinstance(req['data'], str):
options['body'] = req['data'].encode('utf-8')
else:
raise IowebConfigError(
'Invalid type of request data option: %s' %
type(req['data']))
headers['Content-Length'] = len(options['body'])
with self.handle_network_error():
if req['follow_redirect']:
retry_opts = {
'redirect': req['max_redirects'],
'raise_on_redirect': True,
}
else:
retry_opts = {
'redirect': False,
'raise_on_redirect': False,
}
self.urllib3_response = pool.urlopen(
req.method(),
req['url'],
headers=headers,
# total - set to None to remove this constraint
# and fall back on other counts.
retries=Retry(
total=None,
connect=False,
read=False,
**retry_opts,
),
timeout=Timeout(
connect=req['connect_timeout'],
read=req['timeout'],
),
preload_content=False,
decode_content=req['decode_content'],
**options)
def test_timeout(self):
# Requests should time out when expected
block_event = Event()
ready_event = self.start_basic_handler(block_send=block_event, num=6)
# Pool-global timeout
timeout = Timeout(read=SHORT_TIMEOUT)
pool = HTTPConnectionPool(self.host,
self.port,
timeout=timeout,
retries=False)
wait_for_socket(ready_event)
conn = pool._get_conn()
with self.assertRaises(ReadTimeoutError) as cmgr:
pool._make_request(conn, 'GET', '/')
self.assertEqual(cmgr.exception.args[0].split()[-1],
'timeout=%s)' % timeout.read_timeout)
pool._put_conn(conn)
block_event.set() # Release request
wait_for_socket(ready_event)
block_event.clear()
with self.assertRaises(ReadTimeoutError) as cmgr:
pool.request('GET', '/')
self.assertEqual(cmgr.exception.args[0].split()[-1],
'timeout=%s)' % timeout.read_timeout)
block_event.set() # Release request
# Request-specific timeouts should raise errors
pool = HTTPConnectionPool(self.host,
self.port,
timeout=LONG_TIMEOUT,
retries=False)
conn = pool._get_conn()
wait_for_socket(ready_event)
now = time.time()
with self.assertRaises(ReadTimeoutError) as cmgr:
pool._make_request(conn, 'GET', '/', timeout=timeout)
self.assertEqual(cmgr.exception.args[0].split()[-1],
'timeout=%s)' % timeout.read_timeout)
delta = time.time() - now
block_event.set() # Release request
self.assertTrue(
delta < LONG_TIMEOUT,
"timeout was pool-level LONG_TIMEOUT rather than request-level SHORT_TIMEOUT"
)
pool._put_conn(conn)
wait_for_socket(ready_event)
now = time.time()
with self.assertRaises(ReadTimeoutError) as cmgr:
pool.request('GET', '/', timeout=timeout)
self.assertEqual(cmgr.exception.args[0].split()[-1],
'timeout=%s)' % timeout.read_timeout)
delta = time.time() - now
self.assertTrue(
delta < LONG_TIMEOUT,
"timeout was pool-level LONG_TIMEOUT rather than request-level SHORT_TIMEOUT"
)
block_event.set() # Release request
# Timeout int/float passed directly to request and _make_request should
# raise a request timeout
wait_for_socket(ready_event)
with self.assertRaises(ReadTimeoutError) as cmgr:
pool.request('GET', '/', timeout=SHORT_TIMEOUT)
self.assertEqual(cmgr.exception.args[0].split()[-1],
'timeout=%s)' % SHORT_TIMEOUT)
block_event.set() # Release request
wait_for_socket(ready_event)
conn = pool._new_conn()
# FIXME: This assert flakes sometimes. Not sure why.
with self.assertRaises(ReadTimeoutError) as cmgr:
pool._make_request(conn, 'GET', '/', timeout=SHORT_TIMEOUT)
block_event.set() # Release request
self.assertEqual(cmgr.exception.args[0].split()[-1],
'timeout=%s)' % SHORT_TIMEOUT)
def test_invalid_timeouts(self, kwargs, message):
with pytest.raises(ValueError, match=message):
Timeout(**kwargs)
try:
from urllib3 import disable_warnings, PoolManager
from urllib3.util.timeout import Timeout
except ImportError:
ver = version_info[0] if version_info[0] >= 3 else ""
raise ("\n * Package urllib3 not installed. Please install the package urllib3 before continue.\n"
+ " Example: \n"
+ " # apt-get install python%s-pip ; easy_install%s urllib3\n" % (ver, ver))
from urllib3 import disable_warnings, PoolManager
from urllib3.util.timeout import Timeout
disable_warnings()
timeout = Timeout(connect=3.0, read=6.0)
pool = PoolManager(timeout=timeout, cert_reqs='CERT_NONE')
user_agents = ["Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:38.0) Gecko/20100101 Firefox/38.0",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36",
"Mozilla/5.0 (Windows NT 5.1; rv:40.0) Gecko/20100101 Firefox/40.0",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)",
"Mozilla/5.0 (compatible; MSIE 6.0; Windows NT 5.1)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20100101 Firefox/31.0",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36",
"Opera/9.80 (Windows NT 6.2; Win64; x64) Presto/2.12.388 Version/12.17",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0",
def check_vul(url):
"""
Test if a GET to a URL is successful
:param url: The URL to test
:return: A dict with the exploit type as the keys, and the HTTP status code as the value
"""
if gl_args.mode == 'auto-scan' or gl_args.mode == 'file-scan':
timeout = Timeout(connect=1.0, read=3.0)
pool = PoolManager(timeout=timeout, retries=1, cert_reqs='CERT_NONE')
else:
timeout = Timeout(connect=3.0, read=6.0)
pool = PoolManager(timeout=timeout, cert_reqs='CERT_NONE')
url_check = parse_url(url)
if '443' in str(url_check.port) and url_check.scheme != 'https':
url = "https://" + str(url_check.host) + ":" + str(url_check.port)
print(GREEN + "\n ** Checking Host: %s **\n" % url)
headers = {
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Connection": "keep-alive",
"User-Agent": user_agents[randint(0,
len(user_agents) - 1)]
}
paths = {
"jmx-console":
"/jmx-console/HtmlAdaptor?action=inspectMBean&name=jboss.system:type=ServerInfo",
"web-console": "/web-console/ServerInfo.jsp",
"JMXInvokerServlet": "/invoker/JMXInvokerServlet",
"admin-console": "/admin-console/"
}
for i in paths.keys():
if gl_interrupted: break
try:
print(GREEN + " * Checking %s: \t" % i + ENDC),
r = pool.request('HEAD',
url + str(paths[i]),
redirect=False,
headers=headers)
paths[i] = r.status
if paths[i] in (301, 302, 303, 307, 308):
url_redirect = r.get_redirect_location()
print(GREEN +
"[ REDIRECT ]\n * The server sent a redirect to: %s\n" %
url_redirect)
elif paths[i] == 200 or paths[i] == 500:
if i == "admin-console":
print(RED + "[ EXPOSED ]" + ENDC)
else:
print(RED + "[ VULNERABLE ]" + ENDC)
else:
print(GREEN + "[ OK ]")
except:
print(RED +
"\n * An error occurred while connecting to the host %s\n" %
url + ENDC)
paths[i] = 505
return paths
def test_timeout_str(self):
timeout = Timeout(connect=1, read=2, total=3)
assert str(timeout) == "Timeout(connect=1, read=2, total=3)"
timeout = Timeout(connect=1, read=None, total=3)
assert str(timeout) == "Timeout(connect=1, read=None, total=3)"
def post(self, url, data, timeout=None):
"""Request an URL.
Args:
url (:obj:`str`): The web location we want to retrieve.
data (dict[str, str|int]): A dict of key/value pairs. Note: On py2.7 value is unicode.
timeout (:obj:`int` | :obj:`float`): If this value is specified, use it as the read
timeout from the server (instead of the one specified during creation of the
connection pool).
Returns:
A JSON object.
"""
urlopen_kwargs = {}
if timeout is not None:
urlopen_kwargs['timeout'] = Timeout(read=timeout,
connect=self._connect_timeout)
# Are we uploading files?
files = False
for key, val in data.copy().items():
if isinstance(val, InputFile):
# Convert the InputFile to urllib3 field format
data[key] = val.field_tuple
files = True
elif isinstance(val, (float, int)):
# Urllib3 doesn't like floats it seems
data[key] = str(val)
elif key == 'media':
# One media or multiple
if isinstance(val, InputMedia):
# Attach and set val to attached name
data[key] = val.to_json()
if isinstance(val.media, InputFile):
data[val.media.attach] = val.media.field_tuple
else:
# Attach and set val to attached name for all
media = []
for m in val:
media.append(m.to_dict())
if isinstance(m.media, InputFile):
data[m.media.attach] = m.media.field_tuple
data[key] = json.dumps(media)
files = True
# Use multipart upload if we're uploading files, otherwise use JSON
if files:
result = self._request_wrapper('POST',
url,
fields=data,
**urlopen_kwargs)
else:
result = self._request_wrapper(
'POST',
url,
body=json.dumps(data).encode('utf-8'),
headers={'Content-Type': 'application/json'},
**urlopen_kwargs)
return self._parse(result)
