def test_connection_refused(self): # Does the pool retry if there is no listener on the port? # Note: Socket server is not started until after the test. pool = HTTPConnectionPool(self.host, self.port) with self.assertRaises(MaxRetryError): pool.request('GET', '/') self._start_server(lambda x: None)
def test_bad_connect(self): pool = HTTPConnectionPool('badhost.invalid', self.port) try: pool.request('GET', '/', retries=5) self.fail("should raise timeout exception here") except MaxRetryError as e: self.assertTrue(isinstance(e.reason, ProtocolError), e.reason)
def test_timeout(self): url = '/sleep?seconds=0.005' timeout = 0.001 # Pool-global timeout pool = HTTPConnectionPool(self.host, self.port, timeout=timeout) conn = pool._get_conn() with self.assertRaises(SocketTimeout): pool._make_request(conn, 'GET', url) pool._put_conn(conn) with self.assertRaises(TimeoutError): pool.request('GET', url) # Request-specific timeout pool = HTTPConnectionPool(self.host, self.port, timeout=0.5) conn = pool._get_conn() with self.assertRaises(SocketTimeout): pool._make_request(conn, 'GET', url, timeout=timeout) pool._put_conn(conn) with self.assertRaises(TimeoutError): pool.request('GET', url, timeout=timeout)
def test_bad_connect(self): pool = HTTPConnectionPool('badhost.invalid', self.port) try: pool.request('GET', '/', retries=5) self.fail("should raise timeout exception here") except MaxRetryError as e: self.assertEqual(type(e.reason), NewConnectionError)
def test_headers_are_sent_with_the_original_case(self): headers = {'foo': 'bar', 'bAz': 'quux'} parsed_headers = {} def socket_handler(listener): sock = listener.accept()[0] buf = b'' while not buf.endswith(b'\r\n\r\n'): buf += sock.recv(65536) headers_list = [header for header in buf.split(b'\r\n')[1:] if header] for header in headers_list: (key, value) = header.split(b': ') parsed_headers[key.decode('ascii')] = value.decode('ascii') sock.send(( 'HTTP/1.1 204 No Content\r\n' 'Content-Length: 0\r\n' '\r\n').encode('utf-8')) sock.close() self._start_server(socket_handler) expected_headers = {'Accept-Encoding': 'identity', 'Host': '{0}:{1}'.format(self.host, self.port)} expected_headers.update(headers) pool = HTTPConnectionPool(self.host, self.port, retries=False) pool.request('GET', '/', headers=HTTPHeaderDict(headers)) self.assertEqual(expected_headers, parsed_headers)
def test_timeout(self): url = "/sleep?seconds=0.005" timeout = util.Timeout(read=0.001) # Pool-global timeout pool = HTTPConnectionPool(self.host, self.port, timeout=timeout) conn = pool._get_conn() self.assertRaises(ReadTimeoutError, pool._make_request, conn, "GET", url) pool._put_conn(conn) self.assertRaises(ReadTimeoutError, pool.request, "GET", url) # Request-specific timeouts should raise errors pool = HTTPConnectionPool(self.host, self.port, timeout=0.5) conn = pool._get_conn() self.assertRaises(ReadTimeoutError, pool._make_request, conn, "GET", url, timeout=timeout) pool._put_conn(conn) self.assertRaises(ReadTimeoutError, pool.request, "GET", url, timeout=timeout) # Timeout int/float passed directly to request and _make_request should # raise a request timeout self.assertRaises(ReadTimeoutError, pool.request, "GET", url, timeout=0.001) conn = pool._new_conn() self.assertRaises(ReadTimeoutError, pool._make_request, conn, "GET", url, timeout=0.001) pool._put_conn(conn) # Timeout int/float passed directly to _make_request should not raise a # request timeout if it's a high value pool.request("GET", url, timeout=5)
def test_headers_are_sent_with_the_original_case(self): headers = {"foo": "bar", "bAz": "quux"} parsed_headers = {} def socket_handler(listener): sock = listener.accept()[0] buf = b"" while not buf.endswith(b"\r\n\r\n"): buf += sock.recv(65536) headers_list = [header for header in buf.split(b"\r\n")[1:] if header] for header in headers_list: (key, value) = header.split(b": ") parsed_headers[key.decode()] = value.decode() # Send incomplete message (note Content-Length) sock.send(("HTTP/1.1 204 No Content\r\n" "Content-Length: 0\r\n" "\r\n").encode("utf-8")) sock.close() self._start_server(socket_handler) expected_headers = {"Accept-Encoding": "identity", "Host": "{0}:{1}".format(self.host, self.port)} expected_headers.update(headers) pool = HTTPConnectionPool(self.host, self.port, retries=False) pool.request("GET", "/", headers=HTTPHeaderDict(headers)) self.assertEqual(expected_headers, parsed_headers)
def test_release_conn_parameter(self): MAXSIZE = 5 pool = HTTPConnectionPool(self.host, self.port, maxsize=MAXSIZE) assert pool.pool.qsize() == MAXSIZE # Make request without releasing connection pool.request("GET", "/", release_conn=False, preload_content=False) assert pool.pool.qsize() == MAXSIZE - 1
def test_timeout(self): pool = HTTPConnectionPool(self.host, self.port, timeout=0.01) try: pool.request('GET', '/sleep', fields={'seconds': '0.02'}) self.fail("Failed to raise TimeoutError exception") except TimeoutError: pass
def test_source_address_error(self): for addr in INVALID_SOURCE_ADDRESSES: pool = HTTPConnectionPool(self.host, self.port, source_address=addr, retries=False) with pytest.raises(NewConnectionError): pool.request("GET", "/source_address?{0}".format(addr))
def test_release_conn_parameter(self): MAXSIZE=5 pool = HTTPConnectionPool(self.host, self.port, maxsize=MAXSIZE) self.assertEqual(pool.pool.qsize(), MAXSIZE) # Make request without releasing connection pool.request('GET', '/', release_conn=False, preload_content=False) self.assertEqual(pool.pool.qsize(), MAXSIZE-1)
def test_release_conn_parameter(self): MAXSIZE = 5 pool = HTTPConnectionPool(self.host, self.port, maxsize=MAXSIZE) self.assertEqual(pool.pool.qsize(), MAXSIZE) # Make request without releasing connection pool.request('GET', '/', release_conn=False, preload_content=False) self.assertEqual(pool.pool.qsize(), MAXSIZE - 1)
def test_keepalive_close(self): pool = HTTPConnectionPool(self.host, self.port, block=True, maxsize=1, timeout=2) self.addCleanup(pool.close) r = pool.request('GET', '/keepalive?close=1', retries=0, headers={ "Connection": "close", }) self.assertEqual(pool.num_connections, 1) # The dummyserver will have responded with Connection:close, # and httplib will properly cleanup the socket. # We grab the HTTPConnection object straight from the Queue, # because _get_conn() is where the check & reset occurs # pylint: disable-msg=W0212 conn = pool.pool.get() self.assertEqual(conn._sock, None) pool._put_conn(conn) # Now with keep-alive r = pool.request('GET', '/keepalive?close=0', retries=0, headers={ "Connection": "keep-alive", }) # The dummyserver responded with Connection:keep-alive, the connection # persists. conn = pool.pool.get() self.assertNotEqual(conn._sock, None) pool._put_conn(conn) # Another request asking the server to close the connection. This one # should get cleaned up for the next request. r = pool.request('GET', '/keepalive?close=1', retries=0, headers={ "Connection": "close", }) self.assertEqual(r.status, 200) conn = pool.pool.get() self.assertEqual(conn._sock, None) pool._put_conn(conn) # Next request r = pool.request('GET', '/keepalive?close=0')
def test_source_address_error(self): for addr in INVALID_SOURCE_ADDRESSES: pool = HTTPConnectionPool(self.host, self.port, source_address=addr, retries=False) # FIXME: This assert flakes sometimes. Not sure why. with pytest.raises(NewConnectionError): pool.request("GET", "/source_address?{0}".format(addr))
def test_connection_count_bigpool(self): http_pool = HTTPConnectionPool(self.host, self.port, maxsize=16) http_pool.request('GET', '/') http_pool.request('GET', '/') http_pool.request('GET', '/') self.assertEqual(http_pool.num_connections, 1) self.assertEqual(http_pool.num_requests, 3)
def test_connection_error_retries(self): """ ECONNREFUSED error should raise a connection error, with retries """ port = find_unused_port() pool = HTTPConnectionPool(self.host, port) try: pool.request('GET', '/', retries=Retry(connect=3)) self.fail("Should have failed with a connection error.") except MaxRetryError as e: self.assertEqual(type(e.reason), NewConnectionError)
def test_connection_count(self): pool = HTTPConnectionPool(self.host, self.port, maxsize=1) pool.request('GET', '/') pool.request('GET', '/') pool.request('GET', '/') self.assertEqual(pool.num_connections, 1) self.assertEqual(pool.num_requests, 3)
def test_connection_error_retries(self): """ ECONNREFUSED error should raise a connection error, with retries """ port = find_unused_port() pool = HTTPConnectionPool(self.host, port) try: pool.request('GET', '/', retries=Retry(connect=3)) self.fail("Should have failed with a connection error.") except MaxRetryError as e: self.assertTrue(isinstance(e.reason, ProtocolError)) self.assertEqual(e.reason.args[1].errno, errno.ECONNREFUSED)
def test_connection_count_bigpool(self): http_pool = HTTPConnectionPool(self.host, self.port, maxsize=16) self.addCleanup(http_pool.close) http_pool.request("GET", "/") http_pool.request("GET", "/") http_pool.request("GET", "/") self.assertEqual(http_pool.num_connections, 1) self.assertEqual(http_pool.num_requests, 3)
def test_connection_count(self): pool = HTTPConnectionPool(self.host, self.port, maxsize=1) self.addCleanup(pool.close) pool.request("GET", "/") pool.request("GET", "/") pool.request("GET", "/") self.assertEqual(pool.num_connections, 1) self.assertEqual(pool.num_requests, 3)
def test_lazy_load_twice(self): # This test is sad and confusing. Need to figure out what's # going on with partial reads and socket reuse. pool = HTTPConnectionPool(self.host, self.port, block=True, maxsize=1, timeout=2) payload_size = 1024 * 2 first_chunk = 512 boundary = "foo" req_data = {"count": "a" * payload_size} resp_data = encode_multipart_formdata(req_data, boundary=boundary)[0] req2_data = {"count": "b" * payload_size} resp2_data = encode_multipart_formdata(req2_data, boundary=boundary)[0] r1 = pool.request( "POST", "/echo", fields=req_data, multipart_boundary=boundary, preload_content=False, ) assert r1.read(first_chunk) == resp_data[:first_chunk] try: r2 = pool.request( "POST", "/echo", fields=req2_data, multipart_boundary=boundary, preload_content=False, pool_timeout=0.001, ) # This branch should generally bail here, but maybe someday it will # work? Perhaps by some sort of magic. Consider it a TODO. assert r2.read(first_chunk) == resp2_data[:first_chunk] assert r1.read() == resp_data[first_chunk:] assert r2.read() == resp2_data[first_chunk:] assert pool.num_requests == 2 except EmptyPoolError: assert r1.read() == resp_data[first_chunk:] assert pool.num_requests == 1 assert pool.num_connections == 1
def test_lazy_load_twice(self): # This test is sad and confusing. Need to figure out what's # going on with partial reads and socket reuse. pool = HTTPConnectionPool(self.host, self.port, block=True, maxsize=1, timeout=2) payload_size = 1024 * 2 first_chunk = 512 boundary = 'foo' req_data = {'count': 'a' * payload_size} resp_data = encode_multipart_formdata(req_data, boundary=boundary)[0] req2_data = {'count': 'b' * payload_size} resp2_data = encode_multipart_formdata(req2_data, boundary=boundary)[0] r1 = pool.request('POST', '/echo', fields=req_data, multipart_boundary=boundary, preload_content=False) first_data = r1.read(first_chunk) self.assertGreater(len(first_data), 0) self.assertEqual(first_data, resp_data[:len(first_data)]) try: r2 = pool.request('POST', '/echo', fields=req2_data, multipart_boundary=boundary, preload_content=False, pool_timeout=0.001) # This branch should generally bail here, but maybe someday it will # work? Perhaps by some sort of magic. Consider it a TODO. second_data = r2.read(first_chunk) self.assertGreater(len(second_data), 0) self.assertEqual(second_data, resp2_data[:len(second_data)]) self.assertEqual(r1.read(), resp_data[len(first_data):]) self.assertEqual(r2.read(), resp2_data[len(second_data):]) self.assertEqual(pool.num_requests, 2) except EmptyPoolError: self.assertEqual(r1.read(), resp_data[len(first_data):]) self.assertEqual(pool.num_requests, 1) self.assertEqual(pool.num_connections, 1)
def test_timeout_float(self): block_event = Event() ready_event = self.start_basic_handler(block_send=block_event, num=2) # Pool-global timeout pool = HTTPConnectionPool(self.host, self.port, timeout=SHORT_TIMEOUT, retries=False) self.assertRaises(ReadTimeoutError, pool.request, 'GET', '/') block_event.set() # Release block # Shouldn't raise this time ready_event.wait() block_event.set() # Pre-release block pool.request('GET', '/')
def test_timeout(self): url = '/sleep?seconds=0.005' timeout = util.Timeout(read=0.001) # Pool-global timeout pool = HTTPConnectionPool(self.host, self.port, timeout=timeout) conn = pool._get_conn() self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', url) pool._put_conn(conn) self.assertRaises(ReadTimeoutError, pool.request, 'GET', url) # Request-specific timeouts should raise errors pool = HTTPConnectionPool(self.host, self.port, timeout=0.5) conn = pool._get_conn() self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', url, timeout=timeout) pool._put_conn(conn) self.assertRaises(ReadTimeoutError, pool.request, 'GET', url, timeout=timeout) # Timeout int/float passed directly to request and _make_request should # raise a request timeout self.assertRaises(ReadTimeoutError, pool.request, 'GET', url, timeout=0.001) conn = pool._new_conn() self.assertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', url, timeout=0.001) pool._put_conn(conn) # Timeout int/float passed directly to _make_request should not raise a # request timeout if it's a high value pool.request('GET', url, timeout=5)
def test_keepalive_close(self): # NOTE: This used to run against apache.org but it made the test suite # really slow and fail half the time. Setting it to skip until we can # make this run better locally. pool = HTTPConnectionPool(self.host, self.port, block=True, maxsize=1, timeout=2) r = pool.request('GET', '/keepalive?close=1', retries=0, headers={ "Connection": "close", }) self.assertEqual(pool.num_connections, 1) # The dummyserver will have responded with Connection:close, # and httplib will properly cleanup the socket. # We grab the HTTPConnection object straight from the Queue, # because _get_conn() is where the check & reset occurs # pylint: disable-msg=W0212 conn = pool.pool.get() self.assertEqual(conn.sock, None) pool._put_conn(conn) # Now with keep-alive r = pool.request('GET', '/keepalive?close=0', retries=0, headers={ "Connection": "keep-alive", }) # The dummyserver responded with Connection:keep-alive, the connection # persists. conn = pool.pool.get() self.assertNotEqual(conn.sock, None) pool._put_conn(conn) # Another request asking the server to close the connection. This one # should get cleaned up for the next request. r = pool.request('GET', '/keepalive?close=1', retries=0, headers={ "Connection": "close", }) self.assertEqual(r.status, 200) conn = pool.pool.get() self.assertEqual(conn.sock, None) pool._put_conn(conn) # Next request r = pool.request('GET', '/keepalive?close=0')
def test_connection_timeout(self): timed_out = Event() def socket_handler(listener): timed_out.wait() sock = listener.accept()[0] sock.close() self._start_server(socket_handler) pool = HTTPConnectionPool(self.host, self.port, timeout=0.001) with self.assertRaises(TimeoutError): pool.request('GET', '/', retries=0) timed_out.set()
def test_keepalive_close(self): pool = HTTPConnectionPool(self.host, self.port, block=True, maxsize=1, timeout=2) self.addCleanup(pool.close) r = pool.request('GET', '/keepalive?close=1', retries=0, headers={ "Connection": "close", }) self.assertEqual(pool.num_connections, 1) # The dummyserver will have responded with Connection:close, # and httplib will properly cleanup the socket. # We grab the HTTPConnection object straight from the Queue, # because _get_conn() is where the check & reset occurs # pylint: disable-msg=W0212 conn = pool.pool.get() self.assertIsNone(conn.sock) pool._put_conn(conn) # Now with keep-alive r = pool.request('GET', '/keepalive?close=0', retries=0, headers={ "Connection": "keep-alive", }) # The dummyserver responded with Connection:keep-alive, the connection # persists. conn = pool.pool.get() self.assertIsNotNone(conn.sock) pool._put_conn(conn) # Another request asking the server to close the connection. This one # should get cleaned up for the next request. r = pool.request('GET', '/keepalive?close=1', retries=0, headers={ "Connection": "close", }) self.assertEqual(r.status, 200) conn = pool.pool.get() self.assertIsNone(conn.sock) pool._put_conn(conn) # Next request r = pool.request('GET', '/keepalive?close=0')
def test_source_address_ignored(self): # No warning is issued if source_address is omitted. with warnings.catch_warnings(record=True) as w: pool = HTTPConnectionPool(self.host, self.port) assert pool.request('GET', '/source_address').status == 200 assert ( not w or not issubclass(w[-1].category, PythonVersionWarning)) # source_address is ignored in Python 2.6 and older. Warning issued. with warnings.catch_warnings(record=True) as w: for addr in INVALID_SOURCE_ADDRESSES: pool = HTTPConnectionPool( self.host, self.port, source_address=addr) assert pool.request('GET', '/source_address').status == 200 assert issubclass(w[-1].category, PythonVersionWarning)
def test_stream_none_unchunked_response_does_not_hang(self): done_event = Event() def socket_handler(listener): sock = listener.accept()[0] buf = b'' while not buf.endswith(b'\r\n\r\n'): buf += sock.recv(65536) sock.send(b'HTTP/1.1 200 OK\r\n' b'Content-Length: 12\r\n' b'Content-type: text/plain\r\n' b'\r\n' b'hello, world') done_event.wait(5) sock.close() self._start_server(socket_handler) pool = HTTPConnectionPool(self.host, self.port, retries=False) r = pool.request('GET', '/', timeout=1, preload_content=False) # Stream should read to the end. self.assertEqual([b'hello, world'], list(r.stream(None))) done_event.set()
def _test_broken_header_parsing(self, headers): self.start_response_handler( (b"HTTP/1.1 200 OK\r\n" b"Content-Length: 0\r\n" b"Content-type: text/plain\r\n") + b"\r\n".join(headers) + b"\r\n" ) pool = HTTPConnectionPool(self.host, self.port, retries=False) with LogRecorder() as logs: pool.request("GET", "/") for record in logs: if "Failed to parse headers" in record.msg and pool._absolute_url("/") == record.args[0]: return self.fail("Missing log about unparsed headers")
def test_response_headers_are_returned_in_the_original_order(self): # NOTE: Probability this test gives a false negative is 1/(K!) K = 16 # NOTE: Provide headers in non-sorted order (i.e. reversed) # so that if the internal implementation tries to sort them, # a change will be detected. expected_response_headers = [('X-Header-%d' % i, str(i)) for i in reversed(range(K))] def socket_handler(listener): sock = listener.accept()[0] buf = b'' while not buf.endswith(b'\r\n\r\n'): buf += sock.recv(65536) sock.send(b'HTTP/1.1 200 OK\r\n' + b'\r\n'.join([(k.encode('utf8') + b': ' + v.encode('utf8')) for (k, v) in expected_response_headers]) + b'\r\n') sock.close() self._start_server(socket_handler) pool = HTTPConnectionPool(self.host, self.port) r = pool.request('GET', '/', retries=0) actual_response_headers = [(k, v) for (k, v) in r.headers.items() if k.startswith('X-Header-')] self.assertEqual(expected_response_headers, actual_response_headers)
def test_response_headers_are_returned_in_the_original_order(self): # NOTE: Probability this test gives a false negative is 1/(K!) K = 16 # NOTE: Provide headers in non-sorted order (i.e. reversed) # so that if the internal implementation tries to sort them, # a change will be detected. expected_response_headers = [('X-Header-%d' % i, str(i)) for i in reversed(range(K))] def socket_handler(listener): sock = listener.accept()[0] buf = b'' while not buf.endswith(b'\r\n\r\n'): buf += sock.recv(65536) sock.send(b'HTTP/1.1 200 OK\r\n' + b'\r\n'.join([ (k.encode('utf8') + b': ' + v.encode('utf8')) for (k, v) in expected_response_headers ]) + b'\r\n') sock.close() self._start_server(socket_handler) pool = HTTPConnectionPool(self.host, self.port) r = pool.request('GET', '/', retries=0) actual_response_headers = [ (k, v) for (k, v) in r.headers.items() if k.startswith('X-Header-') ] self.assertEqual(expected_response_headers, actual_response_headers)
def main(): try: client = pymongo.MongoClient() db = client[database] col = db[collection] pool = HTTPConnectionPool('games.crossfit.com', maxsize=1) start_time = time.time() for year in YEARS: for comp in COMPS: for gender in GENDERS: results_url = ( "/scores/leaderboard.php?" "competition={}&stage=0&division={}®ion=0&" "numberperpage=100&year={}&showtoggles=1&" "hidedropdowns=1").format(COMPS[comp], GENDERS[gender], year) results_page = pool.request('GET', results_url, preload_content=False) soup = BeautifulSoup(results_page) for athlete in soup.findAll('td', 'name'): scrape_games_athlete(col, athlete, year, comp) print_time_update(year, comp, gender, start_time) finally: print results_url
def test_incomplete_response(self): body = 'Response' partial_body = body[:2] def socket_handler(listener): sock = listener.accept()[0] # Consume request buf = b'' while not buf.endswith(b'\r\n\r\n'): buf = sock.recv(65536) # Send partial response and close socket. sock.send(( 'HTTP/1.1 200 OK\r\n' 'Content-Type: text/plain\r\n' 'Content-Length: %d\r\n' '\r\n' '%s' % (len(body), partial_body)).encode('utf-8') ) sock.close() self._start_server(socket_handler) pool = HTTPConnectionPool(self.host, self.port) response = pool.request('GET', '/', retries=0, preload_content=False) self.assertRaises(ProtocolError, response.read)
def test_source_address_ignored(self): # source_address is ignored in Python 2.6 and older. for addr in INVALID_SOURCE_ADDRESSES: pool = HTTPConnectionPool( self.host, self.port, source_address=addr) r = pool.request('GET', '/source_address') assert r.status == 200
def test_incomplete_response(self): body = 'Response' partial_body = body[:2] def socket_handler(listener): sock = listener.accept()[0] # Consume request buf = b'' while not buf.endswith(b'\r\n\r\n'): buf = sock.recv(65536) # Send partial response and close socket. sock.send(('HTTP/1.1 200 OK\r\n' 'Content-Type: text/plain\r\n' 'Content-Length: %d\r\n' '\r\n' '%s' % (len(body), partial_body)).encode('utf-8')) sock.close() self._start_server(socket_handler) pool = HTTPConnectionPool(self.host, self.port) response = pool.request('GET', '/', retries=0, preload_content=False) self.assertRaises(ProtocolError, response.read)
def test_stream_none_unchunked_response_does_not_hang(self): done_event = Event() def socket_handler(listener): sock = listener.accept()[0] buf = b'' while not buf.endswith(b'\r\n\r\n'): buf += sock.recv(65536) sock.send( b'HTTP/1.1 200 OK\r\n' b'Content-Length: 12\r\n' b'Content-type: text/plain\r\n' b'\r\n' b'hello, world' ) done_event.wait(5) sock.close() self._start_server(socket_handler) pool = HTTPConnectionPool(self.host, self.port, retries=False) self.addCleanup(pool.close) r = pool.request('GET', '/', timeout=1, preload_content=False) # Stream should read to the end. self.assertEqual([b'hello, world'], list(r.stream(None))) done_event.set()
def test_enforce_content_length_no_body(self): done_event = Event() def socket_handler(listener): sock = listener.accept()[0] buf = b'' while not buf.endswith(b'\r\n\r\n'): buf += sock.recv(65536) sock.send( b'HTTP/1.1 200 OK\r\n' b'Content-Length: 22\r\n' b'Content-type: text/plain\r\n' b'\r\n' ) done_event.wait(1) sock.close() self._start_server(socket_handler) conn = HTTPConnectionPool(self.host, self.port, maxsize=1) self.addCleanup(conn.close) # Test stream on 0 length body head_response = conn.request('HEAD', url='/', preload_content=False, enforce_content_length=True) data = [chunk for chunk in head_response.stream(1)] self.assertEqual(len(data), 0) done_event.set()
def test_enforce_content_length_get(self): done_event = Event() def socket_handler(listener): sock = listener.accept()[0] buf = b'' while not buf.endswith(b'\r\n\r\n'): buf += sock.recv(65536) sock.send(b'HTTP/1.1 200 OK\r\n' b'Content-Length: 22\r\n' b'Content-type: text/plain\r\n' b'\r\n' b'hello, world') done_event.wait(1) sock.close() self._start_server(socket_handler) conn = HTTPConnectionPool(self.host, self.port, maxsize=1) # Test stream read when content length less than headers claim get_response = conn.request('GET', url='/', preload_content=False, enforce_content_length=True) data = get_response.stream(100) # Read "good" data before we try to read again. # This won't trigger till generator is exhausted. next(data) self.assertRaises(ProtocolError, next, data) done_event.set()
def _test_broken_header_parsing(self, headers): self.start_response_handler((b'HTTP/1.1 200 OK\r\n' b'Content-Length: 0\r\n' b'Content-type: text/plain\r\n') + b'\r\n'.join(headers) + b'\r\n') pool = HTTPConnectionPool(self.host, self.port, retries=False) with LogRecorder() as logs: pool.request('GET', '/') for record in logs: if 'Failed to parse headers' in record.msg and \ pool._absolute_url('/') == record.args[0]: return self.fail('Missing log about unparsed headers')
class API(object): def __init__(self, client_id: str, client_secret: str, auth_token: str) -> None: self.print_hello_message() self._cid = client_id self._csecret = client_secret self._auth_token = auth_token self.pool = HTTPConnectionPool(HOST_ADDR + ':' + HOST_PORT, timeout=2) @staticmethod def print_hello_message(): logger.info('--------------------------------') logger.info('----- Hello from InsightePy ----') logger.info('--------------------------------') def make_request(self, method: str, url: str, fields: Dict[str, str]) -> Response: # adding user information to field fields['cid'] = self._cid fields['csecret'] = self._csecret fields['authtoken'] = self._auth_token logger.debug('Making request with: {}'.format(fields)) r = self.pool.request(method, ROUTE_PREFIX + url, fields=fields) if r.status == 200: logger.debug('Got raw response: {}'.format(r.data)) try: response = Response(json.loads(r.data)) logger.debug('Built Response: {}'.format(response.__repr__())) return response except Exception as e: logger.error('Encountered error while parsing response: {}'.format(str(e))) logger.error(traceback.format_exc()) else: logger.error('Got a NOT OK Response Code: status={}'.format(r.status)) logger.error(traceback.format_exc()) def say_hello(self): logger.info('Running Say Hello') return self.make_request( 'GET', '/hello', dict() ) def single_extract(self, lang: str, verbatim: str, extractors: List[Extractor] = None) -> Response: """ Extract insight for a single verbatim :param lang: language of the sentence {en/fr/de} :param verbatim: Unicode sentence :param extractors: list of feature extractors :return: dict response from Compute Engine """ logger.info('Running Single Extract on: {}'.format(verbatim)) return self.make_request( 'GET', '/extract', dict( verbatim=verbatim, lang=lang, extractors=json.dumps([_.to_dict() for _ in extractors] if extractors else []) ) )
def test_enforce_content_length_no_body(self): done_event = Event() def socket_handler(listener): sock = listener.accept()[0] buf = b'' while not buf.endswith(b'\r\n\r\n'): buf += sock.recv(65536) sock.send(b'HTTP/1.1 200 OK\r\n' b'Content-Length: 22\r\n' b'Content-type: text/plain\r\n' b'\r\n') done_event.wait(1) sock.close() self._start_server(socket_handler) conn = HTTPConnectionPool(self.host, self.port, maxsize=1) self.addCleanup(conn.close) # Test stream on 0 length body head_response = conn.request('HEAD', url='/', preload_content=False, enforce_content_length=True) data = [chunk for chunk in head_response.stream(1)] self.assertEqual(len(data), 0) done_event.set()
def test_source_address(self): for addr in VALID_SOURCE_ADDRESSES: pool = HTTPConnectionPool(self.host, self.port, source_address=addr, retries=False) r = pool.request('GET', '/source_address') assert r.data == b(addr[0])
def test_httplib_headers_case_insensitive(self): self.start_response_handler( b"HTTP/1.1 200 OK\r\n" b"Content-Length: 0\r\n" b"Content-type: text/plain\r\n" b"\r\n" ) pool = HTTPConnectionPool(self.host, self.port, retries=False) HEADERS = {"Content-Length": "0", "Content-type": "text/plain"} r = pool.request("GET", "/") self.assertEqual(HEADERS, dict(r.headers.items())) # to preserve case sensitivity
def test_source_address(self): for addr in VALID_SOURCE_ADDRESSES: pool = HTTPConnectionPool(self.host, self.port, source_address=addr, retries=False) r = pool.request('GET', '/source_address') assert r.data == b(addr[0]), ( "expected the response to contain the source address {addr}, " "but was {data}".format(data=r.data, addr=b(addr[0])))
def test_lazy_load_twice(self): # This test is sad and confusing. Need to figure out what's # going on with partial reads and socket reuse. pool = HTTPConnectionPool(self.host, self.port, block=True, maxsize=1, timeout=2) payload_size = 1024 * 2 first_chunk = 512 boundary = "foo" req_data = {"count": "a" * payload_size} resp_data = encode_multipart_formdata(req_data, boundary=boundary)[0] req2_data = {"count": "b" * payload_size} resp2_data = encode_multipart_formdata(req2_data, boundary=boundary)[0] r1 = pool.request("POST", "/echo", fields=req_data, multipart_boundary=boundary, preload_content=False) self.assertEqual(r1.read(first_chunk), resp_data[:first_chunk]) try: r2 = pool.request( "POST", "/echo", fields=req2_data, multipart_boundary=boundary, preload_content=False, pool_timeout=0.001, ) # This branch should generally bail here, but maybe someday it will # work? Perhaps by some sort of magic. Consider it a TODO. self.assertEqual(r2.read(first_chunk), resp2_data[:first_chunk]) self.assertEqual(r1.read(), resp_data[first_chunk:]) self.assertEqual(r2.read(), resp2_data[first_chunk:]) self.assertEqual(pool.num_requests, 2) except EmptyPoolError: self.assertEqual(r1.read(), resp_data[first_chunk:]) self.assertEqual(pool.num_requests, 1) self.assertEqual(pool.num_connections, 1)
def test_keepalive(self): pool = HTTPConnectionPool(self.host, self.port, block=True, maxsize=1) r = pool.request('GET', '/keepalive?close=0') r = pool.request('GET', '/keepalive?close=0') self.assertEqual(r.status, 200) self.assertEqual(pool.num_connections, 1) self.assertEqual(pool.num_requests, 2)
def test_empty_head_response_does_not_hang(self): self.start_response_handler( b"HTTP/1.1 200 OK\r\n" b"Content-Length: 256\r\n" b"Content-type: text/plain\r\n" b"\r\n" ) pool = HTTPConnectionPool(self.host, self.port, retries=False) r = pool.request("HEAD", "/", timeout=1, preload_content=False) # stream will use the read method here. self.assertEqual([], list(r.stream()))
def test_timeout_float(self): block_event = Event() ready_event = self.start_basic_handler(block_send=block_event, num=2) # Pool-global timeout pool = HTTPConnectionPool(self.host, self.port, timeout=SHORT_TIMEOUT, retries=False) self.addCleanup(pool.close) wait_for_socket(ready_event) self.assertRaises(ReadTimeoutError, pool.request, "GET", "/") block_event.set() # Release block # Shouldn't raise this time wait_for_socket(ready_event) block_event.set() # Pre-release block pool.request("GET", "/")
def _test_broken_header_parsing(self, headers): self.start_response_handler(( b'HTTP/1.1 200 OK\r\n' b'Content-Length: 0\r\n' b'Content-type: text/plain\r\n' ) + b'\r\n'.join(headers) + b'\r\n' ) pool = HTTPConnectionPool(self.host, self.port, retries=False) with LogRecorder() as logs: pool.request('GET', '/') for record in logs: if 'Failed to parse headers' in record.msg and \ pool._absolute_url('/') == record.args[0]: return self.fail('Missing log about unparsed headers')