def test_fieldname_string_enforcement():
with pytest.raises(TypeError): #@UndefinedVariable
Headers({3: 3})
h = Headers()
with pytest.raises(TypeError): #@UndefinedVariable
h[3] = 5
with pytest.raises(TypeError): #@UndefinedVariable
h.add(3, 4)
with pytest.raises(TypeError): #@UndefinedVariable
del h[3]
def test_fieldname_string_enforcement():
with pytest.raises(Exception):
Headers({3: 3})
h = Headers()
with pytest.raises(Exception):
h[3] = 5
with pytest.raises(Exception):
h.add(3, 4)
with pytest.raises(Exception):
del h[3]
def test_fieldname_string_enforcement():
with pytest.raises(Exception):
Headers({3: 3})
h = Headers()
with pytest.raises(Exception):
h[3] = 5
with pytest.raises(Exception):
h.add(3, 4)
with pytest.raises(Exception):
del h[3]
def test_fieldname_string_enforcement():
with pytest.raises(TypeError): #@UndefinedVariable
Headers({3: 3})
h = Headers()
with pytest.raises(TypeError): #@UndefinedVariable
h[3] = 5
with pytest.raises(TypeError): #@UndefinedVariable
h.add(3, 4)
with pytest.raises(TypeError): #@UndefinedVariable
del h[3]
def test_copy():
rnd_txt = lambda length: ''.join(random.choice(string.ascii_letters) for _ in xrange(length))
h = Headers((rnd_txt(10), rnd_txt(50)) for _ in xrange(100))
c = h.copy()
assert h is not c
assert len(h) == len(c)
assert set(h.keys()) == set(c.keys())
assert h == c
assert type(h) is type(c)
for _ in xrange(100):
rnd_key = rnd_txt(9)
c[rnd_key] = rnd_txt(10)
assert rnd_key in c
assert rnd_key not in h
def test_copy():
rnd_txt = lambda length: ''.join(
random.choice(string.ascii_letters) for _ in xrange(length))
h = Headers((rnd_txt(10), rnd_txt(50)) for _ in xrange(100))
c = h.copy()
assert h is not c
assert len(h) == len(c)
assert set(h.keys()) == set(c.keys())
assert h == c
assert type(h) is type(c)
for _ in xrange(100):
rnd_key = rnd_txt(9)
c[rnd_key] = rnd_txt(10)
assert rnd_key in c
assert rnd_key not in h
def test_create_from_list():
h = Headers([('ab', 'A'), ('cd', 'B'), ('cookie', 'C'), ('cookie', 'D'),
('cookie', 'E')])
assert len(h) == 5
assert 'ab' in h
assert len(h['cookie']) == 3
assert h['cookie'][0] == 'C'
assert h['cookie'][-1] == 'E'
def test_compat_dict():
h = Headers(D='asdf')
h.add('E', 'd')
h.add('E', 'f')
h.add('Cookie', 'd')
h.add('Cookie', 'e')
h.add('Cookie', 'f')
d = h.compatible_dict()
for x in ('Cookie', 'D', 'E'):
assert x in d
assert d['D'] == 'asdf'
assert d['E'] == 'd, f'
assert d['Cookie'] == 'd, e, f'
def test_compat_dict():
h = Headers(D='asdf')
h.add('E', 'd')
h.add('E', 'f')
h.add('Cookie', 'd')
h.add('Cookie', 'e')
h.add('Cookie', 'f')
d = h.compatible_dict()
for x in ('Cookie', 'D', 'E'):
assert x in d
assert d['D'] == 'asdf'
assert d['E'] == 'd, f'
assert d['Cookie'] == 'd, e, f'
def test_compat_dict():
h = Headers(D="asdf")
h.add("E", "d")
h.add("E", "f")
h.add("Cookie", "d")
h.add("Cookie", "e")
h.add("Cookie", "f")
d = h.compatible_dict()
for x in ("Cookie", "D", "E"):
assert x in d
assert d["D"] == "asdf"
assert d["E"] == "d, f"
assert d["Cookie"] == "d, e, f"
def test_case_insensitivity():
h = Headers({'Content-Type': 'text/plain'})
h.add('Content-Encoding', 'utf8')
for val in ('content-type', 'content-encoding'):
assert val.upper() in h
assert val.lower() in h
assert val.capitalize() in h
assert h.get(val.lower()) == h.get(val.upper()) == h.get(
val.capitalize())
del h[val.upper()]
assert val.lower() not in h
def test_case_insensitivity():
h = Headers({'Content-Type': 'text/plain'})
h.add('Content-Encoding', 'utf8')
for val in ('content-type', 'content-encoding'):
assert val.upper() in h
assert val.lower() in h
assert val.capitalize() in h
assert h.get(val.lower()) == h.get(val.upper()) == h.get(val.capitalize())
del h[val.upper()]
assert val.lower() not in h
def test_case_insensitivity():
h = Headers({"Content-Type": "text/plain"})
h.add("Content-Encoding", "utf8")
for val in ("content-type", "content-encoding"):
assert val.upper() in h
assert val.lower() in h
assert val.capitalize() in h
assert h.get(val.lower()) == h.get(val.upper()) == h.get(val.capitalize())
del h[val.upper()]
assert val.lower() not in h
def test_create_from_dict():
h = Headers(dict(ab=1, cd=2, ef=3, gh=4))
assert len(h) == 4
assert 'ab' in h
def test_create_from_kwargs():
h = Headers(ab=1, cd=2, ef=3, gh=4)
assert len(h) == 4
assert 'ab' in h
def test_create_from_iterator():
h = Headers((x, x * 5) for x in string.ascii_lowercase)
assert len(h) == len(string.ascii_lowercase)
def test_header_replace():
d = Headers()
d['Content-Type'] = "text/plain"
d['content-type'] = "text/html"
assert d['content-type'] == "text/html"
class HTTPClient(object):
HTTP_11 = 'HTTP/1.1'
HTTP_10 = 'HTTP/1.0'
BLOCK_SIZE = 1024 * 4 # 4KB
DEFAULT_HEADERS = Headers({
'User-Agent': 'python/gevent-http-client-' + __version__
})
@classmethod
def from_url(cls, url, **kw):
if not isinstance(url, URL):
url = URL(url)
enable_ssl = url.scheme == PROTO_HTTPS
if not enable_ssl:
kw.pop('ssl_options', None)
return cls(url.host, port=url.port, ssl=enable_ssl, **kw)
def __init__(self, host, port=None, headers={},
block_size=BLOCK_SIZE,
connection_timeout=ConnectionPool.DEFAULT_CONNECTION_TIMEOUT,
network_timeout=ConnectionPool.DEFAULT_NETWORK_TIMEOUT,
disable_ipv6=False,
concurrency=1, ssl_options=None, ssl=False, insecure=False,
proxy_host=None, proxy_port=None, version=HTTP_11,
headers_type=Headers):
self.host = host
self.port = port
connection_host = self.host
connection_port = self.port
if proxy_host is not None:
assert proxy_port is not None, \
'you have to provide proxy_port if you set proxy_host'
self.use_proxy = True
connection_host = proxy_host
connection_port = proxy_port
else:
self.use_proxy = False
if ssl and ssl_options is None:
ssl_options = {}
if ssl_options is not None:
self.ssl = True
if not self.port:
self.port = 443
if not connection_port:
connection_port = self.port
self._connection_pool = SSLConnectionPool(
connection_host, connection_port, size=concurrency,
ssl_options=ssl_options,
insecure=insecure,
network_timeout=network_timeout,
connection_timeout=connection_timeout,
disable_ipv6=disable_ipv6)
else:
self.ssl = False
if not self.port:
self.port = 80
if not connection_port:
connection_port = self.port
self._connection_pool = ConnectionPool(
connection_host, connection_port,
size=concurrency,
network_timeout=network_timeout,
connection_timeout=connection_timeout,
disable_ipv6=disable_ipv6)
self.version = version
self.headers_type = headers_type
self.default_headers = headers_type()
self.default_headers.update(self.DEFAULT_HEADERS)
self.default_headers.update(headers)
self.block_size = block_size
self._base_url_string = str(self.get_base_url())
def get_base_url(self):
url = URL()
url.host = self.host
url.port = self.port
url.scheme = self.ssl and PROTO_HTTPS or PROTO_HTTP
return url
def close(self):
self._connection_pool.close()
def _build_request(self, method, request_uri, body="", headers={}):
header_fields = self.headers_type()
header_fields.update(self.default_headers)
header_fields.update(headers)
if self.version == self.HTTP_11 and HEADER_HOST not in header_fields:
host_port = self.host
if self.port not in (80, 443):
host_port += HOST_PORT_SEP + str(self.port)
header_fields[HEADER_HOST] = host_port
if body and HEADER_CONTENT_LENGTH not in header_fields:
header_fields[HEADER_CONTENT_LENGTH] = len(body)
request_url = request_uri
if self.use_proxy:
base_url = self._base_url_string
if request_uri.startswith(SLASH):
base_url = base_url[:-1]
request_url = base_url + request_url
elif not request_url.startswith((SLASH, PROTO_HTTP)):
request_url = SLASH + request_url
elif request_url.startswith(PROTO_HTTP):
if request_url.startswith(self._base_url_string):
request_url = request_url[len(self._base_url_string)-1:]
else:
raise ValueError("Invalid host in URL")
request = method + WHITESPACE + request_url + WHITESPACE + self.version + CRLF
for field, value in header_fields.iteritems():
request += field + FIELD_VALUE_SEP + str(value) + CRLF
request += CRLF
return request
def request(self, method, request_uri, body=b"", headers={}):
request = self._build_request(
method.upper(), request_uri, body=body, headers=headers)
attempts_left = self._connection_pool.size + 1
while 1:
sock = self._connection_pool.get_socket()
try:
sock.sendall(request)
except gevent.socket.error as e:
self._connection_pool.release_socket(sock)
if e.errno == errno.ECONNRESET and attempts_left > 0:
attempts_left -= 1
continue
raise e
if body:
sock.sendall(body)
try:
response = HTTPSocketPoolResponse(sock, self._connection_pool,
block_size=self.block_size, method=method.upper(), headers_type=self.headers_type)
except HTTPConnectionClosed as e:
# connection is released by the response itself
if attempts_left > 0:
attempts_left -= 1
continue
raise e
else:
response._sent_request = request
return response
def get(self, request_uri, headers={}):
return self.request(METHOD_GET, request_uri, headers=headers)
def head(self, request_uri, headers={}):
return self.request(METHOD_HEAD, request_uri, headers=headers)
def post(self, request_uri, body=u'', headers={}):
return self.request(METHOD_POST, request_uri, body=body, headers=headers)
def put(self, request_uri, body=u'', headers={}):
return self.request(METHOD_PUT, request_uri, body=body, headers=headers)
def delete(self, request_uri, body=u'', headers={}):
return self.request(METHOD_DELETE, request_uri, body=body, headers=headers)
def test_compatibility_with_previous_API_write():
h = Headers()
h['asdf'] = 'jklm'
h['asdf'] = 'dfdf'
# Lists only if necessary
assert h['asdf'] == 'dfdf'
class HTTPClient(object):
HTTP_11 = 'HTTP/1.1'
HTTP_10 = 'HTTP/1.0'
BLOCK_SIZE = 1024 * 4 # 4KB
DEFAULT_HEADERS = Headers(
{'User-Agent': 'python/gevent-http-client-' + __version__})
@classmethod
def from_url(cls, url, **kw):
if not isinstance(url, URL):
url = URL(url)
enable_ssl = url.scheme == PROTO_HTTPS
if not enable_ssl:
kw.pop('ssl_options', None)
return cls(url.host, port=url.port, ssl=enable_ssl, **kw)
def __init__(self,
host,
port=None,
headers={},
block_size=BLOCK_SIZE,
connection_timeout=ConnectionPool.DEFAULT_CONNECTION_TIMEOUT,
network_timeout=ConnectionPool.DEFAULT_NETWORK_TIMEOUT,
disable_ipv6=False,
concurrency=1,
ssl=False,
ssl_options=None,
ssl_context_factory=None,
insecure=False,
proxy_host=None,
proxy_port=None,
version=HTTP_11,
headers_type=Headers):
self.host = host
self.port = port
connection_host = self.host
connection_port = self.port
if proxy_host is not None:
assert proxy_port is not None, \
'you have to provide proxy_port if you set proxy_host'
self.use_proxy = True
connection_host = proxy_host
connection_port = proxy_port
else:
self.use_proxy = False
if ssl and ssl_options is None:
ssl_options = {}
if ssl_options is not None:
self.ssl = True
if not self.port:
self.port = 443
if not connection_port:
connection_port = self.port
# Import SSL as late as possible, fail hard with Import Error
from geventhttpclient.connectionpool import SSLConnectionPool
self._connection_pool = SSLConnectionPool(
connection_host,
connection_port,
size=concurrency,
ssl_options=ssl_options,
ssl_context_factory=ssl_context_factory,
insecure=insecure,
network_timeout=network_timeout,
connection_timeout=connection_timeout,
disable_ipv6=disable_ipv6)
else:
self.ssl = False
if not self.port:
self.port = 80
if not connection_port:
connection_port = self.port
self._connection_pool = ConnectionPool(
connection_host,
connection_port,
size=concurrency,
network_timeout=network_timeout,
connection_timeout=connection_timeout,
disable_ipv6=disable_ipv6)
self.version = version
self.headers_type = headers_type
self.default_headers = headers_type()
self.default_headers.update(self.DEFAULT_HEADERS)
self.default_headers.update(headers)
self.block_size = block_size
self._base_url_string = str(self.get_base_url())
def get_base_url(self):
url = URL()
url.host = self.host
url.port = self.port
url.scheme = self.ssl and PROTO_HTTPS or PROTO_HTTP
return url
def close(self):
self._connection_pool.close()
# Like urllib2, try to treat the body as a file if we can't determine the
# file length with `len()`
def _get_body_length(self, body):
try:
return len(body)
except TypeError:
try:
return os.fstat(body.fileno()).st_size
except (AttributeError, OSError):
return None
def _build_request(self, method, request_uri, body="", headers={}):
header_fields = self.headers_type()
header_fields.update(self.default_headers)
header_fields.update(headers)
if self.version == self.HTTP_11 and HEADER_HOST not in header_fields:
host_port = self.host
if self.port not in (80, 443):
host_port += HOST_PORT_SEP + str(self.port)
header_fields[HEADER_HOST] = host_port
if body and HEADER_CONTENT_LENGTH not in header_fields:
body_length = self._get_body_length(body)
if body_length:
header_fields[HEADER_CONTENT_LENGTH] = body_length
request_url = request_uri
if self.use_proxy:
base_url = self._base_url_string
if request_uri.startswith(SLASH):
base_url = base_url[:-1]
request_url = base_url + request_url
elif not request_url.startswith((SLASH, PROTO_HTTP)):
request_url = SLASH + request_url
elif request_url.startswith(PROTO_HTTP):
if request_url.startswith(self._base_url_string):
request_url = request_url[len(self._base_url_string) - 1:]
else:
raise ValueError("Invalid host in URL")
request = method + WHITESPACE + request_url + WHITESPACE + self.version + CRLF
for field, value in header_fields.iteritems():
request += field + FIELD_VALUE_SEP + str(value) + CRLF
request += CRLF
return request
def request(self, method, request_uri, body=b"", headers={}):
if isinstance(body, six.text_type):
body = body.encode('utf-8')
request = self._build_request(method.upper(),
request_uri,
body=body,
headers=headers)
attempts_left = self._connection_pool.size + 1
while 1:
sock = self._connection_pool.get_socket()
try:
_request = request.encode()
if body:
if isinstance(body, six.binary_type):
sock.sendall(_request + body)
else:
sock.sendall(_request)
# TODO: Support non file-like iterables, e.g. `(u"string1", u"string2")`.
if six.PY3:
sock.sendfile(body)
else:
while True:
chunk = body.read(65536)
if not chunk:
break
sock.sendall(chunk)
else:
sock.sendall(_request)
except gevent.socket.error as e:
self._connection_pool.release_socket(sock)
if (e.errno == errno.ECONNRESET
or e.errno == errno.EPIPE) and attempts_left > 0:
attempts_left -= 1
continue
raise e
try:
response = HTTPSocketPoolResponse(
sock,
self._connection_pool,
block_size=self.block_size,
method=method.upper(),
headers_type=self.headers_type)
except HTTPConnectionClosed as e:
# connection is released by the response itself
if attempts_left > 0:
attempts_left -= 1
continue
raise e
else:
response._sent_request = request
return response
def get(self, request_uri, headers={}):
return self.request(METHOD_GET, request_uri, headers=headers)
def head(self, request_uri, headers={}):
return self.request(METHOD_HEAD, request_uri, headers=headers)
def post(self, request_uri, body=u'', headers={}):
return self.request(METHOD_POST,
request_uri,
body=body,
headers=headers)
def put(self, request_uri, body=u'', headers={}):
return self.request(METHOD_PUT,
request_uri,
body=body,
headers=headers)
def delete(self, request_uri, body=u'', headers={}):
return self.request(METHOD_DELETE,
request_uri,
body=body,
headers=headers)
class HTTPClient(object):
HTTP_11 = 'HTTP/1.1'
HTTP_10 = 'HTTP/1.0'
BLOCK_SIZE = 1024 * 4 # 4KB
DEFAULT_HEADERS = Headers({
'User-Agent': 'python/gevent-http-client-' + __version__
})
@classmethod
def from_url(cls, url, **kw):
if not isinstance(url, URL):
url = URL(url)
enable_ssl = url.scheme == 'https'
if not enable_ssl:
kw.pop('ssl_options', None)
return cls(url.host, port=url.port, ssl=enable_ssl, **kw)
def __init__(self, host, port=None, headers={},
block_size=BLOCK_SIZE,
connection_timeout=ConnectionPool.DEFAULT_CONNECTION_TIMEOUT,
network_timeout=ConnectionPool.DEFAULT_NETWORK_TIMEOUT,
disable_ipv6=False,
concurrency=1, ssl_options=None, ssl=False,
proxy_host=None, proxy_port=None, version=HTTP_11,
headers_type=Headers):
self.host = host
self.port = port
connection_host = self.host
connection_port = self.port
if proxy_host is not None:
assert proxy_port is not None, \
'you have to provide proxy_port if you set proxy_host'
self.use_proxy = True
connection_host = proxy_host
connection_port = proxy_port
else:
self.use_proxy = False
if ssl and ssl_options is None:
ssl_options = {}
if ssl_options is not None:
self.ssl = True
self.port = connection_port = self.port or 443
self._connection_pool = SSLConnectionPool(
connection_host, connection_port, size=concurrency,
ssl_options=ssl_options,
network_timeout=network_timeout,
connection_timeout=connection_timeout,
disable_ipv6=disable_ipv6)
else:
self.ssl = False
self.port = connection_port = self.port or 80
self._connection_pool = ConnectionPool(
connection_host, connection_port,
size=concurrency,
network_timeout=network_timeout,
connection_timeout=connection_timeout,
disable_ipv6=disable_ipv6)
self.version = version
self.headers_type = headers_type
self.default_headers = headers_type()
self.default_headers.update(self.DEFAULT_HEADERS)
self.default_headers.update(headers)
self.block_size = block_size
self._base_url_string = str(self.get_base_url())
def get_base_url(self):
url = URL()
url.host = self.host
url.port = self.port
url.scheme = self.ssl and 'https' or 'http'
return url
def close(self):
self._connection_pool.close()
def _build_request(self, method, request_uri, body="", headers={}):
header_fields = self.headers_type()
header_fields.update(self.default_headers)
header_fields.update(headers)
if self.version == self.HTTP_11 and 'Host' not in header_fields:
host_port = self.host
if self.port not in (80, 443):
host_port += ":" + str(self.port)
header_fields['Host'] = host_port
if body and 'Content-Length' not in header_fields:
header_fields['Content-Length'] = len(body)
request_url = request_uri
if self.use_proxy:
base_url = self._base_url_string
if request_uri.startswith('/'):
base_url = base_url[:-1]
request_url = base_url + request_url
elif not request_url.startswith(('/', 'http')):
request_url = '/' + request_url
elif request_url.startswith('http'):
if request_url.startswith(self._base_url_string):
request_url = request_url[len(self._base_url_string)-1:]
else:
raise ValueError("Invalid host in URL")
request = method + " " + request_url + " " + self.version + "\r\n"
for field, value in header_fields.iteritems():
request += field + ': ' + str(value) + "\r\n"
request += "\r\n"
if body:
request += body
return request
def request(self, method, request_uri, body=b"", headers={}):
request = self._build_request(
method.upper(), request_uri, body=body, headers=headers)
attempts_left = self._connection_pool.size + 1
while 1:
sock = self._connection_pool.get_socket()
try:
sock.sendall(request)
except gevent.socket.error as e: #@UndefinedVariable
self._connection_pool.release_socket(sock)
if e.errno == errno.ECONNRESET and attempts_left > 0:
attempts_left -= 1
continue
raise e
try:
ret = HTTPSocketPoolResponse(sock, self._connection_pool,
block_size=self.block_size, method=method.upper(), headers_type=self.headers_type)
except HTTPConnectionClosed as e:
# connection is released by the response itself
if attempts_left > 0:
attempts_left -= 1
continue
raise e
else:
ret._sent_request = request
return ret
def get(self, request_uri, headers={}):
return self.request('GET', request_uri, headers=headers)
def post(self, request_uri, body=u'', headers={}):
return self.request('POST', request_uri, body=body, headers=headers)
def put(self, request_uri, body=u'', headers={}):
return self.request('PUT', request_uri, body=body, headers=headers)
def delete(self, request_uri, body=u'', headers={}):
return self.request('DELETE', request_uri, body=body, headers=headers)
