def test_wsgi_environ(path=None, method=None, headers=None, extra=None,
secure=False, loop=None, body=None):
'''An function to create a WSGI environment dictionary for testing.
:param url: the resource in the ``PATH_INFO``.
:param method: the ``REQUEST_METHOD``.
:param headers: optional request headers
:params secure: a secure connection?
:param extra: additional dictionary of parameters to add.
:return: a valid WSGI environ dictionary.
'''
parser = http_parser(kind=0)
method = (method or 'GET').upper()
path = iri_to_uri(path or '/')
request_headers = Headers(headers, kind='client')
# Add Host if not available
parsed = urlparse(path)
if 'host' not in request_headers:
if not parsed.netloc:
path = '%s%s' % ('https://:443' if secure else 'http://:80', path)
else:
request_headers['host'] = parsed.netloc
#
data = '%s %s HTTP/1.1\r\n\r\n' % (method, path)
data = data.encode('latin1')
parser.execute(data, len(data))
#
stream = io.BytesIO(body or b'')
return wsgi_environ(stream, parser, request_headers,
('127.0.0.1', 8060), '255.0.1.2:8080',
Headers(), https=secure, extra=extra)
def test_wsgi_environ(path='/', method=None, headers=None, extra=None,
secure=False, loop=None):
'''An function to create a WSGI environment dictionary for testing.
:param url: the resource in the ``PATH_INFO``.
:param method: the ``REQUEST_METHOD``.
:param headers: optional request headers
:params secure: a secure connection?
:param extra: additional dictionary of parameters to add.
:return: a valid WSGI environ dictionary.
'''
parser = http_parser(kind=0)
method = (method or 'GET').upper()
path = iri_to_uri(path)
data = '%s %s HTTP/1.1\r\n\r\n' % (method, path)
data = data.encode('latin1')
parser.execute(data, len(data))
request_headers = Headers(headers, kind='client')
# Add Host if not available
parsed = urlparse(path)
if parsed.netloc and 'host' not in request_headers:
request_headers['host'] = parsed.netloc
#
headers = Headers()
stream = StreamReader(request_headers, parser)
extra = extra or {}
extra['pulsar.connection'] = FakeConnection(loop=loop)
return wsgi_environ(stream, ('127.0.0.1', 8060), '777.777.777.777:8080',
headers, https=secure, extra=extra)
def test_remove_header(self):
h = Headers([('Content-type', 'text/html')])
self.assertEqual(len(h), 1)
self.assertEqual(h.remove_header('foo'), None)
self.assertEqual(h.remove_header('content-length'), None)
self.assertEqual(h.remove_header('content-type'), ['text/html'])
self.assertEqual(len(h), 0)
def test_wsgi_environ(url='/',
method=None,
headers=None,
extra=None,
secure=False):
'''An function to create a WSGI environment dictionary for testing.
:param url: the resource in the ``PATH_INFO``.
:param method: the ``REQUEST_METHOD``.
:param headers: optional request headers
:params secure: a secure connection?
:param extra: additional dictionary of parameters to add.
:return: a valid WSGI environ dictionary.
'''
parser = http_parser(kind=0)
method = (method or 'GET').upper()
data = '%s %s HTTP/1.1\r\n\r\n' % (method, url)
data = data.encode('utf-8')
parser.execute(data, len(data))
request_headers = Headers(headers, kind='client')
headers = Headers()
stream = StreamReader(request_headers, parser)
return wsgi_environ(stream, ('127.0.0.1', 8060),
'777.777.777.777:8080',
request_headers,
headers,
https=secure,
extra=extra)
def test_remove_header(self):
h = Headers([('Content-type', 'text/html')])
self.assertEqual(len(h), 1)
self.assertEqual(h.remove_header('foo'), None)
self.assertEqual(h.remove_header('content-length'), None)
self.assertEqual(h.remove_header('content-type'), ['text/html'])
self.assertEqual(len(h), 0)
def __init__(self, client, url, method, inp_params=None, headers=None,
data=None, files=None, json=None, history=None, auth=None,
charset=None, max_redirects=10, source_address=None,
allow_redirects=False, decompress=True, version=None,
wait_continue=False, websocket_handler=None, cookies=None,
params=None, stream=False, proxies=None, verify=True,
**ignored):
self.client = client
self.method = method.upper()
self.inp_params = inp_params or {}
self.unredirected_headers = Headers()
self.history = history
self.wait_continue = wait_continue
self.max_redirects = max_redirects
self.allow_redirects = allow_redirects
self.charset = charset or 'utf-8'
self.version = version
self.decompress = decompress
self.websocket_handler = websocket_handler
self.source_address = source_address
self.stream = stream
self.verify = verify
self.new_parser()
if auth and not isinstance(auth, Auth):
auth = HTTPBasicAuth(*auth)
self.auth = auth
self.headers = client._get_headers(headers)
self.url = full_url(url, params, method=self.method)
self.body = self._encode_body(data, files, json)
self._set_proxy(proxies, ignored)
cookies = cookiejar_from_dict(client.cookies, cookies)
if cookies:
cookies.add_cookie_header(self)
def __init__(self, wsgi_callable, cfg, server_software=None, loop=None):
super(HttpServerResponse, self).__init__(loop=loop)
self.wsgi_callable = wsgi_callable
self.cfg = cfg
self.parser = http_parser(kind=0)
self.headers = Headers()
self.keep_alive = False
self.SERVER_SOFTWARE = server_software or self.SERVER_SOFTWARE
def test_init_int(self):
h = Headers(kind=1)
self.assertEqual(h.kind, 'server')
self.assertEqual(h.kind_number, 1)
h = Headers(kind=0)
self.assertEqual(h.kind, 'client')
self.assertEqual(h.kind_number, 0)
h = Headers(kind=56)
self.assertEqual(h.kind, 'both')
self.assertEqual(h.kind_number, 2)
def test_override(self):
h = Headers([('Accept-encoding', 'gzip'),
('Accept-encoding', 'deflate'), ('Accept', '*/*')],
kind=2)
h.override([('Accept-encoding', 'gzip2'),
('Accept-encoding', 'deflate2'), ('Accept', 'text/html'),
('Accept', '*/*; q=0.8')])
self.assertEqual(len(h), 2)
self.assertEqual(h['accept-encoding'], 'gzip2, deflate2')
self.assertEqual(h['accept'], 'text/html, */*; q=0.8')
def __init__(self, status=None, content=None, response_headers=None,
content_type=None, encoding=None, environ=None):
self.environ = environ
self.status_code = status or self.DEFAULT_STATUS_CODE
self.encoding = encoding
self.cookies = SimpleCookie()
self.headers = Headers(response_headers, kind='server')
self.content = content
if content_type is not None:
self.content_type = content_type
def __init__(self,
client,
url,
method,
inp_params=None,
headers=None,
data=None,
files=None,
history=None,
charset=None,
encode_multipart=True,
multipart_boundary=None,
source_address=None,
allow_redirects=False,
max_redirects=10,
decompress=True,
version=None,
wait_continue=False,
websocket_handler=None,
cookies=None,
urlparams=None,
**ignored):
self.client = client
self._data = None
self.files = files
self.urlparams = urlparams
self.inp_params = inp_params or {}
self.unredirected_headers = Headers(kind='client')
self.method = method.upper()
self.full_url = url
if urlparams:
self._encode_url(urlparams)
self.set_proxy(None)
self.history = history
self.wait_continue = wait_continue
self.max_redirects = max_redirects
self.allow_redirects = allow_redirects
self.charset = charset or 'utf-8'
self.version = version
self.decompress = decompress
self.encode_multipart = encode_multipart
self.multipart_boundary = multipart_boundary
self.websocket_handler = websocket_handler
self.source_address = source_address
self.new_parser()
if self._scheme in tls_schemes:
self._ssl = client.ssl_context(**ignored)
self.headers = client.get_headers(self, headers)
cookies = cookiejar_from_dict(client.cookies, cookies)
if cookies:
cookies.add_cookie_header(self)
self.unredirected_headers['host'] = host_no_default_port(
self._scheme, self._netloc)
client.set_proxy(self)
self.data = data
def test_cookies(self):
h = Headers()
cookies = SimpleCookie({'bla': 'foo', 'pippo': 'pluto'})
self.assertEqual(len(cookies), 2)
for c in cookies.values():
v = c.OutputString()
h.add_header('Set-Cookie', v)
h = str(h)
self.assertTrue(
h in ('Set-Cookie: bla=foo\r\nSet-Cookie: pippo=pluto\r\n\r\n',
'Set-Cookie: pippo=pluto\r\nSet-Cookie: bla=foo\r\n\r\n'))
def test_override(self):
h = Headers([('Accept-encoding', 'gzip'),
('Accept-encoding', 'deflate'),
('Accept', '*/*')], kind=2)
h.override([('Accept-encoding', 'gzip2'),
('Accept-encoding', 'deflate2'),
('Accept', 'text/html'),
('Accept', '*/*; q=0.8')])
self.assertEqual(len(h), 2)
self.assertEqual(h['accept-encoding'], 'gzip2, deflate2')
self.assertEqual(h['accept'], 'text/html, */*; q=0.8')
def test_client_header(self):
h = Headers()
self.assertEqual(len(h), 0)
h['content-type'] = 'text/html'
self.assertEqual(h.get_all('content-type'), ['text/html'])
self.assertEqual(len(h), 1)
h['server'] = 'bla'
self.assertEqual(len(h), 2)
del h['content-type']
self.assertEqual(len(h), 1)
self.assertEqual(h.get_all('content-type', []), [])
def test_cookies(self):
h = Headers()
cookies = SimpleCookie({'bla': 'foo', 'pippo': 'pluto'})
self.assertEqual(len(cookies), 2)
for c in cookies.values():
v = c.OutputString()
h.add_header('Set-Cookie', v)
h = str(h)
self.assertTrue(
h in ('Set-Cookie: bla=foo\r\nSet-Cookie: pippo=pluto\r\n\r\n',
'Set-Cookie: pippo=pluto\r\nSet-Cookie: bla=foo\r\n\r\n'))
def testClientHeader(self):
h = Headers(kind='client')
self.assertEqual(h.kind, 'client')
self.assertEqual(len(h), 0)
h['content-type'] = 'text/html'
self.assertEqual(h.get_all('content-type'), ['text/html'])
self.assertEqual(len(h), 1)
h['server'] = 'bla'
self.assertEqual(len(h), 1)
del h['content-type']
self.assertEqual(len(h), 0)
self.assertEqual(h.get_all('content-type', []), [])
def test_remove_header_value(self):
h = Headers([('Accept-encoding', 'gzip'),
('Accept-encoding', 'deflate'),
('Accept', '*/*')], kind=2)
self.assertEqual(len(h), 2)
self.assertEqual(h['accept-encoding'], 'gzip, deflate')
self.assertEqual(h.remove_header('accept-encoding', 'x'), None)
self.assertEqual(h['accept-encoding'], 'gzip, deflate')
self.assertEqual(h.remove_header('accept-encoding', 'deflate'),
'deflate')
self.assertEqual(len(h), 2)
self.assertEqual(h['accept-encoding'], 'gzip')
def testClientHeader(self):
h = Headers(kind='client')
self.assertEqual(h.kind, 'client')
self.assertEqual(len(h), 0)
h['content-type'] = 'text/html'
self.assertEqual(h.get_all('content-type'), ['text/html'])
self.assertEqual(len(h), 1)
h['server'] = 'bla'
self.assertEqual(len(h), 1)
del h['content-type']
self.assertEqual(len(h), 0)
self.assertEqual(h.get_all('content-type', []), [])
def test_remove_header_value(self):
h = Headers([('Accept-encoding', 'gzip'),
('Accept-encoding', 'deflate'), ('Accept', '*/*')],
kind=2)
self.assertEqual(len(h), 2)
self.assertEqual(h['accept-encoding'], 'gzip, deflate')
self.assertEqual(h.remove_header('accept-encoding', 'x'), None)
self.assertEqual(h['accept-encoding'], 'gzip, deflate')
self.assertEqual(h.remove_header('accept-encoding', 'deflate'),
'deflate')
self.assertEqual(len(h), 2)
self.assertEqual(h['accept-encoding'], 'gzip')
def get_headers(self, request, headers=None):
# Returns a :class:`Header` obtained from combining
# :attr:`headers` with *headers*. Can handle websocket requests.
if request.scheme in ('ws', 'wss'):
d = Headers(
(('Connection', 'Upgrade'), ('Upgrade', 'websocket'),
('Sec-WebSocket-Version', str(max(SUPPORTED_VERSIONS))),
('Sec-WebSocket-Key', self.websocket_key),
('user-agent', self.client_version)),
kind='client')
else:
d = self.headers.copy()
if headers:
d.override(headers)
return d
def test_CacheControl(self):
headers = Headers()
c = CacheControl()
self.assertFalse(c.private)
self.assertFalse(c.maxage)
c(headers)
self.assertEqual(headers['cache-control'], 'no-cache')
c = CacheControl(maxage=3600)
c(headers)
self.assertEqual(headers['cache-control'], 'max-age=3600, public')
c = CacheControl(maxage=3600, private=True)
c(headers)
self.assertEqual(headers['cache-control'], 'max-age=3600, private')
c = CacheControl(maxage=3600, must_revalidate=True)
c(headers)
self.assertEqual(headers['cache-control'],
'max-age=3600, public, must-revalidate')
c = CacheControl(maxage=3600, proxy_revalidate=True)
c(headers)
self.assertEqual(headers['cache-control'],
'max-age=3600, public, proxy-revalidate')
c = CacheControl(maxage=3600, proxy_revalidate=True,
nostore=True)
c(headers)
self.assertEqual(headers['cache-control'],
'no-store, no-cache, must-revalidate, max-age=0')
def data_received(self, data):
'''Implements :meth:`~.ProtocolConsumer.data_received` method.
Once we have a full HTTP message, build the wsgi ``environ`` and
delegate the response to the :func:`wsgi_callable` function.
'''
parser = self.parser
processed = parser.execute(data, len(data))
if not self._stream and parser.is_headers_complete():
headers = Headers(parser.get_headers(), kind='client')
self._stream = StreamReader(headers, parser, self.transport)
self._response(self.wsgi_environ())
#
if parser.is_message_complete():
#
# Stream has the whole body
if not self._stream.on_message_complete.done():
self._stream.on_message_complete.set_result(None)
if processed < len(data):
if not self._buffer:
self._buffer = data[processed:]
self.bind_event('post_request', self._new_request)
else:
self._buffer += data[processed:]
#
elif processed < len(data):
# This is a parsing error, the client must have sent
# bogus data
raise ProtocolError
def testHeaderBytes(self):
h = Headers(kind=None)
h['content-type'] = 'text/html'
h['server'] = 'bla'
self.assertTrue(repr(h).startswith('both '))
self.assertEqual(bytes(h), b'Server: bla\r\n'
b'Content-Type: text/html\r\n\r\n')
def __init__(self, client, url, method, inp_params=None, headers=None,
data=None, files=None, json=None, history=None, auth=None,
charset=None, max_redirects=10, source_address=None,
allow_redirects=False, decompress=True, version=None,
wait_continue=False, websocket_handler=None, cookies=None,
params=None, stream=False, proxies=None, verify=True,
**ignored):
self.client = client
self.method = method.upper()
self.inp_params = inp_params or {}
self.unredirected_headers = Headers(kind='client')
self.history = history
self.wait_continue = wait_continue
self.max_redirects = max_redirects
self.allow_redirects = allow_redirects
self.charset = charset or 'utf-8'
self.version = version
self.decompress = decompress
self.websocket_handler = websocket_handler
self.source_address = source_address
self.stream = stream
self.verify = verify
self.new_parser()
if auth and not isinstance(auth, Auth):
auth = HTTPBasicAuth(*auth)
self.auth = auth
self.headers = client._get_headers(headers)
self.url = full_url(url, params, method=self.method)
self.body = self._encode_body(data, files, json)
self._set_proxy(proxies, ignored)
cookies = cookiejar_from_dict(client.cookies, cookies)
if cookies:
cookies.add_cookie_header(self)
def get_headers(self, request, headers=None):
# Returns a :class:`Header` obtained from combining
# :attr:`headers` with *headers*. Can handle websocket requests.
if request.scheme in ('ws', 'wss'):
d = Headers((
('Connection', 'Upgrade'),
('Upgrade', 'websocket'),
('Sec-WebSocket-Version', str(max(SUPPORTED_VERSIONS))),
('Sec-WebSocket-Key', self.websocket_key),
('user-agent', self.client_version)
), kind='client')
else:
d = self.headers.copy()
if headers:
d.override(headers)
return d
def __init__(self, url, username, password, **kwargs):
super(GerritSession, self).__init__(
headers=Headers([('Content-Type', 'application/json')]),
**kwargs
)
self.url = url
self.auth = HTTPDigestAuth(username, password)
def __init__(self,
status=None,
content=None,
response_headers=None,
content_type=None,
encoding=None,
environ=None,
can_store_cookies=True):
self.environ = environ
self.status_code = status or self.DEFAULT_STATUS_CODE
self.encoding = encoding
self.cookies = SimpleCookie()
self.headers = Headers(response_headers or ())
self.content = content
self._can_store_cookies = can_store_cookies
if content_type is not None:
self.content_type = content_type
def request_headers(self, environ):
'''Modify request headers via the list of :attr:`headers_middleware`.
The returned headers will be sent to the target uri.'''
headers = Headers(kind='client')
for k in environ:
if k.startswith('HTTP_'):
head = k[5:].replace('_','-')
headers[head] = environ[k]
for head in ENVIRON_HEADERS:
k = head.replace('-','_').upper()
v = environ.get(k)
if v:
headers[head] = v
headers.update(self.headers)
for middleware in self.headers_middleware:
middleware(environ, headers)
return headers
def __init__(self, wsgi_callable, cfg, server_software=None):
super(HttpServerResponse, self).__init__()
self.wsgi_callable = wsgi_callable
self.cfg = cfg
self.parser = http_parser(kind=0)
self.headers = Headers()
self.keep_alive = False
self.SERVER_SOFTWARE = server_software or self.SERVER_SOFTWARE
def test_multiple_entry(self):
h = Headers([('Connection', 'Keep-Alive'),
('Accept-Encoding', 'identity'),
('Accept-Encoding', 'deflate'),
('Accept-Encoding', 'compress'),
('Accept-Encoding', 'gzip')],
kind='client')
accept = h['accept-encoding']
self.assertEqual(accept, 'identity, deflate, compress, gzip')
def request_start_response(self,
method,
path,
HTTP_ACCEPT=None,
headers=None,
data=None,
json=None,
content_type=None,
token=None,
oauth=None,
jwt=None,
cookie=None,
params=None,
**extra):
method = method.upper()
extra['REQUEST_METHOD'] = method.upper()
path = path or '/'
extra['HTTP_ACCEPT'] = HTTP_ACCEPT or '*/*'
extra['pulsar.connection'] = mock.MagicMock()
heads = []
if headers:
heads.extend(headers)
if json is not None:
content_type = 'application/json'
assert not data
data = json
if content_type:
heads.append(('content-type', content_type))
if token:
heads.append(('Authorization', 'Bearer %s' % token))
elif oauth:
heads.append(('Authorization', 'OAuth %s' % oauth))
elif jwt:
heads.append(('Authorization', 'JWT %s' % jwt))
if cookie:
heads.append(('Cookie', cookie))
if params:
path = full_url(path, params)
# Encode data
if (method in ENCODE_BODY_METHODS and data is not None
and not isinstance(data, bytes)):
content_type = Headers(heads).get('content-type')
if content_type is None:
data, content_type = encode_multipart_formdata(data)
heads.append(('content-type', content_type))
elif content_type == 'application/json':
data = _json.dumps(data).encode('utf-8')
request = self.app.wsgi_request(path=path,
headers=heads,
body=data,
**extra)
request.environ['SERVER_NAME'] = 'localhost'
start_response = mock.MagicMock()
return request, start_response
def __init__(
self,
client,
url,
method,
inp_params=None,
headers=None,
data=None,
files=None,
history=None,
charset=None,
encode_multipart=True,
multipart_boundary=None,
source_address=None,
allow_redirects=False,
max_redirects=10,
decompress=True,
version=None,
wait_continue=False,
websocket_handler=None,
cookies=None,
urlparams=None,
**ignored
):
self.client = client
self._data = None
self.files = files
self.urlparams = urlparams
self.inp_params = inp_params or {}
self.unredirected_headers = Headers(kind="client")
self.method = method.upper()
self.full_url = url
if urlparams:
self._encode_url(urlparams)
self.set_proxy(None)
self.history = history
self.wait_continue = wait_continue
self.max_redirects = max_redirects
self.allow_redirects = allow_redirects
self.charset = charset or "utf-8"
self.version = version
self.decompress = decompress
self.encode_multipart = encode_multipart
self.multipart_boundary = multipart_boundary
self.websocket_handler = websocket_handler
self.source_address = source_address
self.new_parser()
if self._scheme in tls_schemes:
self._ssl = client.ssl_context(**ignored)
self.headers = client.get_headers(self, headers)
cookies = cookiejar_from_dict(client.cookies, cookies)
if cookies:
cookies.add_cookie_header(self)
self.unredirected_headers["host"] = host_no_default_port(self._scheme, self._netloc)
client.set_proxy(self)
self.data = data
def __init__(self, status=None, content=None, response_headers=None,
content_type=None, encoding=None, environ=None):
self.environ = environ
self.status_code = status or self.DEFAULT_STATUS_CODE
self.encoding = encoding
self.cookies = SimpleCookie()
self.headers = Headers(response_headers, kind='server')
self.content = content
if content_type is not None:
self.content_type = content_type
def get_headers(self, request, headers=None):
# Returns a :class:`Header` obtained from combining
# :attr:`headers` with *headers*. Can handle websocket requests.
if request.scheme in ("ws", "wss"):
d = Headers(
(
("Connection", "Upgrade"),
("Upgrade", "websocket"),
("Sec-WebSocket-Version", str(max(SUPPORTED_VERSIONS))),
("Sec-WebSocket-Key", self.websocket_key),
("user-agent", self.client_version),
),
kind="client",
)
else:
d = self.headers.copy()
if headers:
d.override(headers)
return d
def test_non_standard_request_headers(self):
h = Headers(kind='client')
h['accept'] = 'text/html'
self.assertEqual(len(h), 1)
h['server'] = 'bla'
self.assertEqual(len(h), 1)
h['proxy-connection'] = 'keep-alive'
self.assertEqual(len(h), 2)
headers = str(h)
self.assertTrue('Proxy-Connection:' in headers)
def __init__(self, status=None, content=None, response_headers=None,
content_type=None, encoding=None, environ=None,
can_store_cookies=True):
self.environ = environ
self.status_code = status or self.DEFAULT_STATUS_CODE
self.encoding = encoding
self.cookies = SimpleCookie()
self.headers = Headers(response_headers or ())
self.content = content
self._can_store_cookies = can_store_cookies
if content_type is not None:
self.content_type = content_type
def __init__(self, status=None, content=None, response_headers=None,
content_type=None, encoding=None, environ=None,
can_store_cookies=True):
self.environ = environ
self.status_code = status or self.DEFAULT_STATUS_CODE
self.encoding = encoding
self.cookies = SimpleCookie()
self.headers = Headers(response_headers, kind='server')
self.content = content
self._can_store_cookies = can_store_cookies
if content_type is not None:
self.content_type = content_type
if environ:
cookie = environ.get('HTTP_COOKIE')
if cookie:
self.cookies.load(cookie)
def request_headers(self, environ):
'''Fill request headers from the environ dictionary and
modify them via the list of :attr:`headers_middleware`.
The returned headers will be sent to the target uri.
'''
headers = Headers(kind='client')
for k in environ:
if k.startswith('HTTP_'):
head = k[5:].replace('_', '-')
headers[head] = environ[k]
for head in ENVIRON_HEADERS:
k = head.replace('-', '_').upper()
v = environ.get(k)
if v:
headers[head] = v
return headers
def data_processed(self, response, exc=None, **kw):
'''Receive data from the requesting HTTP client.'''
status = response.get_status()
if status == '100 Continue':
stream = self.environ.get('wsgi.input') or io.BytesIO()
body = yield stream.read()
response.transport.write(body)
if response.parser.is_headers_complete():
if self._headers is None:
headers = self.remove_hop_headers(response.headers)
self._headers = Headers(headers, kind='server')
# start the response
self.start_response(status, list(self._headers))
body = response.recv_body()
if response.parser.is_message_complete():
self._done = True
self.queue.put_nowait(body)
def __init__(
self,
status=None,
content=None,
response_headers=None,
content_type=None,
encoding=None,
environ=None,
start_response=None,
):
super(WsgiResponse, self).__init__(environ, start_response)
self.status_code = status or self.DEFAULT_STATUS_CODE
self.encoding = encoding
self.cookies = SimpleCookie()
self.headers = Headers(response_headers, kind="server")
self.content = content
if content_type is not None:
self.content_type = content_type
def data_received(self, data):
'''Implements :meth:`~.ProtocolConsumer.data_received` method.
Once we have a full HTTP message, build the wsgi ``environ`` and
delegate the response to the :func:`wsgi_callable` function.
'''
p = self.parser
if p.execute(bytes(data), len(data)) == len(data):
if self._request_headers is None and p.is_headers_complete():
self._request_headers = Headers(p.get_headers(), kind='client')
stream = StreamReader(self._request_headers, p, self.transport)
self.bind_event('data_processed', stream.data_processed)
environ = self.wsgi_environ(stream)
self.event_loop. async (self._response(environ))
else:
# This is a parsing error, the client must have sent
# bogus data
raise ProtocolError
def __init__(self, client, url, method, inp_params=None, headers=None,
data=None, files=None, history=None, auth=None,
charset=None, encode_multipart=True, multipart_boundary=None,
source_address=None, allow_redirects=False, max_redirects=10,
decompress=True, version=None, wait_continue=False,
websocket_handler=None, cookies=None, urlparams=None,
stream=False, proxies=None, verify=True, **ignored):
self.client = client
self._data = None
self.files = files
self.urlparams = urlparams
self.inp_params = inp_params or {}
self.unredirected_headers = Headers(kind='client')
self.method = method.upper()
self.full_url = url
if urlparams:
self._encode_url(urlparams)
self.history = history
self.wait_continue = wait_continue
self.max_redirects = max_redirects
self.allow_redirects = allow_redirects
self.charset = charset or 'utf-8'
self.version = version
self.decompress = decompress
self.encode_multipart = encode_multipart
self.multipart_boundary = multipart_boundary
self.websocket_handler = websocket_handler
self.source_address = source_address
self.stream = stream
self.verify = verify
self.new_parser()
if self._scheme in tls_schemes:
self._ssl = client.ssl_context(verify=self.verify, **ignored)
if auth and not isinstance(auth, Auth):
auth = HTTPBasicAuth(*auth)
self.auth = auth
self.headers = client.get_headers(self, headers)
cookies = cookiejar_from_dict(client.cookies, cookies)
if cookies:
cookies.add_cookie_header(self)
self.unredirected_headers['host'] = host_no_default_port(self._scheme,
self._netloc)
self.data = data
self._set_proxy(proxies)
def data_received(self, data):
'''Implements :meth:`~.ProtocolConsumer.data_received` method.
Once we have a full HTTP message, build the wsgi ``environ`` and
delegate the response to the :func:`wsgi_callable` function.
'''
parser = self.parser
processed = parser.execute(data, len(data))
if parser.is_headers_complete():
if not self._body_reader:
headers = Headers(parser.get_headers(), kind='client')
self._body_reader = HttpBodyReader(headers,
parser,
self.transport,
self.cfg.stream_buffer,
loop=self._loop)
ensure_future(self._response(self.wsgi_environ()),
loop=self._loop)
body = parser.recv_body()
if body:
self._body_reader.feed_data(body)
#
if parser.is_message_complete():
#
self._body_reader.feed_eof()
if processed < len(data):
if not self._buffer:
self._buffer = data[processed:]
self.bind_event('post_request', self._new_request)
else:
self._buffer += data[processed:]
#
elif processed < len(data):
# This is a parsing error, the client must have sent
# bogus data
raise ProtocolError
def headers(self):
if not hasattr(self, '_headers'):
if self.parser and self.parser.is_headers_complete():
self._headers = Headers(self.parser.get_headers())
return getattr(self, '_headers', None)
class HttpServerResponse(ProtocolConsumer):
'''Server side WSGI :class:`.ProtocolConsumer`.
.. attribute:: wsgi_callable
The wsgi callable handling requests.
'''
_status = None
_headers_sent = None
_stream = None
_buffer = None
SERVER_SOFTWARE = pulsar.SERVER_SOFTWARE
ONE_TIME_EVENTS = ProtocolConsumer.ONE_TIME_EVENTS + ('on_headers',)
def __init__(self, wsgi_callable, cfg, server_software=None):
super(HttpServerResponse, self).__init__()
self.wsgi_callable = wsgi_callable
self.cfg = cfg
self.parser = http_parser(kind=0)
self.headers = Headers()
self.keep_alive = False
self.SERVER_SOFTWARE = server_software or self.SERVER_SOFTWARE
def data_received(self, data):
'''Implements :meth:`~.ProtocolConsumer.data_received` method.
Once we have a full HTTP message, build the wsgi ``environ`` and
delegate the response to the :func:`wsgi_callable` function.
'''
parser = self.parser
processed = parser.execute(data, len(data))
if not self._stream and parser.is_headers_complete():
headers = Headers(parser.get_headers(), kind='client')
self._stream = StreamReader(headers, parser, self.transport)
self._response(self.wsgi_environ())
#
done = parser.is_message_complete()
if done and not self._stream.on_message_complete.done():
self._stream.on_message_complete.set_result(None)
#
if processed < len(data):
if not done:
# This is a parsing error, the client must have sent
# bogus data
raise ProtocolError
else:
if not self._buffer:
self._buffer = data[processed:]
self.bind_event('post_request', self._new_request)
else:
self._buffer += data[processed:]
@property
def status(self):
return self._status
@property
def upgrade(self):
return self.headers.get('upgrade')
@property
def chunked(self):
return self.headers.get('Transfer-Encoding') == 'chunked'
@property
def content_length(self):
c = self.headers.get('Content-Length')
if c:
return int(c)
@property
def version(self):
return self.parser.get_version()
def start_response(self, status, response_headers, exc_info=None):
'''WSGI compliant ``start_response`` callable, see pep3333_.
The application may call start_response more than once, if and only
if the ``exc_info`` argument is provided.
More precisely, it is a fatal error to call ``start_response`` without
the ``exc_info`` argument if start_response has already been called
within the current invocation of the application.
:parameter status: an HTTP ``status`` string like ``200 OK`` or
``404 Not Found``.
:parameter response_headers: a list of ``(header_name, header_value)``
tuples. It must be a Python list. Each header_name must be a valid
HTTP header field-name (as defined by RFC 2616_, Section 4.2),
without a trailing colon or other punctuation.
:parameter exc_info: optional python ``sys.exc_info()`` tuple.
This argument should be supplied by the application only if
``start_response`` is being called by an error handler.
:return: The :meth:`write` method.
``HOP_HEADERS`` are not considered but no error is raised.
.. _pep3333: http://www.python.org/dev/peps/pep-3333/
.. _2616: http://www.faqs.org/rfcs/rfc2616.html
'''
if exc_info:
try:
if self._headers_sent:
# if exc_info is provided, and the HTTP headers have
# already been sent, start_response must raise an error,
# and should re-raise using the exc_info tuple
reraise(*exc_info)
finally:
# Avoid circular reference
exc_info = None
elif self._status:
# Headers already set. Raise error
raise HttpException("Response headers already set!")
self._status = status
if type(response_headers) is not list:
raise TypeError("Headers must be a list of name/value tuples")
for header, value in response_headers:
if header.lower() in HOP_HEADERS:
# These features are the exclusive province of this class,
# this should be considered a fatal error for an application
# to attempt sending them, but we don't raise an error,
# just log a warning
self.logger.warning('Application passing hop header "%s"',
header)
continue
self.headers.add_header(header, value)
return self.write
def write(self, data, force=False):
'''The write function returned by the :meth:`start_response` method.
Required by the WSGI specification.
:param data: bytes to write
:param force: Optional flag used internally.
'''
if not self._headers_sent:
tosend = self.get_headers()
self._headers_sent = tosend.flat(self.version, self.status)
self.fire_event('on_headers')
self.transport.write(self._headers_sent)
if data:
if self.chunked:
chunks = []
while len(data) >= MAX_CHUNK_SIZE:
chunk, data = data[:MAX_CHUNK_SIZE], data[MAX_CHUNK_SIZE:]
chunks.append(chunk_encoding(chunk))
if data:
chunks.append(chunk_encoding(data))
self.transport.write(b''.join(chunks))
else:
self.transport.write(data)
elif force and self.chunked:
self.transport.write(chunk_encoding(data))
########################################################################
## INTERNALS
@in_loop
def _response(self, environ):
exc_info = None
response = None
done = False
while not done:
done = True
try:
if exc_info is None:
if 'SERVER_NAME' not in environ:
raise HttpException(status=400)
response = self.wsgi_callable(environ, self.start_response)
else:
response = handle_wsgi_error(environ, exc_info)
#
if isinstance(response, Future):
response = yield response
#
if exc_info:
self.start_response(response.status,
response.get_headers(), exc_info)
#
for chunk in response:
if isinstance(chunk, Future):
chunk = yield chunk
self.write(chunk)
#
# make sure we write headers
self.write(b'', True)
except IOError: # client disconnected, end this connection
self.finished()
except Exception as exc:
if wsgi_request(environ).cache.handle_wsgi_error:
self.keep_alive = False
self.connection.close()
self.finished()
else:
done = False
exc_info = sys.exc_info()
else:
if not self.keep_alive:
self.connection.close()
self.finished()
finally:
if hasattr(response, 'close'):
try:
response.close()
except Exception:
self.logger.exception(
'Error while closing wsgi iterator')
def is_chunked(self):
'''Check if the response uses chunked transfer encoding.
Only use chunked responses when the client is speaking HTTP/1.1
or newer and there was no Content-Length header set.
'''
if (self.version <= (1, 0) or
self._status == '200 Connection established' or
has_empty_content(int(self.status[:3]))):
return False
elif self.headers.get('Transfer-Encoding') == 'chunked':
return True
else:
return self.content_length is None
def get_headers(self):
'''Get the headers to send to the client.
'''
if not self._status:
# we are sending headers but the start_response was not called
raise HttpException('Headers not set.')
headers = self.headers
# Set chunked header if needed
if self.is_chunked():
headers['Transfer-Encoding'] = 'chunked'
headers.pop('content-length', None)
else:
headers.pop('Transfer-Encoding', None)
if self.keep_alive:
self.keep_alive = keep_alive_with_status(self._status, headers)
if not self.keep_alive:
headers['connection'] = 'close'
return headers
def wsgi_environ(self):
#return a the WSGI environ dictionary
transport = self.transport
https = True if is_tls(transport.get_extra_info('socket')) else False
multiprocess = (self.cfg.concurrency == 'process')
environ = wsgi_environ(self._stream,
transport.get_extra_info('sockname'),
self.address, self.headers,
self.SERVER_SOFTWARE,
https=https,
extra={'pulsar.connection': self.connection,
'pulsar.cfg': self.cfg,
'wsgi.multiprocess': multiprocess})
self.keep_alive = keep_alive(self.headers, self.parser.get_version())
self.headers.update([('Server', self.SERVER_SOFTWARE),
('Date', format_date_time(time.time()))])
return environ
def _new_request(self, response):
connection = response._connection
if not connection.closed:
connection.data_received(response._buffer)
return connection._current_consumer
return response
class HttpRequest(RequestBase):
'''An :class:`HttpClient` request for an HTTP resource.
This class has a similar interface to :class:`urllib.request.Request`.
:param files: optional dictionary of name, file-like-objects.
:param allow_redirects: allow the response to follow redirects.
.. attribute:: method
The request method
.. attribute:: version
HTTP version for this request, usually ``HTTP/1.1``
.. attribute:: encode_multipart
If ``True`` (default), defaults POST data as ``multipart/form-data``.
Pass ``encode_multipart=False`` to default to
``application/x-www-form-urlencoded``. In any case, this parameter is
overwritten by passing the correct content-type header.
.. attribute:: history
List of past :class:`.HttpResponse` (collected during redirects).
.. attribute:: wait_continue
if ``True``, the :class:`HttpRequest` includes the
``Expect: 100-Continue`` header.
'''
CONNECT = 'CONNECT'
_proxy = None
_ssl = None
_tunnel = None
def __init__(self, client, url, method, inp_params=None, headers=None,
data=None, files=None, history=None,
charset=None, encode_multipart=True, multipart_boundary=None,
source_address=None, allow_redirects=False, max_redirects=10,
decompress=True, version=None, wait_continue=False,
websocket_handler=None, cookies=None, urlparams=None,
**ignored):
self.client = client
self._data = None
self.files = files
self.urlparams = urlparams
self.inp_params = inp_params or {}
self.unredirected_headers = Headers(kind='client')
self.method = method.upper()
self.full_url = url
if urlparams:
self._encode_url(urlparams)
self.set_proxy(None)
self.history = history
self.wait_continue = wait_continue
self.max_redirects = max_redirects
self.allow_redirects = allow_redirects
self.charset = charset or 'utf-8'
self.version = version
self.decompress = decompress
self.encode_multipart = encode_multipart
self.multipart_boundary = multipart_boundary
self.websocket_handler = websocket_handler
self.source_address = source_address
self.new_parser()
if self._scheme in tls_schemes:
self._ssl = client.ssl_context(**ignored)
self.headers = client.get_headers(self, headers)
cookies = cookiejar_from_dict(client.cookies, cookies)
if cookies:
cookies.add_cookie_header(self)
self.unredirected_headers['host'] = host_no_default_port(self._scheme,
self._netloc)
client.set_proxy(self)
self.data = data
@property
def address(self):
'''``(host, port)`` tuple of the HTTP resource'''
return self._tunnel.address if self._tunnel else (self.host, self.port)
@property
def target_address(self):
return (self.host, int(self.port))
@property
def ssl(self):
'''Context for TLS connections.
If this is a tunneled request and the tunnel connection is not yet
established, it returns ``None``.
'''
if not self._tunnel:
return self._ssl
@property
def key(self):
return (self.scheme, self.host, self.port)
@property
def proxy(self):
'''Proxy server for this request.'''
return self._proxy
@property
def netloc(self):
if self._proxy:
return self._proxy.netloc
else:
return self._netloc
def __repr__(self):
return self.first_line()
__str__ = __repr__
@property
def full_url(self):
'''Full url of endpoint'''
return urlunparse((self._scheme, self._netloc, self.path,
self.params, self.query, self.fragment))
@full_url.setter
def full_url(self, url):
self._scheme, self._netloc, self.path, self.params,\
self.query, self.fragment = urlparse(url)
if not self._netloc and self.method == 'CONNECT':
self._scheme, self._netloc, self.path, self.params,\
self.query, self.fragment = urlparse('http://%s' % url)
@property
def data(self):
'''Body of request'''
return self._data
@data.setter
def data(self, data):
self._data = self._encode_data(data)
def first_line(self):
if not self._proxy and self.method != self.CONNECT:
url = urlunparse(('', '', self.path or '/', self.params,
self.query, self.fragment))
else:
url = self.full_url
return '%s %s %s' % (self.method, url, self.version)
def new_parser(self):
self.parser = self.client.http_parser(kind=1,
decompress=self.decompress,
method=self.method)
def set_proxy(self, scheme, *host):
if not host and scheme is None:
self.scheme = self._scheme
self._set_hostport(self._scheme, self._netloc)
else:
le = 2 + len(host)
if not le == 3:
raise TypeError(
'set_proxy() takes exactly three arguments (%s given)'
% le)
if not self._ssl:
self.scheme = scheme
self._set_hostport(scheme, host[0])
self._proxy = scheme_host(scheme, host[0])
else:
self._tunnel = HttpTunnel(self, scheme, host[0])
def _set_hostport(self, scheme, host):
self._tunnel = None
self._proxy = None
self.host, self.port = get_hostport(scheme, host)
def encode(self):
'''The bytes representation of this :class:`HttpRequest`.
Called by :class:`HttpResponse` when it needs to encode this
:class:`HttpRequest` before sending it to the HTTP resource.
'''
# Call body before fist_line in case the query is changes.
first_line = self.first_line()
body = self.data
if body and self.wait_continue:
self.headers['expect'] = '100-continue'
body = None
headers = self.headers
if self.unredirected_headers:
headers = self.unredirected_headers.copy()
headers.update(self.headers)
buffer = [first_line.encode('ascii'), b'\r\n', bytes(headers)]
if body:
buffer.append(body)
return b''.join(buffer)
def add_header(self, key, value):
self.headers[key] = value
def has_header(self, header_name):
'''Check ``header_name`` is in this request headers.
'''
return (header_name in self.headers or
header_name in self.unredirected_headers)
def get_header(self, header_name, default=None):
'''Retrieve ``header_name`` from this request headers.
'''
return self.headers.get(
header_name, self.unredirected_headers.get(header_name, default))
def remove_header(self, header_name):
'''Remove ``header_name`` from this request.
'''
self.headers.pop(header_name, None)
self.unredirected_headers.pop(header_name, None)
def add_unredirected_header(self, header_name, header_value):
self.unredirected_headers[header_name] = header_value
# INTERNAL ENCODING METHODS
def _encode_data(self, data):
body = None
if self.method in ENCODE_URL_METHODS:
self.files = None
self._encode_url(data)
elif isinstance(data, bytes):
assert self.files is None, ('data cannot be bytes when files are '
'present')
body = data
elif isinstance(data, str):
assert self.files is None, ('data cannot be string when files are '
'present')
body = to_bytes(data, self.charset)
elif data or self.files:
if self.files:
body, content_type = self._encode_files(data)
else:
body, content_type = self._encode_params(data)
# set files to None, Important!
self.files = None
self.headers['Content-Type'] = content_type
if body:
self.headers['content-length'] = str(len(body))
elif 'expect' not in self.headers:
self.headers.pop('content-length', None)
self.headers.pop('content-type', None)
return body
def _encode_url(self, data):
query = self.query
if data:
data = native_str(data)
if isinstance(data, str):
data = parse_qsl(data)
else:
data = mapping_iterator(data)
query = parse_qsl(query)
query.extend(data)
query = urlencode(query)
self.query = query
def _encode_files(self, data):
fields = []
for field, val in mapping_iterator(data or ()):
if (isinstance(val, str) or isinstance(val, bytes) or
not hasattr(val, '__iter__')):
val = [val]
for v in val:
if v is not None:
if not isinstance(v, bytes):
v = str(v)
fields.append((field.decode('utf-8') if
isinstance(field, bytes) else field,
v.encode('utf-8') if isinstance(v, str)
else v))
for (k, v) in mapping_iterator(self.files):
# support for explicit filename
ft = None
if isinstance(v, (tuple, list)):
if len(v) == 2:
fn, fp = v
else:
fn, fp, ft = v
else:
fn = guess_filename(v) or k
fp = v
if isinstance(fp, bytes):
fp = BytesIO(fp)
elif isinstance(fp, str):
fp = StringIO(fp)
if ft:
new_v = (fn, fp.read(), ft)
else:
new_v = (fn, fp.read())
fields.append((k, new_v))
#
return encode_multipart_formdata(fields, charset=self.charset)
def _encode_params(self, data):
content_type = self.headers.get('content-type')
# No content type given, chose one
if not content_type:
if self.encode_multipart:
content_type = MULTIPART_FORM_DATA
else:
content_type = FORM_URL_ENCODED
if content_type in JSON_CONTENT_TYPES:
body = json.dumps(data).encode(self.charset)
elif content_type == FORM_URL_ENCODED:
body = urlencode(data).encode(self.charset)
elif content_type == MULTIPART_FORM_DATA:
body, content_type = encode_multipart_formdata(
data, boundary=self.multipart_boundary, charset=self.charset)
else:
raise ValueError("Don't know how to encode body for %s" %
content_type)
return body, content_type
class WsgiResponse(object):
'''A WSGI response.
Instances are callable using the standard WSGI call and, importantly,
iterable::
response = WsgiResponse(200)
A :class:`WsgiResponse` is an iterable over bytes to send back to the
requesting client.
.. attribute:: status_code
Integer indicating the HTTP status, (i.e. 200)
.. attribute:: response
String indicating the HTTP status (i.e. 'OK')
.. attribute:: status
String indicating the HTTP status code and response (i.e. '200 OK')
.. attribute:: content_type
The content type of this response. Can be ``None``.
.. attribute:: headers
The :class:`pulsar.utils.httpurl.Headers` container for this response.
.. attribute:: environ
The dictionary of WSGI environment if passed to the constructor.
'''
_started = False
DEFAULT_STATUS_CODE = 200
def __init__(self, status=None, content=None, response_headers=None,
content_type=None, encoding=None, environ=None):
self.environ = environ
self.status_code = status or self.DEFAULT_STATUS_CODE
self.encoding = encoding
self.cookies = SimpleCookie()
self.headers = Headers(response_headers, kind='server')
self.content = content
if content_type is not None:
self.content_type = content_type
@property
def started(self):
return self._started
@property
def path(self):
if self.environ:
return self.environ.get('PATH_INFO', '')
@property
def method(self):
if self.environ:
return self.environ.get('REQUEST_METHOD')
@property
def connection(self):
if self.environ:
return self.environ.get('pulsar.connection')
def _get_content(self):
return self._content
def _set_content(self, content):
if not self._started:
if content is None:
content = ()
elif ispy3k:
if isinstance(content, str):
content = content.encode(self.encoding or 'utf-8')
else: # pragma nocover
if isinstance(content, unicode):
content = content.encode(self.encoding or 'utf-8')
if isinstance(content, bytes):
content = (content,)
self._content = content
else:
raise RuntimeError('Cannot set content. Already iterated')
content = property(_get_content, _set_content)
def _get_content_type(self):
return self.headers.get('content-type')
def _set_content_type(self, typ):
if typ:
self.headers['content-type'] = typ
else:
self.headers.pop('content-type', None)
content_type = property(_get_content_type, _set_content_type)
def length(self):
if not self.is_streamed:
return reduce(lambda x, y: x+len(y), self.content, 0)
@property
def response(self):
return responses.get(self.status_code)
@property
def status(self):
return '%s %s' % (self.status_code, self.response)
def __str__(self):
return self.status
def __repr__(self):
return '%s(%s)' % (self.__class__.__name__, self)
@property
def is_streamed(self):
"""If the response is streamed (the response is not an iterable with
length information) this property is `True`. In this case streamed
means that there is no information about the number of iterations.
This is usually `True` if a generator is passed to the response object."""
return is_streamed(self.content)
def __iter__(self):
if self._started:
raise RuntimeError('WsgiResponse can be iterated once only')
self._started = True
if is_streamed(self.content):
return wsgi_encoder(self.content, self.encoding or 'utf-8')
else:
return iter(self.content)
def __len__(self):
return len(self.content)
def set_cookie(self, key, **kwargs):
"""
Sets a cookie.
``expires`` can be a string in the correct format or a
``datetime.datetime`` object in UTC. If ``expires`` is a datetime
object then ``max_age`` will be calculated.
"""
set_cookie(self.cookies, key, **kwargs)
def delete_cookie(self, key, path='/', domain=None):
set_cookie(self.cookies, key, max_age=0, path=path, domain=domain,
expires='Thu, 01-Jan-1970 00:00:00 GMT')
def get_headers(self):
headers = self.headers
if has_empty_content(self.status_code, self.method):
headers.pop('content-type', None)
headers.pop('content-length', None)
self._content = ()
else:
if not self.is_streamed:
cl = 0
for c in self.content:
cl += len(c)
if cl == 0 and self.content_type in JSON_CONTENT_TYPES:
self._content = (b'{}',)
cl = len(self._content[0])
headers['Content-Length'] = str(cl)
if not self.content_type:
headers['Content-Type'] = 'text/plain'
for c in self.cookies.values():
headers['Set-Cookie'] = c.OutputString()
return list(headers)
def has_header(self, header):
return header in self.headers
__contains__ = has_header
def __setitem__(self, header, value):
self.headers[header] = value
def __getitem__(self, header):
return self.headers[header]
def test_add_header_with_params(self):
h = Headers()
h.add_header('content-type', 'text/html', charset=DEFAULT_CHARSET)
self.assertEqual(h['content-type'], 'text/html; charset=ISO-8859-1')
class WsgiResponse(WsgiResponseGenerator):
"""A WSGI response wrapper initialized by a WSGI request middleware.
Instances are callable using the standard WSGI call::
response = WsgiResponse(200)
response(environ, start_response)
A :class:`WsgiResponse` is an iterable over bytes to send back to the requesting
client.
.. attribute:: status_code
Integer indicating the HTTP status, (i.e. 200)
.. attribute:: response
String indicating the HTTP status (i.e. 'OK')
.. attribute:: status
String indicating the HTTP status code and response (i.e. '200 OK')
.. attribute:: environ
The dictionary of WSGI environment if passed to the constructor.
"""
_started = False
DEFAULT_STATUS_CODE = 200
def __init__(
self,
status=None,
content=None,
response_headers=None,
content_type=None,
encoding=None,
environ=None,
start_response=None,
):
super(WsgiResponse, self).__init__(environ, start_response)
self.status_code = status or self.DEFAULT_STATUS_CODE
self.encoding = encoding
self.cookies = SimpleCookie()
self.headers = Headers(response_headers, kind="server")
self.content = content
if content_type is not None:
self.content_type = content_type
@property
def started(self):
return self._started
@property
def path(self):
if self.environ:
return self.environ.get("PATH_INFO", "")
@property
def method(self):
if self.environ:
return self.environ.get("REQUEST_METHOD")
@property
def connection(self):
if self.environ:
return self.environ.get("pulsar.connection")
def _get_content(self):
return self._content
def _set_content(self, content):
if not self._started:
if content is None:
content = ()
elif ispy3k: # what a f*****g pain
if isinstance(content, str):
content = bytes(content, "latin-1")
else: # pragma nocover
if isinstance(content, unicode):
content = bytes(content, "latin-1")
if isinstance(content, bytes):
content = (content,)
self._content = content
else:
raise RuntimeError("Cannot set content. Already iterated")
content = property(_get_content, _set_content)
def _get_content_type(self):
return self.headers.get("content-type")
def _set_content_type(self, typ):
if typ:
self.headers["content-type"] = typ
else:
self.headers.pop("content-type", None)
content_type = property(_get_content_type, _set_content_type)
def __call__(self, environ, start_response, exc_info=None):
"""Make sure the headers are set."""
if not exc_info:
for rm in self.middleware:
try:
rm(environ, self)
except:
LOGGER.error("Exception in response middleware", exc_info=True)
start_response(self.status, self.get_headers(), exc_info=exc_info)
return self
def start(self):
self.__call__(self.environ, self.start_response)
def length(self):
if not self.is_streamed:
return reduce(lambda x, y: x + len(y), self.content, 0)
@property
def response(self):
return responses.get(self.status_code)
@property
def status(self):
return "%s %s" % (self.status_code, self.response)
def __str__(self):
return self.status
def __repr__(self):
return "%s(%s)" % (self.__class__.__name__, self)
@property
def is_streamed(self):
"""If the response is streamed (the response is not an iterable with
length information) this property is `True`. In this case streamed
means that there is no information about the number of iterations.
This is usually `True` if a generator is passed to the response object."""
return is_streamed(self.content)
def __iter__(self):
if self._started:
raise RuntimeError("WsgiResponse can be iterated only once")
self._started = True
if self.is_streamed:
return wsgi_iterator(self.content, self.encoding)
else:
return iter(self.content)
def __len__(self):
return len(self.content)
def set_cookie(self, key, **kwargs):
"""
Sets a cookie.
``expires`` can be a string in the correct format or a
``datetime.datetime`` object in UTC. If ``expires`` is a datetime
object then ``max_age`` will be calculated.
"""
set_cookie(self.cookies, key, **kwargs)
def delete_cookie(self, key, path="/", domain=None):
set_cookie(self.cookies, key, max_age=0, path=path, domain=domain, expires="Thu, 01-Jan-1970 00:00:00 GMT")
def get_headers(self):
headers = self.headers
if has_empty_content(self.status_code, self.method):
headers.pop("content-type", None)
headers.pop("content-length", None)
elif not self.is_streamed:
cl = 0
for c in self.content:
cl += len(c)
headers["Content-Length"] = str(cl)
for c in self.cookies.values():
headers["Set-Cookie"] = c.OutputString()
return list(headers)
class HttpClient(AbstractClient):
'''A client for HTTP/HTTPS servers.
It handles pool of asynchronous connections.
:param encode_multipart: optional flag for setting the
:attr:`encode_multipart` attribute
:param pool_size: set the :attr:`pool_size` attribute.
:param store_cookies: set the :attr:`store_cookies` attribute
.. attribute:: headers
Default headers for this :class:`HttpClient`.
Default: :attr:`DEFAULT_HTTP_HEADERS`.
.. attribute:: cookies
Default cookies for this :class:`HttpClient`.
.. attribute:: store_cookies
If ``True`` it remebers response cookies and send them back to
serves.
Default: ``True``
.. attribute:: timeout
Default timeout for requests. If None or 0, no timeout on requests
.. attribute:: encode_multipart
Flag indicating if body data is by default encoded using the
``multipart/form-data`` or ``application/x-www-form-urlencoded``
encoding.
It can be overwritten during a :meth:`request`.
Default: ``True``
.. attribute:: proxy_info
Dictionary of proxy servers for this client.
.. attribute:: pool_size
The size of a pool of connection for a given host.
.. attribute:: connection_pools
Dictionary of connection pools for different hosts
.. attribute:: DEFAULT_HTTP_HEADERS
Default headers for this :class:`HttpClient`
'''
MANY_TIMES_EVENTS = ('connection_made', 'pre_request', 'on_headers',
'post_request', 'connection_lost')
protocol_factory = partial(Connection, HttpResponse)
allow_redirects = False
max_redirects = 10
'''Maximum number of redirects.
It can be overwritten on :meth:`request`.'''
connection_pool = Pool
'''Connection :class:`.Pool` factory
'''
client_version = pulsar.SERVER_SOFTWARE
'''String for the ``User-Agent`` header.'''
version = 'HTTP/1.1'
'''Default HTTP request version for this :class:`HttpClient`.
It can be overwritten on :meth:`request`.'''
DEFAULT_HTTP_HEADERS = Headers([('Connection', 'Keep-Alive'),
('Accept', '*/*'),
('Accept-Encoding', 'deflate'),
('Accept-Encoding', 'gzip')],
kind='client')
DEFAULT_TUNNEL_HEADERS = Headers([('Connection', 'Keep-Alive'),
('Proxy-Connection', 'Keep-Alive')],
kind='client')
request_parameters = ('encode_multipart', 'max_redirects', 'decompress',
'allow_redirects', 'multipart_boundary', 'version',
'websocket_handler')
# Default hosts not affected by proxy settings. This can be overwritten
# by specifying the "no" key in the proxy_info dictionary
no_proxy = set(('localhost', platform.node()))
def __init__(self,
proxy_info=None,
cache=None,
headers=None,
encode_multipart=True,
multipart_boundary=None,
keyfile=None,
certfile=None,
cert_reqs=CERT_NONE,
ca_certs=None,
cookies=None,
store_cookies=True,
max_redirects=10,
decompress=True,
version=None,
websocket_handler=None,
parser=None,
trust_env=True,
loop=None,
client_version=None,
timeout=None,
pool_size=10,
frame_parser=None):
super().__init__(loop)
self.client_version = client_version or self.client_version
self.connection_pools = {}
self.pool_size = pool_size
self.trust_env = trust_env
self.timeout = timeout
self.store_cookies = store_cookies
self.max_redirects = max_redirects
self.cookies = cookiejar_from_dict(cookies)
self.decompress = decompress
self.version = version or self.version
dheaders = self.DEFAULT_HTTP_HEADERS.copy()
dheaders['user-agent'] = self.client_version
if headers:
dheaders.override(headers)
self.headers = dheaders
self.tunnel_headers = self.DEFAULT_TUNNEL_HEADERS.copy()
self.proxy_info = dict(proxy_info or ())
if not self.proxy_info and self.trust_env:
self.proxy_info = get_environ_proxies()
if 'no' not in self.proxy_info:
self.proxy_info['no'] = ','.join(self.no_proxy)
self.encode_multipart = encode_multipart
self.multipart_boundary = multipart_boundary or choose_boundary()
self.websocket_handler = websocket_handler
self.https_defaults = {
'keyfile': keyfile,
'certfile': certfile,
'cert_reqs': cert_reqs,
'ca_certs': ca_certs
}
self.http_parser = parser or http_parser
self.frame_parser = frame_parser or websocket.frame_parser
# Add hooks
self.bind_event('pre_request', Tunneling(self._loop))
self.bind_event('on_headers', handle_101)
self.bind_event('on_headers', handle_100)
self.bind_event('on_headers', handle_cookies)
self.bind_event('post_request', handle_redirect)
@property
def websocket_key(self):
if not hasattr(self, '_websocket_key'):
self._websocket_key = native_str(b64encode(os.urandom(16)),
DEFAULT_CHARSET)
return self._websocket_key
def connect(self, address):
if isinstance(address, tuple):
address = ':'.join(('%s' % v for v in address))
return self.request('CONNECT', address)
def get(self, url, **kwargs):
'''Sends a GET request and returns a :class:`HttpResponse` object.
:params url: url for the new :class:`HttpRequest` object.
:param \*\*kwargs: Optional arguments for the :meth:`request` method.
'''
kwargs.setdefault('allow_redirects', True)
return self.request('GET', url, **kwargs)
def options(self, url, **kwargs):
'''Sends a OPTIONS request and returns a :class:`HttpResponse` object.
:params url: url for the new :class:`HttpRequest` object.
:param \*\*kwargs: Optional arguments for the :meth:`request` method.
'''
kwargs.setdefault('allow_redirects', True)
return self.request('OPTIONS', url, **kwargs)
def head(self, url, **kwargs):
'''Sends a HEAD request and returns a :class:`HttpResponse` object.
:params url: url for the new :class:`HttpRequest` object.
:param \*\*kwargs: Optional arguments for the :meth:`request` method.
'''
return self.request('HEAD', url, **kwargs)
def post(self, url, **kwargs):
'''Sends a POST request and returns a :class:`HttpResponse` object.
:params url: url for the new :class:`HttpRequest` object.
:param \*\*kwargs: Optional arguments for the :meth:`request` method.
'''
return self.request('POST', url, **kwargs)
def put(self, url, **kwargs):
'''Sends a PUT request and returns a :class:`HttpResponse` object.
:params url: url for the new :class:`HttpRequest` object.
:param \*\*kwargs: Optional arguments for the :meth:`request` method.
'''
return self.request('PUT', url, **kwargs)
def patch(self, url, **kwargs):
'''Sends a PATCH request and returns a :class:`HttpResponse` object.
:params url: url for the new :class:`HttpRequest` object.
:param \*\*kwargs: Optional arguments for the :meth:`request` method.
'''
return self.request('PATCH', url, **kwargs)
def delete(self, url, **kwargs):
'''Sends a DELETE request and returns a :class:`HttpResponse` object.
:params url: url for the new :class:`HttpRequest` object.
:param \*\*kwargs: Optional arguments for the :meth:`request` method.
'''
return self.request('DELETE', url, **kwargs)
def request(self, method, url, timeout=None, **params):
'''Constructs and sends a request to a remote server.
It returns a :class:`.Future` which results in a
:class:`HttpResponse` object.
:param method: request method for the :class:`HttpRequest`.
:param url: URL for the :class:`HttpRequest`.
:parameter response: optional pre-existing :class:`HttpResponse` which
starts a new request (for redirects, digest authentication and
so forth).
:param params: optional parameters for the :class:`HttpRequest`
initialisation.
:rtype: a :class:`.Future`
'''
response = self._request(method, url, **params)
if timeout is None:
timeout = self.timeout
if timeout:
response = wait_for(response, timeout, loop=self._loop)
if not self._loop.is_running():
return self._loop.run_until_complete(response)
else:
return response
def _request(self, method, url, **params):
nparams = params.copy()
nparams.update(((name, getattr(self, name))
for name in self.request_parameters
if name not in params))
request = HttpRequest(self, url, method, params, **nparams)
pool = self.connection_pools.get(request.key)
if pool is None:
host, port = request.address
pool = self.connection_pool(partial(self._connect, host, port,
request.ssl),
pool_size=self.pool_size,
loop=self._loop)
self.connection_pools[request.key] = pool
conn = yield from pool.connect()
with conn:
consumer = conn.current_consumer()
# bind request-specific events
consumer.bind_events(**request.inp_params)
consumer.start(request)
response = yield from consumer.on_finished
if response is not None:
consumer = response
if consumer.request_again:
if isinstance(consumer.request_again, Exception):
raise consumer.request_again
elif isinstance(consumer.request_again, ProtocolConsumer):
consumer = consumer.request_again
headers = consumer.headers
if (not headers or not headers.has('connection', 'keep-alive')
or consumer.status_code == 101):
conn.detach()
if isinstance(consumer.request_again, tuple):
method, url, params = consumer.request_again
consumer = yield from self._request(method, url, **params)
return consumer
def close(self, async=True):
'''Close all connections.
Fire the ``finish`` :ref:`one time event <one-time-event>` once done.
Return the :class:`.Future` fired by the ``finish`` event.
'''
for p in self.connection_pools.values():
p.close()
self.connection_pools.clear()
class HttpRequest(RequestBase):
"""An :class:`HttpClient` request for an HTTP resource.
This class has a similar interface to :class:`urllib.request.Request`.
:param files: optional dictionary of name, file-like-objects.
:param allow_redirects: allow the response to follow redirects.
.. attribute:: method
The request method
.. attribute:: version
HTTP version for this request, usually ``HTTP/1.1``
.. attribute:: encode_multipart
If ``True`` (default), defaults POST data as ``multipart/form-data``.
Pass ``encode_multipart=False`` to default to
``application/x-www-form-urlencoded``. In any case, this parameter is
overwritten by passing the correct content-type header.
.. attribute:: history
List of past :class:`.HttpResponse` (collected during redirects).
.. attribute:: wait_continue
if ``True``, the :class:`HttpRequest` includes the
``Expect: 100-Continue`` header.
.. attribute:: stream
Allow for streaming body
"""
_proxy = None
_ssl = None
_tunnel = None
_write_done = False
def __init__(self, client, url, method, inp_params=None, headers=None,
data=None, files=None, history=None, auth=None,
charset=None, encode_multipart=True, multipart_boundary=None,
source_address=None, allow_redirects=False, max_redirects=10,
decompress=True, version=None, wait_continue=False,
websocket_handler=None, cookies=None, urlparams=None,
stream=False, proxies=None, verify=True, **ignored):
self.client = client
self._data = None
self.files = files
self.urlparams = urlparams
self.inp_params = inp_params or {}
self.unredirected_headers = Headers(kind='client')
self.method = method.upper()
self.full_url = url
if urlparams:
self._encode_url(urlparams)
self.history = history
self.wait_continue = wait_continue
self.max_redirects = max_redirects
self.allow_redirects = allow_redirects
self.charset = charset or 'utf-8'
self.version = version
self.decompress = decompress
self.encode_multipart = encode_multipart
self.multipart_boundary = multipart_boundary
self.websocket_handler = websocket_handler
self.source_address = source_address
self.stream = stream
self.verify = verify
self.new_parser()
if self._scheme in tls_schemes:
self._ssl = client.ssl_context(verify=self.verify, **ignored)
if auth and not isinstance(auth, Auth):
auth = HTTPBasicAuth(*auth)
self.auth = auth
self.headers = client.get_headers(self, headers)
cookies = cookiejar_from_dict(client.cookies, cookies)
if cookies:
cookies.add_cookie_header(self)
self.unredirected_headers['host'] = host_no_default_port(self._scheme,
self._netloc)
self.data = data
self._set_proxy(proxies)
@property
def address(self):
"""``(host, port)`` tuple of the HTTP resource
"""
return self._tunnel.address if self._tunnel else (self.host, self.port)
@property
def target_address(self):
return (self.host, int(self.port))
@property
def ssl(self):
"""Context for TLS connections.
If this is a tunneled request and the tunnel connection is not yet
established, it returns ``None``.
"""
if not self._tunnel:
return self._ssl
@property
def key(self):
tunnel = self._tunnel.full_url if self._tunnel else None
return (self.scheme, self.host, self.port, tunnel, self.verify)
@property
def proxy(self):
"""Proxy server for this request.
"""
return self._proxy
@property
def tunnel(self):
"""Tunnel for this request.
"""
return self._tunnel
@property
def netloc(self):
if self._proxy:
return self._proxy.netloc
else:
return self._netloc
def __repr__(self):
return self.first_line()
__str__ = __repr__
@property
def full_url(self):
"""Full url of endpoint
"""
return urlunparse((self._scheme, self._netloc, self.path,
self.params, self.query, self.fragment))
@full_url.setter
def full_url(self, url):
self._scheme, self._netloc, self.path, self.params,\
self.query, self.fragment = urlparse(url)
if not self._netloc and self.method == 'CONNECT':
self._scheme, self._netloc, self.path, self.params,\
self.query, self.fragment = urlparse('http://%s' % url)
@property
def data(self):
"""Body of request
"""
return self._data
@data.setter
def data(self, data):
self._data = self._encode_data(data)
def first_line(self):
if self._proxy:
if self.method == 'CONNECT':
url = self._netloc
else:
url = self.full_url
else:
url = urlunparse(('', '', self.path or '/', self.params,
self.query, self.fragment))
return '%s %s %s' % (self.method, url, self.version)
def new_parser(self):
self.parser = self.client.http_parser(kind=1,
decompress=self.decompress,
method=self.method)
def is_chunked(self):
return self.data and 'content-length' not in self.headers
def encode(self):
"""The bytes representation of this :class:`HttpRequest`.
Called by :class:`HttpResponse` when it needs to encode this
:class:`HttpRequest` before sending it to the HTTP resource.
"""
# Call body before fist_line in case the query is changes.
first_line = self.first_line()
if self.data and self.wait_continue:
self.headers['expect'] = '100-continue'
headers = self.headers
if self.unredirected_headers:
headers = self.unredirected_headers.copy()
headers.update(self.headers)
buffer = [first_line.encode('ascii'), b'\r\n', bytes(headers)]
return b''.join(buffer)
def add_header(self, key, value):
self.headers[key] = value
def has_header(self, header_name):
"""Check ``header_name`` is in this request headers.
"""
return (header_name in self.headers or
header_name in self.unredirected_headers)
def get_header(self, header_name, default=None):
"""Retrieve ``header_name`` from this request headers.
"""
return self.headers.get(
header_name, self.unredirected_headers.get(header_name, default))
def remove_header(self, header_name):
"""Remove ``header_name`` from this request.
"""
val1 = self.headers.pop(header_name, None)
val2 = self.unredirected_headers.pop(header_name, None)
return val1 or val2
def add_unredirected_header(self, header_name, header_value):
self.unredirected_headers[header_name] = header_value
def write_body(self, transport):
assert not self._write_done, 'Body already sent'
self._write_done = True
if not self.data:
return
if is_streamed(self.data):
ensure_future(self._write_streamed_data(transport),
loop=transport._loop)
else:
self._write_body_data(transport, self.data, True)
# INTERNAL ENCODING METHODS
def _encode_data(self, data):
body = None
if self.method in ENCODE_URL_METHODS:
self.files = None
self._encode_url(data)
elif isinstance(data, bytes):
assert self.files is None, ('data cannot be bytes when files are '
'present')
body = data
elif isinstance(data, str):
assert self.files is None, ('data cannot be string when files are '
'present')
body = to_bytes(data, self.charset)
elif data and is_streamed(data):
assert self.files is None, ('data cannot be an iterator when '
'files are present')
if 'content-type' not in self.headers:
self.headers['content-type'] = 'application/octet-stream'
if 'content-length' not in self.headers:
self.headers['transfer-encoding'] = 'chunked'
return data
elif data or self.files:
if self.files:
body, content_type = self._encode_files(data)
else:
body, content_type = self._encode_params(data)
# set files to None, Important!
self.files = None
self.headers['Content-Type'] = content_type
if body:
self.headers['content-length'] = str(len(body))
elif 'expect' not in self.headers:
self.headers.pop('content-length', None)
self.headers.pop('content-type', None)
return body
def _encode_url(self, data):
query = self.query
if data:
data = native_str(data)
if isinstance(data, str):
data = parse_qsl(data)
else:
data = mapping_iterator(data)
query = parse_qsl(query)
query.extend(data)
query = urlencode(query)
self.query = query
def _encode_files(self, data):
fields = []
for field, val in mapping_iterator(data or ()):
if (isinstance(val, str) or isinstance(val, bytes) or
not hasattr(val, '__iter__')):
val = [val]
for v in val:
if v is not None:
if not isinstance(v, bytes):
v = str(v)
fields.append((field.decode('utf-8') if
isinstance(field, bytes) else field,
v.encode('utf-8') if isinstance(v, str)
else v))
for (k, v) in mapping_iterator(self.files):
# support for explicit filename
ft = None
if isinstance(v, (tuple, list)):
if len(v) == 2:
fn, fp = v
else:
fn, fp, ft = v
else:
fn = guess_filename(v) or k
fp = v
if isinstance(fp, bytes):
fp = BytesIO(fp)
elif isinstance(fp, str):
fp = StringIO(fp)
if ft:
new_v = (fn, fp.read(), ft)
else:
new_v = (fn, fp.read())
fields.append((k, new_v))
#
return encode_multipart_formdata(fields, charset=self.charset)
def _encode_params(self, data):
content_type = self.headers.get('content-type')
# No content type given, chose one
if not content_type:
if self.encode_multipart:
content_type = MULTIPART_FORM_DATA
else:
content_type = FORM_URL_ENCODED
if content_type in JSON_CONTENT_TYPES:
body = json.dumps(data).encode(self.charset)
elif content_type == FORM_URL_ENCODED:
body = urlencode(data).encode(self.charset)
elif content_type == MULTIPART_FORM_DATA:
body, content_type = encode_multipart_formdata(
data, boundary=self.multipart_boundary, charset=self.charset)
else:
raise ValueError("Don't know how to encode body for %s" %
content_type)
return body, content_type
def _write_body_data(self, transport, data, finish=False):
if self.is_chunked():
data = http_chunks(data, finish)
elif data:
data = (data,)
else:
return
for chunk in data:
transport.write(chunk)
@asyncio.coroutine
def _write_streamed_data(self, transport):
for data in self.data:
if isawaitable(data):
data = yield from data
self._write_body_data(transport, data)
self._write_body_data(transport, b'', True)
# PROXY INTERNALS
def _set_proxy(self, proxies):
request_proxies = self.client.proxies.copy()
if proxies:
request_proxies.update(proxies)
self.proxies = request_proxies
self.scheme = self._scheme
self._set_hostport(self._scheme, self._netloc)
#
if self.scheme in request_proxies:
hostonly = self.host
no_proxy = [n for n in request_proxies.get('no', '').split(',')
if n]
if not any(map(hostonly.endswith, no_proxy)):
url = request_proxies[self.scheme]
p = urlparse(url)
if not p.scheme:
raise ValueError('Could not understand proxy %s' % url)
scheme = p.scheme
host = p.netloc
if not self._ssl:
self.scheme = scheme
self._set_hostport(scheme, host)
self._proxy = scheme_host(scheme, host)
else:
self._tunnel = HttpTunnel(self, scheme, host)
def _set_hostport(self, scheme, host):
self._tunnel = None
self._proxy = None
self.host, self.port = get_hostport(scheme, host)
class WsgiResponse:
"""A WSGI response.
Instances are callable using the standard WSGI call and, importantly,
iterable::
response = WsgiResponse(200)
A :class:`WsgiResponse` is an iterable over bytes to send back to the
requesting client.
.. attribute:: status_code
Integer indicating the HTTP status, (i.e. 200)
.. attribute:: response
String indicating the HTTP status (i.e. 'OK')
.. attribute:: status
String indicating the HTTP status code and response (i.e. '200 OK')
.. attribute:: content_type
The content type of this response. Can be ``None``.
.. attribute:: headers
The :class:`.Headers` container for this response.
.. attribute:: environ
The dictionary of WSGI environment if passed to the constructor.
.. attribute:: cookies
A python :class:`SimpleCookie` container of cookies included in the
request as well as cookies set during the response.
"""
_iterated = False
_started = False
DEFAULT_STATUS_CODE = 200
def __init__(self, status=None, content=None, response_headers=None,
content_type=None, encoding=None, environ=None,
can_store_cookies=True):
self.environ = environ
self.status_code = status or self.DEFAULT_STATUS_CODE
self.encoding = encoding
self.cookies = SimpleCookie()
self.headers = Headers(response_headers, kind='server')
self.content = content
self._can_store_cookies = can_store_cookies
if content_type is not None:
self.content_type = content_type
@property
def started(self):
return self._started
@property
def iterated(self):
return self._iterated
@property
def path(self):
if self.environ:
return self.environ.get('PATH_INFO', '')
@property
def method(self):
if self.environ:
return self.environ.get('REQUEST_METHOD')
@property
def connection(self):
if self.environ:
return self.environ.get('pulsar.connection')
@property
def content(self):
return self._content
@content.setter
def content(self, content):
if not self._iterated:
if content is None:
content = ()
else:
if isinstance(content, str):
if not self.encoding: # use utf-8 if not set
self.encoding = 'utf-8'
content = content.encode(self.encoding)
if isinstance(content, bytes):
content = (content,)
self._content = content
else:
raise RuntimeError('Cannot set content. Already iterated')
def _get_content_type(self):
return self.headers.get('content-type')
def _set_content_type(self, typ):
if typ:
self.headers['content-type'] = typ
else:
self.headers.pop('content-type', None)
content_type = property(_get_content_type, _set_content_type)
@property
def response(self):
return responses.get(self.status_code)
@property
def status(self):
return '%s %s' % (self.status_code, self.response)
def __str__(self):
return self.status
def __repr__(self):
return '%s(%s)' % (self.__class__.__name__, self)
@property
def is_streamed(self):
"""Check if the response is streamed.
A streamed response is an iterable with no length information.
In this case streamed means that there is no information about
the number of iterations.
This is usually `True` if a generator is passed to the response object.
"""
try:
len(self.content)
except TypeError:
return True
return False
def can_set_cookies(self):
if self.status_code < 400:
return self._can_store_cookies
def length(self):
if not self.is_streamed:
return reduce(lambda x, y: x+len(y), self.content, 0)
def start(self, start_response):
assert not self._started
self._started = True
return start_response(self.status, self.get_headers())
def __iter__(self):
if self._iterated:
raise RuntimeError('WsgiResponse can be iterated once only')
self._started = True
self._iterated = True
if self.is_streamed:
return wsgi_encoder(self.content, self.encoding or 'utf-8')
else:
return iter(self.content)
def close(self):
"""Close this response, required by WSGI
"""
if self.is_streamed:
if hasattr(self.content, 'close'):
self.content.close()
def set_cookie(self, key, **kwargs):
"""
Sets a cookie.
``expires`` can be a string in the correct format or a
``datetime.datetime`` object in UTC. If ``expires`` is a datetime
object then ``max_age`` will be calculated.
"""
set_cookie(self.cookies, key, **kwargs)
def delete_cookie(self, key, path='/', domain=None):
set_cookie(self.cookies, key, max_age=0, path=path, domain=domain,
expires='Thu, 01-Jan-1970 00:00:00 GMT')
def get_headers(self):
"""The list of headers for this response
"""
headers = self.headers
if has_empty_content(self.status_code):
headers.pop('content-type', None)
headers.pop('content-length', None)
self._content = ()
else:
if not self.is_streamed:
cl = 0
for c in self.content:
cl += len(c)
if cl == 0 and self.content_type in JSON_CONTENT_TYPES:
self._content = (b'{}',)
cl = len(self._content[0])
headers['Content-Length'] = str(cl)
ct = self.content_type
# content type encoding available
if self.encoding:
ct = ct or 'text/plain'
if 'charset=' not in ct:
ct = '%s; charset=%s' % (ct, self.encoding)
if ct:
headers['Content-Type'] = ct
if self.method == HEAD:
self._content = ()
if self.can_set_cookies():
for c in self.cookies.values():
headers.add_header('Set-Cookie', c.OutputString())
return list(headers)
def has_header(self, header):
return header in self.headers
__contains__ = has_header
def __setitem__(self, header, value):
self.headers[header] = value
def __getitem__(self, header):
return self.headers[header]
class HttpServerResponse(ProtocolConsumer):
'''Server side WSGI :class:`.ProtocolConsumer`.
.. attribute:: wsgi_callable
The wsgi callable handling requests.
'''
_status = None
_headers_sent = None
_body_reader = None
_buffer = None
_logger = LOGGER
SERVER_SOFTWARE = pulsar.SERVER_SOFTWARE
ONE_TIME_EVENTS = ProtocolConsumer.ONE_TIME_EVENTS + ('on_headers',)
def __init__(self, wsgi_callable, cfg, server_software=None, loop=None):
super().__init__(loop=loop)
self.wsgi_callable = wsgi_callable
self.cfg = cfg
self.parser = http_parser(kind=0)
self.headers = Headers()
self.keep_alive = False
self.SERVER_SOFTWARE = server_software or self.SERVER_SOFTWARE
@property
def headers_sent(self):
'''Available once the headers have been sent to the client.
These are the bytes representing the first response line and
the headers
'''
return self._headers_sent
def data_received(self, data):
'''Implements :meth:`~.ProtocolConsumer.data_received` method.
Once we have a full HTTP message, build the wsgi ``environ`` and
delegate the response to the :func:`wsgi_callable` function.
'''
parser = self.parser
processed = parser.execute(data, len(data))
if parser.is_headers_complete():
if not self._body_reader:
headers = Headers(parser.get_headers(), kind='client')
self._body_reader = HttpBodyReader(headers,
parser,
self.transport,
loop=self._loop)
self._response(self.wsgi_environ())
body = parser.recv_body()
if body:
self._body_reader.feed_data(body)
#
if parser.is_message_complete():
#
self._body_reader.feed_eof()
if processed < len(data):
if not self._buffer:
self._buffer = data[processed:]
self.bind_event('post_request', self._new_request)
else:
self._buffer += data[processed:]
#
elif processed < len(data):
# This is a parsing error, the client must have sent
# bogus data
raise ProtocolError
@property
def status(self):
return self._status
@property
def upgrade(self):
return self.headers.get('upgrade')
@property
def chunked(self):
return self.headers.get('Transfer-Encoding') == 'chunked'
@property
def content_length(self):
c = self.headers.get('Content-Length')
if c:
return int(c)
@property
def version(self):
return self.parser.get_version()
def start_response(self, status, response_headers, exc_info=None):
'''WSGI compliant ``start_response`` callable, see pep3333_.
The application may call start_response more than once, if and only
if the ``exc_info`` argument is provided.
More precisely, it is a fatal error to call ``start_response`` without
the ``exc_info`` argument if start_response has already been called
within the current invocation of the application.
:parameter status: an HTTP ``status`` string like ``200 OK`` or
``404 Not Found``.
:parameter response_headers: a list of ``(header_name, header_value)``
tuples. It must be a Python list. Each header_name must be a valid
HTTP header field-name (as defined by RFC 2616_, Section 4.2),
without a trailing colon or other punctuation.
:parameter exc_info: optional python ``sys.exc_info()`` tuple.
This argument should be supplied by the application only if
``start_response`` is being called by an error handler.
:return: The :meth:`write` method.
``HOP_HEADERS`` are not considered but no error is raised.
.. _pep3333: http://www.python.org/dev/peps/pep-3333/
.. _2616: http://www.faqs.org/rfcs/rfc2616.html
'''
if exc_info:
try:
if self._headers_sent:
# if exc_info is provided, and the HTTP headers have
# already been sent, start_response must raise an error,
# and should re-raise using the exc_info tuple
reraise(*exc_info)
finally:
# Avoid circular reference
exc_info = None
elif self._status:
# Headers already set. Raise error
raise HttpException("Response headers already set!")
self._status = status
if type(response_headers) is not list:
raise TypeError("Headers must be a list of name/value tuples")
for header, value in response_headers:
if header.lower() in HOP_HEADERS:
# These features are the exclusive province of this class,
# this should be considered a fatal error for an application
# to attempt sending them, but we don't raise an error,
# just log a warning
self.logger.warning('Application passing hop header "%s"',
header)
continue
self.headers.add_header(header, value)
return self.write
def write(self, data, force=False):
'''The write function returned by the :meth:`start_response` method.
Required by the WSGI specification.
:param data: bytes to write
:param force: Optional flag used internally
:return: a :class:`~asyncio.Future` or the number of bytes written
'''
write = super().write
chunks = []
if not self._headers_sent:
tosend = self.get_headers()
self._headers_sent = tosend.flat(self.version, self.status)
self.fire_event('on_headers')
chunks.append(self._headers_sent)
if data:
if self.chunked:
chunks.extend(http_chunks(data))
else:
chunks.append(data)
elif force and self.chunked:
chunks.extend(http_chunks(data, True))
if chunks:
return write(b''.join(chunks))
########################################################################
# INTERNALS
@task
def _response(self, environ):
exc_info = None
response = None
done = False
alive = self.cfg.keep_alive or 15
while not done:
done = True
try:
if exc_info is None:
if (not environ.get('HTTP_HOST') and
environ['SERVER_PROTOCOL'] != 'HTTP/1.0'):
raise BadRequest
response = self.wsgi_callable(environ, self.start_response)
if isfuture(response):
response = yield from wait_for(response, alive)
else:
response = handle_wsgi_error(environ, exc_info)
if isfuture(response):
response = yield from wait_for(response, alive)
#
if exc_info:
self.start_response(response.status,
response.get_headers(), exc_info)
#
# Do the actual writing
loop = self._loop
start = loop.time()
for chunk in response:
if isfuture(chunk):
chunk = yield from wait_for(chunk, alive)
start = loop.time()
result = self.write(chunk)
if isfuture(result):
yield from wait_for(result, alive)
start = loop.time()
else:
time_in_loop = loop.time() - start
if time_in_loop > MAX_TIME_IN_LOOP:
self.logger.debug(
'Released the event loop after %.3f seconds',
time_in_loop)
yield None
start = loop.time()
#
# make sure we write headers and last chunk if needed
self.write(b'', True)
# client disconnected, end this connection
except (IOError, AbortWsgi):
self.finished()
except Exception:
if wsgi_request(environ).cache.handle_wsgi_error:
self.keep_alive = False
self._write_headers()
self.connection.close()
self.finished()
else:
done = False
exc_info = sys.exc_info()
else:
log_wsgi_info(self.logger.info, environ, self.status)
self.finished()
if not self.keep_alive:
self.logger.debug('No keep alive, closing connection %s',
self.connection)
self.connection.close()
finally:
if hasattr(response, 'close'):
try:
response.close()
except Exception:
self.logger.exception(
'Error while closing wsgi iterator')
def is_chunked(self):
'''Check if the response uses chunked transfer encoding.
Only use chunked responses when the client is speaking HTTP/1.1
or newer and there was no Content-Length header set.
'''
if (self.version <= (1, 0) or
self._status == '200 Connection established' or
has_empty_content(int(self.status[:3]))):
return False
elif self.headers.get('Transfer-Encoding') == 'chunked':
return True
else:
return self.content_length is None
def get_headers(self):
'''Get the headers to send to the client.
'''
if not self._status:
# we are sending headers but the start_response was not called
raise HttpException('Headers not set.')
headers = self.headers
# Set chunked header if needed
if self.is_chunked():
headers['Transfer-Encoding'] = 'chunked'
headers.pop('content-length', None)
else:
headers.pop('Transfer-Encoding', None)
if self.keep_alive:
self.keep_alive = keep_alive_with_status(self._status, headers)
if not self.keep_alive:
headers['connection'] = 'close'
return headers
def wsgi_environ(self):
# return a the WSGI environ dictionary
transport = self.transport
https = True if is_tls(transport.get_extra_info('socket')) else False
multiprocess = (self.cfg.concurrency == 'process')
environ = wsgi_environ(self._body_reader,
self.parser,
self._body_reader.headers,
transport.get_extra_info('sockname'),
self.address,
self.headers,
self.SERVER_SOFTWARE,
https=https,
extra={'pulsar.connection': self.connection,
'pulsar.cfg': self.cfg,
'wsgi.multiprocess': multiprocess})
self.keep_alive = keep_alive(self.headers, self.parser.get_version(),
environ['REQUEST_METHOD'])
self.headers.update([('Server', self.SERVER_SOFTWARE),
('Date', format_date_time(time.time()))])
return environ
def _new_request(self, _, exc=None):
connection = self._connection
connection.data_received(self._buffer)
def _write_headers(self):
if not self._headers_sent:
if self.content_length:
self.headers['Content-Length'] = '0'
self.write(b'')
class WsgiResponse:
"""A WSGI response.
Instances are callable using the standard WSGI call and, importantly,
iterable::
response = WsgiResponse(200)
A :class:`WsgiResponse` is an iterable over bytes to send back to the
requesting client.
.. attribute:: status_code
Integer indicating the HTTP status, (i.e. 200)
.. attribute:: response
String indicating the HTTP status (i.e. 'OK')
.. attribute:: status
String indicating the HTTP status code and response (i.e. '200 OK')
.. attribute:: content_type
The content type of this response. Can be ``None``.
.. attribute:: headers
The :class:`.Headers` container for this response.
.. attribute:: environ
The dictionary of WSGI environment if passed to the constructor.
.. attribute:: cookies
A python :class:`SimpleCookie` container of cookies included in the
request as well as cookies set during the response.
"""
_iterated = False
_started = False
DEFAULT_STATUS_CODE = 200
def __init__(self,
status=None,
content=None,
response_headers=None,
content_type=None,
encoding=None,
environ=None,
can_store_cookies=True):
self.environ = environ
self.status_code = status or self.DEFAULT_STATUS_CODE
self.encoding = encoding
self.cookies = SimpleCookie()
self.headers = Headers(response_headers, kind='server')
self.content = content
self._can_store_cookies = can_store_cookies
if content_type is not None:
self.content_type = content_type
@property
def started(self):
return self._started
@property
def iterated(self):
return self._iterated
@property
def path(self):
if self.environ:
return self.environ.get('PATH_INFO', '')
@property
def method(self):
if self.environ:
return self.environ.get('REQUEST_METHOD')
@property
def connection(self):
if self.environ:
return self.environ.get('pulsar.connection')
@property
def content(self):
return self._content
@content.setter
def content(self, content):
if not self._iterated:
if content is None:
content = ()
else:
if isinstance(content, str):
if not self.encoding: # use utf-8 if not set
self.encoding = 'utf-8'
content = content.encode(self.encoding)
if isinstance(content, bytes):
content = (content, )
self._content = content
else:
raise RuntimeError('Cannot set content. Already iterated')
def _get_content_type(self):
return self.headers.get('content-type')
def _set_content_type(self, typ):
if typ:
self.headers['content-type'] = typ
else:
self.headers.pop('content-type', None)
content_type = property(_get_content_type, _set_content_type)
@property
def response(self):
return responses.get(self.status_code)
@property
def status(self):
return '%s %s' % (self.status_code, self.response)
def __str__(self):
return self.status
def __repr__(self):
return '%s(%s)' % (self.__class__.__name__, self)
@property
def is_streamed(self):
"""Check if the response is streamed.
A streamed response is an iterable with no length information.
In this case streamed means that there is no information about
the number of iterations.
This is usually `True` if a generator is passed to the response object.
"""
try:
len(self.content)
except TypeError:
return True
return False
def can_set_cookies(self):
if self.status_code < 400:
return self._can_store_cookies
def length(self):
if not self.is_streamed:
return reduce(lambda x, y: x + len(y), self.content, 0)
def start(self, start_response):
assert not self._started
self._started = True
return start_response(self.status, self.get_headers())
def __iter__(self):
if self._iterated:
raise RuntimeError('WsgiResponse can be iterated once only')
self._started = True
self._iterated = True
if self.is_streamed:
return wsgi_encoder(self.content, self.encoding or 'utf-8')
else:
return iter(self.content)
def close(self):
"""Close this response, required by WSGI
"""
if self.is_streamed:
if hasattr(self.content, 'close'):
self.content.close()
def set_cookie(self, key, **kwargs):
"""
Sets a cookie.
``expires`` can be a string in the correct format or a
``datetime.datetime`` object in UTC. If ``expires`` is a datetime
object then ``max_age`` will be calculated.
"""
set_cookie(self.cookies, key, **kwargs)
def delete_cookie(self, key, path='/', domain=None):
set_cookie(self.cookies,
key,
max_age=0,
path=path,
domain=domain,
expires='Thu, 01-Jan-1970 00:00:00 GMT')
def get_headers(self):
"""The list of headers for this response
"""
headers = self.headers
if has_empty_content(self.status_code):
headers.pop('content-type', None)
headers.pop('content-length', None)
self._content = ()
else:
if not self.is_streamed:
cl = 0
for c in self.content:
cl += len(c)
if cl == 0 and self.content_type in JSON_CONTENT_TYPES:
self._content = (b'{}', )
cl = len(self._content[0])
headers['Content-Length'] = str(cl)
ct = self.content_type
# content type encoding available
if self.encoding:
ct = ct or 'text/plain'
if 'charset=' not in ct:
ct = '%s; charset=%s' % (ct, self.encoding)
if ct:
headers['Content-Type'] = ct
if self.method == HEAD:
self._content = ()
if self.can_set_cookies():
for c in self.cookies.values():
headers.add_header('Set-Cookie', c.OutputString())
return list(headers)
def has_header(self, header):
return header in self.headers
__contains__ = has_header
def __setitem__(self, header, value):
self.headers[header] = value
def __getitem__(self, header):
return self.headers[header]
class HttpServerResponse(ProtocolConsumer):
"""Server side WSGI :class:`.ProtocolConsumer`.
.. attribute:: wsgi_callable
The wsgi callable handling requests.
"""
_status = None
_headers_sent = None
_stream = None
_buffer = None
_logger = LOGGER
SERVER_SOFTWARE = pulsar.SERVER_SOFTWARE
ONE_TIME_EVENTS = ProtocolConsumer.ONE_TIME_EVENTS + ("on_headers",)
def __init__(self, wsgi_callable, cfg, server_software=None, loop=None):
super().__init__(loop=loop)
self.wsgi_callable = wsgi_callable
self.cfg = cfg
self.parser = http_parser(kind=0)
self.headers = Headers()
self.keep_alive = False
self.SERVER_SOFTWARE = server_software or self.SERVER_SOFTWARE
@property
def headers_sent(self):
"""Available once the headers have been sent to the client.
These are the bytes representing the first response line and
the headers
"""
return self._headers_sent
def data_received(self, data):
"""Implements :meth:`~.ProtocolConsumer.data_received` method.
Once we have a full HTTP message, build the wsgi ``environ`` and
delegate the response to the :func:`wsgi_callable` function.
"""
parser = self.parser
processed = parser.execute(data, len(data))
if not self._stream and parser.is_headers_complete():
headers = Headers(parser.get_headers(), kind="client")
self._stream = StreamReader(headers, parser, self.transport)
self._response(self.wsgi_environ())
#
if parser.is_message_complete():
#
# Stream has the whole body
if not self._stream.on_message_complete.done():
self._stream.on_message_complete.set_result(None)
if processed < len(data):
if not self._buffer:
self._buffer = data[processed:]
self.bind_event("post_request", self._new_request)
else:
self._buffer += data[processed:]
#
elif processed < len(data):
# This is a parsing error, the client must have sent
# bogus data
raise ProtocolError
@property
def status(self):
return self._status
@property
def upgrade(self):
return self.headers.get("upgrade")
@property
def chunked(self):
return self.headers.get("Transfer-Encoding") == "chunked"
@property
def content_length(self):
c = self.headers.get("Content-Length")
if c:
return int(c)
@property
def version(self):
return self.parser.get_version()
def start_response(self, status, response_headers, exc_info=None):
"""WSGI compliant ``start_response`` callable, see pep3333_.
The application may call start_response more than once, if and only
if the ``exc_info`` argument is provided.
More precisely, it is a fatal error to call ``start_response`` without
the ``exc_info`` argument if start_response has already been called
within the current invocation of the application.
:parameter status: an HTTP ``status`` string like ``200 OK`` or
``404 Not Found``.
:parameter response_headers: a list of ``(header_name, header_value)``
tuples. It must be a Python list. Each header_name must be a valid
HTTP header field-name (as defined by RFC 2616_, Section 4.2),
without a trailing colon or other punctuation.
:parameter exc_info: optional python ``sys.exc_info()`` tuple.
This argument should be supplied by the application only if
``start_response`` is being called by an error handler.
:return: The :meth:`write` method.
``HOP_HEADERS`` are not considered but no error is raised.
.. _pep3333: http://www.python.org/dev/peps/pep-3333/
.. _2616: http://www.faqs.org/rfcs/rfc2616.html
"""
if exc_info:
try:
if self._headers_sent:
# if exc_info is provided, and the HTTP headers have
# already been sent, start_response must raise an error,
# and should re-raise using the exc_info tuple
reraise(*exc_info)
finally:
# Avoid circular reference
exc_info = None
elif self._status:
# Headers already set. Raise error
raise HttpException("Response headers already set!")
self._status = status
if type(response_headers) is not list:
raise TypeError("Headers must be a list of name/value tuples")
for header, value in response_headers:
if header.lower() in HOP_HEADERS:
# These features are the exclusive province of this class,
# this should be considered a fatal error for an application
# to attempt sending them, but we don't raise an error,
# just log a warning
self.logger.warning('Application passing hop header "%s"', header)
continue
self.headers.add_header(header, value)
return self.write
def write(self, data, force=False):
"""The write function returned by the :meth:`start_response` method.
Required by the WSGI specification.
:param data: bytes to write
:param force: Optional flag used internally
:return: a :class:`~asyncio.Future` or the number of bytes written
"""
write = super().write
chunks = []
if not self._headers_sent:
tosend = self.get_headers()
self._headers_sent = tosend.flat(self.version, self.status)
self.fire_event("on_headers")
chunks.append(self._headers_sent)
if data:
if self.chunked:
while len(data) >= MAX_CHUNK_SIZE:
chunk, data = data[:MAX_CHUNK_SIZE], data[MAX_CHUNK_SIZE:]
chunks.append(chunk_encoding(chunk))
if data:
chunks.append(chunk_encoding(data))
else:
chunks.append(data)
elif force and self.chunked:
chunks.append(chunk_encoding(data))
if chunks:
return write(b"".join(chunks))
########################################################################
# INTERNALS
@task
def _response(self, environ):
exc_info = None
response = None
done = False
alive = self.cfg.keep_alive or 15
while not done:
done = True
try:
if exc_info is None:
if "SERVER_NAME" not in environ:
raise HttpException(status=400)
response = self.wsgi_callable(environ, self.start_response)
if isfuture(response):
response = yield from wait_for(response, alive)
else:
response = handle_wsgi_error(environ, exc_info)
if isfuture(response):
response = yield from wait_for(response, alive)
#
if exc_info:
self.start_response(response.status, response.get_headers(), exc_info)
#
# Do the actual writing
loop = self._loop
start = loop.time()
for chunk in response:
if isfuture(chunk):
chunk = yield from wait_for(chunk, alive)
start = loop.time()
result = self.write(chunk)
if isfuture(result):
yield from wait_for(result, alive)
start = loop.time()
else:
time_in_loop = loop.time() - start
if time_in_loop > MAX_TIME_IN_LOOP:
self.logger.debug("Released the event loop after %.3f seconds", time_in_loop)
yield None
start = loop.time()
#
# make sure we write headers and last chunk if needed
self.write(b"", True)
except IOError: # client disconnected, end this connection
self.finished()
except Exception:
if wsgi_request(environ).cache.handle_wsgi_error:
self.keep_alive = False
self._write_headers()
self.connection.close()
self.finished()
else:
done = False
exc_info = sys.exc_info()
else:
if not self.keep_alive:
self.connection.close()
self.finished()
log_wsgi_info(self.logger.info, environ, self.status)
finally:
if hasattr(response, "close"):
try:
response.close()
except Exception:
self.logger.exception("Error while closing wsgi iterator")
def is_chunked(self):
"""Check if the response uses chunked transfer encoding.
Only use chunked responses when the client is speaking HTTP/1.1
or newer and there was no Content-Length header set.
"""
if (
self.version <= (1, 0)
or self._status == "200 Connection established"
or has_empty_content(int(self.status[:3]))
):
return False
elif self.headers.get("Transfer-Encoding") == "chunked":
return True
else:
return self.content_length is None
def get_headers(self):
"""Get the headers to send to the client.
"""
if not self._status:
# we are sending headers but the start_response was not called
raise HttpException("Headers not set.")
headers = self.headers
# Set chunked header if needed
if self.is_chunked():
headers["Transfer-Encoding"] = "chunked"
headers.pop("content-length", None)
else:
headers.pop("Transfer-Encoding", None)
if self.keep_alive:
self.keep_alive = keep_alive_with_status(self._status, headers)
if not self.keep_alive:
headers["connection"] = "close"
return headers
def wsgi_environ(self):
# return a the WSGI environ dictionary
transport = self.transport
https = True if is_tls(transport.get_extra_info("socket")) else False
multiprocess = self.cfg.concurrency == "process"
environ = wsgi_environ(
self._stream,
self.parser,
self._stream.headers,
transport.get_extra_info("sockname"),
self.address,
self.headers,
self.SERVER_SOFTWARE,
https=https,
extra={"pulsar.connection": self.connection, "pulsar.cfg": self.cfg, "wsgi.multiprocess": multiprocess},
)
self.keep_alive = keep_alive(self.headers, self.parser.get_version())
self.headers.update([("Server", self.SERVER_SOFTWARE), ("Date", format_date_time(time.time()))])
return environ
def _new_request(self, _, exc=None):
connection = self._connection
connection.data_received(self._buffer)
def _write_headers(self):
if not self._headers_sent:
if self.content_length:
self.headers["Content-Length"] = "0"
self.write(b"")
