def get_agent(reactor, connect_timeout=None, tcp_nodelay=False):
"""Return appropriate agent based on whether an http_proxy is used or not.
:param connect_timeout: connection timeout in seconds
:type connect_timeout: float
:param tcp_nodelay: flag to enable tcp_nodelay for request
:type tcp_nodelay: boolean
:returns: :class:`twisted.web.client.ProxyAgent` when an http_proxy
environment variable is present, :class:`twisted.web.client.Agent`
otherwise.
"""
# TODO: Would be nice to have https_proxy support too.
http_proxy = os.environ.get('http_proxy')
pool = None
if tcp_nodelay:
from fido._client import HTTPConnectionPoolOverride
pool = HTTPConnectionPoolOverride(reactor=reactor, persistent=False)
if http_proxy is None:
return _twisted_web_client().Agent(
reactor,
connectTimeout=connect_timeout,
pool=pool,
)
parse_result = urlparse(http_proxy)
http_proxy_endpoint = TCP4ClientEndpoint(reactor,
parse_result.hostname,
parse_result.port or 80,
timeout=connect_timeout)
return _twisted_web_client().ProxyAgent(http_proxy_endpoint, pool=pool)
def connectionLost(self, reason):
"""
:param reason:
twisted.web.client.ResponseDone indicates that all bytes from the
response have been successfully delivered
twisted.web.client.PotentialDataLoss if it cannot be
determined if the entire response body has been delivered.
This only occurs when making requests to HTTP servers which do
not set Content-Length or a Transfer-Encoding in the response
twisted.web.client.ResponseFailed indicates that some bytes from
the response were lost. The reasons attribute of the exception
may provide more specific indications as to why.
For more info see https://twistedmatrix.com/documents/9.0.0/api/\
twisted.web.client.Response.html
"""
if (reason.check(_twisted_web_client().ResponseDone)
or reason.check(_twisted_web_client().PotentialDataLoss)):
self.finished.callback(
Response(
code=self.response.code,
headers=self.response.headers,
body=self.buffer.getvalue(),
reason=self.response.phrase,
))
else:
self.finished.errback(reason)
def _build_body_producer(body, headers):
"""
Prepares the body and the headers for the twisted http request performed
by the Twisted Agent.
:param body: request body, MUST be of type bytes.
:returns: a Twisted FileBodyProducer object as required by Twisted Agent
"""
if not body:
return None, headers
# body must be of bytes type.
bodyProducer = _twisted_web_client().FileBodyProducer(io.BytesIO(body))
# content-length needs to be removed because it was computed based on
# body but body is now being processed by twisted FileBodyProducer
# causing content-length to lose meaning and break the client.
# FileBodyProducer will take care of re-computing length and re-adding
# a new content-length header later.
twisted_headers = dict((key, value)
for (key, value) in six.iteritems(headers)
if key.lower() != 'content-length')
return bodyProducer, twisted_headers
def handle_timeout_errors(error):
"""
This errback handles twisted timeout errors and wraps them as fido
exceptions so that users don't need to import twisted APIs (reactor
initialization issues) and dig into Twisted documentation and code.
From the user's perspective and for sanity of usage is better to raise
the friendlier fido.exception errors.
"""
if error.check(_twisted_web_client().ResponseNeverReceived):
if error.value.reasons[0].check(CancelledError):
raise HTTPTimeoutError(
"Connection was closed by fido because the server took "
"more than timeout={timeout} seconds to "
"send the response".format(timeout=timeout)
)
elif error.check(ConnectError):
raise TCPConnectionError(
"Connection was closed by Twisted Agent because there was "
"a problem establishing the connection or the "
"connect_timeout={connect_timeout} was reached."
.format(connect_timeout=connect_timeout)
)
return error
def connectionLost(self, reason):
"""
:param reason:
twisted.web.client.ResponseDone indicates that all bytes from the
response have been successfully delivered
twisted.web.client.PotentialDataLoss if it cannot be
determined if the entire response body has been delivered.
This only occurs when making requests to HTTP servers which do
not set Content-Length or a Transfer-Encoding in the response
twisted.web.client.ResponseFailed indicates that some bytes from
the response were lost. The reasons attribute of the exception
may provide more specific indications as to why.
For more info see https://twistedmatrix.com/documents/9.0.0/api/\
twisted.web.client.Response.html
"""
if (reason.check(_twisted_web_client().ResponseDone) or
reason.check(_twisted_web_client().PotentialDataLoss)):
content_encoding = self.response.headers.getRawHeaders(
'Content-Encoding', [])
body = self.buffer.getvalue()
if 'gzip' in content_encoding and self.decompress_gzip:
try:
body = zlib.decompress(body, GZIP_WINDOW_SIZE)
except zlib.error as e:
self.finished.errback(
Failure(
exc_type=GzipDecompressionError,
exc_value=GzipDecompressionError(e),
exc_tb=sys.exc_info()[2],
),
)
return
self.finished.callback(
Response(
code=self.response.code,
headers=self.response.headers,
body=body,
reason=self.response.phrase,
)
)
else:
self.finished.errback(reason)
def _twisted_web_client():
from fido._client import _twisted_web_client
return _twisted_web_client()