def test_hasHeaderTrue(self):
"""
Check that L{Headers.hasHeader} returns C{True} when the given header
is found.
"""
h = Headers()
h.setRawHeaders(u"test\u00E1", [u"lemur"])
self.assertTrue(h.hasHeader(u"test\u00E1"))
self.assertTrue(h.hasHeader(u"Test\u00E1"))
self.assertTrue(h.hasHeader(b"test\xe1"))
self.assertTrue(h.hasHeader(b"Test\xe1"))
def test_hasHeaderTrue(self):
"""
Check that L{Headers.hasHeader} returns C{True} when the given header
is found.
"""
h = Headers()
h.setRawHeaders("test\u00E1", ["lemur"])
self.assertTrue(h.hasHeader("test\u00E1"))
self.assertTrue(h.hasHeader("Test\u00E1"))
self.assertTrue(h.hasHeader(b"test\xe1"))
self.assertTrue(h.hasHeader(b"Test\xe1"))
def test_nameNotEncodable(self):
"""
Passing L{unicode} to any function that takes a header name will encode
said header name as ISO-8859-1, and if it cannot be encoded, it will
raise a L{UnicodeDecodeError}.
"""
h = Headers()
# Only these two functions take names
with self.assertRaises(UnicodeEncodeError):
h.setRawHeaders(u"\u2603", [u"val"])
with self.assertRaises(UnicodeEncodeError):
h.hasHeader(u"\u2603")
def test_nameNotEncodable(self):
"""
Passing L{unicode} to any function that takes a header name will encode
said header name as ISO-8859-1, and if it cannot be encoded, it will
raise a L{UnicodeDecodeError}.
"""
h = Headers()
# Only these two functions take names
with self.assertRaises(UnicodeEncodeError):
h.setRawHeaders("\u2603", ["val"])
with self.assertRaises(UnicodeEncodeError):
h.hasHeader("\u2603")
def test_removeHeader(self):
"""
Check that L{Headers.removeHeader} removes the given header.
"""
h = Headers()
h.setRawHeaders(b"foo", [b"lemur"])
self.assertTrue(h.hasHeader(b"foo"))
h.removeHeader(b"foo")
self.assertFalse(h.hasHeader(b"foo"))
h.setRawHeaders(b"bar", [b"panda"])
self.assertTrue(h.hasHeader(b"bar"))
h.removeHeader(b"Bar")
self.assertFalse(h.hasHeader(b"bar"))
def test_setRawHeaders(self):
"""
L{Headers.setRawHeaders} sets the header values for the given
header name to the sequence of strings, encoded.
"""
rawValue = ["value1", "value2"]
rawEncodedValue = [b"value1", b"value2"]
h = Headers()
h.setRawHeaders("test", rawValue)
self.assertTrue(h.hasHeader(b"test"))
self.assertTrue(h.hasHeader(b"Test"))
self.assertTrue(h.hasHeader("test"))
self.assertTrue(h.hasHeader("Test"))
self.assertEqual(h.getRawHeaders("test"), rawValue)
self.assertEqual(h.getRawHeaders(b"test"), rawEncodedValue)
def test_removeHeader(self):
"""
Check that L{Headers.removeHeader} removes the given header.
"""
h = Headers()
h.setRawHeaders("foo", ["lemur"])
self.assertTrue(h.hasHeader("foo"))
h.removeHeader("foo")
self.assertFalse(h.hasHeader("foo"))
h.setRawHeaders("bar", ["panda"])
self.assertTrue(h.hasHeader("bar"))
h.removeHeader("Bar")
self.assertFalse(h.hasHeader("bar"))
def test_setRawHeaders(self):
"""
L{Headers.setRawHeaders} sets the header values for the given
header name to the sequence of strings, encoded.
"""
rawValue = [u"value1", u"value2"]
rawEncodedValue = [b"value1", b"value2"]
h = Headers()
h.setRawHeaders("test", rawValue)
self.assertTrue(h.hasHeader(b"test"))
self.assertTrue(h.hasHeader(b"Test"))
self.assertTrue(h.hasHeader("test"))
self.assertTrue(h.hasHeader("Test"))
self.assertEqual(h.getRawHeaders("test"), rawValue)
self.assertEqual(h.getRawHeaders(b"test"), rawEncodedValue)
def request(self, method, uri, headers=None, body_producer=None):
"""Issue a new request to the wrapped agent.
Args:
method (bytes): The HTTP method to use.
uri (bytes): The url to download from.
headers (t.w.h.Headers, optional): Any extra headers to send.
body_producer (t.w.i.IBodyProducer, optional): Request body data.
Returns:
Deferred: The filename of the of the downloaded file.
"""
if headers is None:
headers = Headers()
if not headers.hasHeader(b'User-Agent'):
version = get_version()
user_agent = 'Deluge/%s (https://deluge-torrent.org)' % version
headers.addRawHeader('User-Agent', user_agent)
d = self.agent.request(method=method,
uri=uri,
headers=headers,
bodyProducer=body_producer)
d.addCallback(self.request_callback)
return d
def _requestWithEndpoint(self, key, endpoint, method, parsedURI, headers,
bodyProducer, requestPath):
"""
Issue a new request, given the endpoint and the path sent as part of
the request.
"""
# Create minimal headers, if necessary:
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
#headers = headers.copy() # not supported in twisted <= 11.1, and it doesn't affects us
headers.addRawHeader(
'host',
self._computeHostValue(parsedURI.scheme, parsedURI.host,
parsedURI.port))
d = self._pool.getConnection(key, endpoint)
def cbConnected(proto):
return proto.request(
Request(method,
requestPath,
headers,
bodyProducer,
persistent=self._pool.persistent))
d.addCallback(cbConnected)
return d
def _connectAndRequest(self,
method,
uri,
headers,
bodyProducer,
requestPath=None):
"""
Internal helper to make the request.
@param requestPath: If specified, the path to use for the request
instead of the path extracted from C{uri}.
"""
scheme, host, port, path = _parse(uri)
if requestPath is None:
requestPath = path
d = self._connect(scheme, host, port)
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
headers = headers.copy()
headers.addRawHeader('host',
self._computeHostValue(scheme, host, port))
def cbConnected(proto):
return proto.request(
Request(method, requestPath, headers, bodyProducer))
d.addCallback(cbConnected)
return d
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a new request to the wrapped L{Agent}.
Send a I{Cookie} header if a cookie for C{uri} is stored in
L{CookieAgent.cookieJar}. Cookies are automatically extracted and
stored from requests.
If a C{'cookie'} header appears in C{headers} it will override the
automatic cookie header obtained from the cookie jar.
@see: L{Agent.request}
"""
if headers is None:
headers = Headers()
lastRequest = _FakeUrllib2Request(uri)
# Setting a cookie header explicitly will disable automatic request
# cookies.
if not headers.hasHeader('cookie'):
self.cookieJar.add_cookie_header(lastRequest)
cookieHeader = lastRequest.get_header('Cookie', None)
if cookieHeader is not None:
headers = headers.copy()
headers.addRawHeader('cookie', cookieHeader)
d = self._agent.request(method, uri, headers, bodyProducer)
d.addCallback(self._extractCookies, lastRequest)
return d
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a new request to the wrapped L{Agent}.
Send a I{Cookie} header if a cookie for C{uri} is stored in
L{CookieAgent.cookieJar}. Cookies are automatically extracted and
stored from requests.
If a C{'cookie'} header appears in C{headers} it will override the
automatic cookie header obtained from the cookie jar.
@see: L{Agent.request}
"""
if headers is None:
headers = Headers()
lastRequest = _FakeUrllib2Request(uri)
# Setting a cookie header explicitly will disable automatic request
# cookies.
if not headers.hasHeader('cookie'):
self.cookieJar.add_cookie_header(lastRequest)
cookieHeader = lastRequest.get_header('Cookie', None)
if cookieHeader is not None:
headers = headers.copy()
headers.addRawHeader('cookie', cookieHeader)
d = self._agent.request(method, uri, headers, bodyProducer)
d.addCallback(self._extractCookies, lastRequest)
return d
def test_nameEncoding(self):
"""
Passing L{unicode} to any function that takes a header name will encode
said header name as ISO-8859-1.
"""
h = Headers()
# We set it using a Unicode string.
h.setRawHeaders(u"\u00E1", [b"foo"])
# It's encoded to the ISO-8859-1 value, which we can use to access it
self.assertTrue(h.hasHeader(b"\xe1"))
self.assertEqual(h.getRawHeaders(b"\xe1"), [b'foo'])
# We can still access it using the Unicode string..
self.assertTrue(h.hasHeader(u"\u00E1"))
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a new request via the configured proxy.
"""
scheme, host, port, path = _parse(uri)
request_path = uri
d = self._connect(scheme, host, port)
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
# This is a lot of copying. It might be nice if there were a bit
# less.
headers = Headers(dict(headers.getAllRawHeaders()))
headers.addRawHeader('host',
self._computeHostValue(scheme, host, port))
def cbConnected(proto):
# NOTE: For the proxy case the path should be the full URI.
return proto.request(
Request(method, request_path, headers, bodyProducer))
d.addCallback(cbConnected)
return d
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a new request via the configured proxy.
"""
if version >= Version('twisted', 13, 1, 0):
parsed_uri = _URI.getFromBytes(uri)
scheme = parsed_uri.scheme
host = parsed_uri.host
port = parsed_uri.port
else:
scheme, host, port, path = _parse(uri)
request_path = uri
d = self._connect(scheme, host, port)
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
# This is a lot of copying. It might be nice if there were a bit
# less.
headers = Headers(dict(headers.getAllRawHeaders()))
headers.addRawHeader(
'host', self._computeHostValue(scheme, host, port))
def cbConnected(proto):
# NOTE: For the proxy case the path should be the full URI.
return proto.request(
Request(method, request_path, headers, bodyProducer))
d.addCallback(cbConnected)
return d
def test_nameEncoding(self):
"""
Passing L{unicode} to any function that takes a header name will encode
said header name as ISO-8859-1.
"""
h = Headers()
# We set it using a Unicode string.
h.setRawHeaders("\u00E1", [b"foo"])
# It's encoded to the ISO-8859-1 value, which we can use to access it
self.assertTrue(h.hasHeader(b"\xe1"))
self.assertEqual(h.getRawHeaders(b"\xe1"), [b"foo"])
# We can still access it using the Unicode string..
self.assertTrue(h.hasHeader("\u00E1"))
def request(self, method, uri, headers=None, bodyProducer=None):
parsedURI = client._parse(uri)
host_addr = address.IPv4Address('TCP', parsedURI.host, parsedURI.port)
# ripped from _AgentBase._requestWithEndpoint
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
headers = headers.copy()
headers.addRawHeader(
'host', self._computeHostValue(parsedURI.scheme, parsedURI.host,
parsedURI.port))
request = client.Request(method, parsedURI.path, headers, bodyProducer,
persistent=False)
c = ClientProtocol(request)
# ouch
self.root.putChild('', self.root)
server = Site(self.root).buildProtocol(self.addr)
loopbackAsync(server, c, host_addr, self.addr)
return c.response.addBoth(self._done, c)
def request(self, method, uri, headers=None, bodyProducer=None):
if headers is None:
headers = Headers()
else:
headers = headers.copy()
if not headers.hasHeader('user-agent'):
headers.addRawHeader('user-agent', self.user_agent)
return self.agent.request(method, uri, headers, bodyProducer)
class _FakeUrllib2Request(object):
"""
A fake C{urllib2.Request} object for C{cookielib} to work with.
@see: U{http://docs.python.org/library/urllib2.html#request-objects}
@type uri: C{str}
@ivar uri: Request URI.
@type headers: L{twisted.web.http_headers.Headers}
@ivar headers: Request headers.
@type type: C{str}
@ivar type: The scheme of the URI.
@type host: C{str}
@ivar host: The host[:port] of the URI.
@since: 11.1
"""
def __init__(self, uri):
self.uri = uri
self.headers = Headers()
self.type, rest = splittype(self.uri)
self.host, rest = splithost(rest)
def has_header(self, header):
return self.headers.hasHeader(header)
def add_unredirected_header(self, name, value):
self.headers.addRawHeader(name, value)
def get_full_url(self):
return self.uri
def get_header(self, name, default=None):
headers = self.headers.getRawHeaders(name, default)
if headers is not None:
return headers[0]
return None
def get_host(self):
return self.host
def get_type(self):
return self.type
def is_unverifiable(self):
# In theory this shouldn't be hardcoded.
return False
def test_rawHeadersValueEncoding(self):
"""
Passing L{unicode} to L{Headers.setRawHeaders} will encode the name as
ISO-8859-1 and values as UTF-8.
"""
h = Headers()
h.setRawHeaders(u"\u00E1", [u"\u2603", b"foo"])
self.assertTrue(h.hasHeader(b"\xe1"))
self.assertEqual(h.getRawHeaders(b"\xe1"), [b'\xe2\x98\x83', b'foo'])
def test_hasHeaderTrue(self):
"""
Check that L{Headers.hasHeader} returns C{True} when the given header
is found.
"""
h = Headers()
h.setRawHeaders("test", ["lemur"])
self.assertTrue(h.hasHeader("test"))
self.assertTrue(h.hasHeader("Test"))
def test_rawHeadersValueEncoding(self):
"""
Passing L{unicode} to L{Headers.setRawHeaders} will encode the name as
ISO-8859-1 and values as UTF-8.
"""
h = Headers()
h.setRawHeaders("\u00E1", ["\u2603", b"foo"])
self.assertTrue(h.hasHeader(b"\xe1"))
self.assertEqual(h.getRawHeaders(b"\xe1"), [b"\xe2\x98\x83", b"foo"])
class _FakeUrllib2Request(object):
"""
A fake C{urllib2.Request} object for C{cookielib} to work with.
@see: U{http://docs.python.org/library/urllib2.html#request-objects}
@type uri: C{str}
@ivar uri: Request URI.
@type headers: L{twisted.web.http_headers.Headers}
@ivar headers: Request headers.
@type type: C{str}
@ivar type: The scheme of the URI.
@type host: C{str}
@ivar host: The host[:port] of the URI.
@since: 11.1
"""
def __init__(self, uri):
self.uri = uri
self.headers = Headers()
self.type, rest = splittype(self.uri)
self.host, rest = splithost(rest)
def has_header(self, header):
return self.headers.hasHeader(header)
def add_unredirected_header(self, name, value):
self.headers.addRawHeader(name, value)
def get_full_url(self):
return self.uri
def get_header(self, name, default=None):
headers = self.headers.getRawHeaders(name, default)
if headers is not None:
return headers[0]
return None
def get_host(self):
return self.host
def get_type(self):
return self.type
def is_unverifiable(self):
# In theory this shouldn't be hardcoded.
return False
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a new request.
@param method: The request method to send.
@type method: C{str}
@param uri: The request URI send.
@type uri: C{str}
@param scheme: A string like C{'http'} or C{'https'} (the only two
supported values) to use to determine how to establish the
connection.
@param host: A C{str} giving the hostname which will be connected to in
order to issue a request.
@param port: An C{int} giving the port number the connection will be on.
@param path: A C{str} giving the path portion of the request URL.
@param headers: The request headers to send. If no I{Host} header is
included, one will be added based on the request URI.
@type headers: L{Headers}
@param bodyProducer: An object which will produce the request body or,
if the request body is to be empty, L{None}.
@type bodyProducer: L{IBodyProducer} provider
@return: A L{Deferred} which fires with the result of the request (a
L{Response} instance), or fails if there is a problem setting up a
connection over which to issue the request. It may also fail with
L{SchemeNotSupported} if the scheme of the given URI is not
supported.
@rtype: L{Deferred}
"""
scheme, host, port, path = _parse(uri)
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
# This is a lot of copying. It might be nice if there were a bit
# less.
headers = Headers(dict(headers.getAllRawHeaders()))
headers.addRawHeader(
'host', self._computeHostValue(scheme, host, port))
if self.persistent:
sem = self._semaphores.get((scheme, host, port))
if sem is None:
sem = DeferredSemaphore(self.maxConnectionsPerHostName)
self._semaphores[scheme, host, port] = sem
return sem.run(self._request, method, scheme, host, port, path,
headers, bodyProducer)
else:
return self._request(
method, scheme, host, port, path, headers, bodyProducer)
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a new request.
@param method: The request method to send.
@type method: C{str}
@param uri: The request URI send.
@type uri: C{str}
@param headers: The request headers to send. If no I{Host} header is
included, one will be added based on the request URI.
@type headers: L{Headers}
@param bodyProducer: An object which will produce the request body or,
if the request body is to be empty, L{None}.
@type bodyProducer: L{IBodyProducer} provider
@return: A L{Deferred} which fires with the result of the request (a
L{Response} instance), or fails if there is a problem setting up a
connection over which to issue the request. It may also fail with
L{SchemeNotSupported} if the scheme of the given URI is not
supported.
@rtype: L{Deferred}
"""
scheme, host, port, path = _parse(uri)
if scheme != 'http':
return defer.fail(
SchemeNotSupported("Unsupported scheme: %r" % (scheme, )))
cc = ClientCreator(self._reactor, self._protocol)
d = cc.connectTCP(host, port)
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
# This is a lot of copying. It might be nice if there were a bit
# less.
headers = Headers(dict(headers.getAllRawHeaders()))
headers.addRawHeader('host',
self._computeHostValue(scheme, host, port))
def cbConnected(proto):
return proto.request(Request(method, path, headers, bodyProducer))
d.addCallback(cbConnected)
return d
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a new request.
@param method: The request method to send.
@type method: C{str}
@param uri: The request URI send.
@type uri: C{str}
@param headers: The request headers to send. If no I{Host} header is
included, one will be added based on the request URI.
@type headers: L{Headers}
@param bodyProducer: An object which will produce the request body or,
if the request body is to be empty, L{None}.
@type bodyProducer: L{IBodyProducer} provider
@return: A L{Deferred} which fires with the result of the request (a
L{Response} instance), or fails if there is a problem setting up a
connection over which to issue the request. It may also fail with
L{SchemeNotSupported} if the scheme of the given URI is not
supported.
@rtype: L{Deferred}
"""
scheme, host, port, path = _parse(uri)
if scheme != 'http':
return defer.fail(SchemeNotSupported(
"Unsupported scheme: %r" % (scheme,)))
cc = ClientCreator(self._reactor, self._protocol)
d = cc.connectTCP(host, port)
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
# This is a lot of copying. It might be nice if there were a bit
# less.
headers = Headers(dict(headers.getAllRawHeaders()))
headers.addRawHeader(
'host', self._computeHostValue(scheme, host, port))
def cbConnected(proto):
return proto.request(Request(method, path, headers, bodyProducer))
d.addCallback(cbConnected)
return d
def _requestWithEndpoint(self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath):
"""
Issue a new request, given the endpoint and the path sent as part of
the request.
"""
# Create minimal headers, if necessary:
if headers is None:
headers = Headers()
if not headers.hasHeader("host"):
# headers = headers.copy() # not supported in twisted <= 11.1, and it doesn't affects us
headers.addRawHeader("host", self._computeHostValue(parsedURI.scheme, parsedURI.host, parsedURI.port))
d = self._pool.getConnection(key, endpoint)
def cbConnected(proto):
return proto.request(Request(method, requestPath, headers, bodyProducer, persistent=self._pool.persistent))
d.addCallback(cbConnected)
return d
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a new request via the configured proxy.
"""
request_path = uri
d = self._connect(None, None, None)
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
# This is a lot of copying. It might be nice if there were a bit
# less.
headers = Headers(dict(headers.getAllRawHeaders()))
headers.addRawHeader(
'host', self._computeHostValue(scheme, host, port))
def cbConnected(proto):
# NOTE: For the proxy case the path should be the full URI.
return proto.request(Request(method, request_path, headers, bodyProducer))
d.addCallback(cbConnected)
return d
class _FakeUrllib2Request(object):
"""
A fake C{urllib2.Request} object for C{cookielib} to work with.
@type uri: C{str}
@ivar uri: Request URI.
@type headers: L{twisted.web.http_headers.Headers}
@ivar headers: Request headers.
@since: 11.1
"""
def __init__(self, uri):
self.uri = uri
self.headers = Headers()
def has_header(self, header):
return self.headers.hasHeader(header)
def add_unredirected_header(self, name, value):
self.headers.addRawHeader(name, value)
def get_full_url(self):
return self.uri
def get_header(self, name, default=None):
headers = self.headers.getRawHeaders(name, default)
if headers is not None:
return headers[0]
return None
def is_unverifiable(self):
# In theory this shouldn't be hardcoded.
return False
def _requestWithEndpoint(self, key, endpoint, method, parsedURI,
headers, bodyProducer, requestPath):
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
headers = headers.copy()
headers.addRawHeader(
'host', self._computeHostValue(parsedURI.scheme,
parsedURI.host, parsedURI.port))
d = self._pool.getConnection(key, endpoint)
def cbConnected(proto):
return proto.request(
SigningRequest._construct(self._signData.makeSigner(),
method, requestPath, headers,
bodyProducer,
persistent=self._pool.persistent,
parsedURI=parsedURI))
d.addCallback(cbConnected)
return d
def _connectAndRequest(self, method, uri, headers, bodyProducer,
requestPath=None):
"""
Internal helper to make the request.
@param requestPath: If specified, the path to use for the request
instead of the path extracted from C{uri}.
"""
scheme, host, port, path = _parse(uri)
if requestPath is None:
requestPath = path
d = self._connect(scheme, host, port)
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
headers = headers.copy()
headers.addRawHeader(
'host', self._computeHostValue(scheme, host, port))
def cbConnected(proto):
return proto.request(
Request(method, requestPath, headers, bodyProducer))
d.addCallback(cbConnected)
return d
def request(self, method, uri, headers=None, bodyProducer=None):
"""
:param method: HTTP method (GET/POST/etc).
:type method: bytes
:param uri: Absolute URI to be retrieved.
:type uri: bytes
:param headers: HTTP headers to send with the request, or None to
send no extra headers.
:type headers: twisted.web.http_headers.Headers, None
:param bodyProducer: An object which can generate bytes to make up the
body of this request (for example, the properly encoded contents of
a file for a file upload). Or None if the request is to have
no body.
:type bodyProducer: twisted.web.iweb.IBodyProducer, None
:returns a deferred that fires when the header of the response has
been received (regardless of the response status code). Fails if
there is any problem which prevents that response from being received
(including problems that prevent the request from being sent).
:rtype: Deferred[twisted.web.iweb.IResponse]
"""
parsed_uri = URI.fromBytes(uri, defaultPort=-1)
res = yield self._route_matrix_uri(parsed_uri)
# set up the TLS connection params
#
# XXX disabling TLS is really only supported here for the benefit of the
# unit tests. We should make the UTs cope with TLS rather than having to make
# the code support the unit tests.
if self._tls_client_options_factory is None:
tls_options = None
else:
tls_options = self._tls_client_options_factory.get_options(
res.tls_server_name.decode("ascii")
)
# make sure that the Host header is set correctly
if headers is None:
headers = Headers()
else:
headers = headers.copy()
if not headers.hasHeader(b'host'):
headers.addRawHeader(b'host', res.host_header)
class EndpointFactory(object):
@staticmethod
def endpointForURI(_uri):
ep = LoggingHostnameEndpoint(
self._reactor, res.target_host, res.target_port,
)
if tls_options is not None:
ep = wrapClientTLS(tls_options, ep)
return ep
agent = Agent.usingEndpointFactory(self._reactor, EndpointFactory(), self._pool)
res = yield agent.request(method, uri, headers, bodyProducer)
defer.returnValue(res)
def request(
self,
method: bytes,
uri: bytes,
headers: Optional[Headers] = None,
bodyProducer: Optional[IBodyProducer] = None,
) -> Generator[defer.Deferred, Any, IResponse]:
"""
Args:
method: HTTP method: GET/POST/etc
uri: Absolute URI to be retrieved
headers:
HTTP headers to send with the request, or None to send no extra headers.
bodyProducer:
An object which can generate bytes to make up the
body of this request (for example, the properly encoded contents of
a file for a file upload). Or None if the request is to have
no body.
Returns:
Deferred[twisted.web.iweb.IResponse]:
fires when the header of the response has been received (regardless of the
response status code). Fails if there is any problem which prevents that
response from being received (including problems that prevent the request
from being sent).
"""
# We use urlparse as that will set `port` to None if there is no
# explicit port.
parsed_uri = urllib.parse.urlparse(uri)
# There must be a valid hostname.
assert parsed_uri.hostname
# If this is a matrix:// URI check if the server has delegated matrix
# traffic using well-known delegation.
#
# We have to do this here and not in the endpoint as we need to rewrite
# the host header with the delegated server name.
delegated_server = None
if (parsed_uri.scheme == b"matrix"
and not _is_ip_literal(parsed_uri.hostname)
and not parsed_uri.port):
well_known_result = yield defer.ensureDeferred(
self._well_known_resolver.get_well_known(parsed_uri.hostname))
delegated_server = well_known_result.delegated_server
if delegated_server:
# Ok, the server has delegated matrix traffic to somewhere else, so
# lets rewrite the URL to replace the server with the delegated
# server name.
uri = urllib.parse.urlunparse((
parsed_uri.scheme,
delegated_server,
parsed_uri.path,
parsed_uri.params,
parsed_uri.query,
parsed_uri.fragment,
))
parsed_uri = urllib.parse.urlparse(uri)
# We need to make sure the host header is set to the netloc of the
# server and that a user-agent is provided.
if headers is None:
request_headers = Headers()
else:
request_headers = headers.copy()
if not request_headers.hasHeader(b"host"):
request_headers.addRawHeader(b"host", parsed_uri.netloc)
if not request_headers.hasHeader(b"user-agent"):
request_headers.addRawHeader(b"user-agent", self.user_agent)
res = yield make_deferred_yieldable(
self._agent.request(method, uri, request_headers, bodyProducer))
return res
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Args:
method (bytes): HTTP method: GET/POST/etc
uri (bytes): Absolute URI to be retrieved
headers (twisted.web.http_headers.Headers|None):
HTTP headers to send with the request, or None to
send no extra headers.
bodyProducer (twisted.web.iweb.IBodyProducer|None):
An object which can generate bytes to make up the
body of this request (for example, the properly encoded contents of
a file for a file upload). Or None if the request is to have
no body.
Returns:
Deferred[twisted.web.iweb.IResponse]:
fires when the header of the response has been received (regardless of the
response status code). Fails if there is any problem which prevents that
response from being received (including problems that prevent the request
from being sent).
"""
parsed_uri = URI.fromBytes(uri, defaultPort=-1)
res = yield self._route_matrix_uri(parsed_uri)
# set up the TLS connection params
#
# XXX disabling TLS is really only supported here for the benefit of the
# unit tests. We should make the UTs cope with TLS rather than having to make
# the code support the unit tests.
if self._tls_client_options_factory is None:
tls_options = None
else:
tls_options = self._tls_client_options_factory.get_options(
res.tls_server_name.decode("ascii"))
# make sure that the Host header is set correctly
if headers is None:
headers = Headers()
else:
headers = headers.copy()
if not headers.hasHeader(b"host"):
headers.addRawHeader(b"host", res.host_header)
class EndpointFactory(object):
@staticmethod
def endpointForURI(_uri):
ep = LoggingHostnameEndpoint(self._reactor, res.target_host,
res.target_port)
if tls_options is not None:
ep = wrapClientTLS(tls_options, ep)
return ep
agent = Agent.usingEndpointFactory(self._reactor, EndpointFactory(),
self._pool)
res = yield make_deferred_yieldable(
agent.request(method, uri, headers, bodyProducer))
return res
