def _connect(self, scheme, host, port):
"""
Connect to the given host and port, using a transport selected based on
scheme.
@param scheme: A string like C{'http'} or C{'https'} (the only two
supported values) to use to determine how to establish the
connection.
@param host: A C{str} giving the hostname which will be connected to in
order to issue a request.
@param port: An C{int} giving the port number the connection will be on.
@return: A L{Deferred} which fires with a connected instance of
C{self._protocol}.
"""
cc = ClientCreator(self._reactor, self._protocol)
if scheme == 'http':
d = cc.connectTCP(host, port)
elif scheme == 'https':
d = cc.connectSSL(host, port, self._wrapContextFactory(host, port))
else:
d = defer.fail(
SchemeNotSupported("Unsupported scheme: %r" % (scheme, )))
return d
def _getEndpoint(self, scheme, host, port):
"""
Get an endpoint for the given host and port, using a transport
selected based on scheme.
@param scheme: A string like C{'http'} or C{'https'} (the only two
supported values) to use to determine how to establish the
connection.
@param host: A C{str} giving the hostname which will be connected to in
order to issue a request.
@param port: An C{int} giving the port number the connection will be
on.
@return: An endpoint which can be used to connect to given address.
"""
kwargs = {}
if self._connectTimeout is not None:
kwargs['timeout'] = self._connectTimeout
kwargs['bindAddress'] = self._bindAddress
if scheme == 'http':
return TCP4ClientEndpoint(self._reactor, host, port, **kwargs)
elif scheme == 'https':
return SSL4ClientEndpoint(self._reactor, host, port,
self._wrapContextFactory(host, port),
**kwargs)
else:
raise SchemeNotSupported("Unsupported scheme: %r" % (scheme, ))
def _getEndpoint(self, scheme, host, port):
"""
Get Tor endpoint for the given host and port, using a transport
selected based on scheme. Currently only supports http... but
perhaps we'll fix https later.
@param scheme: The string C{'http'} (currently the only two supported value)
to use to determine how to establish the connection.
@param host: A C{str} giving the hostname which will be connected to in
order to issue a request.
@param port: An C{int} giving the port number the connection will be
on.
@return: An endpoint which can be used to connect to given address.
"""
kwargs = {}
if self._connectTimeout is not None:
kwargs['timeout'] = self._connectTimeout
kwargs['bindAddress'] = self._bindAddress
endpointDescriptor = self._makeEndpointDescriptor(host, port)
if scheme == 'http':
return clientFromString(self._reactor, endpointDescriptor)
else:
raise SchemeNotSupported("Unsupported scheme: %r" % (scheme, ))
def _getEndpoint(self):
"""Return the endpoint to use for the underlying connection."""
args = [self._reactor, self._host, self._port]
if self._scheme == "ws":
endpointClass = TCP4ClientEndpoint
elif self._scheme == "wss":
endpointClass = SSL4ClientEndpoint
args.append(self._sslContextFactory)
else:
raise SchemeNotSupported("Unsupported scheme: %r" % self._scheme)
return endpointClass(*args, timeout=self._timeout)
def _getEndpoint(self, scheme, host, port):
kwargs = {}
if self._connectTimeout is not None:
kwargs['timeout'] = self._connectTimeout
kwargs['bindAddress'] = self._bindAddress
if scheme == 'http':
return TCP4ClientEndpoint(self._reactor, host, port, **kwargs)
elif scheme == 'shttp':
return SOCKS5ClientEndpoint(self._reactor, self._sockhost,
self._sockport, host, port, config.socksoptimisticdata, **kwargs)
elif scheme == 'https':
return SSL4ClientEndpoint(self._reactor, host, port,
self._wrapContextFactory(host, port),
**kwargs)
else:
raise SchemeNotSupported("Unsupported scheme: %r" % (scheme,))
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a new request.
@param method: The request method to send.
@type method: C{str}
@param uri: The request URI send.
@type uri: C{str}
@param headers: The request headers to send. If no I{Host} header is
included, one will be added based on the request URI.
@type headers: L{Headers}
@param bodyProducer: An object which will produce the request body or,
if the request body is to be empty, L{None}.
@type bodyProducer: L{IBodyProducer} provider
@return: A L{Deferred} which fires with the result of the request (a
L{Response} instance), or fails if there is a problem setting up a
connection over which to issue the request. It may also fail with
L{SchemeNotSupported} if the scheme of the given URI is not
supported.
@rtype: L{Deferred}
"""
scheme, host, port, path = _parse(uri)
if scheme != 'http':
return defer.fail(
SchemeNotSupported("Unsupported scheme: %r" % (scheme, )))
cc = ClientCreator(self._reactor, self._protocol)
d = cc.connectTCP(host, port)
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
# This is a lot of copying. It might be nice if there were a bit
# less.
headers = Headers(dict(headers.getAllRawHeaders()))
headers.addRawHeader('host',
self._computeHostValue(scheme, host, port))
def cbConnected(proto):
return proto.request(Request(method, path, headers, bodyProducer))
d.addCallback(cbConnected)
return d
def endpointForURI(self, uri):
"""
Create an endpoint that represents an in-memory connection to
a URI.
Note: This always creates a
:class:`~twisted.internet.endpoints.TCP4ClientEndpoint` on the
assumption :class:`RequestTraversalAgent` ignores everything
about the endpoint but its port.
:param uri: The URI to connect to.
:type uri: :class:`~twisted.web.client.URI`
:return: The endpoint.
:rtype: An
:class:`~twisted.internet.interfaces.IStreamClientEndpoint`
provider.
"""
if uri.scheme not in {b'http', b'https'}:
raise SchemeNotSupported("Unsupported scheme: %r" % (uri.scheme, ))
return TCP4ClientEndpoint(self.reactor, "127.0.0.1", uri.port)
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a request to the server indicated by the given uri.
Supports `http` and `https` schemes.
An existing connection from the connection pool may be used or a new one may be
created.
See also: twisted.web.iweb.IAgent.request
Args:
method (bytes): The request method to use, such as `GET`, `POST`, etc
uri (bytes): The location of the resource to request.
headers (Headers|None): Extra headers to send with the request
bodyProducer (IBodyProducer|None): An object which can generate bytes to
make up the body of this request (for example, the properly encoded
contents of a file for a file upload). Or, None if the request is to
have no body.
Returns:
Deferred[IResponse]: completes when the header of the response has
been received (regardless of the response status code).
Can fail with:
SchemeNotSupported: if the uri is not http or https
twisted.internet.error.TimeoutError if the server we are connecting
to (proxy or destination) does not accept a connection before
connectTimeout.
... other things too.
"""
uri = uri.strip()
if not _VALID_URI.match(uri):
raise ValueError("Invalid URI {!r}".format(uri))
parsed_uri = URI.fromBytes(uri)
pool_key = (parsed_uri.scheme, parsed_uri.host, parsed_uri.port)
request_path = parsed_uri.originForm
should_skip_proxy = False
if self.no_proxy is not None:
should_skip_proxy = proxy_bypass_environment(
parsed_uri.host.decode(),
proxies={"no": self.no_proxy},
)
if (parsed_uri.scheme == b"http" and self.http_proxy_endpoint
and not should_skip_proxy):
# Cache *all* connections under the same key, since we are only
# connecting to a single destination, the proxy:
pool_key = ("http-proxy", self.http_proxy_endpoint)
endpoint = self.http_proxy_endpoint
request_path = uri
elif (parsed_uri.scheme == b"https" and self.https_proxy_endpoint
and not should_skip_proxy):
connect_headers = Headers()
# Determine whether we need to set Proxy-Authorization headers
if self.https_proxy_creds:
# Set a Proxy-Authorization header
connect_headers.addRawHeader(
b"Proxy-Authorization",
self.https_proxy_creds.as_proxy_authorization_value(),
)
endpoint = HTTPConnectProxyEndpoint(
self.proxy_reactor,
self.https_proxy_endpoint,
parsed_uri.host,
parsed_uri.port,
headers=connect_headers,
)
else:
# not using a proxy
endpoint = HostnameEndpoint(self._reactor, parsed_uri.host,
parsed_uri.port,
**self._endpoint_kwargs)
logger.debug("Requesting %s via %s", uri, endpoint)
if parsed_uri.scheme == b"https":
tls_connection_creator = self._policy_for_https.creatorForNetloc(
parsed_uri.host, parsed_uri.port)
endpoint = wrapClientTLS(tls_connection_creator, endpoint)
elif parsed_uri.scheme == b"http":
pass
else:
return defer.fail(
Failure(
SchemeNotSupported("Unsupported scheme: %r" %
(parsed_uri.scheme, ))))
return self._requestWithEndpoint(pool_key, endpoint, method,
parsed_uri, headers, bodyProducer,
request_path)
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a request to the server indicated by the given uri.
Supports `http` and `https` schemes.
An existing connection from the connection pool may be used or a new one may be
created.
See also: twisted.web.iweb.IAgent.request
Args:
method (bytes): The request method to use, such as `GET`, `POST`, etc
uri (bytes): The location of the resource to request.
headers (Headers|None): Extra headers to send with the request
bodyProducer (IBodyProducer|None): An object which can generate bytes to
make up the body of this request (for example, the properly encoded
contents of a file for a file upload). Or, None if the request is to
have no body.
Returns:
Deferred[IResponse]: completes when the header of the response has
been received (regardless of the response status code).
"""
uri = uri.strip()
if not _VALID_URI.match(uri):
raise ValueError("Invalid URI {!r}".format(uri))
parsed_uri = URI.fromBytes(uri)
pool_key: tuple = (parsed_uri.scheme, parsed_uri.host, parsed_uri.port)
request_path = parsed_uri.originForm
if parsed_uri.scheme == b"http" and self.proxy_endpoint:
# Cache *all* connections under the same key, since we are only
# connecting to a single destination, the proxy:
pool_key = ("http-proxy", self.proxy_endpoint)
endpoint = self.proxy_endpoint
request_path = uri
elif parsed_uri.scheme == b"https" and self.proxy_endpoint:
endpoint = HTTPConnectProxyEndpoint(
self._reactor,
self.proxy_endpoint,
parsed_uri.host,
parsed_uri.port,
self._proxy_auth,
)
else:
# not using a proxy
endpoint = HostnameEndpoint(self._reactor, parsed_uri.host,
parsed_uri.port,
**self._endpoint_kwargs)
logger.debug("Requesting %s via %s", uri, endpoint)
if parsed_uri.scheme == b"https":
tls_connection_creator = self._policy_for_https.creatorForNetloc(
parsed_uri.host, parsed_uri.port)
endpoint = wrapClientTLS(tls_connection_creator, endpoint)
elif parsed_uri.scheme == b"http":
pass
else:
return defer.fail(
Failure(
SchemeNotSupported("Unsupported scheme: %r" %
(parsed_uri.scheme, ))))
return self._requestWithEndpoint(pool_key, endpoint, method,
parsed_uri, headers, bodyProducer,
request_path)
def endpointForURI(self, uri):
if uri.scheme not in {b'http', b'https'}:
raise SchemeNotSupported("Unsupported scheme: %r" % (uri.scheme, ))
return TCP4ClientEndpoint(self.reactor, "127.0.0.1", uri.port)