def _getEndpoint(self, scheme, host, port):
if scheme not in ('http', 'https'):
raise SchemeNotSupported('unsupported scheme', scheme)
endpoint = self.endpointFactory(host, port, self.proxyEndpoint,
**self.endpointArgs)
if scheme == 'https':
endpoint = self._tlsWrapper(self._wrapContextFactory(host, port),
endpoint)
return endpoint
def _getEndpoint(self, scheme, host, port):
if scheme not in ('http', 'https'):
raise SchemeNotSupported('unsupported scheme', scheme)
endpoint = self.endpointFactory(
host, port, self.proxyEndpoint, **self.endpointArgs)
if scheme == 'https':
if hasattr(self, '_wrapContextFactory'):
tlsPolicy = self._wrapContextFactory(host, port)
elif hasattr(self, '_policyForHTTPS'):
tlsPolicy = self._policyForHTTPS.creatorForNetloc(host, port)
else:
raise NotImplementedError("can't figure out how to make a context factory")
endpoint = self._tlsWrapper(tlsPolicy, endpoint)
return endpoint
def _getEndpoint(self, scheme, host, port):
if scheme not in (b'http', b'https'):
raise SchemeNotSupported('unsupported scheme', scheme)
endpoint = self.endpointFactory(host, port, self.proxyEndpoint,
**self.endpointArgs)
if scheme == b'https':
if _twisted_12_1 <= twisted.version < _twisted_14_0:
tlsPolicy = self._wrapContextFactory(host, port)
elif _twisted_14_0 <= twisted.version:
tlsPolicy = self._policyForHTTPS.creatorForNetloc(host, port)
else:
raise NotImplementedError(
"can't figure out how to make a context factory")
endpoint = self._tlsWrapper(tlsPolicy, endpoint)
return endpoint
def endpointForURI(self, uri):
if uri.scheme not in (b'http', b'https'):
raise SchemeNotSupported('unsupported scheme', uri.scheme)
endpoint = TCP4ClientEndpoint(self.reactor,
host=uri.host.decode(),
port=uri.port,
bindAddress=self._bindAddress,
timeout=self._connectTimeout)
factory = self.endpointFactory(self.reactor, endpoint,
self.proxy_config)
if uri.scheme == b'https':
tlsPolicy = self._policyForHTTPS.creatorForNetloc(
uri.host, uri.port)
factory = self._tlsWrapper(tlsPolicy, factory)
return factory
def _getEndpoint(self, parsedURI, host=None, port=None):
try:
host, port = parsedURI.host, parsedURI.port
scheme = parsedURI.scheme
except AttributeError:
scheme = parsedURI
if scheme not in ('http', 'https'):
raise SchemeNotSupported('unsupported scheme', scheme)
endpoint = OnionRoutedTCPClientEndpoint(host, port, self.state,
self.path)
if scheme == 'https':
if hasattr(self, '_wrapContextFactory'):
tls_policy = self._wrapContextFactory(host, port)
elif hasattr(self, '_policyForHTTPS'):
tls_policy = self._policyForHTTPS.creatorForNetloc(host, port)
else:
raise NotImplementedError(
"can't figure out how to make a context factory")
endpoint = self._tlsWrapper(tls_policy, endpoint)
return endpoint
def _get_agent(self, request, timeout):
bindAddress = request.meta.get('bindaddress') or self._bindAddress
proxy = request.meta.get('proxy', '').lower()
timeout = request.meta.pop('proxy_timeout', timeout)
if proxy:
parsed_proxy = dsnparse3.parse(proxy)
if parsed_proxy.scheme in ('http', 'https'):
return super(ScrapyAgent, self)._get_agent(request, timeout)
if parsed_proxy.scheme not in ('socks4', 'socks4a', 'socks5',
'socks5h'):
raise SchemeNotSupported('unsupported scheme',
parsed_proxy.scheme)
return ProxyAgent(reactor=reactor,
proxy_scheme=parsed_proxy.scheme,
proxy_host=parsed_proxy.host,
proxy_port=parsed_proxy.port,
proxy_username=parsed_proxy.username,
proxy_password=parsed_proxy.password,
contextFactory=self._contextFactory,
bindAddress=bindAddress)
return super(ScrapyAgent, self)._get_agent(request, timeout)