def should_bypass_proxies(url, no_proxy=None): """ Yet another requests.should_bypass_proxies Test if proxies should not be used for a particular url. """ parsed = urlparse(url) # special cases if parsed.hostname in [None, '']: return True # special cases if no_proxy in [None, '']: return False if no_proxy == '*': return True no_proxy = no_proxy.lower().replace(' ', '') entries = (host for host in no_proxy.split(',') if host) if is_ipv4(parsed.hostname): for item in entries: if in_ipv4net(parsed.hostname, item): return True return proxy_bypass_environment(parsed.hostname, {'no': no_proxy})
async def _do_connect(self, protocol_factory: IProtocolFactory) -> IProtocol: first_exception = None server_list = await self._resolve_server() for server in server_list: host = server.host port = server.port should_skip_proxy = False if self.no_proxy is not None: should_skip_proxy = proxy_bypass_environment( host.decode(), proxies={"no": self.no_proxy}, ) endpoint: IStreamClientEndpoint try: if self._https_proxy_endpoint and not should_skip_proxy: logger.debug( "Connecting to %s:%i via %s", host.decode("ascii"), port, self._https_proxy_endpoint, ) endpoint = HTTPConnectProxyEndpoint( self._reactor, self._https_proxy_endpoint, host, port, proxy_creds=self._https_proxy_creds, ) else: logger.debug("Connecting to %s:%i", host.decode("ascii"), port) # not using a proxy endpoint = HostnameEndpoint(self._reactor, host, port) if self._tls_options: endpoint = wrapClientTLS(self._tls_options, endpoint) result = await make_deferred_yieldable( endpoint.connect(protocol_factory)) return result except Exception as e: logger.info("Failed to connect to %s:%i: %s", host.decode("ascii"), port, e) if not first_exception: first_exception = e # We return the first failure because that's probably the most interesting. if first_exception: raise first_exception # This shouldn't happen as we should always have at least one host/port # to try and if that doesn't work then we'll have an exception. raise Exception("Failed to resolve server %r" % (self._parsed_uri.netloc, ))
def request(self, method, uri, headers=None, bodyProducer=None): """ Issue a request to the server indicated by the given uri. Supports `http` and `https` schemes. An existing connection from the connection pool may be used or a new one may be created. See also: twisted.web.iweb.IAgent.request Args: method (bytes): The request method to use, such as `GET`, `POST`, etc uri (bytes): The location of the resource to request. headers (Headers|None): Extra headers to send with the request bodyProducer (IBodyProducer|None): An object which can generate bytes to make up the body of this request (for example, the properly encoded contents of a file for a file upload). Or, None if the request is to have no body. Returns: Deferred[IResponse]: completes when the header of the response has been received (regardless of the response status code). Can fail with: SchemeNotSupported: if the uri is not http or https twisted.internet.error.TimeoutError if the server we are connecting to (proxy or destination) does not accept a connection before connectTimeout. ... other things too. """ uri = uri.strip() if not _VALID_URI.match(uri): raise ValueError("Invalid URI {!r}".format(uri)) parsed_uri = URI.fromBytes(uri) pool_key = (parsed_uri.scheme, parsed_uri.host, parsed_uri.port) request_path = parsed_uri.originForm should_skip_proxy = False if self.no_proxy is not None: should_skip_proxy = proxy_bypass_environment( parsed_uri.host.decode(), proxies={"no": self.no_proxy}, ) if (parsed_uri.scheme == b"http" and self.http_proxy_endpoint and not should_skip_proxy): # Cache *all* connections under the same key, since we are only # connecting to a single destination, the proxy: pool_key = ("http-proxy", self.http_proxy_endpoint) endpoint = self.http_proxy_endpoint request_path = uri elif (parsed_uri.scheme == b"https" and self.https_proxy_endpoint and not should_skip_proxy): connect_headers = Headers() # Determine whether we need to set Proxy-Authorization headers if self.https_proxy_creds: # Set a Proxy-Authorization header connect_headers.addRawHeader( b"Proxy-Authorization", self.https_proxy_creds.as_proxy_authorization_value(), ) endpoint = HTTPConnectProxyEndpoint( self.proxy_reactor, self.https_proxy_endpoint, parsed_uri.host, parsed_uri.port, headers=connect_headers, ) else: # not using a proxy endpoint = HostnameEndpoint(self._reactor, parsed_uri.host, parsed_uri.port, **self._endpoint_kwargs) logger.debug("Requesting %s via %s", uri, endpoint) if parsed_uri.scheme == b"https": tls_connection_creator = self._policy_for_https.creatorForNetloc( parsed_uri.host, parsed_uri.port) endpoint = wrapClientTLS(tls_connection_creator, endpoint) elif parsed_uri.scheme == b"http": pass else: return defer.fail( Failure( SchemeNotSupported("Unsupported scheme: %r" % (parsed_uri.scheme, )))) return self._requestWithEndpoint(pool_key, endpoint, method, parsed_uri, headers, bodyProducer, request_path)