def test_select_proxies(url, expected): """Make sure we can select per-host proxies correctly.""" proxies = { 'http': 'http://http.proxy', 'http://some.host': 'http://some.host.proxy' } assert select_proxy(url, proxies) == expected
def test_override_env_proxy_and_go_direct(self, monkeypatch): """Ensure that it's possible to ignore environment proxy settings for a request.""" monkeypatch.setenv('HTTP_PROXY', 'http://env') sess = requests.Session() settings = sess.merge_environment_settings(arbitrary_url, {'http': None}, False, False, False) assert settings['proxies'] == {} # requests.session.merge_setting removes entries with None value. assert not select_proxy(arbitrary_url, settings['proxies'])
def test_environment_proxies(self, monkeypatch, request_url, expected_proxies, expected_proxy_selection): """Test usage of proxy settings from environment variables, including host exclusions using `NO_PROXY`.""" monkeypatch.setenv('HTTP_PROXY', 'http://env') monkeypatch.setenv('NO_PROXY', 'a.local, .b.local') sess = requests.Session() settings = sess.merge_environment_settings(request_url, {}, False, False, False) assert settings['proxies'] == expected_proxies assert select_proxy(request_url, settings['proxies']) == expected_proxy_selection
def test_supercede_environment_settings(self, monkeypatch): monkeypatch.setenv('HTTP_PROXY', 'http://env') sess = requests.Session() settings = sess.merge_environment_settings(arbitrary_url, {'http': 'http://other'}, False, False, False) assert settings['proxies'] == {'http': 'http://other'} assert select_proxy(arbitrary_url, settings['proxies']) == 'http://other'
def test_override_env_proxy_and_go_direct(self, monkeypatch): """Ensure that it's possible to ignore environment proxy settings for a request.""" monkeypatch.setenv("HTTP_PROXY", "http://env") sess = requests.Session() settings = sess.merge_environment_settings(arbitrary_url, {"http": None}, False, False, False) settings["proxies"].pop("travis_apt", None) assert settings["proxies"] == { } # requests.session.merge_setting removes entries with None value. assert not select_proxy(arbitrary_url, settings["proxies"])
def test_environment_proxies(self, monkeypatch, request_url, expected_proxies, expected_proxy_selection): """Test usage of proxy settings from environment variables, including host exclusions using `NO_PROXY`.""" monkeypatch.setenv("HTTP_PROXY", "http://env") monkeypatch.setenv("NO_PROXY", "a.local, .b.local") sess = requests.Session() settings = sess.merge_environment_settings(request_url, {}, False, False, False) settings["proxies"].pop("travis_apt", None) assert settings["proxies"] == expected_proxies assert select_proxy(request_url, settings["proxies"]) == expected_proxy_selection
def _get_curl_connection(self, url, proxies=None): """Returns a new CURL connection to handle the request to a given URL. Args: url (str): the URL of the request being sent. proxies (dict, optional): A Requests-style dictionary of proxies used on this request. Returns: CURLConnectionPool: a connection pool that is capable of handling the given request. """ proxy_url = select_proxy(url, proxies) if proxy_url: pool = self._pool_provider.get_pool_for_proxied_url(proxy_url, url) else: pool = self._pool_provider.get_pool_for_url(url) return pool
def send(self, stream=False, timeout=None, verify=True, cert=None, proxies=None): request = self.request connect_timeout, self.read_timeout = parse_timeout(timeout) self.stream_body = stream # set connect timeout with stack_context.ExceptionStackContext(self._handle_exception): if connect_timeout: self._timeout = self.io_loop.call_later( connect_timeout, stack_context.wrap( functools.partial(self._on_timeout, 'while connecting'))) # set proxy related info proxy = select_proxy(request.url, proxies) self.headers = request.headers.copy() if proxy: proxy = prepend_scheme_if_needed(proxy, 'http') parsed = urlparse(proxy) scheme, host, port = parsed.scheme, proxy, parsed.port port = port or (443 if scheme == 'https' else 80) self.start_line = RequestStartLine(request.method, request.url, '') self.headers.update(get_proxy_headers(proxy)) else: host, port = None, None self.start_line = request.start_line self.tcp_client.connect(request.host, request.port, af=request.af, ssl_options=self._get_ssl_options( request, verify, cert), max_buffer_size=self.max_buffer_size, source_ip=host, source_port=port, callback=self._on_connect)
def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple :param verify: (optional) Whether to verify SSL certificates. :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. """ # setup proxy object proxy = select_proxy(request.url,proxies) if proxy: if not self._my_proxy_obj: proxy = proxy[proxy.find('//')+2:] host,port = proxy.split(':') self._my_proxy_obj = Proxy(Proxy.Type.HTTP,InetSocketAddress(str(host),int(port))) else: self._my_proxy_obj = None # build the request #print 'request.method',request.method -- ignored #print 'request.body',request.body -- ignored #url = self.request_url(request, proxies) -- ignored - just the params u = URL(request.url) conn = u.openConnection(self._my_proxy_obj) if self._my_proxy_obj else u.openConnection() conn.setAllowUserInteraction(False) conn.setDoInput(True); self.add_headers(request) for k,v in request.headers.iteritems(): conn.addRequestProperty(k,v) # make the request! try: conn.connect() except java.net.ConnectException, e: raise ConnectionError(e.getMessage())
def test_select_proxies(url, expected, proxies): """Make sure we can select per-host proxies correctly.""" assert select_proxy(url, proxies) == expected
def _send_side_effect(request, **kw): assert select_proxy(request.url, kw['proxies']) == '192.168.50.100:1234' raise ConnectionError()
def test_select_proxies(url, expected): """Make sure we can select per-host proxies correctly.""" proxies = {'http': 'http://http.proxy', 'http://some.host': 'http://some.host.proxy'} assert select_proxy(url, proxies) == expected
def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None): """Sends Request object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple :param verify: (optional) Whether to verify SSL certificates. :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: trip.adapters.MessageDelegate """ if isinstance(timeout, tuple): try: connect_timeout, read_timeout = timeout except ValueError as e: # this may raise a string formatting error. err = ("Invalid timeout {0}. Pass a (connect, read) " "timeout tuple, or a single float to set " "both timeouts to the same value".format(timeout)) raise ValueError(err) else: connect_timeout, read_timeout = timeout, timeout timeout_reason = {} if connect_timeout: timeout_reason['reason'] = 'while connecting' self.io_loop.add_timeout( self.io_loop.time() + connect_timeout, stack_context.wrap( functools.partial(self._on_timeout, timeout_reason))) proxy = select_proxy(request.url, proxies) if proxy: host, port = (proxy.split(':') + [80])[:2] port = int(port) start_line = RequestStartLine(request.method, request.url, '') else: host, port = None, None start_line = request.start_line s = yield self.tcp_client.connect(request.host, request.port, af=request.af, ssl_options=self._get_ssl_options( request, verify, cert), max_buffer_size=self.max_buffer_size, source_ip=host, source_port=port) if not timeout_reason or timeout_reason.get('reason'): s.set_nodelay(True) timeout_reason.clear() else: raise gen.Return( Timeout(timeout_reason.get('error', 'unknown'), request=request)) connection = HTTPConnection( s, HTTP1ConnectionParameters(no_keep_alive=True, max_header_size=self.max_header_size, max_body_size=self.max_body_size, decompress=request.decompress)) if read_timeout: timeout_reason['reason'] = 'during request' self.io_loop.add_timeout( self.io_loop.time() + connect_timeout, stack_context.wrap( functools.partial(self._on_timeout, timeout_reason))) connection.write_headers(start_line, request.headers) if request.body is not None: connection.write(request.body) #TODO: partial sending connection.finish() future = Future() def handle_response(response): if isinstance(response, Exception): future.set_exception(response) else: future.set_result(response) resp = MessageDelegate(request, connection, handle_response, stream) headers_received = yield connection.read_headers(resp) if not stream and headers_received: yield connection.read_body(resp) if not timeout_reason or timeout_reason.get('reason'): timeout_reason.clear() resp = yield future raise gen.Return(resp) else: raise gen.Return( Timeout(timeout_reason.get('error', 'unknown'), request=request))
def add_headers(self, request, **kwargs): auth = select_proxy(request.url, self.servers) if auth is not None: username, password = auth value = _basic_auth_str(username, password) request.headers.setdefault('Authorization', value)
def request_url(self, request, proxies): proxy = select_proxy(request.url, proxies) if (not proxy.lower().startswith('socks') and request.url.lower().startswith('https://')): return urldefragauth(request.url) return super().request_url(request, proxies)