Example #1
0
def test_select_proxies(url, expected):
    """Make sure we can select per-host proxies correctly."""
    proxies = {
        'http': 'http://http.proxy',
        'http://some.host': 'http://some.host.proxy'
    }
    assert select_proxy(url, proxies) == expected
Example #2
0
 def test_override_env_proxy_and_go_direct(self, monkeypatch):
     """Ensure that it's possible to ignore environment proxy settings for a request."""
     monkeypatch.setenv('HTTP_PROXY', 'http://env')
     sess = requests.Session()
     settings = sess.merge_environment_settings(arbitrary_url, {'http': None}, False, False, False)
     assert settings['proxies'] == {}  # requests.session.merge_setting removes entries with None value.
     assert not select_proxy(arbitrary_url, settings['proxies'])
Example #3
0
 def test_override_env_proxy_and_go_direct(self, monkeypatch):
     """Ensure that it's possible to ignore environment proxy settings for a request."""
     monkeypatch.setenv('HTTP_PROXY', 'http://env')
     sess = requests.Session()
     settings = sess.merge_environment_settings(arbitrary_url, {'http': None}, False, False, False)
     assert settings['proxies'] == {}  # requests.session.merge_setting removes entries with None value.
     assert not select_proxy(arbitrary_url, settings['proxies'])
Example #4
0
 def test_environment_proxies(self, monkeypatch, request_url, expected_proxies, expected_proxy_selection):
     """Test usage of proxy settings from environment variables, including host exclusions using `NO_PROXY`."""
     monkeypatch.setenv('HTTP_PROXY', 'http://env')
     monkeypatch.setenv('NO_PROXY', 'a.local, .b.local')
     sess = requests.Session()
     settings = sess.merge_environment_settings(request_url, {}, False, False, False)
     assert settings['proxies'] == expected_proxies
     assert select_proxy(request_url, settings['proxies']) == expected_proxy_selection
Example #5
0
 def test_environment_proxies(self, monkeypatch, request_url, expected_proxies, expected_proxy_selection):
     """Test usage of proxy settings from environment variables, including host exclusions using `NO_PROXY`."""
     monkeypatch.setenv('HTTP_PROXY', 'http://env')
     monkeypatch.setenv('NO_PROXY', 'a.local, .b.local')
     sess = requests.Session()
     settings = sess.merge_environment_settings(request_url, {}, False, False, False)
     assert settings['proxies'] == expected_proxies
     assert select_proxy(request_url, settings['proxies']) == expected_proxy_selection
Example #6
0
 def test_supercede_environment_settings(self, monkeypatch):
     monkeypatch.setenv('HTTP_PROXY', 'http://env')
     sess = requests.Session()
     settings = sess.merge_environment_settings(arbitrary_url,
                                                {'http': 'http://other'},
                                                False, False, False)
     assert settings['proxies'] == {'http': 'http://other'}
     assert select_proxy(arbitrary_url,
                         settings['proxies']) == 'http://other'
Example #7
0
 def test_override_env_proxy_and_go_direct(self, monkeypatch):
     """Ensure that it's possible to ignore environment proxy settings for a request."""
     monkeypatch.setenv("HTTP_PROXY", "http://env")
     sess = requests.Session()
     settings = sess.merge_environment_settings(arbitrary_url,
                                                {"http": None}, False,
                                                False, False)
     settings["proxies"].pop("travis_apt", None)
     assert settings["proxies"] == {
     }  # requests.session.merge_setting removes entries with None value.
     assert not select_proxy(arbitrary_url, settings["proxies"])
Example #8
0
 def test_environment_proxies(self, monkeypatch, request_url,
                              expected_proxies, expected_proxy_selection):
     """Test usage of proxy settings from environment variables, including host exclusions using `NO_PROXY`."""
     monkeypatch.setenv("HTTP_PROXY", "http://env")
     monkeypatch.setenv("NO_PROXY", "a.local, .b.local")
     sess = requests.Session()
     settings = sess.merge_environment_settings(request_url, {}, False,
                                                False, False)
     settings["proxies"].pop("travis_apt", None)
     assert settings["proxies"] == expected_proxies
     assert select_proxy(request_url,
                         settings["proxies"]) == expected_proxy_selection
Example #9
0
    def _get_curl_connection(self, url, proxies=None):
        """Returns a new CURL connection to handle the request to a given URL.

        Args:
            url (str): the URL of the request being sent.
            proxies (dict, optional): A Requests-style dictionary of proxies used on this request.

        Returns:
            CURLConnectionPool: a connection pool that is capable of handling the given request.
        """
        proxy_url = select_proxy(url, proxies)

        if proxy_url:
            pool = self._pool_provider.get_pool_for_proxied_url(proxy_url, url)
        else:
            pool = self._pool_provider.get_pool_for_url(url)

        return pool
Example #10
0
    def send(self,
             stream=False,
             timeout=None,
             verify=True,
             cert=None,
             proxies=None):
        request = self.request
        connect_timeout, self.read_timeout = parse_timeout(timeout)
        self.stream_body = stream

        # set connect timeout
        with stack_context.ExceptionStackContext(self._handle_exception):
            if connect_timeout:
                self._timeout = self.io_loop.call_later(
                    connect_timeout,
                    stack_context.wrap(
                        functools.partial(self._on_timeout,
                                          'while connecting')))

            # set proxy related info
            proxy = select_proxy(request.url, proxies)
            self.headers = request.headers.copy()
            if proxy:
                proxy = prepend_scheme_if_needed(proxy, 'http')
                parsed = urlparse(proxy)
                scheme, host, port = parsed.scheme, proxy, parsed.port
                port = port or (443 if scheme == 'https' else 80)
                self.start_line = RequestStartLine(request.method, request.url,
                                                   '')
                self.headers.update(get_proxy_headers(proxy))
            else:
                host, port = None, None
                self.start_line = request.start_line

            self.tcp_client.connect(request.host,
                                    request.port,
                                    af=request.af,
                                    ssl_options=self._get_ssl_options(
                                        request, verify, cert),
                                    max_buffer_size=self.max_buffer_size,
                                    source_ip=host,
                                    source_port=port,
                                    callback=self._on_connect)
Example #11
0
    def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None):
        """Sends PreparedRequest object. Returns Response object.

        :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
        :param stream: (optional) Whether to stream the request content.
        :param timeout: (optional) How long to wait for the server to send
            data before giving up, as a float, or a :ref:`(connect timeout,
            read timeout) <timeouts>` tuple.
        :type timeout: float or tuple
        :param verify: (optional) Whether to verify SSL certificates.
        :param cert: (optional) Any user-provided SSL certificate to be trusted.
        :param proxies: (optional) The proxies dictionary to apply to the request.
        """

        # setup proxy object
        proxy = select_proxy(request.url,proxies)
        if proxy:
            if not self._my_proxy_obj:
                proxy = proxy[proxy.find('//')+2:]
                host,port = proxy.split(':')
                self._my_proxy_obj = Proxy(Proxy.Type.HTTP,InetSocketAddress(str(host),int(port)))
        else:
            self._my_proxy_obj = None

        # build the request
        #print 'request.method',request.method -- ignored
        #print 'request.body',request.body -- ignored
        #url = self.request_url(request, proxies) -- ignored - just the params

        u = URL(request.url)
        conn = u.openConnection(self._my_proxy_obj) if self._my_proxy_obj else u.openConnection()
        conn.setAllowUserInteraction(False)
        conn.setDoInput(True);

        self.add_headers(request)
        for k,v in request.headers.iteritems():
            conn.addRequestProperty(k,v)

        # make the request!
        try: conn.connect()
        except java.net.ConnectException, e:
            raise ConnectionError(e.getMessage())
Example #12
0
def test_select_proxies(url, expected, proxies):
    """Make sure we can select per-host proxies correctly."""
    assert select_proxy(url, proxies) == expected
Example #13
0
 def _send_side_effect(request, **kw):
     assert select_proxy(request.url,
                         kw['proxies']) == '192.168.50.100:1234'
     raise ConnectionError()
Example #14
0
def test_select_proxies(url, expected, proxies):
    """Make sure we can select per-host proxies correctly."""
    assert select_proxy(url, proxies) == expected
Example #15
0
def test_select_proxies(url, expected):
    """Make sure we can select per-host proxies correctly."""
    proxies = {'http': 'http://http.proxy',
               'http://some.host': 'http://some.host.proxy'}
    assert select_proxy(url, proxies) == expected
Example #16
0
    def send(self,
             request,
             stream=False,
             timeout=None,
             verify=True,
             cert=None,
             proxies=None):
        """Sends Request object. Returns Response object.

        :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
        :param stream: (optional) Whether to stream the request content.
        :param timeout: (optional) How long to wait for the server to send
            data before giving up, as a float, or a :ref:`(connect timeout,
            read timeout) <timeouts>` tuple.
        :type timeout: float or tuple
        :param verify: (optional) Whether to verify SSL certificates.
        :param cert: (optional) Any user-provided SSL certificate to be trusted.
        :param proxies: (optional) The proxies dictionary to apply to the request.
        :rtype: trip.adapters.MessageDelegate
        """
        if isinstance(timeout, tuple):
            try:
                connect_timeout, read_timeout = timeout
            except ValueError as e:
                # this may raise a string formatting error.
                err = ("Invalid timeout {0}. Pass a (connect, read) "
                       "timeout tuple, or a single float to set "
                       "both timeouts to the same value".format(timeout))
                raise ValueError(err)
        else:
            connect_timeout, read_timeout = timeout, timeout

        timeout_reason = {}
        if connect_timeout:
            timeout_reason['reason'] = 'while connecting'
            self.io_loop.add_timeout(
                self.io_loop.time() + connect_timeout,
                stack_context.wrap(
                    functools.partial(self._on_timeout, timeout_reason)))

        proxy = select_proxy(request.url, proxies)
        if proxy:
            host, port = (proxy.split(':') + [80])[:2]
            port = int(port)
            start_line = RequestStartLine(request.method, request.url, '')
        else:
            host, port = None, None
            start_line = request.start_line

        s = yield self.tcp_client.connect(request.host,
                                          request.port,
                                          af=request.af,
                                          ssl_options=self._get_ssl_options(
                                              request, verify, cert),
                                          max_buffer_size=self.max_buffer_size,
                                          source_ip=host,
                                          source_port=port)

        if not timeout_reason or timeout_reason.get('reason'):
            s.set_nodelay(True)
            timeout_reason.clear()
        else:
            raise gen.Return(
                Timeout(timeout_reason.get('error', 'unknown'),
                        request=request))

        connection = HTTPConnection(
            s,
            HTTP1ConnectionParameters(no_keep_alive=True,
                                      max_header_size=self.max_header_size,
                                      max_body_size=self.max_body_size,
                                      decompress=request.decompress))

        if read_timeout:
            timeout_reason['reason'] = 'during request'
            self.io_loop.add_timeout(
                self.io_loop.time() + connect_timeout,
                stack_context.wrap(
                    functools.partial(self._on_timeout, timeout_reason)))

        connection.write_headers(start_line, request.headers)
        if request.body is not None:
            connection.write(request.body)  #TODO: partial sending
        connection.finish()

        future = Future()

        def handle_response(response):
            if isinstance(response, Exception):
                future.set_exception(response)
            else:
                future.set_result(response)

        resp = MessageDelegate(request, connection, handle_response, stream)

        headers_received = yield connection.read_headers(resp)

        if not stream and headers_received:
            yield connection.read_body(resp)

        if not timeout_reason or timeout_reason.get('reason'):
            timeout_reason.clear()
            resp = yield future
            raise gen.Return(resp)
        else:
            raise gen.Return(
                Timeout(timeout_reason.get('error', 'unknown'),
                        request=request))
Example #17
0
 def add_headers(self, request, **kwargs):
     auth = select_proxy(request.url, self.servers)
     if auth is not None:
         username, password = auth
         value = _basic_auth_str(username, password)
         request.headers.setdefault('Authorization', value)
Example #18
0
 def test_supercede_environment_settings(self, monkeypatch):
     monkeypatch.setenv('HTTP_PROXY', 'http://env')
     sess = requests.Session()
     settings = sess.merge_environment_settings(arbitrary_url, {'http': 'http://other'}, False, False, False)
     assert settings['proxies'] == {'http': 'http://other'}
     assert select_proxy(arbitrary_url, settings['proxies']) == 'http://other'
Example #19
0
 def request_url(self, request, proxies):
     proxy = select_proxy(request.url, proxies)
     if (not proxy.lower().startswith('socks') and
        request.url.lower().startswith('https://')):
         return urldefragauth(request.url)
     return super().request_url(request, proxies)