Beispiel #1
0
class HttpRequest(RequestBase):
    '''An :class:`HttpClient` request for an HTTP resource.

    This class has a similar interface to :class:`urllib.request.Request`.

    :param files: optional dictionary of name, file-like-objects.
    :param allow_redirects: allow the response to follow redirects.

    .. attribute:: method

        The request method

    .. attribute:: version

        HTTP version for this request, usually ``HTTP/1.1``

    .. attribute:: encode_multipart

        If ``True`` (default), defaults POST data as ``multipart/form-data``.
        Pass ``encode_multipart=False`` to default to
        ``application/x-www-form-urlencoded``. In any case, this parameter is
        overwritten by passing the correct content-type header.

    .. attribute:: history

        List of past :class:`.HttpResponse` (collected during redirects).

    .. attribute:: wait_continue

        if ``True``, the :class:`HttpRequest` includes the
        ``Expect: 100-Continue`` header.

    '''
    CONNECT = 'CONNECT'
    _proxy = None
    _ssl = None
    _tunnel = None

    def __init__(self,
                 client,
                 url,
                 method,
                 inp_params=None,
                 headers=None,
                 data=None,
                 files=None,
                 history=None,
                 charset=None,
                 encode_multipart=True,
                 multipart_boundary=None,
                 source_address=None,
                 allow_redirects=False,
                 max_redirects=10,
                 decompress=True,
                 version=None,
                 wait_continue=False,
                 websocket_handler=None,
                 cookies=None,
                 urlparams=None,
                 **ignored):
        self.client = client
        self._data = None
        self.files = files
        self.urlparams = urlparams
        self.inp_params = inp_params or {}
        self.unredirected_headers = Headers(kind='client')
        self.method = method.upper()
        self.full_url = url
        if urlparams:
            self._encode_url(urlparams)
        self.set_proxy(None)
        self.history = history
        self.wait_continue = wait_continue
        self.max_redirects = max_redirects
        self.allow_redirects = allow_redirects
        self.charset = charset or 'utf-8'
        self.version = version
        self.decompress = decompress
        self.encode_multipart = encode_multipart
        self.multipart_boundary = multipart_boundary
        self.websocket_handler = websocket_handler
        self.source_address = source_address
        self.new_parser()
        if self._scheme in tls_schemes:
            self._ssl = client.ssl_context(**ignored)
        self.headers = client.get_headers(self, headers)
        cookies = cookiejar_from_dict(client.cookies, cookies)
        if cookies:
            cookies.add_cookie_header(self)
        self.unredirected_headers['host'] = host_no_default_port(
            self._scheme, self._netloc)
        client.set_proxy(self)
        self.data = data

    @property
    def address(self):
        '''``(host, port)`` tuple of the HTTP resource'''
        return self._tunnel.address if self._tunnel else (self.host, self.port)

    @property
    def target_address(self):
        return (self.host, int(self.port))

    @property
    def ssl(self):
        '''Context for TLS connections.

        If this is a tunneled request and the tunnel connection is not yet
        established, it returns ``None``.
        '''
        if not self._tunnel:
            return self._ssl

    @property
    def key(self):
        return (self.scheme, self.host, self.port)

    @property
    def proxy(self):
        '''Proxy server for this request.'''
        return self._proxy

    @property
    def netloc(self):
        if self._proxy:
            return self._proxy.netloc
        else:
            return self._netloc

    def __repr__(self):
        return self.first_line()

    __str__ = __repr__

    @property
    def full_url(self):
        '''Full url of endpoint'''
        return urlunparse((self._scheme, self._netloc, self.path, self.params,
                           self.query, self.fragment))

    @full_url.setter
    def full_url(self, url):
        self._scheme, self._netloc, self.path, self.params,\
            self.query, self.fragment = urlparse(url)
        if not self._netloc and self.method == 'CONNECT':
            self._scheme, self._netloc, self.path, self.params,\
                self.query, self.fragment = urlparse('http://%s' % url)

    @property
    def data(self):
        '''Body of request'''
        return self._data

    @data.setter
    def data(self, data):
        self._data = self._encode_data(data)

    def first_line(self):
        if not self._proxy and self.method != self.CONNECT:
            url = urlunparse(('', '', self.path
                              or '/', self.params, self.query, self.fragment))
        else:
            url = self.full_url
        return '%s %s %s' % (self.method, url, self.version)

    def new_parser(self):
        self.parser = self.client.http_parser(kind=1,
                                              decompress=self.decompress,
                                              method=self.method)

    def set_proxy(self, scheme, *host):
        if not host and scheme is None:
            self.scheme = self._scheme
            self._set_hostport(self._scheme, self._netloc)
        else:
            le = 2 + len(host)
            if not le == 3:
                raise TypeError(
                    'set_proxy() takes exactly three arguments (%s given)' %
                    le)
            if not self._ssl:
                self.scheme = scheme
                self._set_hostport(scheme, host[0])
                self._proxy = scheme_host(scheme, host[0])
            else:
                self._tunnel = HttpTunnel(self, scheme, host[0])

    def _set_hostport(self, scheme, host):
        self._tunnel = None
        self._proxy = None
        self.host, self.port = get_hostport(scheme, host)

    def encode(self):
        '''The bytes representation of this :class:`HttpRequest`.

        Called by :class:`HttpResponse` when it needs to encode this
        :class:`HttpRequest` before sending it to the HTTP resource.
        '''
        # Call body before fist_line in case the query is changes.
        first_line = self.first_line()
        body = self.data
        if body and self.wait_continue:
            self.headers['expect'] = '100-continue'
            body = None
        headers = self.headers
        if self.unredirected_headers:
            headers = self.unredirected_headers.copy()
            headers.update(self.headers)
        buffer = [first_line.encode('ascii'), b'\r\n', bytes(headers)]
        if body:
            buffer.append(body)
        return b''.join(buffer)

    def add_header(self, key, value):
        self.headers[key] = value

    def has_header(self, header_name):
        '''Check ``header_name`` is in this request headers.
        '''
        return (header_name in self.headers
                or header_name in self.unredirected_headers)

    def get_header(self, header_name, default=None):
        '''Retrieve ``header_name`` from this request headers.
        '''
        return self.headers.get(
            header_name, self.unredirected_headers.get(header_name, default))

    def remove_header(self, header_name):
        '''Remove ``header_name`` from this request.
        '''
        self.headers.pop(header_name, None)
        self.unredirected_headers.pop(header_name, None)

    def add_unredirected_header(self, header_name, header_value):
        self.unredirected_headers[header_name] = header_value

    # INTERNAL ENCODING METHODS
    def _encode_data(self, data):
        body = None
        if self.method in ENCODE_URL_METHODS:
            self.files = None
            self._encode_url(data)
        elif isinstance(data, bytes):
            assert self.files is None, ('data cannot be bytes when files are '
                                        'present')
            body = data
        elif isinstance(data, str):
            assert self.files is None, ('data cannot be string when files are '
                                        'present')
            body = to_bytes(data, self.charset)
        elif data or self.files:
            if self.files:
                body, content_type = self._encode_files(data)
            else:
                body, content_type = self._encode_params(data)
            # set files to None, Important!
            self.files = None
            self.headers['Content-Type'] = content_type
        if body:
            self.headers['content-length'] = str(len(body))
        elif 'expect' not in self.headers:
            self.headers.pop('content-length', None)
            self.headers.pop('content-type', None)
        return body

    def _encode_url(self, data):
        query = self.query
        if data:
            data = native_str(data)
            if isinstance(data, str):
                data = parse_qsl(data)
            else:
                data = mapping_iterator(data)
            query = parse_qsl(query)
            query.extend(data)
            query = urlencode(query)
        self.query = query

    def _encode_files(self, data):
        fields = []
        for field, val in mapping_iterator(data or ()):
            if (isinstance(val, str) or isinstance(val, bytes)
                    or not hasattr(val, '__iter__')):
                val = [val]
            for v in val:
                if v is not None:
                    if not isinstance(v, bytes):
                        v = str(v)
                    fields.append(
                        (field.decode('utf-8')
                         if isinstance(field, bytes) else field,
                         v.encode('utf-8') if isinstance(v, str) else v))
        for (k, v) in mapping_iterator(self.files):
            # support for explicit filename
            ft = None
            if isinstance(v, (tuple, list)):
                if len(v) == 2:
                    fn, fp = v
                else:
                    fn, fp, ft = v
            else:
                fn = guess_filename(v) or k
                fp = v
            if isinstance(fp, bytes):
                fp = BytesIO(fp)
            elif isinstance(fp, str):
                fp = StringIO(fp)
            if ft:
                new_v = (fn, fp.read(), ft)
            else:
                new_v = (fn, fp.read())
            fields.append((k, new_v))
        #
        return encode_multipart_formdata(fields, charset=self.charset)

    def _encode_params(self, data):
        content_type = self.headers.get('content-type')
        # No content type given, chose one
        if not content_type:
            if self.encode_multipart:
                content_type = MULTIPART_FORM_DATA
            else:
                content_type = FORM_URL_ENCODED

        if content_type in JSON_CONTENT_TYPES:
            body = json.dumps(data).encode(self.charset)
        elif content_type == FORM_URL_ENCODED:
            body = urlencode(data).encode(self.charset)
        elif content_type == MULTIPART_FORM_DATA:
            body, content_type = encode_multipart_formdata(
                data, boundary=self.multipart_boundary, charset=self.charset)
        else:
            raise ValueError("Don't know how to encode body for %s" %
                             content_type)
        return body, content_type
Beispiel #2
0
class HttpRequest(RequestBase):
    """An :class:`HttpClient` request for an HTTP resource.

    This class has a similar interface to :class:`urllib.request.Request`.

    :param files: optional dictionary of name, file-like-objects.
    :param allow_redirects: allow the response to follow redirects.

    .. attribute:: method

        The request method

    .. attribute:: version

        HTTP version for this request, usually ``HTTP/1.1``

    .. attribute:: history

        List of past :class:`.HttpResponse` (collected during redirects).

    .. attribute:: wait_continue

        if ``True``, the :class:`HttpRequest` includes the
        ``Expect: 100-Continue`` header.

    .. attribute:: stream

        Allow for streaming body

    """
    _proxy = None
    _ssl = None
    _tunnel = None
    _write_done = False

    def __init__(self, client, url, method, inp_params=None, headers=None,
                 data=None, files=None, json=None, history=None, auth=None,
                 charset=None, max_redirects=10, source_address=None,
                 allow_redirects=False, decompress=True, version=None,
                 wait_continue=False, websocket_handler=None, cookies=None,
                 params=None, stream=False, proxies=None, verify=True,
                 **ignored):
        self.client = client
        self.method = method.upper()
        self.inp_params = inp_params or {}
        self.unredirected_headers = Headers()
        self.history = history
        self.wait_continue = wait_continue
        self.max_redirects = max_redirects
        self.allow_redirects = allow_redirects
        self.charset = charset or 'utf-8'
        self.version = version
        self.decompress = decompress
        self.websocket_handler = websocket_handler
        self.source_address = source_address
        self.stream = stream
        self.verify = verify
        self.new_parser()
        if auth and not isinstance(auth, Auth):
            auth = HTTPBasicAuth(*auth)
        self.auth = auth
        self.headers = client._get_headers(headers)
        self.url = full_url(url, params, method=self.method)
        self.body = self._encode_body(data, files, json)
        self._set_proxy(proxies, ignored)
        cookies = cookiejar_from_dict(client.cookies, cookies)
        if cookies:
            cookies.add_cookie_header(self)

    @property
    def _loop(self):
        return self.client._loop

    @property
    def address(self):
        """``(host, port)`` tuple of the HTTP resource
        """
        return self._tunnel.address if self._tunnel else super().address

    @property
    def ssl(self):
        """Context for TLS connections.

        If this is a tunneled request and the tunnel connection is not yet
        established, it returns ``None``.
        """
        if not self._tunnel:
            return self._ssl

    @property
    def key(self):
        tunnel = self._tunnel.url if self._tunnel else None
        scheme, host, port = scheme_host_port(self._proxy or self.url)
        return scheme, host, port, tunnel, self.verify

    @property
    def proxy(self):
        """Proxy server for this request.
        """
        return self._proxy

    @property
    def tunnel(self):
        """Tunnel for this request.
        """
        return self._tunnel

    def __repr__(self):
        return self.first_line()
    __str__ = __repr__

    def first_line(self):
        p = urlparse(self.url)
        if self._proxy:
            if self.method == 'CONNECT':
                url = p.netloc
            else:
                url = self.url
        else:
            url = urlunparse(('', '', p.path or '/', p.params,
                              p.query, p.fragment))
        return '%s %s %s' % (self.method, url, self.version)

    def new_parser(self):
        self.parser = self.client.http_parser(
            kind=1, decompress=self.decompress
        )
        return self.parser

    def is_chunked(self):
        return self.body and 'content-length' not in self.headers

    def encode(self):
        """The bytes representation of this :class:`HttpRequest`.

        Called by :class:`HttpResponse` when it needs to encode this
        :class:`HttpRequest` before sending it to the HTTP resource.
        """
        # Call body before fist_line in case the query is changes.
        first_line = self.first_line()
        if self.body and self.wait_continue:
            self.headers['expect'] = '100-continue'
        headers = self.headers
        if self.unredirected_headers:
            headers = self.unredirected_headers.copy()
            headers.update(self.headers)
        buffer = [first_line.encode('ascii'), b'\r\n',  bytes(headers)]
        return b''.join(buffer)

    def add_header(self, key, value):
        self.headers[key] = value

    def has_header(self, header_name):
        """Check ``header_name`` is in this request headers.
        """
        return (header_name in self.headers or
                header_name in self.unredirected_headers)

    def get_header(self, header_name, default=None):
        """Retrieve ``header_name`` from this request headers.
        """
        return self.headers.get(
            header_name, self.unredirected_headers.get(header_name, default))

    def remove_header(self, header_name):
        """Remove ``header_name`` from this request.
        """
        val1 = self.headers.pop(header_name, None)
        val2 = self.unredirected_headers.pop(header_name, None)
        return val1 or val2

    def add_unredirected_header(self, header_name, header_value):
        self.unredirected_headers[header_name] = header_value

    def write_body(self, transport):
        assert not self._write_done, 'Body already sent'
        self._write_done = True
        if not self.body:
            return
        if is_streamed(self.body):
            ensure_future(self._write_streamed_data(transport),
                          loop=self._loop)
        else:
            self._write_body_data(transport, self.body, True)

    # INTERNAL ENCODING METHODS
    def _encode_body(self, data, files, json):
        body = None
        if isinstance(data, (str, bytes)):
            if files:
                raise ValueError('data cannot be a string or bytes when '
                                 'files are present')
            body = to_bytes(data, self.charset)
        elif data and is_streamed(data):
            if files:
                raise ValueError('data cannot be an iterator when '
                                 'files are present')
            if 'content-length' not in self.headers:
                self.headers['transfer-encoding'] = 'chunked'
            return data
        elif data or files:
            if files:
                body, content_type = self._encode_files(data, files)
            else:
                body, content_type = self._encode_params(data)
            self.headers['Content-Type'] = content_type
        elif json:
            body = _json.dumps(json).encode(self.charset)
            self.headers['Content-Type'] = 'application/json'

        if body:
            self.headers['content-length'] = str(len(body))

        return body

    def _encode_files(self, data, files):
        fields = []
        for field, val in mapping_iterator(data or ()):
            if (isinstance(val, str) or isinstance(val, bytes) or
                    not hasattr(val, '__iter__')):
                val = [val]
            for v in val:
                if v is not None:
                    if not isinstance(v, bytes):
                        v = str(v)
                    fields.append((field.decode('utf-8') if
                                   isinstance(field, bytes) else field,
                                   v.encode('utf-8') if isinstance(v, str)
                                   else v))
        for (k, v) in mapping_iterator(files):
            # support for explicit filename
            ft = None
            if isinstance(v, (tuple, list)):
                if len(v) == 2:
                    fn, fp = v
                else:
                    fn, fp, ft = v
            else:
                fn = guess_filename(v) or k
                fp = v
            if isinstance(fp, bytes):
                fp = BytesIO(fp)
            elif isinstance(fp, str):
                fp = StringIO(fp)
            if ft:
                new_v = (fn, fp.read(), ft)
            else:
                new_v = (fn, fp.read())
            fields.append((k, new_v))
        #
        return encode_multipart_formdata(fields, charset=self.charset)

    def _encode_params(self, params):
        content_type = self.headers.get('content-type')
        # No content type given, chose one
        if not content_type:
            content_type = FORM_URL_ENCODED

        if hasattr(params, 'read'):
            params = params.read()

        if content_type in JSON_CONTENT_TYPES:
            body = _json.dumps(params)
        elif content_type == FORM_URL_ENCODED:
            body = urlencode(tuple(split_url_params(params)))
        elif content_type == MULTIPART_FORM_DATA:
            body, content_type = encode_multipart_formdata(
                params, charset=self.charset)
        else:
            body = params
        return to_bytes(body, self.charset), content_type

    def _write_body_data(self, transport, data, finish=False):
        if self.is_chunked():
            data = http_chunks(data, finish)
        elif data:
            data = (data,)
        else:
            return
        for chunk in data:
            transport.write(chunk)

    async def _write_streamed_data(self, transport):
        for data in self.body:
            if isawaitable(data):
                data = await data
            self._write_body_data(transport, data)
        self._write_body_data(transport, b'', True)

    # PROXY INTERNALS
    def _set_proxy(self, proxies, ignored):
        url = urlparse(self.url)
        self.unredirected_headers['host'] = host_no_default_port(url.scheme,
                                                                 url.netloc)
        if url.scheme in tls_schemes:
            self._ssl = self.client._ssl_context(verify=self.verify, **ignored)

        request_proxies = self.client.proxies.copy()
        if proxies:
            request_proxies.update(proxies)
        self.proxies = request_proxies
        #
        if url.scheme in request_proxies:
            host, port = get_hostport(url.scheme, url.netloc)
            no_proxy = [n for n in request_proxies.get('no', '').split(',')
                        if n]
            if not any(map(host.endswith, no_proxy)):
                url = request_proxies[url.scheme]
                if not self._ssl:
                    self._proxy = url
                else:
                    self._tunnel = HttpTunnel(self, url)
Beispiel #3
0
class HttpRequest(RequestBase):
    '''An :class:`HttpClient` request for an HTTP resource.

    This class has a similar interface to :class:`urllib.request.Request`.

    :param files: optional dictionary of name, file-like-objects.
    :param allow_redirects: allow the response to follow redirects.

    .. attribute:: method

        The request method

    .. attribute:: version

        HTTP version for this request, usually ``HTTP/1.1``

    .. attribute:: encode_multipart

        If ``True`` (default), defaults POST data as ``multipart/form-data``.
        Pass ``encode_multipart=False`` to default to
        ``application/x-www-form-urlencoded``. In any case, this parameter is
        overwritten by passing the correct content-type header.

    .. attribute:: history

        List of past :class:`.HttpResponse` (collected during redirects).

    .. attribute:: wait_continue

        if ``True``, the :class:`HttpRequest` includes the
        ``Expect: 100-Continue`` header.

    '''
    CONNECT = 'CONNECT'
    _proxy = None
    _ssl = None
    _tunnel = None

    def __init__(self, client, url, method, inp_params=None, headers=None,
                 data=None, files=None, history=None,
                 charset=None, encode_multipart=True, multipart_boundary=None,
                 source_address=None, allow_redirects=False, max_redirects=10,
                 decompress=True, version=None, wait_continue=False,
                 websocket_handler=None, cookies=None, urlparams=None,
                 **ignored):
        self.client = client
        self._data = None
        self.files = files
        self.urlparams = urlparams
        self.inp_params = inp_params or {}
        self.unredirected_headers = Headers(kind='client')
        self.method = method.upper()
        self.full_url = url
        if urlparams:
            self._encode_url(urlparams)
        self.set_proxy(None)
        self.history = history
        self.wait_continue = wait_continue
        self.max_redirects = max_redirects
        self.allow_redirects = allow_redirects
        self.charset = charset or 'utf-8'
        self.version = version
        self.decompress = decompress
        self.encode_multipart = encode_multipart
        self.multipart_boundary = multipart_boundary
        self.websocket_handler = websocket_handler
        self.source_address = source_address
        self.new_parser()
        if self._scheme in tls_schemes:
            self._ssl = client.ssl_context(**ignored)
        self.headers = client.get_headers(self, headers)
        cookies = cookiejar_from_dict(client.cookies, cookies)
        if cookies:
            cookies.add_cookie_header(self)
        self.unredirected_headers['host'] = host_no_default_port(self._scheme,
                                                                 self._netloc)
        client.set_proxy(self)
        self.data = data

    @property
    def address(self):
        '''``(host, port)`` tuple of the HTTP resource'''
        return self._tunnel.address if self._tunnel else (self.host, self.port)

    @property
    def target_address(self):
        return (self.host, int(self.port))

    @property
    def ssl(self):
        '''Context for TLS connections.

        If this is a tunneled request and the tunnel connection is not yet
        established, it returns ``None``.
        '''
        if not self._tunnel:
            return self._ssl

    @property
    def key(self):
        return (self.scheme, self.host, self.port)

    @property
    def proxy(self):
        '''Proxy server for this request.'''
        return self._proxy

    @property
    def netloc(self):
        if self._proxy:
            return self._proxy.netloc
        else:
            return self._netloc

    def __repr__(self):
        return self.first_line()
    __str__ = __repr__

    @property
    def full_url(self):
        '''Full url of endpoint'''
        return urlunparse((self._scheme, self._netloc, self.path,
                           self.params, self.query, self.fragment))

    @full_url.setter
    def full_url(self, url):
        self._scheme, self._netloc, self.path, self.params,\
            self.query, self.fragment = urlparse(url)
        if not self._netloc and self.method == 'CONNECT':
            self._scheme, self._netloc, self.path, self.params,\
                self.query, self.fragment = urlparse('http://%s' % url)

    @property
    def data(self):
        '''Body of request'''
        return self._data

    @data.setter
    def data(self, data):
        self._data = self._encode_data(data)

    def first_line(self):
        if not self._proxy and self.method != self.CONNECT:
            url = urlunparse(('', '', self.path or '/', self.params,
                              self.query, self.fragment))
        else:
            url = self.full_url
        return '%s %s %s' % (self.method, url, self.version)

    def new_parser(self):
        self.parser = self.client.http_parser(kind=1,
                                              decompress=self.decompress,
                                              method=self.method)

    def set_proxy(self, scheme, *host):
        if not host and scheme is None:
            self.scheme = self._scheme
            self._set_hostport(self._scheme, self._netloc)
        else:
            le = 2 + len(host)
            if not le == 3:
                raise TypeError(
                    'set_proxy() takes exactly three arguments (%s given)'
                    % le)
            if not self._ssl:
                self.scheme = scheme
                self._set_hostport(scheme, host[0])
                self._proxy = scheme_host(scheme, host[0])
            else:
                self._tunnel = HttpTunnel(self, scheme, host[0])

    def _set_hostport(self, scheme, host):
        self._tunnel = None
        self._proxy = None
        self.host, self.port = get_hostport(scheme, host)

    def encode(self):
        '''The bytes representation of this :class:`HttpRequest`.

        Called by :class:`HttpResponse` when it needs to encode this
        :class:`HttpRequest` before sending it to the HTTP resource.
        '''
        # Call body before fist_line in case the query is changes.
        first_line = self.first_line()
        body = self.data
        if body and self.wait_continue:
            self.headers['expect'] = '100-continue'
            body = None
        headers = self.headers
        if self.unredirected_headers:
            headers = self.unredirected_headers.copy()
            headers.update(self.headers)
        buffer = [first_line.encode('ascii'), b'\r\n',  bytes(headers)]
        if body:
            buffer.append(body)
        return b''.join(buffer)

    def add_header(self, key, value):
        self.headers[key] = value

    def has_header(self, header_name):
        '''Check ``header_name`` is in this request headers.
        '''
        return (header_name in self.headers or
                header_name in self.unredirected_headers)

    def get_header(self, header_name, default=None):
        '''Retrieve ``header_name`` from this request headers.
        '''
        return self.headers.get(
            header_name, self.unredirected_headers.get(header_name, default))

    def remove_header(self, header_name):
        '''Remove ``header_name`` from this request.
        '''
        self.headers.pop(header_name, None)
        self.unredirected_headers.pop(header_name, None)

    def add_unredirected_header(self, header_name, header_value):
        self.unredirected_headers[header_name] = header_value

    # INTERNAL ENCODING METHODS
    def _encode_data(self, data):
        body = None
        if self.method in ENCODE_URL_METHODS:
            self.files = None
            self._encode_url(data)
        elif isinstance(data, bytes):
            assert self.files is None, ('data cannot be bytes when files are '
                                        'present')
            body = data
        elif isinstance(data, str):
            assert self.files is None, ('data cannot be string when files are '
                                        'present')
            body = to_bytes(data, self.charset)
        elif data or self.files:
            if self.files:
                body, content_type = self._encode_files(data)
            else:
                body, content_type = self._encode_params(data)
            # set files to None, Important!
            self.files = None
            self.headers['Content-Type'] = content_type
        if body:
            self.headers['content-length'] = str(len(body))
        elif 'expect' not in self.headers:
            self.headers.pop('content-length', None)
            self.headers.pop('content-type', None)
        return body

    def _encode_url(self, data):
        query = self.query
        if data:
            data = native_str(data)
            if isinstance(data, str):
                data = parse_qsl(data)
            else:
                data = mapping_iterator(data)
            query = parse_qsl(query)
            query.extend(data)
            query = urlencode(query)
        self.query = query

    def _encode_files(self, data):
        fields = []
        for field, val in mapping_iterator(data or ()):
            if (isinstance(val, str) or isinstance(val, bytes) or
                    not hasattr(val, '__iter__')):
                val = [val]
            for v in val:
                if v is not None:
                    if not isinstance(v, bytes):
                        v = str(v)
                    fields.append((field.decode('utf-8') if
                                   isinstance(field, bytes) else field,
                                   v.encode('utf-8') if isinstance(v, str)
                                   else v))
        for (k, v) in mapping_iterator(self.files):
            # support for explicit filename
            ft = None
            if isinstance(v, (tuple, list)):
                if len(v) == 2:
                    fn, fp = v
                else:
                    fn, fp, ft = v
            else:
                fn = guess_filename(v) or k
                fp = v
            if isinstance(fp, bytes):
                fp = BytesIO(fp)
            elif isinstance(fp, str):
                fp = StringIO(fp)
            if ft:
                new_v = (fn, fp.read(), ft)
            else:
                new_v = (fn, fp.read())
            fields.append((k, new_v))
        #
        return encode_multipart_formdata(fields, charset=self.charset)

    def _encode_params(self, data):
        content_type = self.headers.get('content-type')
        # No content type given, chose one
        if not content_type:
            if self.encode_multipart:
                content_type = MULTIPART_FORM_DATA
            else:
                content_type = FORM_URL_ENCODED

        if content_type in JSON_CONTENT_TYPES:
            body = json.dumps(data).encode(self.charset)
        elif content_type == FORM_URL_ENCODED:
            body = urlencode(data).encode(self.charset)
        elif content_type == MULTIPART_FORM_DATA:
            body, content_type = encode_multipart_formdata(
                data, boundary=self.multipart_boundary, charset=self.charset)
        else:
            raise ValueError("Don't know how to encode body for %s" %
                             content_type)
        return body, content_type
Beispiel #4
0
class HttpRequest(RequestBase):
    """An :class:`HttpClient` request for an HTTP resource.

    This class has a similar interface to :class:`urllib.request.Request`.

    :param files: optional dictionary of name, file-like-objects.
    :param allow_redirects: allow the response to follow redirects.

    .. attribute:: method

        The request method

    .. attribute:: version

        HTTP version for this request, usually ``HTTP/1.1``

    .. attribute:: encode_multipart

        If ``True`` (default), defaults POST data as ``multipart/form-data``.
        Pass ``encode_multipart=False`` to default to
        ``application/x-www-form-urlencoded``. In any case, this parameter is
        overwritten by passing the correct content-type header.

    .. attribute:: history

        List of past :class:`.HttpResponse` (collected during redirects).

    .. attribute:: wait_continue

        if ``True``, the :class:`HttpRequest` includes the
        ``Expect: 100-Continue`` header.

    .. attribute:: stream

        Allow for streaming body

    """
    _proxy = None
    _ssl = None
    _tunnel = None
    _write_done = False

    def __init__(self, client, url, method, inp_params=None, headers=None,
                 data=None, files=None, history=None, auth=None,
                 charset=None, encode_multipart=True, multipart_boundary=None,
                 source_address=None, allow_redirects=False, max_redirects=10,
                 decompress=True, version=None, wait_continue=False,
                 websocket_handler=None, cookies=None, urlparams=None,
                 stream=False, proxies=None, verify=True, **ignored):
        self.client = client
        self._data = None
        self.files = files
        self.urlparams = urlparams
        self.inp_params = inp_params or {}
        self.unredirected_headers = Headers(kind='client')
        self.method = method.upper()
        self.full_url = url
        if urlparams:
            self._encode_url(urlparams)
        self.history = history
        self.wait_continue = wait_continue
        self.max_redirects = max_redirects
        self.allow_redirects = allow_redirects
        self.charset = charset or 'utf-8'
        self.version = version
        self.decompress = decompress
        self.encode_multipart = encode_multipart
        self.multipart_boundary = multipart_boundary
        self.websocket_handler = websocket_handler
        self.source_address = source_address
        self.stream = stream
        self.verify = verify
        self.new_parser()
        if self._scheme in tls_schemes:
            self._ssl = client.ssl_context(verify=self.verify, **ignored)
        if auth and not isinstance(auth, Auth):
            auth = HTTPBasicAuth(*auth)
        self.auth = auth
        self.headers = client.get_headers(self, headers)
        cookies = cookiejar_from_dict(client.cookies, cookies)
        if cookies:
            cookies.add_cookie_header(self)
        self.unredirected_headers['host'] = host_no_default_port(self._scheme,
                                                                 self._netloc)
        self.data = data
        self._set_proxy(proxies)

    @property
    def address(self):
        """``(host, port)`` tuple of the HTTP resource
        """
        return self._tunnel.address if self._tunnel else (self.host, self.port)

    @property
    def target_address(self):
        return (self.host, int(self.port))

    @property
    def ssl(self):
        """Context for TLS connections.

        If this is a tunneled request and the tunnel connection is not yet
        established, it returns ``None``.
        """
        if not self._tunnel:
            return self._ssl

    @property
    def key(self):
        tunnel = self._tunnel.full_url if self._tunnel else None
        return (self.scheme, self.host, self.port, tunnel, self.verify)

    @property
    def proxy(self):
        """Proxy server for this request.
        """
        return self._proxy

    @property
    def tunnel(self):
        """Tunnel for this request.
        """
        return self._tunnel

    @property
    def netloc(self):
        if self._proxy:
            return self._proxy.netloc
        else:
            return self._netloc

    def __repr__(self):
        return self.first_line()
    __str__ = __repr__

    @property
    def full_url(self):
        """Full url of endpoint
        """
        return urlunparse((self._scheme, self._netloc, self.path,
                           self.params, self.query, self.fragment))

    @full_url.setter
    def full_url(self, url):
        self._scheme, self._netloc, self.path, self.params,\
            self.query, self.fragment = urlparse(url)
        if not self._netloc and self.method == 'CONNECT':
            self._scheme, self._netloc, self.path, self.params,\
                self.query, self.fragment = urlparse('http://%s' % url)

    @property
    def data(self):
        """Body of request
        """
        return self._data

    @data.setter
    def data(self, data):
        self._data = self._encode_data(data)

    def first_line(self):
        if self._proxy:
            if self.method == 'CONNECT':
                url = self._netloc
            else:
                url = self.full_url
        else:
            url = urlunparse(('', '', self.path or '/', self.params,
                              self.query, self.fragment))
        return '%s %s %s' % (self.method, url, self.version)

    def new_parser(self):
        self.parser = self.client.http_parser(kind=1,
                                              decompress=self.decompress,
                                              method=self.method)

    def is_chunked(self):
        return self.data and 'content-length' not in self.headers

    def encode(self):
        """The bytes representation of this :class:`HttpRequest`.

        Called by :class:`HttpResponse` when it needs to encode this
        :class:`HttpRequest` before sending it to the HTTP resource.
        """
        # Call body before fist_line in case the query is changes.
        first_line = self.first_line()
        if self.data and self.wait_continue:
            self.headers['expect'] = '100-continue'
        headers = self.headers
        if self.unredirected_headers:
            headers = self.unredirected_headers.copy()
            headers.update(self.headers)
        buffer = [first_line.encode('ascii'), b'\r\n',  bytes(headers)]
        return b''.join(buffer)

    def add_header(self, key, value):
        self.headers[key] = value

    def has_header(self, header_name):
        """Check ``header_name`` is in this request headers.
        """
        return (header_name in self.headers or
                header_name in self.unredirected_headers)

    def get_header(self, header_name, default=None):
        """Retrieve ``header_name`` from this request headers.
        """
        return self.headers.get(
            header_name, self.unredirected_headers.get(header_name, default))

    def remove_header(self, header_name):
        """Remove ``header_name`` from this request.
        """
        val1 = self.headers.pop(header_name, None)
        val2 = self.unredirected_headers.pop(header_name, None)
        return val1 or val2

    def add_unredirected_header(self, header_name, header_value):
        self.unredirected_headers[header_name] = header_value

    def write_body(self, transport):
        assert not self._write_done, 'Body already sent'
        self._write_done = True
        if not self.data:
            return
        if is_streamed(self.data):
            ensure_future(self._write_streamed_data(transport),
                          loop=transport._loop)
        else:
            self._write_body_data(transport, self.data, True)

    # INTERNAL ENCODING METHODS
    def _encode_data(self, data):
        body = None
        if self.method in ENCODE_URL_METHODS:
            self.files = None
            self._encode_url(data)
        elif isinstance(data, bytes):
            assert self.files is None, ('data cannot be bytes when files are '
                                        'present')
            body = data
        elif isinstance(data, str):
            assert self.files is None, ('data cannot be string when files are '
                                        'present')
            body = to_bytes(data, self.charset)
        elif data and is_streamed(data):
            assert self.files is None, ('data cannot be an iterator when '
                                        'files are present')
            if 'content-type' not in self.headers:
                self.headers['content-type'] = 'application/octet-stream'
            if 'content-length' not in self.headers:
                self.headers['transfer-encoding'] = 'chunked'
            return data
        elif data or self.files:
            if self.files:
                body, content_type = self._encode_files(data)
            else:
                body, content_type = self._encode_params(data)
            # set files to None, Important!
            self.files = None
            self.headers['Content-Type'] = content_type
        if body:
            self.headers['content-length'] = str(len(body))
        elif 'expect' not in self.headers:
            self.headers.pop('content-length', None)
            self.headers.pop('content-type', None)
        return body

    def _encode_url(self, data):
        query = self.query
        if data:
            data = native_str(data)
            if isinstance(data, str):
                data = parse_qsl(data)
            else:
                data = mapping_iterator(data)
            query = parse_qsl(query)
            query.extend(data)
            query = urlencode(query)
        self.query = query

    def _encode_files(self, data):
        fields = []
        for field, val in mapping_iterator(data or ()):
            if (isinstance(val, str) or isinstance(val, bytes) or
                    not hasattr(val, '__iter__')):
                val = [val]
            for v in val:
                if v is not None:
                    if not isinstance(v, bytes):
                        v = str(v)
                    fields.append((field.decode('utf-8') if
                                   isinstance(field, bytes) else field,
                                   v.encode('utf-8') if isinstance(v, str)
                                   else v))
        for (k, v) in mapping_iterator(self.files):
            # support for explicit filename
            ft = None
            if isinstance(v, (tuple, list)):
                if len(v) == 2:
                    fn, fp = v
                else:
                    fn, fp, ft = v
            else:
                fn = guess_filename(v) or k
                fp = v
            if isinstance(fp, bytes):
                fp = BytesIO(fp)
            elif isinstance(fp, str):
                fp = StringIO(fp)
            if ft:
                new_v = (fn, fp.read(), ft)
            else:
                new_v = (fn, fp.read())
            fields.append((k, new_v))
        #
        return encode_multipart_formdata(fields, charset=self.charset)

    def _encode_params(self, data):
        content_type = self.headers.get('content-type')
        # No content type given, chose one
        if not content_type:
            if self.encode_multipart:
                content_type = MULTIPART_FORM_DATA
            else:
                content_type = FORM_URL_ENCODED

        if content_type in JSON_CONTENT_TYPES:
            body = json.dumps(data).encode(self.charset)
        elif content_type == FORM_URL_ENCODED:
            body = urlencode(data).encode(self.charset)
        elif content_type == MULTIPART_FORM_DATA:
            body, content_type = encode_multipart_formdata(
                data, boundary=self.multipart_boundary, charset=self.charset)
        else:
            raise ValueError("Don't know how to encode body for %s" %
                             content_type)
        return body, content_type

    def _write_body_data(self, transport, data, finish=False):
        if self.is_chunked():
            data = http_chunks(data, finish)
        elif data:
            data = (data,)
        else:
            return
        for chunk in data:
            transport.write(chunk)

    @asyncio.coroutine
    def _write_streamed_data(self, transport):
        for data in self.data:
            if isawaitable(data):
                data = yield from data
            self._write_body_data(transport, data)
        self._write_body_data(transport, b'', True)

    # PROXY INTERNALS
    def _set_proxy(self, proxies):
        request_proxies = self.client.proxies.copy()
        if proxies:
            request_proxies.update(proxies)
        self.proxies = request_proxies
        self.scheme = self._scheme
        self._set_hostport(self._scheme, self._netloc)
        #
        if self.scheme in request_proxies:
            hostonly = self.host
            no_proxy = [n for n in request_proxies.get('no', '').split(',')
                        if n]
            if not any(map(hostonly.endswith, no_proxy)):
                url = request_proxies[self.scheme]
                p = urlparse(url)
                if not p.scheme:
                    raise ValueError('Could not understand proxy %s' % url)
                scheme = p.scheme
                host = p.netloc
                if not self._ssl:
                    self.scheme = scheme
                    self._set_hostport(scheme, host)
                    self._proxy = scheme_host(scheme, host)
                else:
                    self._tunnel = HttpTunnel(self, scheme, host)

    def _set_hostport(self, scheme, host):
        self._tunnel = None
        self._proxy = None
        self.host, self.port = get_hostport(scheme, host)