Ejemplo n.º 1
0
    def _consume_current_data(self, size):
        written = 0

        if self._current_data is None:
            written = self._buffer.write(self.boundary)
            written += self._buffer.write('\r\n')

        elif (self._current_data is not None and
                super_len(self._current_data) > 0):
            written = self._buffer.write(self._current_data.read(size))

        if super_len(self._current_data) == 0:
            written += self._buffer.write('\r\n{0}\r\n'.format(self.boundary))

        return written
Ejemplo n.º 2
0
    def _consume_current_data(self, size):
        """Consume bytes from current field (_current_data, if any) to _buffer.
        If field is finished after operation, add parts separator.
        :returns number of bytes written to _buffer"""
        written = 0

        # File objects need an integer size
        if size is None:
            size = -1

        if self._current_data is not None:
            # and super_len(self._current_data) >= 0):
            # Read from field, copy to _buffer:
            written = self._buffer.write(self._current_data.read(size))
            #print "   written=",written,"  super_len(self._current_data) == ",super_len(self._current_data)
            if super_len(self._current_data) == 0:
                # we'we just finished current field: emit boundary now
                # and reset it to None to forget
                self._current_data = None
                written += self._buffer.write(
                    encode_with('\r\n{0}'.format(self.boundary),
                                self.encoding)
                    )
                # If this is the last separator add -- before \r\n:
                #print "END of field: current_number=",self._current_field_number
                if self._current_field_number == len(self.fields):
                    self._buffer.write(encode_with('--', self.encoding))
                self._buffer.write(encode_with('\r\n', self.encoding))

        return written
    def _load_bytes(self, size):
        self._buffer.smart_truncate()
        amount_to_load = size - super_len(self._buffer)
        bytes_to_append = True

        while amount_to_load > 0 and bytes_to_append:
            bytes_to_append = self._get_bytes()
            amount_to_load -= self._buffer.append(bytes_to_append)
Ejemplo n.º 4
0
 def _calculate_length(self):
     boundary_len = len(self.boundary)  # Length of --{boundary}
     self._len = 0
     for (header, data) in self._fields_list:
         # boundary length + header length + body length + len('\r\n') * 2
         self._len += boundary_len + len(header) + super_len(data) + 4
     # Length of trailing boundary '--{boundary}--\r\n'
     self._len += boundary_len + 4
Ejemplo n.º 5
0
    def test_super_len_io_streams(self):
        """ Ensures that we properly deal with different kinds of IO streams. """
        # uses StringIO or io.StringIO (see import above)
        from io import BytesIO
        from requests.utils import super_len

        assert super_len(StringIO.StringIO()) == 0
        assert super_len(StringIO.StringIO("with so much drama in the LBC")) == 29

        assert super_len(BytesIO()) == 0
        assert super_len(BytesIO(b"it's kinda hard bein' snoop d-o-double-g")) == 40

        try:
            import cStringIO
        except ImportError:
            pass
        else:
            assert super_len(cStringIO.StringIO("but some how, some way...")) == 25
Ejemplo n.º 6
0
 def _calculate_length(self):
     boundary_len = len(self.boundary)  # Length of --{boundary}
     self._len = 0
     for (header, data) in self._fields_list:
         # boundary length + header length + body length + len('\r\n') * 2
         # note that in case header contains non-ascii chars, count must be based on encoded value:
         self._len += boundary_len + len(encode_with(header, self.encoding)) + super_len(data) + 4
     # Length of trailing boundary '--{boundary}--\r\n'
     self._len += boundary_len + 4
Ejemplo n.º 7
0
    def _consume_current_data(self, size):
        written = 0

        if self._current_data is None:
            written = self._buffer.write(self.boundary.encode())
            written += self._buffer.write('\r\n'.encode())

        elif (self._current_data is not None and
                super_len(self._current_data) > 0):
            written = self._buffer.write(self._current_data.read(size))

        if (self._current_data is not None and
                super_len(self._current_data) - self._current_data.tell() == 0 and
                not self.finished):
            written += self._buffer.write(
                '\r\n{0}\r\n'.format(self.boundary).encode()
                )

        return written
Ejemplo n.º 8
0
    def test_super_len_handles_files_raising_weird_errors_in_tell(self, error):
        """If tell() raises errors, assume the cursor is at position zero."""
        class BoomFile(object):
            def __len__(self):
                return 5

            def tell(self):
                raise error()

        assert super_len(BoomFile()) == 0
Ejemplo n.º 9
0
    def _calculate_length(self):
        """
        This uses the parts to calculate the length of the body.

        This returns the calculated length so __len__ can be lazy.
        """
        boundary_len = len(self.boundary)  # Length of --{boundary}
        # boundary length + header length + body length + len('\r\n') * 2
        self._len = sum((boundary_len + super_len(p) + 4) for p in self.parts) + boundary_len + 4
        return self._len
Ejemplo n.º 10
0
    def smart_truncate(self):
        to_be_read = super_len(self)
        already_read = self._get_end() - to_be_read

        if already_read >= to_be_read:
            old_bytes = self.read()
            self.seek(0, 0)
            self.truncate()
            self.write(old_bytes)
            self.seek(0, 0)  # We want to be at the beginning
Ejemplo n.º 11
0
    def test_super_len_tell_ioerror(self, error):
        """Ensure that if tell gives an IOError super_len doesn't fail"""
        class NoLenBoomFile(object):
            def tell(self):
                raise error()

            def seek(self, offset, whence):
                pass

        assert super_len(NoLenBoomFile()) == 0
    def test_stringio(self):
        """Test StringIO wrapper"""
        parent_fd = io.BytesIO(b"0123456789" * 1024)
        parent_fd.seek(0)

        with SubsetIO(parent_fd, 1, 10) as chunked_fd:
            self.assertEqual(super_len(chunked_fd), 10)

            data = chunked_fd.read()

            self.assertEqual(data, b"1234567890")
Ejemplo n.º 13
0
    def bytes_left_to_write(self):
        """Determine if there are bytes left to write.

        :returns: bool -- ``True`` if there are bytes left to write, otherwise
            ``False``
        """
        to_read = 0
        if self.headers_unread:
            to_read += len(self.headers)

        return (to_read + super_len(self.body)) > 0
Ejemplo n.º 14
0
    def _consume_current_data(self, size):
        written = 0

        # File objects need an integer size
        if size is None:
            size = -1

        if self._current_data is None:
            written = self._buffer.write(self.boundary.encode('utf-8'))
            written += self._buffer.write('\r\n'.encode('utf-8'))

        elif (self._current_data is not None and
                super_len(self._current_data) > 0):
            written = self._buffer.write(self._current_data.read(size))

        if super_len(self._current_data) == 0 and not self.finished:
            written += self._buffer.write(
                '\r\n{0}\r\n'.format(self.boundary).encode('utf-8')
                )

        return written
Ejemplo n.º 15
0
    def _calculate_load_amount(self, read_size):
        """This calculates how many bytes need to be added to the buffer.

        When a consumer read's ``x`` from the buffer, there are two cases to
        satisfy:

            1. Enough data in the buffer to return the requested amount
            2. Not enough data

        This function uses the amount of unread bytes in the buffer and
        determines how much the Encoder has to load before it can return the
        requested amount of bytes.

        :param int read_size: the number of bytes the consumer requests
        :returns: int -- the number of bytes that must be loaded into the
            buffer before the read can be satisfied. This will be strictly
            non-negative
        """
        amount = read_size - super_len(self._buffer)
        return amount if amount > 0 else 0
Ejemplo n.º 16
0
    def write_to(self, buffer, size):
        """Write the requested amount of bytes to the buffer provided.

        The number of bytes written may exceed size on the first read since we
        load the headers ambitiously.

        :param CustomBytesIO buffer: buffer we want to write bytes to
        :param int size: number of bytes requested to be written to the buffer
        :returns: int -- number of bytes actually written
        """
        written = 0
        if self.headers_unread:
            written += buffer.append(self.headers)
            self.headers_unread = False

        while super_len(self.body) > 0 and (size == -1 or written < size):
            amount_to_read = size
            if size != -1:
                amount_to_read = size - written
            written += buffer.append(self.body.read(amount_to_read))

        return written
Ejemplo n.º 17
0
 def __init__(self, headers, body):
     self.headers = headers
     self.body = body
     self.headers_unread = True
     self.len = len(self.headers) + super_len(self.body)
Ejemplo n.º 18
0
    def __init__(self,
                 name,
                 *,
                 filename=None,
                 headers=None,
                 content_type=None,
                 file=None,
                 filepath=None,
                 content=None,
                 encoding='utf-8'):
        self.name = quote(name, safe='')
        self.headers = headers or {}
        self.content_length = None
        self._should_close_file = False

        if content is not None:
            if isinstance(content, str):
                content = content.encode(encoding)
        self.content = content

        if filepath is not None:
            file = open(filepath, 'rb')
            self._should_close_file = True
        if file is not None:
            if not isinstance(file, AsyncFile):
                file = AsyncFile(file)
        self.file = file

        if content is None and file is None:
            raise ValueError('Field data must be provided.')
        if content is not None and file is not None:
            raise ValueError("Can't provide both content and file.")

        if content is not None:
            self.content_length = len(content)
            self._body_position = None
        else:
            with file.blocking() as f:
                self.content_length = super_len(f)
                self._body_position = safe_tell(f)

        if filename is None:
            if filepath is None and file is not None:
                filepath = getattr(file, 'name')
            if filepath is not None:
                filename = basename(filepath)
        if filename is not None:
            filename = quote(filename, safe='')
        self.filename = filename

        if content_type is None and filename is not None:
            content_type = mimetypes.guess_type(filename)[0]
        if content_type is not None:
            self.headers['Content-Type'] = content_type

        disposition = ['form-data', f'name="{self.name}"']
        if self.filename is not None:
            disposition.append(f'filename="{self.filename}"')
        self.headers['Content-Disposition'] = '; '.join(disposition)

        self.encoded_headers = encode_headers(self.headers)
Ejemplo n.º 19
0
 def test_file(self, tmpdir, mode, warnings_num, recwarn):
     file_obj = tmpdir.join('test.txt')
     file_obj.write('Test')
     with file_obj.open(mode) as fd:
         assert super_len(fd) == 4
     assert len(recwarn) == warnings_num
Ejemplo n.º 20
0
 def test_io_streams(self, stream, value):
     """Ensures that we properly deal with different kinds of IO streams."""
     assert super_len(stream()) == 0
     assert super_len(stream(value)) == 4
Ejemplo n.º 21
0
 def test_super_len_with_fileno(self):
     with open(__file__, 'rb') as f:
         length = super_len(f)
         file_data = f.read()
     assert length == len(file_data)
Ejemplo n.º 22
0
 def test_super_len_with_tell(self):
     foo = StringIO.StringIO('12345')
     assert super_len(foo) == 5
     foo.read(2)
     assert super_len(foo) == 3
Ejemplo n.º 23
0
    def prepare_body(self, data, files, json=None):
        """Prepares the given HTTP body data."""

        # Check if file, fo, generator, iterator.
        # If not, run through normal process.

        # Nottin' on you.
        body = None
        content_type = None

        if not data and json is not None:
            # urllib3 requires a bytes-like body. Python 2's json.dumps
            # provides this natively, but Python 3 gives a Unicode string.
            content_type = 'application/json'
            body = dumps(json)
            if not isinstance(body, bytes):
                body = body.encode('utf-8')

        is_stream = all([
            hasattr(data, '__iter__'),
            not isinstance(data, ((str, bytes), list, tuple, Mapping))
        ])

        try:
            length = super_len(data)
        except (TypeError, AttributeError, UnsupportedOperation):
            length = None

        if is_stream:
            body = data

            if getattr(body, 'tell', None) is not None:
                # Record the current file position before reading.
                # This will allow us to rewind a file in the event
                # of a redirect.
                try:
                    self._body_position = body.tell()
                except (IOError, OSError):
                    # This differentiates from None, allowing us to catch
                    # a failed `tell()` later when trying to rewind the body
                    self._body_position = object()

            if files:
                raise NotImplementedError(
                    'Streamed bodies and files are mutually exclusive.')

            if length:
                self.headers['Content-Length'] = str(length)
            else:
                self.headers['Transfer-Encoding'] = 'chunked'
        else:
            # Multi-part file uploads.
            if files:
                (body, content_type) = self._encode_files(files, data)
            else:
                if data:
                    body = self._encode_params(data)
                    if isinstance(data, (str, bytes)) or hasattr(data, 'read'):
                        content_type = None
                    else:
                        content_type = 'application/x-www-form-urlencoded'

            self.prepare_content_length(body)

            # Add content-type if it wasn't explicitly provided.
            if content_type and ('content-type' not in self.headers):
                self.headers['Content-Type'] = content_type

        self.body = body
Ejemplo n.º 24
0
 def test_super_len_with_no_matches(self):
     """Ensure that objects without any length methods default to 0"""
     assert super_len(object()) == 0
Ejemplo n.º 25
0
 def __len__(self):
     return super_len(self.fd) - self.fd.tell()
Ejemplo n.º 26
0
    def _request(self, method, url_or_endpoint, **kwargs):
        http_client = AsyncHTTPClient()
        http_client = AsyncHTTPClient()

        if not url_or_endpoint.startswith(('http://', 'https://')):
            api_base_url = kwargs.pop('api_base_url', self.API_BASE_URL)
            url = '{base}{endpoint}'.format(base=api_base_url,
                                            endpoint=url_or_endpoint)
        else:
            url = url_or_endpoint

        headers = {}
        params = kwargs.get('params', {})
        if 'access_token' not in params:
            # 这里需要针对 tornado 特殊处理
            access_token = None
            if self.access_token:
                if not self.expires_at:
                    # user provided access_token, just return it
                    access_token = self.access_token
                else:
                    timestamp = time.time()
                    if self.expires_at - timestamp > 60:
                        access_token = self.access_token

            if not access_token:
                # fetch access
                yield self.fetch_access_token()
                access_token = self.access_token

            params['access_token'] = self.access_token

        params = urlencode(dict((k, to_binary(v)) for k, v in params.items()))
        url = '{0}?{1}'.format(url, params)

        data = kwargs.get('data', {})
        files = kwargs.get('files')
        if files:
            from requests.models import RequestEncodingMixin
            from requests.utils import super_len

            body, content_type = RequestEncodingMixin._encode_files(
                files, data)
            headers['Content-Type'] = content_type
            headers['Content-Length'] = super_len(body)
        else:
            if isinstance(data, dict):
                body = json.dumps(data, ensure_ascii=False)
                body = body.encode('utf-8')
            else:
                body = data

        result_processor = kwargs.pop('result_processor', None)
        timeout = kwargs.get('timeout', self.timeout)
        req = HTTPRequest(url=url,
                          method=method.upper(),
                          headers=headers,
                          body=body if method.upper() != "GET" else None,
                          request_timeout=timeout)
        res = yield http_client.fetch(req)
        if res.error is not None:
            raise WeChatClientException(errcode=None,
                                        errmsg=None,
                                        client=self,
                                        request=req,
                                        response=res)

        result = self._handle_result(res, method, url, result_processor,
                                     **kwargs)
        raise Return(result)
Ejemplo n.º 27
0
 def test_string(self):
     assert super_len('Test') == 4
Ejemplo n.º 28
0
 def test_io_streams(self, stream, value):
     """Ensures that we properly deal with different kinds of IO streams."""
     assert super_len(stream()) == 0
     assert super_len(stream(value)) == 4
Ejemplo n.º 29
0
 def __init__(self, headers, body):
     self.headers = headers
     self.body = body
     self.headers_unread = True
     self.len = len(self.headers) + super_len(self.body)
Ejemplo n.º 30
0
 def test_string(self):
     assert super_len('Test') == 4
Ejemplo n.º 31
0
 def test_super_len_with__len__(self):
     foo = [1,2,3,4]
     len_foo = super_len(foo)
     assert len_foo == 4
Ejemplo n.º 32
0
 def len(self):
     return super_len(self.fd) - self.fd.tell()
Ejemplo n.º 33
0
 def test_super_len_with_tell(self):
     foo = StringIO.StringIO('12345')
     assert super_len(foo) == 5
     foo.read(2)
     assert super_len(foo) == 3
Ejemplo n.º 34
0
 def test_super_len_with_no_matches(self):
     """Ensure that objects without any length methods default to 0"""
     assert super_len(object()) == 0
Ejemplo n.º 35
0
    def test_super_len_with_no__len__(self):
        class LenFile(object):
            def __init__(self):
                self.len = 5

        assert super_len(LenFile()) == 5
Ejemplo n.º 36
0
 def test_super_len_correctly_calculates_len_of_partially_read_file(self):
     """Ensure that we handle partially consumed file like objects."""
     s = StringIO.StringIO()
     s.write('foobarbogus')
     assert super_len(s) == 0
Ejemplo n.º 37
0
 def test_super_len_correctly_calculates_len_of_partially_read_file(self):
     """Ensure that we handle partially consumed file like objects."""
     s = StringIO.StringIO()
     s.write('foobarbogus')
     assert super_len(s) == 0
Ejemplo n.º 38
0
 def test_super_len_with_fileno(self):
     with open(__file__, 'rb') as f:
         length = super_len(f)
         file_data = f.read()
     assert length == len(file_data)
Ejemplo n.º 39
0
    def _request(self, method, url_or_endpoint, **kwargs):
        http_client = AsyncHTTPClient()
        if not url_or_endpoint.startswith(('http://', 'https://')):
            api_base_url = kwargs.pop('api_base_url', self.API_BASE_URL)
            url = '{base}{endpoint}'.format(
                base=api_base_url,
                endpoint=url_or_endpoint
            )
        else:
            url = url_or_endpoint

        headers = {}
        params = kwargs.pop('params', {})
        if 'access_token' not in params:
            params['access_token'] = self.access_token

        params = urlencode(dict((k, to_binary(v)) for k, v in params.items()))
        url = '{0}?{1}'.format(url, params)

        data = kwargs.get('data')
        files = kwargs.get('files')
        if files:
            from requests.models import RequestEncodingMixin
            from requests.utils import super_len

            body, content_type = RequestEncodingMixin._encode_files(
                files,
                data
            )
            headers['Content-Type'] = content_type
            headers['Content-Length'] = super_len(body)
        else:
            if isinstance(data, dict):
                body = json.dumps(data, ensure_ascii=False)
                body = body.encode('utf-8')
            else:
                body = data

        result_processor = kwargs.pop('result_processor', None)
        timeout = kwargs.get('timeout', self.timeout)
        req = HTTPRequest(
            url=url,
            method=method.upper(),
            headers=headers,
            body=body,
            request_timeout=timeout
        )
        res = yield http_client.fetch(req)
        if res.error is not None:
            raise WeChatClientException(
                errcode=None,
                errmsg=None,
                client=self,
                request=req,
                response=res
            )

        result = self._handle_result(
            res, method, url, result_processor, **kwargs
        )
        raise Return(result)
Ejemplo n.º 40
0
 def test_file(self, tmpdir, mode, warnings_num, recwarn):
     file_obj = tmpdir.join('test.txt')
     file_obj.write('Test')
     with file_obj.open(mode) as fd:
         assert super_len(fd) == 4
     assert len(recwarn) == warnings_num
Ejemplo n.º 41
0
 def __len__(self):
     return len(self.headers) + super_len(self.body)
Ejemplo n.º 42
0
    def test_super_len_with_no__len__(self):
        class LenFile(object):
            def __init__(self):
                self.len = 5

        assert super_len(LenFile()) == 5
Ejemplo n.º 43
0
 def test_string(self):
     assert super_len("Test") == 4
Ejemplo n.º 44
0
 def test_string(self):
     assert super_len("Test") == 4
Ejemplo n.º 45
0
 def test_super_len_with__len__(self):
     foo = [1, 2, 3, 4]
     len_foo = super_len(foo)
     assert len_foo == 4