示例#1
0
def _to_textio(fp: IO, mode: str, read_codec: str) -> TextIO:
    if 'b' in mode:
        fp = cast(TextIO, fp)
        # TODO: FIx me
        fp.decoder = read_codec
        fp.native_reader = fp.read
        fp.read = lambda *args: _auto_decode(fp, *args)
    if getattr(fp, 'native_closer', None):
        fp.native_closer = fp.close
        fp.close = lambda *a: _wrapped_close(fp)
    return fp
示例#2
0
def _auto_decode(fp: IO, nbytes: Optional[int] = None) -> str:
    # We have a file opened in binary mode and haven't formally established a decoder.
    # We may need as many as four bytes to determine the encoding.
    # TODO: should the BOM (if present) count in nbytes?
    if not fp.decoder:
        if nbytes and nbytes < 4:
            # Nothing we can do if they're asking for less than 4 bytes
            data = fp.native_reader(nbytes)
            return data.decode() if data else data
        # Use the BOM to figure out what is toing on.  If no BOM, we always go to UTF-8
        bom = fp.native_reader(4)
        if len(bom) < 4:
            return bom.decode(fp.decoder if fp.decoder else 'utf-8')
        fp.decoder = json.detect_encoding(bom)
        if nbytes is None:
            return (bom + fp.native_reader()).decode(fp.decoder)
        else:
            return (bom + fp.native_reader(nbytes - 4)).decode(fp.decoder)
    else:
        return fp.native_reader(nbytes).decode(fp.decoder)