Exemple #1
0
def prepare_rw_output_stream(output):
    """
    Prepare an output stream that supports both reading and writing.
    Intended to be used for writing & updating signed files:
    when producing a signature, we render the PDF to a byte buffer with
    placeholder values for the signature data, or straight to the provided
    output stream if possible.

    More precisely: this function will return the original output stream
    if it is writable, readable and seekable.
    If the ``output`` parameter is ``None``, not readable or not seekable,
    this function will return a :class:`.BytesIO` instance instead.
    If the ``output`` parameter is not ``None`` and not writable,
    :class:`.IOError` will be raised.

    :param output:
        A writable file-like object, or ``None``.
    :return:
        A file-like object that supports reading, writing and seeking.
    """
    if output is None:
        output = BytesIO()
    else:
        # Rationale for the explicit writability check:
        #  If the output buffer is not readable or not seekable, it's
        #  about to be replaced with a BytesIO instance, and in that
        #  case, the write error would only happen *after* the signing
        #  operations are done. We want to avoid that scenario.
        if not output.writable():
            raise IOError("Output buffer is not writable")  # pragma: nocover
        if not output.seekable() or not output.readable():
            output = BytesIO()

    return output
Exemple #2
0
from io import BytesIO, StringIO

bio = BytesIO()
print(bio.readable(), bio.writable(), bio.seekable())
bio.write(b'magede\nPython')
bio.seek(0)
print(bio.readline())
print(bio.getvalue())
bio.close()

sio = StringIO()
print(sio.readable(), sio.writable(), sio.seekable())
sio.write('magedu\nPython')
sio.seek(0)
print(sio.readline())
print(sio.getvalue())
sio.close()

# 二者都是io模块中的类:在内存中,开辟一个文本或者二进制模式的buffer,可以像文件对象一样操作它,
# 当close方法被调用的时候,这个buffer会被释放
# getvalue()获取全部内容,跟文件指针没有关系
# StringIO的好处:一般来说,磁盘的操作比内存的操作要慢的多,内存足够的情况下,
# 一般的优化思路是少落地,减少磁盘IO的过程,可以大大提高程序的运行效率

# 类文件对象:file-like对象,可以像文件对象一样操作
from sys import stdout

f = stdout
print(type(f))
f.write('magedu.com')  # 控制台输出
Exemple #3
0
class _BaseBinaryWrapper:
    def __init__(self, stream: Union[typing.BinaryIO, bytes] = b""):
        if isinstance(stream, bytes) or isinstance(stream, bytearray):
            self.stream = BytesIO(stream)
        else:
            self.stream = stream

    # Wrappings:
    def close(self) -> None:
        return self.stream.close()

    def flush(self) -> None:
        return self.stream.flush()

    def read(self, n: int = -1) -> AnyStr:
        return self.stream.read(n)

    def readable(self) -> bool:
        return self.stream.readable()

    def readline(self, limit: int = -1) -> AnyStr:
        return self.stream.readline(limit)

    def readlines(self, hint: int = -1) -> List[AnyStr]:
        return self.stream.readlines(hint)

    def write(self, s: Union[bytes, bytearray]) -> int:
        return self.stream.write(s)

    def writable(self) -> bool:
        return self.stream.writable()

    def writelines(self, lines: Iterable[AnyStr]) -> None:
        self.stream.writelines(lines)

    def seek(self, offset: int, whence: int = 0) -> int:
        return self.stream.seek(offset, whence)

    def seekable(self) -> bool:
        return self.stream.seekable()

    def tell(self) -> int:
        return self.stream.tell()

    def fileno(self) -> int:
        return self.stream.fileno()

    def __enter__(self):
        self.stream.__enter__()
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.stream.__exit__(exc_type, exc_val, exc_tb)

    # helper functions

    def readall(self):
        self.stream.seek(0)
        return self.stream.read()

    def getvalue(self):
        if isinstance(self.stream, BytesIO):
            return self.stream.getvalue()
        pos = self.stream.tell()
        ret = self.readall()
        self.stream.seek(pos)
        return ret

    def align(self, alignment=4):
        if offset := (self.tell() % alignment):
            self.seek(self.tell() + alignment - offset)
Exemple #4
0
    class Buffer:
        def __init__(self):
            self.stream = BytesIO()
            self.reader = StreamReader(self.stream)
            self.writer = StreamWriter(self.stream)

        def delete_vertices(self, offset, vrtf, count):
            end_offset = offset + vrtf.stride * count
            self.stream.seek(end_offset, SEEK_SET)
            end_data = self.stream.read(-1)
            self.stream.seek(offset, SEEK_SET)
            self.stream.truncate()
            self.stream.writable(end_data)

        def read_vertices(self, offset, vrtf, count,uvscales):
            self.stream.seek(offset, SEEK_SET)
            return [self.read_vertex(vrtf,uvscales) for i in range(count)]

        def read_vertex(self, vrtf, uvscales):
            vertex = Vertex()
            start = self.stream.tell()
            end = start + vrtf.stride
            for declaration in vrtf.declarations:
                u = declaration.usage
                value = self.read_element(declaration,uvscales[declaration.usage_index])
                if u == VertexFormat.USAGE.POSITION: vertex.position = value
                elif u == VertexFormat.USAGE.NORMAL: vertex.normal = value
                elif u == VertexFormat.USAGE.UV:
                    if vertex.uv == None: vertex.uv = []
                    vertex.uv.append(value)
                elif u == VertexFormat.USAGE.BLEND_INDEX: vertex.blend_indices = value
                elif u == VertexFormat.USAGE.BLEND_WEIGHT: vertex.blend_weights = value
                elif u == VertexFormat.USAGE.COLOR: vertex.colour = value
                elif u == VertexFormat.USAGE.TANGENT: vertex.tangent = value
                else:
                    raise Exception("Unknown usage %s", declaration.usage)
            actual = self.stream.tell()
            return vertex

        def write_vertices(self, vrtf, vertices, uvscales=None):
            self.stream.seek(0, SEEK_END)
            offset = self.stream.tell()
            for vertex in vertices: self.write_vertex(vrtf, vertex)

        def write_vertex(self, vrtf, v):
            for declaration in vrtf.declarations:
                u = declaration.usage
                if u == VertexFormat.USAGE.POSITION: data = v.position
                elif u == VertexFormat.USAGE.NORMAL: data = v.normal
                elif u == VertexFormat.USAGE.UV: data = v.uv[vrtf.usage_index]
                elif u == VertexFormat.USAGE.BLEND_INDEX: data = v.blend_indices
                elif u == VertexFormat.USAGE.BLEND_WEIGHT: data = v.blend_weights
                elif u == VertexFormat.USAGE.COLOR: data = v.colour
                elif u == VertexFormat.USAGE.TANGENT: data = v.tangents
                else: raise Exception('Unknown VRTF usage type %i' % u)
                self.write_element(declaration, data)

        def write_element(self, declaration, value):
            pass

        def read_element(self, declaration, uvscale):
            float_count = VertexFormat.FORMAT.float_count(declaration.format)
            value = [0.0] * float_count
            f = declaration.format
            u = declaration.usage
            if u == VertexFormat.USAGE.UV:
                if f == VertexFormat.FORMAT.SHORT2:
                    for i in range(float_count): value[i] = self.reader.i16() * uvscale
                elif f == VertexFormat.FORMAT.SHORT4:
                    shorts = [self.reader.i16() for i in range(4)]
                    assert shorts[2] == 0
                    value = [shorts[0] /0x7FFF, shorts[1]/0x7FFF, shorts[3] /0x1FF]
            elif f in (VertexFormat.FORMAT.FLOAT, VertexFormat.FORMAT.FLOAT2, VertexFormat.FORMAT.FLOAT3,
                       VertexFormat.FORMAT.FLOAT4):
                for i in range(float_count): value[i] = self.reader.f32()
            elif f == VertexFormat.FORMAT.UBYTE4:
                for i in range(float_count): value[i] = self.reader.i8()
            elif f == VertexFormat.FORMAT.COLOR_UBYTE4:
                if u == VertexFormat.USAGE.COLOR:
                    for i in range(float_count): value[i] = self.reader.u8() / 0xFF
                elif u == VertexFormat.USAGE.BLEND_WEIGHT:
                    for i in range(float_count): value[VertexFormat.FORMAT.UBYTE_MAP[i]] = self.reader.u8() / 0xFF
                elif u in (VertexFormat.USAGE.NORMAL, VertexFormat.USAGE.TANGENT):
                    bytes = [self.reader.u8() for i in range(4)]
                    for i in range(float_count - 1):
                        value[i] = -1 if bytes[2 - i] == 0 else ( ((bytes[2 - i] + 1) / 128.0) - 1)
                    determinant = 0.0
                    if not bytes[3]: determinant = -1.0
                    elif bytes[3] == 127.0: determinant = 0.0
                    elif bytes[3] == 255.0: determinant = 1.0
                    else: print("Unexpected handedness %i " % bytes[3])
                    value[float_count - 1] = determinant
                else:
                    raise Exception("Unhandled usage %s for format %s" % (u, f))
            elif f == VertexFormat.FORMAT.SHORT2:
                for i in range(float_count): value[i] = self.reader.i16() / 0xFFFF
            elif f == VertexFormat.FORMAT.SHORT4:
                shorts = [self.reader.i16() for i in range(3)]
                scalar = self.reader.u16()
                if not scalar: scalar = 0x7FFF
                for i in range(float_count): value[i] = float(shorts[i]) / float(scalar)
            elif f == VertexFormat.FORMAT.USHORT4N:
                shorts = [self.reader.i16() for i in range(3)]
                scalar = self.reader.u16()
                if not scalar: scalar = 511
                for i in range(float_count): value[i] = shorts[i] / scalar
            elif f == VertexFormat.FORMAT.UBYTE4:
                data = [self.reader.i8() for i in range(4)]
            else:
                raise Exception("Unhandled format %s" % f)
            return value

        def __del__(self):
            if self.stream != None:
                self.stream.close()
Exemple #5
0
    class Buffer:
        def __init__(self):
            self.stream = BytesIO()
            self.reader = StreamReader(self.stream)
            self.writer = StreamWriter(self.stream)

        def delete_vertices(self, offset, vrtf, count):
            end_offset = offset + vrtf.stride * count
            self.stream.seek(end_offset, SEEK_SET)
            end_data = self.stream.read(-1)
            self.stream.seek(offset, SEEK_SET)
            self.stream.truncate()
            self.stream.writable(end_data)

        def read_vertices(self, offset, vrtf, count, uvscales):
            self.stream.seek(offset, SEEK_SET)
            return [self.read_vertex(vrtf, uvscales) for i in range(count)]

        def read_vertex(self, vrtf, uvscales):
            vertex = Vertex()
            start = self.stream.tell()
            end = start + vrtf.stride
            for declaration in vrtf.declarations:
                u = declaration.usage
                value = self.read_element(declaration,
                                          uvscales[declaration.usage_index])
                if u == VertexFormat.USAGE.POSITION: vertex.position = value
                elif u == VertexFormat.USAGE.NORMAL: vertex.normal = value
                elif u == VertexFormat.USAGE.UV:
                    if vertex.uv == None: vertex.uv = []
                    vertex.uv.append(value)
                elif u == VertexFormat.USAGE.BLEND_INDEX:
                    vertex.blend_indices = value
                elif u == VertexFormat.USAGE.BLEND_WEIGHT:
                    vertex.blend_weights = value
                elif u == VertexFormat.USAGE.COLOR:
                    vertex.colour = value
                elif u == VertexFormat.USAGE.TANGENT:
                    vertex.tangent = value
                else:
                    raise Exception("Unknown usage %s", declaration.usage)
            actual = self.stream.tell()
            return vertex

        def write_vertices(self, vrtf, vertices, uvscales=None):
            self.stream.seek(0, SEEK_END)
            offset = self.stream.tell()
            for vertex in vertices:
                self.write_vertex(vrtf, vertex)

        def write_vertex(self, vrtf, v):
            for declaration in vrtf.declarations:
                u = declaration.usage
                if u == VertexFormat.USAGE.POSITION: data = v.position
                elif u == VertexFormat.USAGE.NORMAL: data = v.normal
                elif u == VertexFormat.USAGE.UV: data = v.uv[vrtf.usage_index]
                elif u == VertexFormat.USAGE.BLEND_INDEX:
                    data = v.blend_indices
                elif u == VertexFormat.USAGE.BLEND_WEIGHT:
                    data = v.blend_weights
                elif u == VertexFormat.USAGE.COLOR:
                    data = v.colour
                elif u == VertexFormat.USAGE.TANGENT:
                    data = v.tangents
                else:
                    raise Exception('Unknown VRTF usage type %i' % u)
                self.write_element(declaration, data)

        def write_element(self, declaration, value):
            pass

        def read_element(self, declaration, uvscale):
            float_count = VertexFormat.FORMAT.float_count(declaration.format)
            value = [0.0] * float_count
            f = declaration.format
            u = declaration.usage
            if u == VertexFormat.USAGE.UV:
                if f == VertexFormat.FORMAT.SHORT2:
                    for i in range(float_count):
                        value[i] = self.reader.i16() * uvscale
                elif f == VertexFormat.FORMAT.SHORT4:
                    shorts = [self.reader.i16() for i in range(4)]
                    assert shorts[2] == 0
                    value = [
                        shorts[0] / 0x7FFF, shorts[1] / 0x7FFF,
                        shorts[3] / 0x1FF
                    ]
            elif f in (VertexFormat.FORMAT.FLOAT, VertexFormat.FORMAT.FLOAT2,
                       VertexFormat.FORMAT.FLOAT3, VertexFormat.FORMAT.FLOAT4):
                for i in range(float_count):
                    value[i] = self.reader.f32()
            elif f == VertexFormat.FORMAT.UBYTE4:
                for i in range(float_count):
                    value[i] = self.reader.i8()
            elif f == VertexFormat.FORMAT.COLOR_UBYTE4:
                if u == VertexFormat.USAGE.COLOR:
                    for i in range(float_count):
                        value[i] = self.reader.u8() / 0xFF
                elif u == VertexFormat.USAGE.BLEND_WEIGHT:
                    for i in range(float_count):
                        value[VertexFormat.FORMAT.
                              UBYTE_MAP[i]] = self.reader.u8() / 0xFF
                elif u in (VertexFormat.USAGE.NORMAL,
                           VertexFormat.USAGE.TANGENT):
                    bytes = [self.reader.u8() for i in range(4)]
                    for i in range(float_count - 1):
                        value[i] = -1 if bytes[2 - i] == 0 else ((
                            (bytes[2 - i] + 1) / 128.0) - 1)
                    determinant = 0.0
                    if not bytes[3]: determinant = -1.0
                    elif bytes[3] == 127.0: determinant = 0.0
                    elif bytes[3] == 255.0: determinant = 1.0
                    else: print("Unexpected handedness %i " % bytes[3])
                    value[float_count - 1] = determinant
                else:
                    raise Exception("Unhandled usage %s for format %s" %
                                    (u, f))
            elif f == VertexFormat.FORMAT.SHORT2:
                for i in range(float_count):
                    value[i] = self.reader.i16() / 0xFFFF
            elif f == VertexFormat.FORMAT.SHORT4:
                shorts = [self.reader.i16() for i in range(3)]
                scalar = self.reader.u16()
                if not scalar: scalar = 0x7FFF
                for i in range(float_count):
                    value[i] = float(shorts[i]) / float(scalar)
            elif f == VertexFormat.FORMAT.USHORT4N:
                shorts = [self.reader.i16() for i in range(3)]
                scalar = self.reader.u16()
                if not scalar: scalar = 511
                for i in range(float_count):
                    value[i] = shorts[i] / scalar
            elif f == VertexFormat.FORMAT.UBYTE4:
                data = [self.reader.i8() for i in range(4)]
            else:
                raise Exception("Unhandled format %s" % f)
            return value

        def __del__(self):
            if self.stream != None:
                self.stream.close()
Exemple #6
0
class VerifiableStream(BinaryIO):
    """A binary stream whose contents can be verified to not have changed.

    The stream does not accept a HMAC key, but generates it randomly as a nonce. While unusual,
    this is intentional -- these streams are meant to be used as part of model serialization,
    where their nonces and HMAC codes are stored in a cryptographically signed metadata file.
    In other words, the HMAC simply ensures that stream's data has not changed, and does not
    guarantee the data's origin -- that's the metadata signature's job.

    The stream is meant to be used in the following sequence:
        - instantiate the stream
        - write all data to the stream (the stream is not readable yet!)
        - call "finalize()" on the stream, saving the returned nonce and HMAC code
        - read data from the stream (the stream is not writable any more!)
    """
    def __init__(self):
        """Create a new VerifiableStream with a random nonce."""
        self._finalized = False
        self._random_nonce = os.urandom(
            16)  # this is bytes, be careful trying to add strings to it
        self._underlying_stream = BytesIO()
        self._hmac_state = hmac.new(self._random_nonce, digestmod=HASHER)

    def _ensure_finalized(self):
        """Raise an error if the stream has not already been finalized."""
        if not self._finalized:
            raise AssertionError(
                "Expected the stream to be finalized, but it was not!")

    def _ensure_not_finalized(self):
        """Raise an error if the stream has already been finalized."""
        if self._finalized:
            raise AssertionError(
                "Expected the stream to not be finalized, but it was!")

    def finalize(self):
        """Calculate the HMAC code for the stream, disable writing and enable reading.

        Returns:
            tuple (nonce, HMAC code)  (both of type string)
        """
        self._ensure_not_finalized()

        self._finalized = True

        nonce_string = _convert_base64_bytes_to_string(self._random_nonce)
        hmac_string = _convert_base64_bytes_to_string(
            self._hmac_state.digest())

        return nonce_string, hmac_string

    # methods for writing require that the stream not be finalized
    def writable(self) -> bool:
        """Return True if the stream is writable, and False otherwise."""
        if self._finalized:
            return False
        else:
            return self._underlying_stream.writable()

    @validate(b=bytes)
    def write(self, b: bytes) -> int:
        """Write the given binary data to the stream, and include it in the HMAC calculation."""
        self._ensure_not_finalized()
        num_bytes = self._underlying_stream.write(b)
        self._hmac_state.update(b)
        return num_bytes

    def writelines(self, lines: Iterable[bytes]) -> None:
        """Write lines to a stream"""
        self._ensure_not_finalized(
        )  # technically done by `write` but doesn't hurt to be safe
        for line in lines:
            self.write(line)
        return None

    # methods for reading require that the stream is finalized
    def readable(self) -> bool:
        """Return True if the stream is readable, and False otherwise."""
        if self._finalized:
            return self._underlying_stream.readable()
        else:
            return False

    def read(self, size=None) -> bytes:
        """Read bytes from stream"""
        self._ensure_finalized()
        return self._underlying_stream.read(size)

    def readall(self) -> bytes:
        """Read lines from stream"""
        raise NotImplementedError(
            "`VerifiablStream` does not implement `readall` since the underlying BtytesIO does not "
            "implement it.")

    def readline(self, size=None) -> bytes:
        """Read a line from stream"""
        self._ensure_finalized()
        return self._underlying_stream.readline(size)

    def readlines(self, size=None) -> List[bytes]:
        """Read lines from stream"""
        self._ensure_finalized()
        return self._underlying_stream.readlines(size)

    def read1(self, size) -> bytes:
        """Read bytes from stream"""
        self._ensure_finalized()
        return self._underlying_stream.read1(size)

    def readinto(self, b) -> Optional[int]:
        """Read bytes into another buffer"""
        self._ensure_finalized()
        return self._underlying_stream.readinto(b)

    def readinto1(self, b) -> Optional[int]:
        """Read bytes into another buffer"""
        self._ensure_finalized()
        return self._underlying_stream.readinto1(b)

    # seeking requires a finalized stream
    def seekable(self):
        """Return True if the read pointer in the stream can be moved, and False otherwise."""
        if self._finalized:
            return self._underlying_stream.seekable()
        else:
            return False

    def seek(self, *args, **kwargs) -> int:
        """Seek to a new position. Return the new position"""
        self._ensure_finalized()
        return self._underlying_stream.seek(*args, **kwargs)

    def truncate(self, size: Optional[int] = ...) -> None:
        """Truncate the stream"""
        raise NotImplementedError(
            "`VerifiableStream` does not support truncation. It is too "
            "complicated to keep track of the hmac digests")

    def close(self):
        """Close the stream, discarding its data. Will raise an error if not finalized yet."""
        if self._finalized:
            return self._underlying_stream.close()
        else:
            raise AssertionError(
                "Attempting to close an unfinalized VerifiableStream. This is "
                "almost certainly a bug.")

    # a bunch of attributes/methods that are always accessible
    def isatty(self) -> bool:
        """Determine whether this is a terminal"""
        return self._underlying_stream.isatty()

    @property
    def closed(self) -> bool:
        """Determine whether the stream is closed"""
        return self._underlying_stream.closed

    def fileno(self) -> int:
        """Return the underlying file descriptor"""
        # this will technically raise UnsuportedOperation, but better to let BytesIO do that
        return self._underlying_stream.fileno()

    def mode(self) -> str:
        """Return the underlying file descriptor"""
        # this doesn't exist for the underlying stream
        raise AssertionError(
            "`VerifiableStream` does not have a mode. This is probably a bug in "
            "something assuming that the stream is a backed by a file")

    def name(self) -> str:
        """Return the underlying file descriptor"""
        # this doesn't exist for the underlying stream
        raise AssertionError(
            "`VerifiableStream` does not have a name. This is probably a bug in "
            "something assuming the stream is a file descriptor")

    def flush(self) -> None:
        """Flush the underlying stream"""
        # this technically does nothing in BytesIO
        return self._underlying_stream.flush()

    def tell(self) -> int:
        """Tell the current position"""
        return self._underlying_stream.tell()

    # context manager methods
    def __enter__(self) -> "VerifiableStream":
        """Enter"""
        return self

    def __exit__(
        self,
        exc_type: Optional[Type[BaseException]],
        exc_val: Optional[BaseException],
        exc_tb: Optional[TracebackType],
    ) -> bool:
        """Exit"""
        return self._underlying_stream.__exit__(exc_type, exc_val, exc_tb)