def prepare_rw_output_stream(output): """ Prepare an output stream that supports both reading and writing. Intended to be used for writing & updating signed files: when producing a signature, we render the PDF to a byte buffer with placeholder values for the signature data, or straight to the provided output stream if possible. More precisely: this function will return the original output stream if it is writable, readable and seekable. If the ``output`` parameter is ``None``, not readable or not seekable, this function will return a :class:`.BytesIO` instance instead. If the ``output`` parameter is not ``None`` and not writable, :class:`.IOError` will be raised. :param output: A writable file-like object, or ``None``. :return: A file-like object that supports reading, writing and seeking. """ if output is None: output = BytesIO() else: # Rationale for the explicit writability check: # If the output buffer is not readable or not seekable, it's # about to be replaced with a BytesIO instance, and in that # case, the write error would only happen *after* the signing # operations are done. We want to avoid that scenario. if not output.writable(): raise IOError("Output buffer is not writable") # pragma: nocover if not output.seekable() or not output.readable(): output = BytesIO() return output
from io import BytesIO, StringIO bio = BytesIO() print(bio.readable(), bio.writable(), bio.seekable()) bio.write(b'magede\nPython') bio.seek(0) print(bio.readline()) print(bio.getvalue()) bio.close() sio = StringIO() print(sio.readable(), sio.writable(), sio.seekable()) sio.write('magedu\nPython') sio.seek(0) print(sio.readline()) print(sio.getvalue()) sio.close() # 二者都是io模块中的类:在内存中,开辟一个文本或者二进制模式的buffer,可以像文件对象一样操作它, # 当close方法被调用的时候,这个buffer会被释放 # getvalue()获取全部内容,跟文件指针没有关系 # StringIO的好处:一般来说,磁盘的操作比内存的操作要慢的多,内存足够的情况下, # 一般的优化思路是少落地,减少磁盘IO的过程,可以大大提高程序的运行效率 # 类文件对象:file-like对象,可以像文件对象一样操作 from sys import stdout f = stdout print(type(f)) f.write('magedu.com') # 控制台输出
class _BaseBinaryWrapper: def __init__(self, stream: Union[typing.BinaryIO, bytes] = b""): if isinstance(stream, bytes) or isinstance(stream, bytearray): self.stream = BytesIO(stream) else: self.stream = stream # Wrappings: def close(self) -> None: return self.stream.close() def flush(self) -> None: return self.stream.flush() def read(self, n: int = -1) -> AnyStr: return self.stream.read(n) def readable(self) -> bool: return self.stream.readable() def readline(self, limit: int = -1) -> AnyStr: return self.stream.readline(limit) def readlines(self, hint: int = -1) -> List[AnyStr]: return self.stream.readlines(hint) def write(self, s: Union[bytes, bytearray]) -> int: return self.stream.write(s) def writable(self) -> bool: return self.stream.writable() def writelines(self, lines: Iterable[AnyStr]) -> None: self.stream.writelines(lines) def seek(self, offset: int, whence: int = 0) -> int: return self.stream.seek(offset, whence) def seekable(self) -> bool: return self.stream.seekable() def tell(self) -> int: return self.stream.tell() def fileno(self) -> int: return self.stream.fileno() def __enter__(self): self.stream.__enter__() return self def __exit__(self, exc_type, exc_val, exc_tb): self.stream.__exit__(exc_type, exc_val, exc_tb) # helper functions def readall(self): self.stream.seek(0) return self.stream.read() def getvalue(self): if isinstance(self.stream, BytesIO): return self.stream.getvalue() pos = self.stream.tell() ret = self.readall() self.stream.seek(pos) return ret def align(self, alignment=4): if offset := (self.tell() % alignment): self.seek(self.tell() + alignment - offset)
class Buffer: def __init__(self): self.stream = BytesIO() self.reader = StreamReader(self.stream) self.writer = StreamWriter(self.stream) def delete_vertices(self, offset, vrtf, count): end_offset = offset + vrtf.stride * count self.stream.seek(end_offset, SEEK_SET) end_data = self.stream.read(-1) self.stream.seek(offset, SEEK_SET) self.stream.truncate() self.stream.writable(end_data) def read_vertices(self, offset, vrtf, count,uvscales): self.stream.seek(offset, SEEK_SET) return [self.read_vertex(vrtf,uvscales) for i in range(count)] def read_vertex(self, vrtf, uvscales): vertex = Vertex() start = self.stream.tell() end = start + vrtf.stride for declaration in vrtf.declarations: u = declaration.usage value = self.read_element(declaration,uvscales[declaration.usage_index]) if u == VertexFormat.USAGE.POSITION: vertex.position = value elif u == VertexFormat.USAGE.NORMAL: vertex.normal = value elif u == VertexFormat.USAGE.UV: if vertex.uv == None: vertex.uv = [] vertex.uv.append(value) elif u == VertexFormat.USAGE.BLEND_INDEX: vertex.blend_indices = value elif u == VertexFormat.USAGE.BLEND_WEIGHT: vertex.blend_weights = value elif u == VertexFormat.USAGE.COLOR: vertex.colour = value elif u == VertexFormat.USAGE.TANGENT: vertex.tangent = value else: raise Exception("Unknown usage %s", declaration.usage) actual = self.stream.tell() return vertex def write_vertices(self, vrtf, vertices, uvscales=None): self.stream.seek(0, SEEK_END) offset = self.stream.tell() for vertex in vertices: self.write_vertex(vrtf, vertex) def write_vertex(self, vrtf, v): for declaration in vrtf.declarations: u = declaration.usage if u == VertexFormat.USAGE.POSITION: data = v.position elif u == VertexFormat.USAGE.NORMAL: data = v.normal elif u == VertexFormat.USAGE.UV: data = v.uv[vrtf.usage_index] elif u == VertexFormat.USAGE.BLEND_INDEX: data = v.blend_indices elif u == VertexFormat.USAGE.BLEND_WEIGHT: data = v.blend_weights elif u == VertexFormat.USAGE.COLOR: data = v.colour elif u == VertexFormat.USAGE.TANGENT: data = v.tangents else: raise Exception('Unknown VRTF usage type %i' % u) self.write_element(declaration, data) def write_element(self, declaration, value): pass def read_element(self, declaration, uvscale): float_count = VertexFormat.FORMAT.float_count(declaration.format) value = [0.0] * float_count f = declaration.format u = declaration.usage if u == VertexFormat.USAGE.UV: if f == VertexFormat.FORMAT.SHORT2: for i in range(float_count): value[i] = self.reader.i16() * uvscale elif f == VertexFormat.FORMAT.SHORT4: shorts = [self.reader.i16() for i in range(4)] assert shorts[2] == 0 value = [shorts[0] /0x7FFF, shorts[1]/0x7FFF, shorts[3] /0x1FF] elif f in (VertexFormat.FORMAT.FLOAT, VertexFormat.FORMAT.FLOAT2, VertexFormat.FORMAT.FLOAT3, VertexFormat.FORMAT.FLOAT4): for i in range(float_count): value[i] = self.reader.f32() elif f == VertexFormat.FORMAT.UBYTE4: for i in range(float_count): value[i] = self.reader.i8() elif f == VertexFormat.FORMAT.COLOR_UBYTE4: if u == VertexFormat.USAGE.COLOR: for i in range(float_count): value[i] = self.reader.u8() / 0xFF elif u == VertexFormat.USAGE.BLEND_WEIGHT: for i in range(float_count): value[VertexFormat.FORMAT.UBYTE_MAP[i]] = self.reader.u8() / 0xFF elif u in (VertexFormat.USAGE.NORMAL, VertexFormat.USAGE.TANGENT): bytes = [self.reader.u8() for i in range(4)] for i in range(float_count - 1): value[i] = -1 if bytes[2 - i] == 0 else ( ((bytes[2 - i] + 1) / 128.0) - 1) determinant = 0.0 if not bytes[3]: determinant = -1.0 elif bytes[3] == 127.0: determinant = 0.0 elif bytes[3] == 255.0: determinant = 1.0 else: print("Unexpected handedness %i " % bytes[3]) value[float_count - 1] = determinant else: raise Exception("Unhandled usage %s for format %s" % (u, f)) elif f == VertexFormat.FORMAT.SHORT2: for i in range(float_count): value[i] = self.reader.i16() / 0xFFFF elif f == VertexFormat.FORMAT.SHORT4: shorts = [self.reader.i16() for i in range(3)] scalar = self.reader.u16() if not scalar: scalar = 0x7FFF for i in range(float_count): value[i] = float(shorts[i]) / float(scalar) elif f == VertexFormat.FORMAT.USHORT4N: shorts = [self.reader.i16() for i in range(3)] scalar = self.reader.u16() if not scalar: scalar = 511 for i in range(float_count): value[i] = shorts[i] / scalar elif f == VertexFormat.FORMAT.UBYTE4: data = [self.reader.i8() for i in range(4)] else: raise Exception("Unhandled format %s" % f) return value def __del__(self): if self.stream != None: self.stream.close()
class Buffer: def __init__(self): self.stream = BytesIO() self.reader = StreamReader(self.stream) self.writer = StreamWriter(self.stream) def delete_vertices(self, offset, vrtf, count): end_offset = offset + vrtf.stride * count self.stream.seek(end_offset, SEEK_SET) end_data = self.stream.read(-1) self.stream.seek(offset, SEEK_SET) self.stream.truncate() self.stream.writable(end_data) def read_vertices(self, offset, vrtf, count, uvscales): self.stream.seek(offset, SEEK_SET) return [self.read_vertex(vrtf, uvscales) for i in range(count)] def read_vertex(self, vrtf, uvscales): vertex = Vertex() start = self.stream.tell() end = start + vrtf.stride for declaration in vrtf.declarations: u = declaration.usage value = self.read_element(declaration, uvscales[declaration.usage_index]) if u == VertexFormat.USAGE.POSITION: vertex.position = value elif u == VertexFormat.USAGE.NORMAL: vertex.normal = value elif u == VertexFormat.USAGE.UV: if vertex.uv == None: vertex.uv = [] vertex.uv.append(value) elif u == VertexFormat.USAGE.BLEND_INDEX: vertex.blend_indices = value elif u == VertexFormat.USAGE.BLEND_WEIGHT: vertex.blend_weights = value elif u == VertexFormat.USAGE.COLOR: vertex.colour = value elif u == VertexFormat.USAGE.TANGENT: vertex.tangent = value else: raise Exception("Unknown usage %s", declaration.usage) actual = self.stream.tell() return vertex def write_vertices(self, vrtf, vertices, uvscales=None): self.stream.seek(0, SEEK_END) offset = self.stream.tell() for vertex in vertices: self.write_vertex(vrtf, vertex) def write_vertex(self, vrtf, v): for declaration in vrtf.declarations: u = declaration.usage if u == VertexFormat.USAGE.POSITION: data = v.position elif u == VertexFormat.USAGE.NORMAL: data = v.normal elif u == VertexFormat.USAGE.UV: data = v.uv[vrtf.usage_index] elif u == VertexFormat.USAGE.BLEND_INDEX: data = v.blend_indices elif u == VertexFormat.USAGE.BLEND_WEIGHT: data = v.blend_weights elif u == VertexFormat.USAGE.COLOR: data = v.colour elif u == VertexFormat.USAGE.TANGENT: data = v.tangents else: raise Exception('Unknown VRTF usage type %i' % u) self.write_element(declaration, data) def write_element(self, declaration, value): pass def read_element(self, declaration, uvscale): float_count = VertexFormat.FORMAT.float_count(declaration.format) value = [0.0] * float_count f = declaration.format u = declaration.usage if u == VertexFormat.USAGE.UV: if f == VertexFormat.FORMAT.SHORT2: for i in range(float_count): value[i] = self.reader.i16() * uvscale elif f == VertexFormat.FORMAT.SHORT4: shorts = [self.reader.i16() for i in range(4)] assert shorts[2] == 0 value = [ shorts[0] / 0x7FFF, shorts[1] / 0x7FFF, shorts[3] / 0x1FF ] elif f in (VertexFormat.FORMAT.FLOAT, VertexFormat.FORMAT.FLOAT2, VertexFormat.FORMAT.FLOAT3, VertexFormat.FORMAT.FLOAT4): for i in range(float_count): value[i] = self.reader.f32() elif f == VertexFormat.FORMAT.UBYTE4: for i in range(float_count): value[i] = self.reader.i8() elif f == VertexFormat.FORMAT.COLOR_UBYTE4: if u == VertexFormat.USAGE.COLOR: for i in range(float_count): value[i] = self.reader.u8() / 0xFF elif u == VertexFormat.USAGE.BLEND_WEIGHT: for i in range(float_count): value[VertexFormat.FORMAT. UBYTE_MAP[i]] = self.reader.u8() / 0xFF elif u in (VertexFormat.USAGE.NORMAL, VertexFormat.USAGE.TANGENT): bytes = [self.reader.u8() for i in range(4)] for i in range(float_count - 1): value[i] = -1 if bytes[2 - i] == 0 else (( (bytes[2 - i] + 1) / 128.0) - 1) determinant = 0.0 if not bytes[3]: determinant = -1.0 elif bytes[3] == 127.0: determinant = 0.0 elif bytes[3] == 255.0: determinant = 1.0 else: print("Unexpected handedness %i " % bytes[3]) value[float_count - 1] = determinant else: raise Exception("Unhandled usage %s for format %s" % (u, f)) elif f == VertexFormat.FORMAT.SHORT2: for i in range(float_count): value[i] = self.reader.i16() / 0xFFFF elif f == VertexFormat.FORMAT.SHORT4: shorts = [self.reader.i16() for i in range(3)] scalar = self.reader.u16() if not scalar: scalar = 0x7FFF for i in range(float_count): value[i] = float(shorts[i]) / float(scalar) elif f == VertexFormat.FORMAT.USHORT4N: shorts = [self.reader.i16() for i in range(3)] scalar = self.reader.u16() if not scalar: scalar = 511 for i in range(float_count): value[i] = shorts[i] / scalar elif f == VertexFormat.FORMAT.UBYTE4: data = [self.reader.i8() for i in range(4)] else: raise Exception("Unhandled format %s" % f) return value def __del__(self): if self.stream != None: self.stream.close()
class VerifiableStream(BinaryIO): """A binary stream whose contents can be verified to not have changed. The stream does not accept a HMAC key, but generates it randomly as a nonce. While unusual, this is intentional -- these streams are meant to be used as part of model serialization, where their nonces and HMAC codes are stored in a cryptographically signed metadata file. In other words, the HMAC simply ensures that stream's data has not changed, and does not guarantee the data's origin -- that's the metadata signature's job. The stream is meant to be used in the following sequence: - instantiate the stream - write all data to the stream (the stream is not readable yet!) - call "finalize()" on the stream, saving the returned nonce and HMAC code - read data from the stream (the stream is not writable any more!) """ def __init__(self): """Create a new VerifiableStream with a random nonce.""" self._finalized = False self._random_nonce = os.urandom( 16) # this is bytes, be careful trying to add strings to it self._underlying_stream = BytesIO() self._hmac_state = hmac.new(self._random_nonce, digestmod=HASHER) def _ensure_finalized(self): """Raise an error if the stream has not already been finalized.""" if not self._finalized: raise AssertionError( "Expected the stream to be finalized, but it was not!") def _ensure_not_finalized(self): """Raise an error if the stream has already been finalized.""" if self._finalized: raise AssertionError( "Expected the stream to not be finalized, but it was!") def finalize(self): """Calculate the HMAC code for the stream, disable writing and enable reading. Returns: tuple (nonce, HMAC code) (both of type string) """ self._ensure_not_finalized() self._finalized = True nonce_string = _convert_base64_bytes_to_string(self._random_nonce) hmac_string = _convert_base64_bytes_to_string( self._hmac_state.digest()) return nonce_string, hmac_string # methods for writing require that the stream not be finalized def writable(self) -> bool: """Return True if the stream is writable, and False otherwise.""" if self._finalized: return False else: return self._underlying_stream.writable() @validate(b=bytes) def write(self, b: bytes) -> int: """Write the given binary data to the stream, and include it in the HMAC calculation.""" self._ensure_not_finalized() num_bytes = self._underlying_stream.write(b) self._hmac_state.update(b) return num_bytes def writelines(self, lines: Iterable[bytes]) -> None: """Write lines to a stream""" self._ensure_not_finalized( ) # technically done by `write` but doesn't hurt to be safe for line in lines: self.write(line) return None # methods for reading require that the stream is finalized def readable(self) -> bool: """Return True if the stream is readable, and False otherwise.""" if self._finalized: return self._underlying_stream.readable() else: return False def read(self, size=None) -> bytes: """Read bytes from stream""" self._ensure_finalized() return self._underlying_stream.read(size) def readall(self) -> bytes: """Read lines from stream""" raise NotImplementedError( "`VerifiablStream` does not implement `readall` since the underlying BtytesIO does not " "implement it.") def readline(self, size=None) -> bytes: """Read a line from stream""" self._ensure_finalized() return self._underlying_stream.readline(size) def readlines(self, size=None) -> List[bytes]: """Read lines from stream""" self._ensure_finalized() return self._underlying_stream.readlines(size) def read1(self, size) -> bytes: """Read bytes from stream""" self._ensure_finalized() return self._underlying_stream.read1(size) def readinto(self, b) -> Optional[int]: """Read bytes into another buffer""" self._ensure_finalized() return self._underlying_stream.readinto(b) def readinto1(self, b) -> Optional[int]: """Read bytes into another buffer""" self._ensure_finalized() return self._underlying_stream.readinto1(b) # seeking requires a finalized stream def seekable(self): """Return True if the read pointer in the stream can be moved, and False otherwise.""" if self._finalized: return self._underlying_stream.seekable() else: return False def seek(self, *args, **kwargs) -> int: """Seek to a new position. Return the new position""" self._ensure_finalized() return self._underlying_stream.seek(*args, **kwargs) def truncate(self, size: Optional[int] = ...) -> None: """Truncate the stream""" raise NotImplementedError( "`VerifiableStream` does not support truncation. It is too " "complicated to keep track of the hmac digests") def close(self): """Close the stream, discarding its data. Will raise an error if not finalized yet.""" if self._finalized: return self._underlying_stream.close() else: raise AssertionError( "Attempting to close an unfinalized VerifiableStream. This is " "almost certainly a bug.") # a bunch of attributes/methods that are always accessible def isatty(self) -> bool: """Determine whether this is a terminal""" return self._underlying_stream.isatty() @property def closed(self) -> bool: """Determine whether the stream is closed""" return self._underlying_stream.closed def fileno(self) -> int: """Return the underlying file descriptor""" # this will technically raise UnsuportedOperation, but better to let BytesIO do that return self._underlying_stream.fileno() def mode(self) -> str: """Return the underlying file descriptor""" # this doesn't exist for the underlying stream raise AssertionError( "`VerifiableStream` does not have a mode. This is probably a bug in " "something assuming that the stream is a backed by a file") def name(self) -> str: """Return the underlying file descriptor""" # this doesn't exist for the underlying stream raise AssertionError( "`VerifiableStream` does not have a name. This is probably a bug in " "something assuming the stream is a file descriptor") def flush(self) -> None: """Flush the underlying stream""" # this technically does nothing in BytesIO return self._underlying_stream.flush() def tell(self) -> int: """Tell the current position""" return self._underlying_stream.tell() # context manager methods def __enter__(self) -> "VerifiableStream": """Enter""" return self def __exit__( self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType], ) -> bool: """Exit""" return self._underlying_stream.__exit__(exc_type, exc_val, exc_tb)