class SandboxInOut(object): """Collect written text, and return it when called.""" class InnerSandboxInOut(object): def __init__(self, printer, _getattr_=None): self.printer = printer self._getattr_ = _getattr_ def write(self, text: str): self.printer.write(text, consumer=False) def read(self): line = self.printer.readline(consumer=True) if len(line) == 0: return None return str(line) def __call__(self, testing=None): return self def _call_print(self, *objects, **kwargs): if kwargs.get('file', None) is None: kwargs['file'] = self else: self._getattr_(kwargs['file'], 'write') print(*objects, **kwargs) def __init__(self, _getattr_=None): self.stream = BufferedRandom(BytesIO()) self.position = 0 def printer(self): return self.InnerSandboxInOut(self) def reader(self): return lambda: self.readline() def write(self, line, consumer=True): if not consumer: self.stream.seek(2) # go to the end of the stream else: self.stream.seek(0, self.position) self.stream.write(bytes(line, 'utf-8')) if consumer: self.position = self.stream.tell() def readline(self, consumer=True): if not consumer: self.stream.seek(2) # go to the end of the stream else: self.stream.seek(0, self.position) line = self.stream.readline() if len(line) == 0: return None if consumer: self.position = self.stream.tell() return str(line.decode('utf-8')).strip() def get_stream(self): return self.stream
def FileWriteInteger(file_handle: io.BufferedRandom, value: int, size: int = INT_VALUE) -> int: if size < CHAR_VALUE or size > INT_VALUE: raise Exception("Size error") file_handle.write( int(value).to_bytes(size, byteorder="little", signed=False)) return size
def FileWriteString(file_handle: io.BufferedRandom, text: str, length: int = -1) -> int: if length != -1: text = text[:length] length = len(text) if file_handle.binflag: file_handle.write(text.encode(file_handle.encoding)) else: file_handle.write(text) return length
def saveWorld(self, world: World, fileIO: io.BufferedRandom, worldManager: WorldManager): # Checking if file size matches! if not (world.sizeX == 256 and world.sizeY == 256 and world.sizeZ == 256): raise WorldFormatError( f"RawWorldFormat - Trying to save world that has invalid world size! Expected: 256, 256, 256! Got: {world.sizeX}, {world.sizeY}, {world.sizeZ}!" ) # Clearing Current Save File fileIO.truncate(0) fileIO.seek(0) # Saving Map To File fileIO.write(world.gzipMap())
def FileWriteDouble(file_handle: io.BufferedRandom, value: float) -> int: file_handle.write(struct.pack("<d", value)) return 8
def FileWriteLong(file_handle: io.BufferedRandom, value: int) -> int: file_handle.write(int(value).to_bytes(8, byteorder="little", signed=False)) return 8
class BinaryCopy(): def __init__(self, chunk_size=256, buffer_size=64 * 1024 * 1024): self.chunk_size = chunk_size self.buffer_size = buffer_size self.fs = BufferedRandom(BytesIO(), buffer_size=buffer_size) self._row_header_struct = Struct("!hiqiq") self._row_int_struct = Struct("!i") self._row_bigint_struct = Struct("!iq") self._row_null_val = pack("!i", -1) def write_binary_header(self): self.fs.write(pack('!11sii', b'PGCOPY\n\xff\r\n\0', 0, 0)) def write_binary_string(self, obj, keyname): try: val = obj[keyname] if val is None: self.fs.write(self._row_null_val) else: val = val.encode() self.fs.write(self._row_int_struct.pack(len(val))) self.fs.write(val) except KeyError as e: self.fs.write(self._row_null_val) def write_binary_bigint(self, val): if val is None: self.fs.write(self._row_null_val) else: self.fs.write(self._row_bigint_struct.pack(8, val)) def write_comment_row(self, row): obj = row[1] self.fs.write( _row_header_struct.pack( 8, 8, int(obj["id"], 36), 8, timestamp_to_pgtimestamp(int(obj["created_utc"])))) # write article id # these have to be written separately because of possible null self.write_binary_bigint(get_article(obj.get("link_id", None))) self.write_binary_bigint(get_sub_id(obj.get("subreddit_id", None))) self.write_binary_bigint(get_parent(obj.get("parent_id", None))) # write strings self.write_binary_string(obj, "author") self.write_binary_string(obj, "subreddit") # write jsonb data data = row[0].encode() self.fs.write(pack("!ib", len(data) + 1, 1)) self.fs.write(data) def write_submission_row(self, row): obj = row[1] self.fs.write( _row_header_struct.pack( 6, 8, int(obj["id"], 36), 8, timestamp_to_pgtimestamp(int(obj["created_utc"])))) # write article id # these have to be written separately because of possible null self.write_binary_bigint(get_sub_id(obj.get("subreddit_id", None))) # write strings self.write_binary_string(obj, "author") self.write_binary_string(obj, "subreddit") # write jsonb data data = row[0].encode() self.fs.write(pack("!ib", len(data) + 1, 1)) self.fs.write(data) def copy_comments(self, conn, table, lines): self.write_binary_header() for l in lines: self.write_comment_row(l) # write end of task self.fs.write(pack('!h', -1)) self.fs.flush() self.fs.seek(0) conn.cursor.copy_expert("copy %s from stdin with binary " % (table), self.fs) self.fs.seek(0) self.fs.truncate() def copy(self, conn, table, lines, thing_type): self.write_binary_header() if thing_type == "comments": for l in lines: self.write_comment_row(l) elif thing_type == "submissions": for l in lines: self.write_submission_row(l) else: raise Exception("Unknown thing type {}".format(thing_type)) # write end of task self.fs.write(pack('!h', -1)) self.fs.flush() self.fs.seek(0) conn.cursor.copy_expert("copy %s from stdin with binary " % (table), self.fs) self.fs.seek(0) self.fs.truncate()
def add_chunk(self, fd: io.BufferedRandom, chunk: bytes) -> None: if not fd: return else: fd.write(chunk)