class SandboxInOut(object):
"""Collect written text, and return it when called."""
class InnerSandboxInOut(object):
def __init__(self, printer, _getattr_=None):
self.printer = printer
self._getattr_ = _getattr_
def write(self, text: str):
self.printer.write(text, consumer=False)
def read(self):
line = self.printer.readline(consumer=True)
if len(line) == 0:
return None
return str(line)
def __call__(self, testing=None):
return self
def _call_print(self, *objects, **kwargs):
if kwargs.get('file', None) is None:
kwargs['file'] = self
else:
self._getattr_(kwargs['file'], 'write')
print(*objects, **kwargs)
def __init__(self, _getattr_=None):
self.stream = BufferedRandom(BytesIO())
self.position = 0
def printer(self):
return self.InnerSandboxInOut(self)
def reader(self):
return lambda: self.readline()
def write(self, line, consumer=True):
if not consumer:
self.stream.seek(2) # go to the end of the stream
else:
self.stream.seek(0, self.position)
self.stream.write(bytes(line, 'utf-8'))
if consumer:
self.position = self.stream.tell()
def readline(self, consumer=True):
if not consumer:
self.stream.seek(2) # go to the end of the stream
else:
self.stream.seek(0, self.position)
line = self.stream.readline()
if len(line) == 0:
return None
if consumer:
self.position = self.stream.tell()
return str(line.decode('utf-8')).strip()
def get_stream(self):
return self.stream
def FileWriteInteger(file_handle: io.BufferedRandom,
value: int,
size: int = INT_VALUE) -> int:
if size < CHAR_VALUE or size > INT_VALUE:
raise Exception("Size error")
file_handle.write(
int(value).to_bytes(size, byteorder="little", signed=False))
return size
def FileWriteString(file_handle: io.BufferedRandom,
text: str,
length: int = -1) -> int:
if length != -1:
text = text[:length]
length = len(text)
if file_handle.binflag:
file_handle.write(text.encode(file_handle.encoding))
else:
file_handle.write(text)
return length
def saveWorld(self, world: World, fileIO: io.BufferedRandom,
worldManager: WorldManager):
# Checking if file size matches!
if not (world.sizeX == 256 and world.sizeY == 256
and world.sizeZ == 256):
raise WorldFormatError(
f"RawWorldFormat - Trying to save world that has invalid world size! Expected: 256, 256, 256! Got: {world.sizeX}, {world.sizeY}, {world.sizeZ}!"
)
# Clearing Current Save File
fileIO.truncate(0)
fileIO.seek(0)
# Saving Map To File
fileIO.write(world.gzipMap())
def FileWriteDouble(file_handle: io.BufferedRandom, value: float) -> int:
file_handle.write(struct.pack("<d", value))
return 8
def FileWriteLong(file_handle: io.BufferedRandom, value: int) -> int:
file_handle.write(int(value).to_bytes(8, byteorder="little", signed=False))
return 8
class BinaryCopy():
def __init__(self, chunk_size=256, buffer_size=64 * 1024 * 1024):
self.chunk_size = chunk_size
self.buffer_size = buffer_size
self.fs = BufferedRandom(BytesIO(), buffer_size=buffer_size)
self._row_header_struct = Struct("!hiqiq")
self._row_int_struct = Struct("!i")
self._row_bigint_struct = Struct("!iq")
self._row_null_val = pack("!i", -1)
def write_binary_header(self):
self.fs.write(pack('!11sii', b'PGCOPY\n\xff\r\n\0', 0, 0))
def write_binary_string(self, obj, keyname):
try:
val = obj[keyname]
if val is None:
self.fs.write(self._row_null_val)
else:
val = val.encode()
self.fs.write(self._row_int_struct.pack(len(val)))
self.fs.write(val)
except KeyError as e:
self.fs.write(self._row_null_val)
def write_binary_bigint(self, val):
if val is None:
self.fs.write(self._row_null_val)
else:
self.fs.write(self._row_bigint_struct.pack(8, val))
def write_comment_row(self, row):
obj = row[1]
self.fs.write(
_row_header_struct.pack(
8, 8, int(obj["id"], 36), 8,
timestamp_to_pgtimestamp(int(obj["created_utc"]))))
# write article id
# these have to be written separately because of possible null
self.write_binary_bigint(get_article(obj.get("link_id", None)))
self.write_binary_bigint(get_sub_id(obj.get("subreddit_id", None)))
self.write_binary_bigint(get_parent(obj.get("parent_id", None)))
# write strings
self.write_binary_string(obj, "author")
self.write_binary_string(obj, "subreddit")
# write jsonb data
data = row[0].encode()
self.fs.write(pack("!ib", len(data) + 1, 1))
self.fs.write(data)
def write_submission_row(self, row):
obj = row[1]
self.fs.write(
_row_header_struct.pack(
6, 8, int(obj["id"], 36), 8,
timestamp_to_pgtimestamp(int(obj["created_utc"]))))
# write article id
# these have to be written separately because of possible null
self.write_binary_bigint(get_sub_id(obj.get("subreddit_id", None)))
# write strings
self.write_binary_string(obj, "author")
self.write_binary_string(obj, "subreddit")
# write jsonb data
data = row[0].encode()
self.fs.write(pack("!ib", len(data) + 1, 1))
self.fs.write(data)
def copy_comments(self, conn, table, lines):
self.write_binary_header()
for l in lines:
self.write_comment_row(l)
# write end of task
self.fs.write(pack('!h', -1))
self.fs.flush()
self.fs.seek(0)
conn.cursor.copy_expert("copy %s from stdin with binary " % (table),
self.fs)
self.fs.seek(0)
self.fs.truncate()
def copy(self, conn, table, lines, thing_type):
self.write_binary_header()
if thing_type == "comments":
for l in lines:
self.write_comment_row(l)
elif thing_type == "submissions":
for l in lines:
self.write_submission_row(l)
else:
raise Exception("Unknown thing type {}".format(thing_type))
# write end of task
self.fs.write(pack('!h', -1))
self.fs.flush()
self.fs.seek(0)
conn.cursor.copy_expert("copy %s from stdin with binary " % (table),
self.fs)
self.fs.seek(0)
self.fs.truncate()
def add_chunk(self, fd: io.BufferedRandom, chunk: bytes) -> None:
if not fd:
return
else:
fd.write(chunk)
