def saveWorld(self, world: World, fileIO: io.BufferedRandom,
worldManager: WorldManager):
# Checking if file size matches!
if not (world.sizeX == 256 and world.sizeY == 256
and world.sizeZ == 256):
raise WorldFormatError(
f"RawWorldFormat - Trying to save world that has invalid world size! Expected: 256, 256, 256! Got: {world.sizeX}, {world.sizeY}, {world.sizeZ}!"
)
# Clearing Current Save File
fileIO.truncate(0)
fileIO.seek(0)
# Saving Map To File
fileIO.write(world.gzipMap())
class BinaryCopy():
def __init__(self, chunk_size=256, buffer_size=64 * 1024 * 1024):
self.chunk_size = chunk_size
self.buffer_size = buffer_size
self.fs = BufferedRandom(BytesIO(), buffer_size=buffer_size)
self._row_header_struct = Struct("!hiqiq")
self._row_int_struct = Struct("!i")
self._row_bigint_struct = Struct("!iq")
self._row_null_val = pack("!i", -1)
def write_binary_header(self):
self.fs.write(pack('!11sii', b'PGCOPY\n\xff\r\n\0', 0, 0))
def write_binary_string(self, obj, keyname):
try:
val = obj[keyname]
if val is None:
self.fs.write(self._row_null_val)
else:
val = val.encode()
self.fs.write(self._row_int_struct.pack(len(val)))
self.fs.write(val)
except KeyError as e:
self.fs.write(self._row_null_val)
def write_binary_bigint(self, val):
if val is None:
self.fs.write(self._row_null_val)
else:
self.fs.write(self._row_bigint_struct.pack(8, val))
def write_comment_row(self, row):
obj = row[1]
self.fs.write(
_row_header_struct.pack(
8, 8, int(obj["id"], 36), 8,
timestamp_to_pgtimestamp(int(obj["created_utc"]))))
# write article id
# these have to be written separately because of possible null
self.write_binary_bigint(get_article(obj.get("link_id", None)))
self.write_binary_bigint(get_sub_id(obj.get("subreddit_id", None)))
self.write_binary_bigint(get_parent(obj.get("parent_id", None)))
# write strings
self.write_binary_string(obj, "author")
self.write_binary_string(obj, "subreddit")
# write jsonb data
data = row[0].encode()
self.fs.write(pack("!ib", len(data) + 1, 1))
self.fs.write(data)
def write_submission_row(self, row):
obj = row[1]
self.fs.write(
_row_header_struct.pack(
6, 8, int(obj["id"], 36), 8,
timestamp_to_pgtimestamp(int(obj["created_utc"]))))
# write article id
# these have to be written separately because of possible null
self.write_binary_bigint(get_sub_id(obj.get("subreddit_id", None)))
# write strings
self.write_binary_string(obj, "author")
self.write_binary_string(obj, "subreddit")
# write jsonb data
data = row[0].encode()
self.fs.write(pack("!ib", len(data) + 1, 1))
self.fs.write(data)
def copy_comments(self, conn, table, lines):
self.write_binary_header()
for l in lines:
self.write_comment_row(l)
# write end of task
self.fs.write(pack('!h', -1))
self.fs.flush()
self.fs.seek(0)
conn.cursor.copy_expert("copy %s from stdin with binary " % (table),
self.fs)
self.fs.seek(0)
self.fs.truncate()
def copy(self, conn, table, lines, thing_type):
self.write_binary_header()
if thing_type == "comments":
for l in lines:
self.write_comment_row(l)
elif thing_type == "submissions":
for l in lines:
self.write_submission_row(l)
else:
raise Exception("Unknown thing type {}".format(thing_type))
# write end of task
self.fs.write(pack('!h', -1))
self.fs.flush()
self.fs.seek(0)
conn.cursor.copy_expert("copy %s from stdin with binary " % (table),
self.fs)
self.fs.seek(0)
self.fs.truncate()
class BASE_WS_DEVICE:
def __init__(self,
target,
password,
init=False,
ssl=False,
auth=False,
capath=CA_PATH,
passphrase=None):
self.ws = None
self.ip = target
if ':' in password:
self.pswd, self.passphrase = password.split(':')
else:
self.pswd = password
self.passphrase = passphrase
if ':' in target:
self.ip, self.port = target.split(':')
self.port = int(self.port)
else:
self.port = 8266
self.hostname = None
self.hostname_mdns = None
self.bytes_sent = 0
self.buff = b''
self.raw_buff = b''
self.prompt = b'>>> '
self.response = ''
self._kbi = '\x03'
self._banner = '\x02'
self._reset = '\x04'
self._hreset = "import machine; machine.reset()\r"
self._traceback = b'Traceback (most recent call last):'
self._flush = b''
self.linebuffer = BufferedRandom(BytesIO())
self._debug = False
self.output = None
self.platform = None
self.connected = False
self.repl_CONN = self.connected
self._ssl = ssl
self._uriprotocol = 'ws'
if ssl:
self._uriprotocol = 'wss'
if init:
ip_now = None
if self.passphrase:
auth = True
ssl = True
self._uriprotocol = 'wss'
if self.port == 8266:
self.port = 8833
if self.ip.endswith('.local'):
self.hostname = self.ip
self.hostname_mdns = self.ip
try:
ip_now = socket.gethostbyname(self.hostname)
except socket.gaierror:
raise DeviceNotFound(
f"WebSocketDevice @ "
f"{self._uriprotocol}:"
f"//{self.ip}:{self.port} is not reachable")
if not ssl:
self._uriprotocol = 'ws'
self.ws = wsclient.connect(f'ws://{self.ip}:{self.port}',
self.pswd)
else:
if self.port == 8266:
self.port = 8833
self._uriprotocol = 'wss'
self.ws = wsclient.connect(f'wss://{self.ip}:{self.port}',
self.pswd,
auth=auth,
capath=capath,
passphrase=self.passphrase)
if self.ws:
self.connected = True
# resolve name, store ip in self.ip
# store mdns name in self.hostname
if ip_now:
self.ip = ip_now
self.repl_CONN = self.connected
else:
raise DeviceNotFound(
f"WebSocketDevice @ "
f"{self._uriprotocol}:"
f"//{self.ip}:{self.port} is not reachable")
def open_wconn(self, ssl=False, auth=False, capath=CA_PATH):
try:
ip_now = None
if self.passphrase:
auth = True
ssl = True
self._uriprotocol = 'wss'
if self.port == 8266:
self.port = 8833
if self.ip.endswith('.local'):
self.hostname = self.ip
self.hostname_mdns = self.ip
try:
ip_now = socket.gethostbyname(self.hostname)
except socket.gaierror:
raise DeviceNotFound(
f"WebSocketDevice @ "
f"{self._uriprotocol}:"
f"//{self.ip}:{self.port} is not reachable")
if not ssl:
self._uriprotocol = 'ws'
if self.port == 8833:
self.port = 8266
self.ws = wsclient.connect(f'ws://{self.ip}:{self.port}',
self.pswd)
else:
self._uriprotocol = 'wss'
if self.port == 8266:
self.port = 8833
if self.passphrase:
auth = True
self.ws = wsclient.connect(f'wss://{self.ip}:{self.port}',
self.pswd,
auth=auth,
capath=capath,
passphrase=self.passphrase)
if self.ws:
self.connected = True
self.repl_CONN = self.connected
if ip_now:
self.ip = ip_now
else:
raise DeviceNotFound(
f"WebSocketDevice @ "
f"{self._uriprotocol}:"
f"//{self.ip}:{self.port} is not reachable")
except sslib.SSLError:
raise sslib.SSLError
except Exception as e:
print(e)
def close_wconn(self):
if self.ws:
self.ws.close()
self.connected = False
if self.hostname_mdns:
self.ip = self.hostname_mdns
self.repl_CONN = self.connected
time.sleep(0.1)
def connect(self, **kargs):
self.open_wconn(**kargs)
def disconnect(self):
self.close_wconn()
@property
def address(self):
return self.ip
@property
def debug(self):
return self._debug
@debug.setter
def debug(self, opt):
assert isinstance(opt, bool)
self._debug = opt
self.ws.debug = opt
def write(self, cmd):
n_bytes = len(bytes(cmd, 'utf-8'))
self.ws.send(cmd)
return n_bytes
def read_all(self):
self.ws.sock.settimeout(None)
try:
self.raw_buff = b''
while self.prompt not in self.raw_buff:
try:
fin, opcode, data = self.ws.read_frame()
self.raw_buff += data
except AttributeError:
pass
return self.raw_buff
except socket.timeout:
return self.raw_buff
def flush(self):
self.ws.sock.settimeout(0.01)
self._flush = b''
self.linebuffer.truncate(0)
self.linebuffer.seek(0)
self.ws.reset_buffers()
while self.ws_readable():
while True:
try:
fin, opcode, data = self.ws.read_frame()
self._flush += data
except socket.timeout:
break
except wsprotocol.NoDataException:
break
self.ws.reset_buffers()
def ws_readable(self):
for i in range(3):
try:
readable, writable, exceptional = select.select([self.ws.sock],
[self.ws.sock],
[self.ws.sock])
if readable:
return True
else:
return False
except Exception as e:
if self.debug:
print(e)
time.sleep(0.01)
def wr_cmd(self,
cmd,
silent=False,
rtn=True,
rtn_resp=False,
long_string=False):
self.output = None
self.response = ''
self.buff = b''
self.flush()
self.bytes_sent = self.write(cmd + '\r')
# time.sleep(0.1)
# self.buff = self.read_all()[self.bytes_sent:]
self.buff = self.read_all()
if self.buff == b'':
# time.sleep(0.1)
self.buff = self.read_all()
# print(self.buff)
# filter command
cmd_filt = bytes(cmd + '\r\n', 'utf-8')
self.buff = self.buff.replace(cmd_filt, b'', 1)
if self._traceback in self.buff:
long_string = True
if long_string:
self.response = self.buff.replace(b'\r', b'').replace(
b'\r\n>>> ', b'').replace(b'>>> ',
b'').decode('utf-8', 'ignore')
else:
self.response = self.buff.replace(b'\r\n', b'').replace(
b'\r\n>>> ', b'').replace(b'>>> ',
b'').decode('utf-8', 'ignore')
if not silent:
if self.response != '\n' and self.response != '':
print(self.response)
else:
self.response = ''
if rtn:
self.get_output()
if self.output == '\n' and self.output == '':
self.output = None
if self.output is None:
if self.response != '' and self.response != '\n':
self.output = self.response
if rtn_resp:
return self.output
def cmd(self, cmd, silent=False, rtn=False, long_string=False):
disconnect_on_end = not self.connected
if not self.connected:
self.open_wconn(ssl=self._ssl, auth=True)
self.wr_cmd(cmd, silent=True, long_string=long_string)
if self.connected:
if disconnect_on_end:
self.close_wconn()
self.get_output()
if not silent:
print(self.response)
if rtn:
return self.output
def reset(self, silent=False, reconnect=True, hr=False):
if not silent:
print('Rebooting device...')
if self.connected:
if not hr:
self.bytes_sent = self.write(self._reset)
else:
self.bytes_sent = self.write(self._hreset)
if self._uriprotocol == 'ws':
time.sleep(0.2)
else:
time.sleep(1)
self.close_wconn()
if reconnect:
time.sleep(1)
while True:
try:
self.open_wconn(ssl=self._ssl, auth=True)
self.wr_cmd(self._banner, silent=True)
break
except Exception:
time.sleep(0.5)
self.ws._close()
self.cmd('')
if not silent:
print('Done!')
else:
self.open_wconn(ssl=self._ssl, auth=True)
if not hr:
self.bytes_sent = self.write(self._reset)
else:
self.bytes_sent = self.write(self._hreset)
if self._uriprotocol == 'ws':
time.sleep(0.2)
else:
time.sleep(1)
self.close_wconn()
if not silent:
print('Done!')
def kbi(self, silent=True, pipe=None, long_string=False):
if self.connected:
if pipe is not None:
self.wr_cmd(self._kbi, silent=silent)
bf_output = self.response.split('Traceback')[0]
traceback = 'Traceback' + self.response.split('Traceback')[1]
if bf_output != '' and bf_output != '\n':
pipe(bf_output)
pipe(traceback, std='stderr')
else:
self.wr_cmd(self._kbi, silent=silent, long_string=long_string)
self.cmd('')
else:
self.cmd(self._kbi, silent=silent)
self.cmd('')
def banner(self, pipe=None):
self.wr_cmd(self._banner, silent=True, long_string=True)
if pipe is None:
print(self.response.replace('\n\n', '\n'))
else:
pipe(self.response.replace('\n\n', '\n'))
def get_output(self):
try:
self.output = ast.literal_eval(self.response)
except Exception:
if 'bytearray' in self.response:
try:
self.output = bytearray(
ast.literal_eval(
self.response.strip().split('bytearray')[1]))
except Exception:
pass
else:
if 'array' in self.response:
try:
arr = ast.literal_eval(
self.response.strip().split('array')[1])
self.output = array(arr[0], arr[1])
except Exception:
pass
pass
