def saveWorld(self, world: World, fileIO: io.BufferedRandom, worldManager: WorldManager): # Checking if file size matches! if not (world.sizeX == 256 and world.sizeY == 256 and world.sizeZ == 256): raise WorldFormatError( f"RawWorldFormat - Trying to save world that has invalid world size! Expected: 256, 256, 256! Got: {world.sizeX}, {world.sizeY}, {world.sizeZ}!" ) # Clearing Current Save File fileIO.truncate(0) fileIO.seek(0) # Saving Map To File fileIO.write(world.gzipMap())
class BinaryCopy(): def __init__(self, chunk_size=256, buffer_size=64 * 1024 * 1024): self.chunk_size = chunk_size self.buffer_size = buffer_size self.fs = BufferedRandom(BytesIO(), buffer_size=buffer_size) self._row_header_struct = Struct("!hiqiq") self._row_int_struct = Struct("!i") self._row_bigint_struct = Struct("!iq") self._row_null_val = pack("!i", -1) def write_binary_header(self): self.fs.write(pack('!11sii', b'PGCOPY\n\xff\r\n\0', 0, 0)) def write_binary_string(self, obj, keyname): try: val = obj[keyname] if val is None: self.fs.write(self._row_null_val) else: val = val.encode() self.fs.write(self._row_int_struct.pack(len(val))) self.fs.write(val) except KeyError as e: self.fs.write(self._row_null_val) def write_binary_bigint(self, val): if val is None: self.fs.write(self._row_null_val) else: self.fs.write(self._row_bigint_struct.pack(8, val)) def write_comment_row(self, row): obj = row[1] self.fs.write( _row_header_struct.pack( 8, 8, int(obj["id"], 36), 8, timestamp_to_pgtimestamp(int(obj["created_utc"])))) # write article id # these have to be written separately because of possible null self.write_binary_bigint(get_article(obj.get("link_id", None))) self.write_binary_bigint(get_sub_id(obj.get("subreddit_id", None))) self.write_binary_bigint(get_parent(obj.get("parent_id", None))) # write strings self.write_binary_string(obj, "author") self.write_binary_string(obj, "subreddit") # write jsonb data data = row[0].encode() self.fs.write(pack("!ib", len(data) + 1, 1)) self.fs.write(data) def write_submission_row(self, row): obj = row[1] self.fs.write( _row_header_struct.pack( 6, 8, int(obj["id"], 36), 8, timestamp_to_pgtimestamp(int(obj["created_utc"])))) # write article id # these have to be written separately because of possible null self.write_binary_bigint(get_sub_id(obj.get("subreddit_id", None))) # write strings self.write_binary_string(obj, "author") self.write_binary_string(obj, "subreddit") # write jsonb data data = row[0].encode() self.fs.write(pack("!ib", len(data) + 1, 1)) self.fs.write(data) def copy_comments(self, conn, table, lines): self.write_binary_header() for l in lines: self.write_comment_row(l) # write end of task self.fs.write(pack('!h', -1)) self.fs.flush() self.fs.seek(0) conn.cursor.copy_expert("copy %s from stdin with binary " % (table), self.fs) self.fs.seek(0) self.fs.truncate() def copy(self, conn, table, lines, thing_type): self.write_binary_header() if thing_type == "comments": for l in lines: self.write_comment_row(l) elif thing_type == "submissions": for l in lines: self.write_submission_row(l) else: raise Exception("Unknown thing type {}".format(thing_type)) # write end of task self.fs.write(pack('!h', -1)) self.fs.flush() self.fs.seek(0) conn.cursor.copy_expert("copy %s from stdin with binary " % (table), self.fs) self.fs.seek(0) self.fs.truncate()
class BASE_WS_DEVICE: def __init__(self, target, password, init=False, ssl=False, auth=False, capath=CA_PATH, passphrase=None): self.ws = None self.ip = target if ':' in password: self.pswd, self.passphrase = password.split(':') else: self.pswd = password self.passphrase = passphrase if ':' in target: self.ip, self.port = target.split(':') self.port = int(self.port) else: self.port = 8266 self.hostname = None self.hostname_mdns = None self.bytes_sent = 0 self.buff = b'' self.raw_buff = b'' self.prompt = b'>>> ' self.response = '' self._kbi = '\x03' self._banner = '\x02' self._reset = '\x04' self._hreset = "import machine; machine.reset()\r" self._traceback = b'Traceback (most recent call last):' self._flush = b'' self.linebuffer = BufferedRandom(BytesIO()) self._debug = False self.output = None self.platform = None self.connected = False self.repl_CONN = self.connected self._ssl = ssl self._uriprotocol = 'ws' if ssl: self._uriprotocol = 'wss' if init: ip_now = None if self.passphrase: auth = True ssl = True self._uriprotocol = 'wss' if self.port == 8266: self.port = 8833 if self.ip.endswith('.local'): self.hostname = self.ip self.hostname_mdns = self.ip try: ip_now = socket.gethostbyname(self.hostname) except socket.gaierror: raise DeviceNotFound( f"WebSocketDevice @ " f"{self._uriprotocol}:" f"//{self.ip}:{self.port} is not reachable") if not ssl: self._uriprotocol = 'ws' self.ws = wsclient.connect(f'ws://{self.ip}:{self.port}', self.pswd) else: if self.port == 8266: self.port = 8833 self._uriprotocol = 'wss' self.ws = wsclient.connect(f'wss://{self.ip}:{self.port}', self.pswd, auth=auth, capath=capath, passphrase=self.passphrase) if self.ws: self.connected = True # resolve name, store ip in self.ip # store mdns name in self.hostname if ip_now: self.ip = ip_now self.repl_CONN = self.connected else: raise DeviceNotFound( f"WebSocketDevice @ " f"{self._uriprotocol}:" f"//{self.ip}:{self.port} is not reachable") def open_wconn(self, ssl=False, auth=False, capath=CA_PATH): try: ip_now = None if self.passphrase: auth = True ssl = True self._uriprotocol = 'wss' if self.port == 8266: self.port = 8833 if self.ip.endswith('.local'): self.hostname = self.ip self.hostname_mdns = self.ip try: ip_now = socket.gethostbyname(self.hostname) except socket.gaierror: raise DeviceNotFound( f"WebSocketDevice @ " f"{self._uriprotocol}:" f"//{self.ip}:{self.port} is not reachable") if not ssl: self._uriprotocol = 'ws' if self.port == 8833: self.port = 8266 self.ws = wsclient.connect(f'ws://{self.ip}:{self.port}', self.pswd) else: self._uriprotocol = 'wss' if self.port == 8266: self.port = 8833 if self.passphrase: auth = True self.ws = wsclient.connect(f'wss://{self.ip}:{self.port}', self.pswd, auth=auth, capath=capath, passphrase=self.passphrase) if self.ws: self.connected = True self.repl_CONN = self.connected if ip_now: self.ip = ip_now else: raise DeviceNotFound( f"WebSocketDevice @ " f"{self._uriprotocol}:" f"//{self.ip}:{self.port} is not reachable") except sslib.SSLError: raise sslib.SSLError except Exception as e: print(e) def close_wconn(self): if self.ws: self.ws.close() self.connected = False if self.hostname_mdns: self.ip = self.hostname_mdns self.repl_CONN = self.connected time.sleep(0.1) def connect(self, **kargs): self.open_wconn(**kargs) def disconnect(self): self.close_wconn() @property def address(self): return self.ip @property def debug(self): return self._debug @debug.setter def debug(self, opt): assert isinstance(opt, bool) self._debug = opt self.ws.debug = opt def write(self, cmd): n_bytes = len(bytes(cmd, 'utf-8')) self.ws.send(cmd) return n_bytes def read_all(self): self.ws.sock.settimeout(None) try: self.raw_buff = b'' while self.prompt not in self.raw_buff: try: fin, opcode, data = self.ws.read_frame() self.raw_buff += data except AttributeError: pass return self.raw_buff except socket.timeout: return self.raw_buff def flush(self): self.ws.sock.settimeout(0.01) self._flush = b'' self.linebuffer.truncate(0) self.linebuffer.seek(0) self.ws.reset_buffers() while self.ws_readable(): while True: try: fin, opcode, data = self.ws.read_frame() self._flush += data except socket.timeout: break except wsprotocol.NoDataException: break self.ws.reset_buffers() def ws_readable(self): for i in range(3): try: readable, writable, exceptional = select.select([self.ws.sock], [self.ws.sock], [self.ws.sock]) if readable: return True else: return False except Exception as e: if self.debug: print(e) time.sleep(0.01) def wr_cmd(self, cmd, silent=False, rtn=True, rtn_resp=False, long_string=False): self.output = None self.response = '' self.buff = b'' self.flush() self.bytes_sent = self.write(cmd + '\r') # time.sleep(0.1) # self.buff = self.read_all()[self.bytes_sent:] self.buff = self.read_all() if self.buff == b'': # time.sleep(0.1) self.buff = self.read_all() # print(self.buff) # filter command cmd_filt = bytes(cmd + '\r\n', 'utf-8') self.buff = self.buff.replace(cmd_filt, b'', 1) if self._traceback in self.buff: long_string = True if long_string: self.response = self.buff.replace(b'\r', b'').replace( b'\r\n>>> ', b'').replace(b'>>> ', b'').decode('utf-8', 'ignore') else: self.response = self.buff.replace(b'\r\n', b'').replace( b'\r\n>>> ', b'').replace(b'>>> ', b'').decode('utf-8', 'ignore') if not silent: if self.response != '\n' and self.response != '': print(self.response) else: self.response = '' if rtn: self.get_output() if self.output == '\n' and self.output == '': self.output = None if self.output is None: if self.response != '' and self.response != '\n': self.output = self.response if rtn_resp: return self.output def cmd(self, cmd, silent=False, rtn=False, long_string=False): disconnect_on_end = not self.connected if not self.connected: self.open_wconn(ssl=self._ssl, auth=True) self.wr_cmd(cmd, silent=True, long_string=long_string) if self.connected: if disconnect_on_end: self.close_wconn() self.get_output() if not silent: print(self.response) if rtn: return self.output def reset(self, silent=False, reconnect=True, hr=False): if not silent: print('Rebooting device...') if self.connected: if not hr: self.bytes_sent = self.write(self._reset) else: self.bytes_sent = self.write(self._hreset) if self._uriprotocol == 'ws': time.sleep(0.2) else: time.sleep(1) self.close_wconn() if reconnect: time.sleep(1) while True: try: self.open_wconn(ssl=self._ssl, auth=True) self.wr_cmd(self._banner, silent=True) break except Exception: time.sleep(0.5) self.ws._close() self.cmd('') if not silent: print('Done!') else: self.open_wconn(ssl=self._ssl, auth=True) if not hr: self.bytes_sent = self.write(self._reset) else: self.bytes_sent = self.write(self._hreset) if self._uriprotocol == 'ws': time.sleep(0.2) else: time.sleep(1) self.close_wconn() if not silent: print('Done!') def kbi(self, silent=True, pipe=None, long_string=False): if self.connected: if pipe is not None: self.wr_cmd(self._kbi, silent=silent) bf_output = self.response.split('Traceback')[0] traceback = 'Traceback' + self.response.split('Traceback')[1] if bf_output != '' and bf_output != '\n': pipe(bf_output) pipe(traceback, std='stderr') else: self.wr_cmd(self._kbi, silent=silent, long_string=long_string) self.cmd('') else: self.cmd(self._kbi, silent=silent) self.cmd('') def banner(self, pipe=None): self.wr_cmd(self._banner, silent=True, long_string=True) if pipe is None: print(self.response.replace('\n\n', '\n')) else: pipe(self.response.replace('\n\n', '\n')) def get_output(self): try: self.output = ast.literal_eval(self.response) except Exception: if 'bytearray' in self.response: try: self.output = bytearray( ast.literal_eval( self.response.strip().split('bytearray')[1])) except Exception: pass else: if 'array' in self.response: try: arr = ast.literal_eval( self.response.strip().split('array')[1]) self.output = array(arr[0], arr[1]) except Exception: pass pass