class SandboxInOut(object): """Collect written text, and return it when called.""" class InnerSandboxInOut(object): def __init__(self, printer, _getattr_=None): self.printer = printer self._getattr_ = _getattr_ def write(self, text: str): self.printer.write(text, consumer=False) def read(self): line = self.printer.readline(consumer=True) if len(line) == 0: return None return str(line) def __call__(self, testing=None): return self def _call_print(self, *objects, **kwargs): if kwargs.get('file', None) is None: kwargs['file'] = self else: self._getattr_(kwargs['file'], 'write') print(*objects, **kwargs) def __init__(self, _getattr_=None): self.stream = BufferedRandom(BytesIO()) self.position = 0 def printer(self): return self.InnerSandboxInOut(self) def reader(self): return lambda: self.readline() def write(self, line, consumer=True): if not consumer: self.stream.seek(2) # go to the end of the stream else: self.stream.seek(0, self.position) self.stream.write(bytes(line, 'utf-8')) if consumer: self.position = self.stream.tell() def readline(self, consumer=True): if not consumer: self.stream.seek(2) # go to the end of the stream else: self.stream.seek(0, self.position) line = self.stream.readline() if len(line) == 0: return None if consumer: self.position = self.stream.tell() return str(line.decode('utf-8')).strip() def get_stream(self): return self.stream
def saveWorld(self, world: World, fileIO: io.BufferedRandom, worldManager: WorldManager): # Checking if file size matches! if not (world.sizeX == 256 and world.sizeY == 256 and world.sizeZ == 256): raise WorldFormatError( f"RawWorldFormat - Trying to save world that has invalid world size! Expected: 256, 256, 256! Got: {world.sizeX}, {world.sizeY}, {world.sizeZ}!" ) # Clearing Current Save File fileIO.truncate(0) fileIO.seek(0) # Saving Map To File fileIO.write(world.gzipMap())
def __init__(self, file_: BufferedRandom, length: int) -> None: # Read the file into a buffer. file_.seek(0, SEEK_END) buf = bytearray(file_.tell()) file_.seek(0, SEEK_SET) file_.readinto(buf) # Overwrite the header, in case it is currently being written to. header = list_header(length) buf[:len(header)] = header # Parse the buffer's contents as list items. super().__init__(namespacify(cbor2.loads(buf))) # Store the file pointer for `extend` calls. self._file = file_
def merge_pdf(pages, target_pdf, img_format='png'): merged_pdf = PyPDF2.PdfFileWriter() log.debug('Merging {} into one PDF document'.format(len(pages))) for page in pages: # reset binary stream position page.seek(0) pdf_page_stream = BufferedRandom(BytesIO()) log.debug('Converting image to PDF') with wandImage(file=page, format=img_format) as image: img_converted = image.convert('pdf') img_converted.save(file=pdf_page_stream) img_converted.close() log.debug('Adding PDF page to merged document') pdf_page_stream.seek(0) merged_pdf.addPage(PyPDF2.PdfFileReader(pdf_page_stream).getPage(0)) merged_pdf.write(open(target_pdf, 'wb'))
def merge_pdf(pages, target_pdf, img_format='png'): merged_pdf = PyPDF2.PdfFileWriter() log.debug('Merging {} into one PDF document'.format(len(pages))) for page in pages: # reset binary stream position page.seek(0) pdf_page_stream = BufferedRandom(BytesIO()) log.debug('Converting image to PDF') with wandImage(file=page, format=img_format) as image: img_converted = image.convert('pdf') img_converted.save(file=pdf_page_stream) img_converted.close() log.debug('Adding PDF page to merged document') pdf_page_stream.seek(0) merged_pdf.addPage(PyPDF2.PdfFileReader(pdf_page_stream).getPage(0)) merged_pdf.write(open(target_pdf, 'wb'))
def split_pdf(src_filename, pdf_res, page_list=None, img_format='png'): pdf_source = PyPDF2.PdfFileReader(open(src_filename, 'rb')) logging.debug('Source PDF {} contains {} pages'.format( src_filename, pdf_source.getNumPages())) pages = [] if not page_list: log.debug('Extracting all pages from PDF') # no pages defined, so create a list of all page numbers page_list = list(range(pdf_source.getNumPages())) else: log.debug('Extracting {} page(s) from PDF'.format(len(page_list))) for page_nr in page_list: log.debug('Processing page {}'.format(page_nr)) page = pdf_source.getPage(page_nr) log.debug('Extracting page from source PDF') # extract single page and save it to a temporary stream pdf_writer = PyPDF2.PdfFileWriter() pdf_writer.addPage(page) pdf_page_stream = BufferedRandom(BytesIO()) pdf_writer.write(pdf_page_stream) # reset the binary stream's position to the beginning pdf_page_stream.seek(0) log.debug('Converting PDF page to image ({})'.format(img_format)) # Define the resolution when opening the intermediate PDF for better quality converted PNGs # http://stackoverflow.com/questions/17314382/improve-quality-of-wand-conversion with wandImage(file=pdf_page_stream, resolution=pdf_res) as pdf_page: image_page = pdf_page.convert(img_format) image_page_stream = BufferedRandom(BytesIO()) image_page.save(file=image_page_stream) pages.append(image_page_stream) return pages
def split_pdf(src_filename, pdf_res, page_list=None, img_format='png'): pdf_source = PyPDF2.PdfFileReader(open(src_filename, 'rb')) logging.debug('Source PDF {} contains {} pages'.format(src_filename, pdf_source.getNumPages())) pages = [] if not page_list: log.debug('Extracting all pages from PDF') # no pages defined, so create a list of all page numbers page_list = list(range(pdf_source.getNumPages())) else: log.debug('Extracting {} page(s) from PDF'.format(len(page_list))) for page_nr in page_list: log.debug('Processing page {}'.format(page_nr)) page = pdf_source.getPage(page_nr) log.debug('Extracting page from source PDF') # extract single page and save it to a temporary stream pdf_writer = PyPDF2.PdfFileWriter() pdf_writer.addPage(page) pdf_page_stream = BufferedRandom(BytesIO()) pdf_writer.write(pdf_page_stream) # reset the binary stream's position to the beginning pdf_page_stream.seek(0) log.debug('Converting PDF page to image ({})'.format(img_format)) # Define the resolution when opening the intermediate PDF for better quality converted PNGs # http://stackoverflow.com/questions/17314382/improve-quality-of-wand-conversion with wandImage(file=pdf_page_stream, resolution=pdf_res) as pdf_page: image_page = pdf_page.convert(img_format) image_page_stream = BufferedRandom(BytesIO()) image_page.save(file=image_page_stream) pages.append(image_page_stream) return pages
def read_frame_data(seekable_binary_stream: io.BufferedRandom): seekable_binary_stream.seek(0) header = seekable_binary_stream.read(2) if header != b'\xff\xd8': raise Exception size = None while size is None: marker = seekable_binary_stream.read(2) # fix marker reading position if marker[0] != 255 and marker[1] == 255: seekable_binary_stream.seek(-1, 1) marker = seekable_binary_stream.read(2) if marker == b'\xff\x00': raise ValueError("jpeg marker not found") if marker in START_OF_FRAME_MARKERS: seekable_binary_stream.seek(3, 1) size = struct.unpack('>HH', seekable_binary_stream.read(4)) components = seekable_binary_stream.read(1)[0] component_scales = [] subsampling = SUPPORTED_COLOR_SPACES.NONE for component in range(components): component_id = seekable_binary_stream.read(1)[0] colorspace_struct = seekable_binary_stream.read(1)[0] h = (colorspace_struct & 0xf0) >> 4 v = colorspace_struct & 0x0f component_scales.append((h, v)) seekable_binary_stream.seek(1, 1) if component_scales[0] == (2, 2): subsampling = SUPPORTED_COLOR_SPACES.YUV420 elif component_scales[0] == (2, 1) or component_scales[0] == (1, 2): subsampling = SUPPORTED_COLOR_SPACES.YUV422 else: subsampling = SUPPORTED_COLOR_SPACES.YUV444 return (size, subsampling) else: frame_len = struct.unpack('>H', seekable_binary_stream.read(2))[0] seekable_binary_stream.seek(frame_len - 2, 1)
def read(self, path, offset, size): buf = None br = None conn = None try: conn, desc = self.session.data_objects.open(path, O_RDONLY) raw = iRODSDataObjectFileRaw(conn, desc) br = BufferedRandom(raw) new_offset = br.seek(offset) if new_offset == offset: buf = br.read(size) finally: if br: br.close() if conn: conn.release(True) return buf
def FileSeek(file_handle: io.BufferedRandom, offset: int, origin: int) -> bool: file_handle.seek(offset, origin) return True
class BinaryCopy(): def __init__(self, chunk_size=256, buffer_size=64 * 1024 * 1024): self.chunk_size = chunk_size self.buffer_size = buffer_size self.fs = BufferedRandom(BytesIO(), buffer_size=buffer_size) self._row_header_struct = Struct("!hiqiq") self._row_int_struct = Struct("!i") self._row_bigint_struct = Struct("!iq") self._row_null_val = pack("!i", -1) def write_binary_header(self): self.fs.write(pack('!11sii', b'PGCOPY\n\xff\r\n\0', 0, 0)) def write_binary_string(self, obj, keyname): try: val = obj[keyname] if val is None: self.fs.write(self._row_null_val) else: val = val.encode() self.fs.write(self._row_int_struct.pack(len(val))) self.fs.write(val) except KeyError as e: self.fs.write(self._row_null_val) def write_binary_bigint(self, val): if val is None: self.fs.write(self._row_null_val) else: self.fs.write(self._row_bigint_struct.pack(8, val)) def write_comment_row(self, row): obj = row[1] self.fs.write( _row_header_struct.pack( 8, 8, int(obj["id"], 36), 8, timestamp_to_pgtimestamp(int(obj["created_utc"])))) # write article id # these have to be written separately because of possible null self.write_binary_bigint(get_article(obj.get("link_id", None))) self.write_binary_bigint(get_sub_id(obj.get("subreddit_id", None))) self.write_binary_bigint(get_parent(obj.get("parent_id", None))) # write strings self.write_binary_string(obj, "author") self.write_binary_string(obj, "subreddit") # write jsonb data data = row[0].encode() self.fs.write(pack("!ib", len(data) + 1, 1)) self.fs.write(data) def write_submission_row(self, row): obj = row[1] self.fs.write( _row_header_struct.pack( 6, 8, int(obj["id"], 36), 8, timestamp_to_pgtimestamp(int(obj["created_utc"])))) # write article id # these have to be written separately because of possible null self.write_binary_bigint(get_sub_id(obj.get("subreddit_id", None))) # write strings self.write_binary_string(obj, "author") self.write_binary_string(obj, "subreddit") # write jsonb data data = row[0].encode() self.fs.write(pack("!ib", len(data) + 1, 1)) self.fs.write(data) def copy_comments(self, conn, table, lines): self.write_binary_header() for l in lines: self.write_comment_row(l) # write end of task self.fs.write(pack('!h', -1)) self.fs.flush() self.fs.seek(0) conn.cursor.copy_expert("copy %s from stdin with binary " % (table), self.fs) self.fs.seek(0) self.fs.truncate() def copy(self, conn, table, lines, thing_type): self.write_binary_header() if thing_type == "comments": for l in lines: self.write_comment_row(l) elif thing_type == "submissions": for l in lines: self.write_submission_row(l) else: raise Exception("Unknown thing type {}".format(thing_type)) # write end of task self.fs.write(pack('!h', -1)) self.fs.flush() self.fs.seek(0) conn.cursor.copy_expert("copy %s from stdin with binary " % (table), self.fs) self.fs.seek(0) self.fs.truncate()
class BASE_WS_DEVICE: def __init__(self, target, password, init=False, ssl=False, auth=False, capath=CA_PATH, passphrase=None): self.ws = None self.ip = target if ':' in password: self.pswd, self.passphrase = password.split(':') else: self.pswd = password self.passphrase = passphrase if ':' in target: self.ip, self.port = target.split(':') self.port = int(self.port) else: self.port = 8266 self.hostname = None self.hostname_mdns = None self.bytes_sent = 0 self.buff = b'' self.raw_buff = b'' self.prompt = b'>>> ' self.response = '' self._kbi = '\x03' self._banner = '\x02' self._reset = '\x04' self._hreset = "import machine; machine.reset()\r" self._traceback = b'Traceback (most recent call last):' self._flush = b'' self.linebuffer = BufferedRandom(BytesIO()) self._debug = False self.output = None self.platform = None self.connected = False self.repl_CONN = self.connected self._ssl = ssl self._uriprotocol = 'ws' if ssl: self._uriprotocol = 'wss' if init: ip_now = None if self.passphrase: auth = True ssl = True self._uriprotocol = 'wss' if self.port == 8266: self.port = 8833 if self.ip.endswith('.local'): self.hostname = self.ip self.hostname_mdns = self.ip try: ip_now = socket.gethostbyname(self.hostname) except socket.gaierror: raise DeviceNotFound( f"WebSocketDevice @ " f"{self._uriprotocol}:" f"//{self.ip}:{self.port} is not reachable") if not ssl: self._uriprotocol = 'ws' self.ws = wsclient.connect(f'ws://{self.ip}:{self.port}', self.pswd) else: if self.port == 8266: self.port = 8833 self._uriprotocol = 'wss' self.ws = wsclient.connect(f'wss://{self.ip}:{self.port}', self.pswd, auth=auth, capath=capath, passphrase=self.passphrase) if self.ws: self.connected = True # resolve name, store ip in self.ip # store mdns name in self.hostname if ip_now: self.ip = ip_now self.repl_CONN = self.connected else: raise DeviceNotFound( f"WebSocketDevice @ " f"{self._uriprotocol}:" f"//{self.ip}:{self.port} is not reachable") def open_wconn(self, ssl=False, auth=False, capath=CA_PATH): try: ip_now = None if self.passphrase: auth = True ssl = True self._uriprotocol = 'wss' if self.port == 8266: self.port = 8833 if self.ip.endswith('.local'): self.hostname = self.ip self.hostname_mdns = self.ip try: ip_now = socket.gethostbyname(self.hostname) except socket.gaierror: raise DeviceNotFound( f"WebSocketDevice @ " f"{self._uriprotocol}:" f"//{self.ip}:{self.port} is not reachable") if not ssl: self._uriprotocol = 'ws' if self.port == 8833: self.port = 8266 self.ws = wsclient.connect(f'ws://{self.ip}:{self.port}', self.pswd) else: self._uriprotocol = 'wss' if self.port == 8266: self.port = 8833 if self.passphrase: auth = True self.ws = wsclient.connect(f'wss://{self.ip}:{self.port}', self.pswd, auth=auth, capath=capath, passphrase=self.passphrase) if self.ws: self.connected = True self.repl_CONN = self.connected if ip_now: self.ip = ip_now else: raise DeviceNotFound( f"WebSocketDevice @ " f"{self._uriprotocol}:" f"//{self.ip}:{self.port} is not reachable") except sslib.SSLError: raise sslib.SSLError except Exception as e: print(e) def close_wconn(self): if self.ws: self.ws.close() self.connected = False if self.hostname_mdns: self.ip = self.hostname_mdns self.repl_CONN = self.connected time.sleep(0.1) def connect(self, **kargs): self.open_wconn(**kargs) def disconnect(self): self.close_wconn() @property def address(self): return self.ip @property def debug(self): return self._debug @debug.setter def debug(self, opt): assert isinstance(opt, bool) self._debug = opt self.ws.debug = opt def write(self, cmd): n_bytes = len(bytes(cmd, 'utf-8')) self.ws.send(cmd) return n_bytes def read_all(self): self.ws.sock.settimeout(None) try: self.raw_buff = b'' while self.prompt not in self.raw_buff: try: fin, opcode, data = self.ws.read_frame() self.raw_buff += data except AttributeError: pass return self.raw_buff except socket.timeout: return self.raw_buff def flush(self): self.ws.sock.settimeout(0.01) self._flush = b'' self.linebuffer.truncate(0) self.linebuffer.seek(0) self.ws.reset_buffers() while self.ws_readable(): while True: try: fin, opcode, data = self.ws.read_frame() self._flush += data except socket.timeout: break except wsprotocol.NoDataException: break self.ws.reset_buffers() def ws_readable(self): for i in range(3): try: readable, writable, exceptional = select.select([self.ws.sock], [self.ws.sock], [self.ws.sock]) if readable: return True else: return False except Exception as e: if self.debug: print(e) time.sleep(0.01) def wr_cmd(self, cmd, silent=False, rtn=True, rtn_resp=False, long_string=False): self.output = None self.response = '' self.buff = b'' self.flush() self.bytes_sent = self.write(cmd + '\r') # time.sleep(0.1) # self.buff = self.read_all()[self.bytes_sent:] self.buff = self.read_all() if self.buff == b'': # time.sleep(0.1) self.buff = self.read_all() # print(self.buff) # filter command cmd_filt = bytes(cmd + '\r\n', 'utf-8') self.buff = self.buff.replace(cmd_filt, b'', 1) if self._traceback in self.buff: long_string = True if long_string: self.response = self.buff.replace(b'\r', b'').replace( b'\r\n>>> ', b'').replace(b'>>> ', b'').decode('utf-8', 'ignore') else: self.response = self.buff.replace(b'\r\n', b'').replace( b'\r\n>>> ', b'').replace(b'>>> ', b'').decode('utf-8', 'ignore') if not silent: if self.response != '\n' and self.response != '': print(self.response) else: self.response = '' if rtn: self.get_output() if self.output == '\n' and self.output == '': self.output = None if self.output is None: if self.response != '' and self.response != '\n': self.output = self.response if rtn_resp: return self.output def cmd(self, cmd, silent=False, rtn=False, long_string=False): disconnect_on_end = not self.connected if not self.connected: self.open_wconn(ssl=self._ssl, auth=True) self.wr_cmd(cmd, silent=True, long_string=long_string) if self.connected: if disconnect_on_end: self.close_wconn() self.get_output() if not silent: print(self.response) if rtn: return self.output def reset(self, silent=False, reconnect=True, hr=False): if not silent: print('Rebooting device...') if self.connected: if not hr: self.bytes_sent = self.write(self._reset) else: self.bytes_sent = self.write(self._hreset) if self._uriprotocol == 'ws': time.sleep(0.2) else: time.sleep(1) self.close_wconn() if reconnect: time.sleep(1) while True: try: self.open_wconn(ssl=self._ssl, auth=True) self.wr_cmd(self._banner, silent=True) break except Exception: time.sleep(0.5) self.ws._close() self.cmd('') if not silent: print('Done!') else: self.open_wconn(ssl=self._ssl, auth=True) if not hr: self.bytes_sent = self.write(self._reset) else: self.bytes_sent = self.write(self._hreset) if self._uriprotocol == 'ws': time.sleep(0.2) else: time.sleep(1) self.close_wconn() if not silent: print('Done!') def kbi(self, silent=True, pipe=None, long_string=False): if self.connected: if pipe is not None: self.wr_cmd(self._kbi, silent=silent) bf_output = self.response.split('Traceback')[0] traceback = 'Traceback' + self.response.split('Traceback')[1] if bf_output != '' and bf_output != '\n': pipe(bf_output) pipe(traceback, std='stderr') else: self.wr_cmd(self._kbi, silent=silent, long_string=long_string) self.cmd('') else: self.cmd(self._kbi, silent=silent) self.cmd('') def banner(self, pipe=None): self.wr_cmd(self._banner, silent=True, long_string=True) if pipe is None: print(self.response.replace('\n\n', '\n')) else: pipe(self.response.replace('\n\n', '\n')) def get_output(self): try: self.output = ast.literal_eval(self.response) except Exception: if 'bytearray' in self.response: try: self.output = bytearray( ast.literal_eval( self.response.strip().split('bytearray')[1])) except Exception: pass else: if 'array' in self.response: try: arr = ast.literal_eval( self.response.strip().split('array')[1]) self.output = array(arr[0], arr[1]) except Exception: pass pass