def FileWriteInteger(file_handle: io.BufferedRandom, value: int, size: int = INT_VALUE) -> int: if size < CHAR_VALUE or size > INT_VALUE: raise Exception("Size error") file_handle.write( int(value).to_bytes(size, byteorder="little", signed=False)) return size
def FileReadString(file_handle: io.BufferedRandom, length: int = -1) -> str: if file_handle.binflag: if length < 1: return "" return file_handle.read( len("a".encode(file_handle.encoding)) * length).decode( file_handle.encoding) return file_handle.readline(length)
def compare(pages_x, pages_y, img_format='png', all_pages=False): # create a list of tuples from corresponding pages of both sources # we assume here that the order in which to compare them is the list order pages = list(zip(pages_x, pages_y)) differences = [] diff_cnt = 0 for page_x, page_y in pages: # reset binary stream position page_x.seek(0) page_y.seek(0) # open the image and convert to RGBA mode img_x = Image.open(page_x).convert('RGBA') img_y = Image.open(page_y).convert('RGBA') diff_xy = ImageChops.difference(img_x, img_y) # check if the images are identical; if so, getbbox() will return None # http://effbot.org/zone/pil-comparing-images.htm if diff_xy.getbbox(): log.debug('Difference found') diff_cnt += 1 # http://stackoverflow.com/questions/18341754/color-in-red-diffrencies-between-two-pictures red_layer = Image.new(diff_xy.mode, diff_xy.size, 'red') # Make a red layer the same size diff_red = ImageChops.multiply(red_layer, diff_xy) comp_xy = ImageChops.blend(diff_red, img_y, 0.7) diff_stream = BufferedRandom(BytesIO()) comp_xy.save(diff_stream, format=img_format) differences.append(diff_stream) elif all_pages: log.debug('Pages are identical') # no difference detected between the 2 images; just save one of the originals diff_stream = BufferedRandom(BytesIO()) img_x.save(diff_stream, format=img_format) differences.append(diff_stream) if all_pages and len(pages_x) != len(pages_y): log.debug('Adding remaining pages (nothing to compare to)') # we need to return a complete set of pages, and both sets contain a different number of pages shortest, longest = ( pages_x, pages_y) if len(pages_x) < len(pages_y) else (pages_y, pages_x) remaining = longest[len(shortest):] for page in remaining: page.seek(0) img = Image.open(page).convert('RGBA') diff_stream = BufferedRandom(BytesIO()) img.save(diff_stream, format=img_format) differences.append(diff_stream) log.debug('{} differences found'.format(diff_cnt)) return differences
def __init__(self, chunk_size=256, buffer_size=64 * 1024 * 1024): self.chunk_size = chunk_size self.buffer_size = buffer_size self.fs = BufferedRandom(BytesIO(), buffer_size=buffer_size) self._row_header_struct = Struct("!hiqiq") self._row_int_struct = Struct("!i") self._row_bigint_struct = Struct("!iq") self._row_null_val = pack("!i", -1)
def FileWriteString(file_handle: io.BufferedRandom, text: str, length: int = -1) -> int: if length != -1: text = text[:length] length = len(text) if file_handle.binflag: file_handle.write(text.encode(file_handle.encoding)) else: file_handle.write(text) return length
def open(self, path, mode='r', errors=None): stream = self.opened.get(path) if not stream: if mode[0] == 'r': raise FileNotFoundError(path) stream = self.opened[path] = BytesIO() stream.seek(0) stream = BufferedRandom(stream) if mode[-1] == 'b' else TextIOWrapper( stream) stream.close = stream.flush return stream
def FileReadInteger(file_handle: io.BufferedRandom, size: int = INT_VALUE) -> int: if size < CHAR_VALUE or size > INT_VALUE: raise Exception("Size error") return int.from_bytes(file_handle.read(size), byteorder="little", signed=False)
def open(self, path, mode, options=None): flags, seek_to_end = { 'r': (O_RDONLY, False), 'r+': (O_RDWR, False), 'w': (O_WRONLY | O_CREAT, False), 'w+': (O_RDWR | O_CREAT, False), 'a': (O_WRONLY | O_CREAT, True), 'a+': (O_RDWR | O_CREAT, True), }[mode] # TODO: Use seek_to_end message_body = FileOpenRequest( objPath=path, createMode=0, openFlags=flags, offset=0, dataSize=-1, numThreads=0, oprType=0, KeyValPair_PI=StringStringMap(options), ) message = iRODSMessage('RODS_API_REQ', msg=message_body, int_info=api_number['DATA_OBJ_OPEN_AN']) conn = self.sess.pool.get_connection() conn.send(message) desc = conn.recv().int_info return BufferedRandom(iRODSDataObjectFileRaw(conn, desc, options))
def do_binary_copy2(conn, table, lines, chunk_size): with BufferedRandom(BytesIO(), buffer_size=(256 * 1024 * 1024)) as fs: #fs = BytesIO() lines_iter = iter(lines) while True: all_lines = list(islice(lines_iter, chunk_size)) if len(all_lines) == 0: break write_binary_header(fs) for l in all_lines: write_binary_row2(fs, l) # write end of task fs.write(pack('!h', -1)) fs.flush() fs.seek(0) #hexdump(fs.raw.getbuffer()) conn.cursor.copy_expert( "copy %s from stdin with binary freeze" % (table), fs) fs.seek(0) fs.truncate()
def download(self, path, to): conn, desc = self.session.data_objects.open(path, O_RDONLY) raw = iRODSDataObjectFileRaw(conn, desc) br = BufferedRandom(raw) try: with open(to, 'w') as wf: while(True): buf = _readLargeBlock(br) if not buf: break wf.write(buf) finally: conn.release(True) br.close() return to
def read(self, path, offset, size): buf = None br = None conn = None try: conn, desc = self.session.data_objects.open(path, O_RDONLY) raw = iRODSDataObjectFileRaw(conn, desc) br = BufferedRandom(raw) new_offset = br.seek(offset) if new_offset == offset: buf = br.read(size) finally: if br: br.close() if conn: conn.release(True) return buf
def __init__(self, fobj, mode='rb', bufsize=-1, close=True): """ :param fobj: Either an integer fileno, or an object supporting the usual :meth:`socket.fileno` method. The file will be put in non-blocking mode. """ if isinstance(fobj, int): fileno = fobj fobj = None else: fileno = fobj.fileno() if not isinstance(fileno, int): raise TypeError('fileno must be int: %r' % fileno) orig_mode = mode mode = (mode or 'rb').replace('b', '') if 'U' in mode: self._translate = True mode = mode.replace('U', '') else: self._translate = False if len(mode) != 1: # Python 3 builtin `open` raises a ValueError for invalid modes; # Python 2 ignores in. In the past, we raised an AssertionError, if __debug__ was # enabled (which it usually was). Match Python 3 because it makes more sense # and because __debug__ may not be enabled raise ValueError('mode can only be [rb, rU, wb], not %r' % (orig_mode, )) self._fobj = fobj self._closed = False self._close = close self.fileio = GreenFileDescriptorIO(fileno, mode, closefd=close) if bufsize < 0: bufsize = self.default_bufsize if mode == 'r': if bufsize == 0: bufsize = 1 elif bufsize == 1: bufsize = self.default_bufsize self.io = BufferedReader(self.fileio, bufsize) elif mode == 'w': if bufsize == 0: bufsize = 1 elif bufsize == 1: bufsize = self.default_bufsize self.io = BufferedWriter(self.fileio, bufsize) else: # QQQ: not used self.io = BufferedRandom(self.fileio, bufsize) if self._translate: self.io = TextIOWrapper(self.io)
def open(self, mode='r'): flag, create_if_not_exists, seek_to_end = { 'r': (O_RDONLY, False, False), 'r+': (O_RDWR, False, False), 'w': (O_WRONLY, True, False), 'w+': (O_RDWR, True, False), 'a': (O_WRONLY, True, True), 'a+': (O_RDWR, True, True), }[mode] # TODO: Actually use create_if_not_exists and seek_to_end conn, desc = self.manager.open(self.path, flag) return BufferedRandom(iRODSDataObjectFileRaw(conn, desc))
def merge_pdf(pages, target_pdf, img_format='png'): merged_pdf = PyPDF2.PdfFileWriter() log.debug('Merging {} into one PDF document'.format(len(pages))) for page in pages: # reset binary stream position page.seek(0) pdf_page_stream = BufferedRandom(BytesIO()) log.debug('Converting image to PDF') with wandImage(file=page, format=img_format) as image: img_converted = image.convert('pdf') img_converted.save(file=pdf_page_stream) img_converted.close() log.debug('Adding PDF page to merged document') pdf_page_stream.seek(0) merged_pdf.addPage(PyPDF2.PdfFileReader(pdf_page_stream).getPage(0)) merged_pdf.write(open(target_pdf, 'wb'))
def split_pdf(src_filename, pdf_res, page_list=None, img_format='png'): pdf_source = PyPDF2.PdfFileReader(open(src_filename, 'rb')) logging.debug('Source PDF {} contains {} pages'.format( src_filename, pdf_source.getNumPages())) pages = [] if not page_list: log.debug('Extracting all pages from PDF') # no pages defined, so create a list of all page numbers page_list = list(range(pdf_source.getNumPages())) else: log.debug('Extracting {} page(s) from PDF'.format(len(page_list))) for page_nr in page_list: log.debug('Processing page {}'.format(page_nr)) page = pdf_source.getPage(page_nr) log.debug('Extracting page from source PDF') # extract single page and save it to a temporary stream pdf_writer = PyPDF2.PdfFileWriter() pdf_writer.addPage(page) pdf_page_stream = BufferedRandom(BytesIO()) pdf_writer.write(pdf_page_stream) # reset the binary stream's position to the beginning pdf_page_stream.seek(0) log.debug('Converting PDF page to image ({})'.format(img_format)) # Define the resolution when opening the intermediate PDF for better quality converted PNGs # http://stackoverflow.com/questions/17314382/improve-quality-of-wand-conversion with wandImage(file=pdf_page_stream, resolution=pdf_res) as pdf_page: image_page = pdf_page.convert(img_format) image_page_stream = BufferedRandom(BytesIO()) image_page.save(file=image_page_stream) pages.append(image_page_stream) return pages
def split_pdf(src_filename, pdf_res, page_list=None, img_format='png'): pdf_source = PyPDF2.PdfFileReader(open(src_filename, 'rb')) logging.debug('Source PDF {} contains {} pages'.format(src_filename, pdf_source.getNumPages())) pages = [] if not page_list: log.debug('Extracting all pages from PDF') # no pages defined, so create a list of all page numbers page_list = list(range(pdf_source.getNumPages())) else: log.debug('Extracting {} page(s) from PDF'.format(len(page_list))) for page_nr in page_list: log.debug('Processing page {}'.format(page_nr)) page = pdf_source.getPage(page_nr) log.debug('Extracting page from source PDF') # extract single page and save it to a temporary stream pdf_writer = PyPDF2.PdfFileWriter() pdf_writer.addPage(page) pdf_page_stream = BufferedRandom(BytesIO()) pdf_writer.write(pdf_page_stream) # reset the binary stream's position to the beginning pdf_page_stream.seek(0) log.debug('Converting PDF page to image ({})'.format(img_format)) # Define the resolution when opening the intermediate PDF for better quality converted PNGs # http://stackoverflow.com/questions/17314382/improve-quality-of-wand-conversion with wandImage(file=pdf_page_stream, resolution=pdf_res) as pdf_page: image_page = pdf_page.convert(img_format) image_page_stream = BufferedRandom(BytesIO()) image_page.save(file=image_page_stream) pages.append(image_page_stream) return pages
def __init__(self, file: BufferedRandom) -> None: """Eine SaveFile Instanz der Generation 4 Parameter ---------------- ```py (BufferedRandom) file ``` Die geöffnete Speicherdatei """ self.File = file self.data = [x for x in file.read()] self.GeneralBlockPosition = self.GetActiveBlock(0, self.SmallBlockEnd) self.GeneralBlockStart = self.GeneralBlockPosition * self.PartitionSize
def __init__(self, fobj, mode='rb', bufsize=-1, close=True): """ :param fobj: Either an integer fileno, or an object supporting the usual :meth:`socket.fileno` method. The file will be put in non-blocking mode. """ if isinstance(fobj, int): fileno = fobj fobj = None else: fileno = fobj.fileno() if not isinstance(fileno, int): raise TypeError('fileno must be int: %r' % fileno) mode = (mode or 'rb').replace('b', '') if 'U' in mode: self._translate = True mode = mode.replace('U', '') else: self._translate = False assert len(mode) == 1, 'mode can only be [rb, rU, wb]' self._fobj = fobj self._closed = False self._close = close self.fileio = GreenFileDescriptorIO(fileno, mode, closefd=close) if bufsize < 0: bufsize = self.default_bufsize if mode == 'r': if bufsize == 0: bufsize = 1 elif bufsize == 1: bufsize = self.default_bufsize self.io = BufferedReader(self.fileio, bufsize) elif mode == 'w': if bufsize == 0: bufsize = 1 elif bufsize == 1: bufsize = self.default_bufsize self.io = BufferedWriter(self.fileio, bufsize) else: # QQQ: not used self.io = BufferedRandom(self.fileio, bufsize) if self._translate: self.io = TextIOWrapper(self.io)
def crop_image(file_object, x, y, w, h, rotate): """ Crop a file request Storage object and return it in png format :param gluon.storage.Storage file_object: File request Storage object :param str x: Cropper X position :param str y: Cropper Y position :param str w: Cropper image width :param str h: Cropper image height :param str rotate: Cropper rotate value :return: File request Storage object :rtype: gluon.storage.Storage """ logger.debug("start cropping image") image = file_object.file image_filename = file_object.filename try: logger.debug("Opening request image") im = Image.open(image).convert('RGBA') except Exception as e: logger.error(e) return file_object logger.debug("Request image open") if rotate: img_tempfile = im.rotate(-to_int(rotate), expand=True) logger.debug("Image rotated") else: img_tempfile = im img_tempfile = img_tempfile.crop( (to_int(x), to_int(y), to_int(w) + to_int(x), to_int(h) + to_int(y))) logger.debug("Image cropped") tempfile_io = BytesIO() img_tempfile.save(tempfile_io, format='PNG') tempfile_io.seek(0) tempfile_io = BufferedRandom(tempfile_io) logger.debug("Image saved to buffer") logger.debug("Image cropping finished") file_object.file = tempfile_io file_object.filename = '.'.join(image_filename.split('.')[:-1]) + ".png" logger.debug("New image file object builded") return file_object
def __init__(self, file_: BufferedRandom, length: int) -> None: # Read the file into a buffer. file_.seek(0, SEEK_END) buf = bytearray(file_.tell()) file_.seek(0, SEEK_SET) file_.readinto(buf) # Overwrite the header, in case it is currently being written to. header = list_header(length) buf[:len(header)] = header # Parse the buffer's contents as list items. super().__init__(namespacify(cbor2.loads(buf))) # Store the file pointer for `extend` calls. self._file = file_
def __init_buffer_io(self, buffering, binary, updating, creating, reading, writing, appending): if buffering < 0: buffering = self.__buffer_size() if buffering == 0: if binary: # logger.debug("Using raw_io as buffer_io for %s ..." % self.uri) return self.raw_io raise ValueError("can't have unbuffered text I/O") if updating: buffered_io = BufferedRandom(self.raw_io, buffering) elif creating or writing or appending: buffered_io = BufferedWriter(self.raw_io, buffering) elif reading: buffered_io = BufferedReader(self.raw_io, buffering) else: raise ValueError("Unknown mode: %r" % self.mode) # logger.debug("Initialized buffer IO for %s" % self.uri) return buffered_io
def saveWorld(self, world: World, fileIO: io.BufferedRandom, worldManager: WorldManager): # Checking if file size matches! if not (world.sizeX == 256 and world.sizeY == 256 and world.sizeZ == 256): raise WorldFormatError( f"RawWorldFormat - Trying to save world that has invalid world size! Expected: 256, 256, 256! Got: {world.sizeX}, {world.sizeY}, {world.sizeZ}!" ) # Clearing Current Save File fileIO.truncate(0) fileIO.seek(0) # Saving Map To File fileIO.write(world.gzipMap())
def __init__(self, fobj, mode='rb', bufsize=-1, close=True): if isinstance(fobj, int): fileno = fobj fobj = None else: fileno = fobj.fileno() if not isinstance(fileno, int): raise TypeError('fileno must be int: %r' % fileno) mode = (mode or 'rb').replace('b', '') if 'U' in mode: self._translate = True mode = mode.replace('U', '') else: self._translate = False assert len(mode) == 1, 'mode can only be [rb, rU, wb]' self._fobj = fobj self._closed = False self._close = close self.fileio = GreenFileDescriptorIO(fileno, mode, closefd=close) if bufsize < 0: bufsize = self.default_bufsize if mode == 'r': if bufsize == 0: bufsize = 1 elif bufsize == 1: bufsize = self.default_bufsize self.io = BufferedReader(self.fileio, bufsize) elif mode == 'w': if bufsize == 0: bufsize = 1 elif bufsize == 1: bufsize = self.default_bufsize self.io = BufferedWriter(self.fileio, bufsize) else: # QQQ: not used self.io = BufferedRandom(self.fileio, bufsize) if self._translate: self.io = TextIOWrapper(self.io)
def open(self, path, mode, options=None): if options is None: options = {} flags, seek_to_end = { 'r': (O_RDONLY, False), 'r+': (O_RDWR, False), 'w': (O_WRONLY | O_CREAT, False), 'w+': (O_RDWR | O_CREAT, False), 'a': (O_WRONLY | O_CREAT, True), 'a+': (O_RDWR | O_CREAT, True), }[mode] # TODO: Use seek_to_end try: oprType = options[kw.OPR_TYPE_KW] except KeyError: oprType = 0 # sanitize options before packing options = {str(key): str(value) for key, value in options.items()} message_body = FileOpenRequest( objPath=path, createMode=0, openFlags=flags, offset=0, dataSize=-1, numThreads=self.sess.numThreads, oprType=oprType, KeyValPair_PI=StringStringMap(options), ) message = iRODSMessage('RODS_API_REQ', msg=message_body, int_info=api_number['DATA_OBJ_OPEN_AN']) conn = self.sess.pool.get_connection() conn.send(message) desc = conn.recv().int_info return BufferedRandom(iRODSDataObjectFileRaw(conn, desc, options))
def FileReadDouble(file_handle: io.BufferedRandom) -> float: return struct.unpack("<d", file_handle.read(8))[0]
def FileTell(file_handle: io.BufferedRandom) -> int: return file_handle.tell()
def FileReadFloat(file_handle: io.BufferedRandom) -> float: return struct.unpack("<f", file_handle.read(4))[0]
def FileReadLong(file_handle: io.BufferedRandom) -> int: return int.from_bytes(file_handle.read(8), byteorder="little", signed=False)
def FileWriteLong(file_handle: io.BufferedRandom, value: int) -> int: file_handle.write(int(value).to_bytes(8, byteorder="little", signed=False)) return 8
def FileFlush(file_handle: io.BufferedRandom) -> None: file_handle.flush()
def FileIsEnding(file_handle: io.BufferedRandom) -> bool: return (file_handle.peek() == 0)
def FileSeek(file_handle: io.BufferedRandom, offset: int, origin: int) -> bool: file_handle.seek(offset, origin) return True
def FileWriteDouble(file_handle: io.BufferedRandom, value: float) -> int: file_handle.write(struct.pack("<d", value)) return 8