def run(self): logging.info("Resolving oplog for host %s:%s to max timestamp: %s" % (self.host, self.port, self.max_end_ts)) try: if self.dump_gzip: tailed_oplog_fh = GzipFile(self.tailed_oplog_file) mongodump_oplog_fh = GzipFile(self.mongodump_oplog_file, 'a+') else: tailed_oplog_fh = open(self.tailed_oplog_file) mongodump_oplog_fh = open(self.mongodump_oplog_file, 'a+') for change in decode_file_iter(tailed_oplog_fh): if 'ts' in change: ts = change['ts'] if ts > self.mongodump_oplog_last_ts or self.mongodump_oplog_last_ts is None: if ts < self.max_end_ts: mongodump_oplog_fh.write(BSON.encode(change)) self.changes += 1 self.last_ts = ts elif ts > self.max_end_ts: break tailed_oplog_fh.close() mongodump_oplog_fh.flush() mongodump_oplog_fh.close() except Exception, e: logging.fatal("Resolving of oplogs failed! Error: %s" % e) raise e
def gzip_generator(string_generator): """Return generator for gzipping given string generator. Example: >>> import StringIO >>> z = ''.join(gzip_generator(iter(['hello,', ' ', 'world!']))) >>> ''.join(gunzip_generator(StringIO.StringIO(z))) 'hello, world!' """ # Use gzip and not zlib to make proper gzip header. buffer = StringIO() gzip = GzipFile(fileobj=buffer, mode='w') # Yield header yield buffer.getvalue() buffer.truncate(0) for string in string_generator: gzip.write(string) gzip.flush() yield buffer.getvalue() buffer.truncate(0) # Flush gzip.close() yield buffer.getvalue()
def compress_string(s): # avg_block_size is acutally the reciporical of the average # intended interflush distance. rnd = Random(s) flushes_remaining = FLUSH_LIMIT if len(s) < AVERAGE_SPAN_BETWEEN_FLUSHES * APPROX_MIN_FLUSHES: avg_block_size = APPROX_MIN_FLUSHES / float(len(s) + 1) else: avg_block_size = 1.0 / AVERAGE_SPAN_BETWEEN_FLUSHES s = StringIO(s) zbuf = BytesIO() zfile = GzipFile(mode='wb', compresslevel=6, fileobj=zbuf) chunk = s.read(MIN_INTERFLUSH_INTERVAL + int(rnd.expovariate(avg_block_size))) while chunk and flushes_remaining: zfile.write(chunk) zfile.flush() flushes_remaining -= 1 chunk = s.read(MIN_INTERFLUSH_INTERVAL + int(rnd.expovariate(avg_block_size))) zfile.write(chunk) zfile.write(s.read()) zfile.close() return zbuf.getvalue()
def to_gzip(data): iobuffer = StringIO() gzip_mem_object = GzipFile(mode='wb', compresslevel=6, fileobj=iobuffer) gzip_mem_object.write(data) gzip_mem_object.flush() gzip_mem_object.close() return iobuffer.getvalue()
class GzipWrap(object): def __init__(self, input, filename=None, compresslevel=1): self.input = iter(input) self.buffer = "" self.zipper = GzipFile(filename, mode="wb", fileobj=self, compresslevel=compresslevel, mtime=0) self.is_done = False def read(self, size=-1): if size is None or size < 0: raise Exception( "Call to GzipWrap with unbound size will result in poor performance" ) # If the buffer already has enough bytes, then simply pop them off of # the beginning and return them. if len(self.buffer) >= size or self.is_done: ret = self.buffer[0:size] self.buffer = self.buffer[size:] return ret # Otherwise, zip the input until we have enough bytes. while True: # Attempt to retrieve the next bytes to write. is_done = False input_size = 0 input_buffer = "" while input_size < GZIP_BUFFER_SIZE: try: s = self.input.next() input_buffer += s input_size = input_size + len(s) except StopIteration: is_done = True break self.zipper.write(input_buffer) if is_done: self.zipper.flush() self.zipper.close() self.is_done = True if len(self.buffer) >= size or is_done: ret = self.buffer[0:size] self.buffer = self.buffer[size:] return ret def flush(self): pass def write(self, data): self.buffer += data def close(self): self.input.close()
def compress_string(s): # avg_block_size is acutally the reciporical of the average # intended interflush distance. rnd = Random(s) flushes_remaining = FLUSH_LIMIT if len(s) < AVERAGE_SPAN_BETWEEN_FLUSHES * APPROX_MIN_FLUSHES: avg_block_size = APPROX_MIN_FLUSHES / float(len(s) + 1) else: avg_block_size = 1.0 / AVERAGE_SPAN_BETWEEN_FLUSHES s = StringIO(s) if isinstance(s, six.text_type) else BytesIO(s) zbuf = BytesIO() zfile = GzipFile(mode='wb', compresslevel=6, fileobj=zbuf) chunk = s.read(MIN_INTERFLUSH_INTERVAL + int(rnd.expovariate(avg_block_size))) while chunk and flushes_remaining: zfile.write(chunk) zfile.flush() flushes_remaining -= 1 chunk = s.read(MIN_INTERFLUSH_INTERVAL + int(rnd.expovariate(avg_block_size))) zfile.write(chunk) zfile.write(s.read()) zfile.close() return zbuf.getvalue()
def write_to(self, out, newline='\x0D\x0A', gzip=False): if gzip: out = GzipFile(fileobj=out) self._write_to(out, newline) if gzip: out.flush() out.close()
def GET(self): web.header('content-encoding', 'gzip') out = StringIO() f = GzipFile(fileobj=out, mode='w', compresslevel=6) name = web.input(name='').name f.write(SITE % name) f.flush() return out.getvalue()
def GET(self): web.header('content-encoding','gzip') out = StringIO() f = GzipFile(fileobj=out, mode='w', compresslevel=6) name = web.input(name='').name f.write(SITE % name) f.flush() return out.getvalue()
def gzip_file(self, fff, compresslevel=6): with closing(StringIO()) as strio: gz = GzipFile(mode='wb', compresslevel=compresslevel, fileobj=strio) gz.write(fff) gz.flush() gz.close() return strio.getvalue()
def GET(self): web.header('content-encoding','gzip') out = StringIO() f = GzipFile(fileobj=out, mode='w', compresslevel=6) name = web.input(name='').name name = name.replace('+', ' ') f.write((SITE % name).replace('f675d2395f243c89', 'correct_horse_battery_staple')) f.flush() return out.getvalue()
class NBTFile(TAG_Compound): """Represents an NBT file object""" def __init__(self, filename=None, mode=None, buffer=None): super(NBTFile,self).__init__() self.__class__.__name__ = "TAG_Compound" self.filename = filename self.type = TAG_Byte(self.id) #make a file object if filename: self.file = GzipFile(filename, mode) elif buffer: self.file = buffer else: self.file = None #parse the file given intitially if self.file: self.parse_file() if filename and 'close' in dir(self.file): self.file.close() self.file = None def parse_file(self, filename=None, buffer=None): if filename: self.file = GzipFile(filename, 'rb') if buffer: self.file = buffer if self.file: type = TAG_Byte(buffer=self.file) if type.value == self.id: name = TAG_String(buffer=self.file).value self._parse_buffer(self.file) self.name = name self.file.close() else: raise ValueError("First record is not a Compound Tag") else: ValueError("need a file!") def write_file(self, filename=None, buffer=None): if buffer: self.file = buffer elif filename: self.file = GzipFile(filename, "wb") elif self.filename: self.file = GzipFile(self.filename, "wb") elif not self.file: raise ValueError("Need to specify either a filename or a file") #Render tree to file TAG_Byte(self.id)._render_buffer(self.file) TAG_String(self.name)._render_buffer(self.file) self._render_buffer(self.file) #make sure the file is complete if 'flush' in dir(self.file): self.file.flush() if filename and 'close' in dir(self.file): self.file.close()
def GET(self): web.header('content-encoding', 'gzip') out = StringIO() f = GzipFile(fileobj=out, mode='w', compresslevel=6) name = web.input(name='').name name = name.replace('+', ' ') f.write((SITE % name).replace('f675d2395f243c89', 'correct_horse_battery_staple')) f.flush() return out.getvalue()
def write_sample_data(self, file, compress=False): if compress: file = GzipFile(mode='w', fileobj=file) for line in fragmented_message_type_8: file.write(bytes(line, "ascii")) file.write(newline) file.write(bytes("garbage data", "ascii")) file.write(newline) file.write(bytes(message_type_1, "ascii")) file.write(newline) file.flush()
def compress_sequence(sequence): buf = StreamingBuffer() zfile = GzipFile(mode='wb', compresslevel=6, fileobj=buf) # Output headers... yield buf.read() for item in sequence: zfile.write(item) zfile.flush() yield buf.read() zfile.close() yield buf.read()
class GzipEncoding(Encoding): def __init__(self, compression_level=6): super() self._encoder_buffer = io.BytesIO() self._encoder = None self._decoder_buffer = io.BytesIO() self._decoder = zlib.decompressobj(16 + zlib.MAX_WBITS) self._compression_level = compression_level def name(self): return 'gzip' def encode(self, data): data = data.encode('utf-8') self._encoder_buffer.write(varint.encode(len(data))) # Don't initialize the encoder before the first call to encode(), since # it writes the gzip header immediately and we need to insert the # message length prior to that happening. if self._encoder is None: self._encoder = GzipFile(fileobj=self._encoder_buffer, mode='wb', compresslevel=self._compression_level) self._encoder.write(data) self._encoder.flush() output = self._encoder_buffer.getvalue() reset_buffer(self._encoder_buffer) return output def decode(self, data): # Decode the varuint prefix off the data first, then smash the remaining # data into the decode buffer and reset it to read any previous tail. prefix_stream = io.BytesIO(data) decoded_bytes = varint.decode_stream(prefix_stream) self._decoder_buffer.write(data[prefix_stream.tell():]) self._decoder_buffer.seek(0) decoded_data = self._decoder.decompress( self._decoder_buffer.getbuffer(), decoded_bytes) reset_buffer(self._decoder_buffer, self._decoder.unconsumed_tail) return decoded_data.decode('utf-8')
def create_metadata_record_bytes( url='http://example.com/', content_type='image/png', date='2016-08-03T10:49:41Z', content=b'', include_block_digest=True): """Build WARC metadata record bits.""" headers = { WarcRecord.TYPE: WarcRecord.METADATA, WarcRecord.URL: url.encode('utf-8'), WarcRecord.CONTENT_TYPE: content_type.encode('utf-8'), WarcRecord.DATE: date.encode('utf-8') } if include_block_digest: hasher = hashlib.sha1(content) block_digest = base64.b32encode(hasher.digest()) headers[WarcRecord.BLOCK_DIGEST] = b'sha1:' + block_digest # XXX - I wish I could use WarcRecord. Current implementation of # WarcRecord.write_to() ignores Warc-Block-Digest passed and writes out # hex-encoded SHA256 calculated from the content. out = io.BytesIO() if False: rec = WarcRecord( headers=headers.items(), content=(content_type.encode('utf-8'), content) ) out = io.BytesIO() rec.write_to(out, gzip=True) return out.getvalue() else: z = GzipFile(fileobj=out, mode='wb') z.write(b'WARC/1.0\r\n') for k, v in headers.items(): z.write(b''.join((k, b': ', v, b'\r\n'))) z.write('Content-Length: {}\r\n'.format(len(content)).encode('ascii')) z.write(b'\r\n') z.write(content) z.write(b'\r\n\r\n') z.flush() z.close() return out.getvalue()
def write_to(self, out, newline=b'\x0D\x0A', gzip=False): if self.content_file is not None: if not self._content_file_valid: raise Exception('cannot write record because content_file has already been used') if gzip: if hasattr(out, 'mode'): out = GzipFile(fileobj=out) else: out = GzipFile(fileobj=out, mode='ab') self._write_to(out, newline) if gzip: out.flush() out.close() if self.content_file is not None: self._content_file_valid = False
def write_to(self, out, newline=b'\x0D\x0A', gzip=False): if self.content_file is not None: if not self._content_file_valid: raise Exception( 'cannot write record because content_file has already been used' ) if gzip: if hasattr(out, 'mode'): out = GzipFile(fileobj=out) else: out = GzipFile(fileobj=out, mode='ab') self._write_to(out, newline) if gzip: out.flush() out.close() if self.content_file is not None: self._content_file_valid = False
class _GzipEncoder(object): def __init__(self, request, preamble): preamble.headers['Vary'] = preamble.headers.get('Vary', '') + ',Accept-Encoding' self._accepted = 'gzip' in request.headers.get('Accept-Encoding', '').replace(' ','').split(',') if self._accepted: self._value = BytesIO() self._file = GzipFile(mode='wb', fileobj=self._value) preamble.headers['Content-Encoding'] = preamble.headers.get('Content-Encoding', '') + ',gzip' def encode(self, data): if self._accepted: self._file.write(data) self._file.flush() data = self._value.getvalue() self._value.truncate(0) self._value.seek(0) return data def finish(self, data): if self._accepted: data = self.encode(data) self._file.close() return data
class Gzipper(object): """ Text stream compressor. """ def __init__(self, filename=None, compress=False): self.compress = compress if not self.compress: self.buffer = '' else: self.io = StringIO() self.zipfile = GzipFile(filename, mode='wb', fileobj=self.io) def read(self): if not self.compress: result = self.buffer self.buffer = '' return result self.zipfile.flush() self.io.seek(0) line = self.io.read() self.io.seek(0) self.io.truncate() return line def write(self, l): if not self.compress: self.buffer += l else: self.zipfile.write(l) def close(self): if not self.compress: return self.buffer self.zipfile.close() self.io.seek(0) return self.io.read()
def compress_sequence(sequence): avg_block_size = 1.0 / AVERAGE_SPAN_BETWEEN_FLUSHES buf = StreamingBuffer() zfile = GzipFile(mode='wb', compresslevel=6, fileobj=buf) # Output headers... yield buf.read() flushes_remaining = FLUSH_LIMIT rnd = None count = None rnd = None for item in sequence: if rnd is None: rnd = Random(hash(item)) count = int(rnd.expovariate(avg_block_size)) chunking_buf = BytesIO(item) chunk = chunking_buf.read(count) while chunk: if count is not None: count -= len(chunk) zfile.write(chunk) if count <= 0: flushes_remaining -= 1 zfile.flush() yield buf.read() if flushes_remaining: count = int(rnd.expovariate(avg_block_size)) else: count = None if count is None: chunk = chunking_buf.read() else: chunk = chunking_buf.read(count) zfile.flush() yield buf.read() if chunk is None: break for item in sequence: zfile.write(chunking_buf.read()) zfile.flush() yield buf.read() zfile.close() yield buf.read()
def compress_sequence(sequence): avg_block_size = 1.0 / AVERAGE_SPAN_BETWEEN_FLUSHES buf = StreamingBuffer() zfile = GzipFile(mode='wb', compresslevel=6, fileobj=buf) # Output headers... yield buf.read() flushes_remaining = FLUSH_LIMIT rnd = None count = None rnd = None for item in sequence: if rnd is None: rnd = Random(0) count = int(rnd.expovariate(avg_block_size)) chunking_buf = BytesIO(item) chunk = chunking_buf.read(count) while chunk: if count is not None: count -= len(chunk) zfile.write(chunk) if count <= 0: flushes_remaining -= 1 zfile.flush() yield buf.read() if flushes_remaining: count = int(rnd.expovariate(avg_block_size)) else: count = None if count is None: chunk = chunking_buf.read() else: chunk = chunking_buf.read(count) zfile.flush() yield buf.read() if chunk is None: break for item in sequence: zfile.write(chunking_buf.read()) zfile.flush() yield buf.read() zfile.close() yield buf.read()
class GZipPipe(StringIO) : """This class implements a compression pipe suitable for asynchronous process. Only one buffer of data is read/compressed at a time. The process doesn't read the whole file at once : This improve performance and prevent hight memory consumption for big files.""" # Size of the internal buffer CHUNCK_SIZE = 1024 def __init__(self, source = None, name = "data") : """Constructor @param source Source data to compress (as a stream/File/Buffer - anything with a read() method) @param name Name of the data within the zip file""" # Source file self.source = source # OEF reached for source ? self.source_eof = False # Buffer self.buffer = "" StringIO.__init__(self) # Inherited constructor # Init ZipFile that writes to us (the StringIO buffer) self.zipfile = GzipFile(name, 'wb', 9, self) def write(self, data) : """The write mzthod shouldn't be called from outside. A GZipFile was created with this current object as a output buffer anbd it fills it whenever we write to it (calling the read method of this object will do it for you) """ self.buffer += data def read(self, size = -1) : """Calling read() on a zip pipe will suck data from the source stream. @param size Maximum size to read - Read whole compressed file if not specified. @return Compressed data""" # Feed the zipped buffer by writing source data to the zip stream while ((len(self.buffer) < size) or (size == -1)) and not self.source_eof : # No source given in input if self.source == None: break # Get a chunk of source data chunk = self.source.read(GZipPipe.CHUNCK_SIZE) # Feed the source zip file (that fills the compressed buffer) self.zipfile.write(chunk) # End of source file ? if (len(chunk) < GZipPipe.CHUNCK_SIZE) : self.source_eof = True self.zipfile.flush() self.zipfile.close() break # We have enough data in the buffer (or source file is EOF): Give it to the output if size == 0: result = "" if size >= 1 : result = self.buffer[0:size] self.buffer = self.buffer[size:] else : # size < 0 : All requested result = self.buffer self.buffer = "" return result
def fork_map( func, arg_list: typing.Iterable, workers: int = 10, tmp_dir=None, gzip=False, gzip_level=4, ) -> typing.Generator[None, typing.Any, None]: result_files = [ NamedTemporaryFile( mode="wb+", delete=False, dir=tmp_dir, prefix="fork-map-", suffix=".results.gz" if gzip else ".results", ) for _ in range(workers) ] if not isinstance(arg_list, list): arg_list = list(arg_list) children = set() freeze() for i in range(workers): pid = os.fork() if pid: # Is parent children.add(pid) continue # Is child if gzip: result_file = GzipFile( fileobj=result_files[i], mode="wb", compresslevel=gzip_level, ) else: result_file = result_files[i] for task in arg_list[i::workers]: try: res = pickle.dumps((func(task), False)) except Exception as e: res = pickle.dumps((e, True)) result_file.write(HEADER.pack(len(res))) result_file.write(res) result_file.flush() return exit(0) unfreeze() while children: pid, code = os.wait() if code: raise RuntimeError( "Child process %d exited with code %r", pid, code, ) children.remove(pid) for i in range(workers): result_files[i].seek(0) if gzip: result_read_files = [ GzipFile(fileobj=fp, mode="rb") for fp in result_files ] else: result_read_files = result_files try: for i in range(len(arg_list)): fp = result_read_files[i % workers] hdr = fp.read(HEADER.size) size = HEADER.unpack(hdr)[0] res, exc = pickle.loads(fp.read(size)) if exc: raise exc yield res finally: for fp in result_files: os.remove(fp.name)
class NBTFile(TAG_Compound): """Represents an NBT file object""" def __init__(self, filename=None, mode=None, buffer=None, fileobj=None): super(NBTFile,self).__init__() self.__class__.__name__ = "TAG_Compound" self.filename = filename self.type = TAG_Byte(self.id) #make a file object if filename: self.file = GzipFile(filename, mode) elif buffer: self.file = buffer elif fileobj: self.file = GzipFile(fileobj=fileobj) else: self.file = None #parse the file given intitially if self.file: self.parse_file() if filename and 'close' in dir(self.file): self.file.close() self.file = None def parse_file(self, filename=None, buffer=None, fileobj=None): if filename: self.file = GzipFile(filename, 'rb') elif buffer: self.file = buffer elif fileobj: self.file = GzipFile(fileobj=fileobj) if self.file: try: type = TAG_Byte(buffer=self.file) if type.value == self.id: name = TAG_String(buffer=self.file).value self._parse_buffer(self.file) self.name = name self.file.close() else: raise MalformedFileError("First record is not a Compound Tag") except StructError as e: raise MalformedFileError("Partial File Parse: file possibly truncated.") else: ValueError("need a file!") def write_file(self, filename=None, buffer=None, fileobj=None): if buffer: self.file = buffer elif filename: self.filename = filename self.file = GzipFile(filename, "wb") elif fileobj: self.file = GzipFile(fileobj=fileobj) elif self.filename: self.file = GzipFile(self.filename, "wb") elif not self.file: raise ValueError("Need to specify either a filename or a file") #Render tree to file TAG_Byte(self.id)._render_buffer(self.file) TAG_String(self.name)._render_buffer(self.file) self._render_buffer(self.file) #make sure the file is complete if 'flush' in dir(self.file): self.file.flush() if self.filename and 'close' in dir(self.file): self.file.close()
def flush(self, lib_mode=FLUSH): self._buffer.flush() GzipFile.flush(self, lib_mode)
class NBTFile(TAG_Compound): """Represent an NBT file object.""" def __init__(self, filename=None, buffer=None, fileobj=None): """ Create a new NBTFile object. Specify either a filename, file object or data buffer. If filename of file object is specified, data should be GZip-compressed. If a data buffer is specified, it is assumed to be uncompressed. If filename is specified, the file is closed after reading and writing. If file object is specified, the caller is responsible for closing the file. """ super(NBTFile, self).__init__() self.filename = filename self.type = TAG_Byte(self.id) closefile = True # make a file object if filename: self.filename = filename self.file = GzipFile(filename, 'rb') elif buffer: if hasattr(buffer, 'name'): self.filename = buffer.name self.file = buffer closefile = False elif fileobj: if hasattr(fileobj, 'name'): self.filename = fileobj.name self.file = GzipFile(fileobj=fileobj) else: self.file = None closefile = False # parse the file given initially if self.file: self.parse_file() if closefile: # Note: GzipFile().close() does NOT close the fileobj, # So we are still responsible for closing that. try: self.file.close() except (AttributeError, IOError): pass self.file = None def parse_file(self, filename=None, buffer=None, fileobj=None): """Completely parse a file, extracting all tags.""" if filename: self.file = GzipFile(filename, 'rb') elif buffer: if hasattr(buffer, 'name'): self.filename = buffer.name self.file = buffer elif fileobj: if hasattr(fileobj, 'name'): self.filename = fileobj.name self.file = GzipFile(fileobj=fileobj) if self.file: try: type = TAG_Byte(buffer=self.file) if type.value == self.id: name = TAG_String(buffer=self.file).value self._parse_buffer(self.file) self.name = name self.file.close() else: raise MalformedFileError( "First record is not a Compound Tag") except StructError as e: raise MalformedFileError( "Partial File Parse: file possibly truncated.") else: raise ValueError( "NBTFile.parse_file(): Need to specify either a " "filename or a file object" ) def write_file(self, filename=None, buffer=None, fileobj=None): """Write this NBT file to a file.""" closefile = True if buffer: self.filename = None self.file = buffer closefile = False elif filename: self.filename = filename self.file = GzipFile(filename, "wb") elif fileobj: self.filename = None self.file = GzipFile(fileobj=fileobj, mode="wb") elif self.filename: self.file = GzipFile(self.filename, "wb") elif not self.file: raise ValueError( "NBTFile.write_file(): Need to specify either a " "filename or a file object" ) # Render tree to file TAG_Byte(self.id)._render_buffer(self.file) TAG_String(self.name)._render_buffer(self.file) self._render_buffer(self.file) # make sure the file is complete try: self.file.flush() except (AttributeError, IOError): pass if closefile: try: self.file.close() except (AttributeError, IOError): pass def __repr__(self): """ Return a string (ascii formated for Python 2, unicode for Python 3) describing the class, name and id for debugging purposes. """ if self.filename: return "<%s(%r) with %s(%r) at 0x%x>" % ( self.__class__.__name__, self.filename, TAG_Compound.__name__, self.name, id(self) ) else: return "<%s with %s(%r) at 0x%x>" % ( self.__class__.__name__, TAG_Compound.__name__, self.name, id(self) )
class NBTFile(TAG_Compound): """Represent an NBT file object.""" def __init__(self, filename=None, buffer=None, fileobj=None): super(NBTFile, self).__init__() self.filename = filename self.type = TAG_Byte(self.id) #make a file object if filename: self.file = GzipFile(filename, 'rb') elif buffer: self.file = buffer elif fileobj: self.file = GzipFile(fileobj=fileobj) else: self.file = None #parse the file given intitially if self.file: self.parse_file() if self.filename and 'close' in dir(self.file): self.file.close() self.file = None def parse_file(self, filename=None, buffer=None, fileobj=None): """Completely parse a file, extracting all tags.""" if filename: self.file = GzipFile(filename, 'rb') elif buffer: self.file = buffer elif fileobj: self.file = GzipFile(fileobj=fileobj) if self.file: try: type = TAG_Byte(buffer=self.file) if type.value == self.id: name = TAG_String(buffer=self.file).value self._parse_buffer(self.file) self.name = name self.file.close() else: raise MalformedFileError("First record is not a Compound Tag") except StructError as e: raise MalformedFileError("Partial File Parse: file possibly truncated.") else: ValueError("need a file!") def write_file(self, filename=None, buffer=None, fileobj=None): """Write this NBT file to a file.""" if buffer: self.filename = None self.file = buffer elif filename: self.filename = filename self.file = GzipFile(filename, "wb") elif fileobj: self.filename = None self.file = GzipFile(fileobj=fileobj, mode="wb") elif self.filename: self.file = GzipFile(self.filename, "wb") elif not self.file: raise ValueError("Need to specify either a filename or a file") #Render tree to file TAG_Byte(self.id)._render_buffer(self.file) TAG_String(self.name)._render_buffer(self.file) self._render_buffer(self.file) #make sure the file is complete if 'flush' in dir(self.file): self.file.flush() if self.filename and 'close' in dir(self.file): self.file.close() def __repr__(self): """ Return a string (ascii formated for Python 2, unicode for Python 3) describing the class, name and id for debugging purposes. """ if self.filename: return "<%s(%r) with %s(%r) at 0x%x>" % (self.__class__.__name__, self.filename, \ TAG_Compound.__name__, self.name, id(self)) else: return "<%s with %s(%r) at 0x%x>" % (self.__class__.__name__, \ TAG_Compound.__name__, self.name, id(self))
class NBTFile(Tag_Compound): """Represent an NBT file object.""" def __init__(self, filename=None, buffer=None, fileobj=None): """ Create a new NBTFile object. Specify either a filename, file object or data buffer. If filename of file object is specified, data should be GZip-compressed. If a data buffer is specified, it is assumed to be uncompressed. If filename is specified, the file is closed after reading and writing. If file object is specified, the caller is responsible for closing the file. """ super(NBTFile, self).__init__() self.filename = filename self.type = TAG_Byte(self.id) closefile = True # make a file object if filename: self.filename = filename self.file = GzipFile(filename, 'rb') elif buffer: if hasattr(buffer, 'name'): self.filename = buffer.name self.file = buffer closefile = False elif fileobj: if hasattr(fileobj, 'name'): self.filename = fileobj.name self.file = GzipFile(fileobj=fileobj) else: self.file = None closefile = False # parse the file given initially if self.file: self.parse_file() if closefile: # Note: GzipFile().close() does NOT close the fileobj, # So we are still responsible for closing that. try: self.file.close() except (AttributeError, IOError): pass self.file = None def parse_file(self, filename=None, buffer=None, fileobj=None): """Completely parse a file, extracting all tags.""" if filename: self.file = GzipFile(filename, 'rb') elif buffer: if hasattr(buffer, 'name'): self.filename = buffer.name self.file = buffer elif fileobj: if hasattr(fileobj, 'name'): self.filename = fileobj.name self.file = GzipFile(fileobj=fileobj) if self.file: try: type = TAG_Byte(buffer=self.file) if type.value == self.id: name = Tag_String(buffer=self.file).value self._parse_buffer(self.file) self.name = name self.file.close() else: raise MalformedFileError( "First record is not a Compound Tag") except StructError as e: raise MalformedFileError( "Partial File Parse: file possibly truncated.") else: raise ValueError("NBTFile.parse_file(): Need to specify either a " "filename or a file object") def write_file(self, filename=None, buffer=None, fileobj=None): """Write this NBT file to a file.""" closefile = True if buffer: self.filename = None self.file = buffer closefile = False elif filename: self.filename = filename self.file = GzipFile(filename, "wb") elif fileobj: self.filename = None self.file = GzipFile(fileobj=fileobj, mode="wb") elif self.filename: self.file = GzipFile(self.filename, "wb") elif not self.file: raise ValueError("NBTFile.write_file(): Need to specify either a " "filename or a file object") # Render tree to file TAG_Byte(self.id)._render_buffer(self.file) Tag_String(self.name)._render_buffer(self.file) self._render_buffer(self.file) # make sure the file is complete try: self.file.flush() except (AttributeError, IOError): pass if closefile: try: self.file.close() except (AttributeError, IOError): pass def __repr__(self): if self.filename: return "<{}({!r}) with {}({!r}) at {:#x}>".format( self.__class__.__name__, self.filename, Tag_Compound.__name__, self.name, id(self)) else: return "<{} with {}({!r}) at {:#x}>".format( self.__class__.__name__, Tag_Compound.__name__, self.name, id(self), )
class GZipPipe(StringIO): """This class implements a compression pipe suitable for asynchronous process. Only one buffer of data is read/compressed at a time. The process doesn't read the whole file at once : This improves performance and prevent hight memory consumption for big files.""" # Size of the internal buffer CHUNCK_SIZE = 1024 def __init__(self, source=None, name="data"): """Constructor @param source Source data to compress (as a stream/File/Buffer - anything with a read() method) @param name Name of the data within the zip file""" # Source file self.source = source # OEF reached for source ? self.source_eof = False # Buffer self.buffer = "" StringIO.__init__(self) # Inherited constructor # Init ZipFile that writes to us (the StringIO buffer) self.zipfile = GzipFile(name, 'wb', 9, self) def write(self, data): """The write mzthod shouldn't be called from outside. A GZipFile was created with this current object as a output buffer anbd it fills it whenever we write to it (calling the read method of this object will do it for you) """ self.buffer += data def read(self, size=-1): """Calling read() on a zip pipe will suck data from the source stream. @param size Maximum size to read - Read whole compressed file if not specified. @return Compressed data""" # Feed the zipped buffer by writing source data to the zip stream while ((len(self.buffer) < size) or (size == -1)) and not self.source_eof: # No source given in input if self.source == None: break # Get a chunk of source data chunk = self.source.read(GZipPipe.CHUNCK_SIZE) # Feed the source zip file (that fills the compressed buffer) self.zipfile.write(chunk) # End of source file ? if (len(chunk) < GZipPipe.CHUNCK_SIZE): self.source_eof = True self.zipfile.flush() self.zipfile.close() break # We have enough data in the buffer (or source file is EOF): Give it to the output if size == 0: result = "" if size >= 1: result = self.buffer[0:size] self.buffer = self.buffer[size:] else: # size < 0 : All requested result = self.buffer self.buffer = "" return result
class NBTFile(TAG_Compound): """Represent an NBT file object.""" def __init__(self, filename=None, buffer=None, fileobj=None): super(NBTFile, self).__init__() self.filename = filename self.type = TAG_Byte(self.tid) closefile = True # make a file object if filename: self.file = GzipFile(filename, 'rb') elif buffer: if hasattr(buffer, 'name'): self.filename = buffer.name self.file = buffer closefile = False elif fileobj: if hasattr(fileobj, 'name'): self.filename = fileobj.name self.file = GzipFile(fileobj=fileobj) else: self.file = None closefile = False # parse the file given initially if self.file: self.parse_file() if closefile: # Note: GzipFile().close() does NOT close the fileobj, # So the caller is still responsible for closing that. try: self.file.close() except (AttributeError, IOError): pass self.file = None def parse_file(self, filename=None, buffer=None, fileobj=None): """Completely parse a file, extracting all tags.""" if filename: self.file = GzipFile(filename, 'rb') elif buffer: if hasattr(buffer, 'name'): self.filename = buffer.name self.file = buffer elif fileobj: if hasattr(fileobj, 'name'): self.filename = fileobj.name self.file = GzipFile(fileobj=fileobj) if self.file: try: type = TAG_Byte(buffer=self.file) if type.value == self.tid: name = TAG_String(buffer=self.file).value self._parse_buffer(self.file) self.name = name self.file.close() else: raise MalformedFileError( "First record is not a Compound Tag") except StructError as e: raise MalformedFileError( "Partial File Parse: file possibly truncated.") else: raise ValueError( "NBTFile.parse_file(): Need to specify either a filename or a file object") def write_file(self, filename=None, buffer=None, fileobj=None): """Write this NBT file to a file.""" closefile = True if buffer: self.filename = None self.file = buffer closefile = False elif filename: self.filename = filename self.file = GzipFile(filename, "wb") elif fileobj: self.filename = None self.file = GzipFile(fileobj=fileobj, mode="wb") elif self.filename: self.file = GzipFile(self.filename, "wb") elif not self.file: raise ValueError("NBTFile.write_file(): Need to specify either a filename or a file object") # Render tree to file TAG_Byte(self.tid)._render_buffer(self.file) TAG_String(self.name)._render_buffer(self.file) self._render_buffer(self.file) # make sure the file is complete try: self.file.flush() except (AttributeError, IOError): pass if closefile: try: self.file.close() except (AttributeError, IOError): pass def __repr__(self): """ Return a string (ascii formated for Python 2, unicode for Python 3) describing the class, name and id for debugging purposes. """ if self.filename: return "<%s(%r) with %s(%r) at 0x%x>" % \ (self.__class__.__name__, self.filename, TAG_Compound.__name__, self.name, id(self)) else: return "<%s with %s(%r) at 0x%x>" % \ (self.__class__.__name__, TAG_Compound.__name__, self.name, id(self))
class NBTFile(TAG_Compound): """Represent an NBT file object.""" def __init__(self, filename=None, buffer=None, fileobj=None): super(NBTFile, self).__init__() self.filename = filename self.type = TAG_Byte(self.id) #make a file object if filename: self.file = GzipFile(filename, 'rb') elif buffer: self.file = buffer elif fileobj: self.file = GzipFile(fileobj=fileobj) else: self.file = None #parse the file given intitially if self.file: self.parse_file() if self.filename and 'close' in dir(self.file): self.file.close() self.file = None def parse_file(self, filename=None, buffer=None, fileobj=None): """Completely parse a file, extracting all tags.""" if filename: self.file = GzipFile(filename, 'rb') elif buffer: self.file = buffer elif fileobj: self.file = GzipFile(fileobj=fileobj) if self.file: try: type = TAG_Byte(buffer=self.file) if type.value == self.id: name = TAG_String(buffer=self.file).value self._parse_buffer(self.file) self.name = name self.file.close() else: raise MalformedFileError( "First record is not a Compound Tag") except StructError as e: raise MalformedFileError( "Partial File Parse: file possibly truncated.") else: ValueError("need a file!") def write_file(self, filename=None, buffer=None, fileobj=None): """Write this NBT file to a file.""" if buffer: self.filename = None self.file = buffer elif filename: self.filename = filename self.file = GzipFile(filename, "wb") elif fileobj: self.filename = None self.file = GzipFile(fileobj=fileobj, mode="wb") elif self.filename: self.file = GzipFile(self.filename, "wb") elif not self.file: raise ValueError("Need to specify either a filename or a file") #Render tree to file TAG_Byte(self.id)._render_buffer(self.file) TAG_String(self.name)._render_buffer(self.file) self._render_buffer(self.file) #make sure the file is complete if 'flush' in dir(self.file): self.file.flush() if self.filename and 'close' in dir(self.file): self.file.close() def __repr__(self): """ Return a string (ascii formated for Python 2, unicode for Python 3) describing the class, name and id for debugging purposes. """ if self.filename: return "<%s(%r) with %s(%r) at 0x%x>" % (self.__class__.__name__, self.filename, \ TAG_Compound.__name__, self.name, id(self)) else: return "<%s with %s(%r) at 0x%x>" % (self.__class__.__name__, \ TAG_Compound.__name__, self.name, id(self))