def _read_lzma2(self, coder, input, level, num_coders): size = self._uncompressed[level] is_last_coder = (level + 1) == num_coders if is_last_coder and not self._folder.solid: dec = pylzma.decompressobj(maxlength=self._start+size, lzma2=True) else: dec = pylzma.decompressobj(lzma2=True) try: return self._read_from_decompressor(coder, dec, input, level, num_coders, with_cache=True) except ValueError: if self._is_encrypted(): raise WrongPasswordError('invalid password') raise
def test_decompression_stream_two(self): # test decompression in two steps decompress = pylzma.decompressobj() data = decompress.decompress(self.plain_with_eos[:10]) data += decompress.decompress(self.plain_with_eos[10:]) data += decompress.flush() self.assertEqual(data, self.plain)
def test_decompression_stream_props(self): # test decompression with properties in separate step decompress = pylzma.decompressobj() data = decompress.decompress(self.plain_with_eos[:5]) data += decompress.decompress(self.plain_with_eos[5:]) data += decompress.flush() self.assertEqual(data, self.plain)
def unlzma(fi, fo, fi_close=True, fo_close=True, bufs=6553500): """ Decompress `fi` into `fo` (`file` or filename) """ if isinstance(fi, str): fi, fi_n = open(fi, "rb"), fi # fi_close = True if isinstance(fo, str): fo, fo_n = open(fo, "wb"), fo # fo_close = True # i.seek(0) # XXXX: better way? # * s.decompress *requires* an `output buffer size`, i.e. size of the # unpacked data, otherwise packed data is stored in internal buffers # and returned on flush (which gets problematic). # * Suggested default is to read by 1 byte and use the default buffer # size. Which gets slow. # * Nearest hax to fix is to use output buffer over 30x (or something) # the size of input buffer. Which is not a nice thing to do, but # works... mostly. # * ... anyway, symptoms: slowdown on decompression down to zero speed, # high memory usage (up to almost size of the uncompressed file), # after which all the decompressed data is written in one go. in_bufs = int(bufs / 100) s = pylzma.decompressobj() while True: tmp = fi.read(in_bufs) if not tmp: break fo.write(s.decompress(tmp, bufs)) fo.write(s.flush()) if fo_close: fo.close() if fi_close: fi.close() return fi, fo
def decomp_worker(in_path, out_path, time_queue): in_file = open(in_path, "rb") out_file = open(out_path, "w+b") start_time = datetime.now() data_size = 0 counter = 0 obj = pylzma.decompressobj() while True: chunk = in_file.read(CHUNK_SIZE) if not chunk: break data_size = data_size + len(chunk) decomp_chunk = obj.decompress(chunk) #print "in decomp : %d %d" % (data_size, len(decomp_chunk)) out_file.write(decomp_chunk) counter = counter + 1 in_file.close() out_file.close() end_time = datetime.now() time_queue.put({'start_time': start_time, 'end_time': end_time}) print "[Decomp] : (%s)-(%s)=(%s) (%d loop, %d bytes)" % ( start_time.strftime('%X'), end_time.strftime('%X'), str(end_time - start_time), counter, data_size)
def test_decompression_stream_reset(self): # test reset decompress = pylzma.decompressobj() data = decompress.decompress(self.plain_with_eos[:10]) decompress.reset() data = decompress.decompress(self.plain_with_eos[:15]) data += decompress.decompress(self.plain_with_eos[15:]) data += decompress.flush() self.assertEqual(data, self.plain)
def _read_lzma(self, coder, input): dec = pylzma.decompressobj(maxlength=self._start+self.size) try: return self._read_from_decompressor(coder, dec, input, checkremaining=True, with_cache=True) except ValueError: if self._is_encrypted(): raise WrongPasswordError('invalid password') raise
def _read_lzma(self, coder, input, level): size = self._uncompressed[level] dec = pylzma.decompressobj(maxlength=self._start+size) try: return self._read_from_decompressor(coder, dec, input, level, checkremaining=True, with_cache=True) except ValueError: if self._is_encrypted(): raise WrongPasswordError('invalid password') raise
def test_decompression_streaming_noeos(self): # test decompressing with one byte at a time... decompress = pylzma.decompressobj(maxlength=len(self.plain)) infile = StringIO(self.plain_without_eos) outfile = StringIO() while 1: data = infile.read(1) if not data: break outfile.write(decompress.decompress(data, 1)) outfile.write(decompress.flush()) self.assertEqual(outfile.getvalue(), self.plain)
def test_decompression_streaming_noeos(self): # test decompressing with one byte at a time... decompress = pylzma.decompressobj(maxlength=len(self.plain)) infile = BytesIO(self.plain_without_eos) outfile = BytesIO() while 1: data = infile.read(1) if not data: break outfile.write(decompress.decompress(data, 1)) outfile.write(decompress.flush()) self.assertEqual(outfile.getvalue(), self.plain)
def de7ZipIt(compressed_file,decompressed_file): # Decomrpess the file (as a stream) to a file (as a stream) i = open(compressed_file, 'rb') o = open(decompressed_file, 'wb') s = pylzma.decompressobj() while True: tmp = i.read(1) if not tmp: break o.write(s.decompress(tmp)) o.close() i.close()
def test_compress_large_stream(self): # decompress large block of repeating data, stream version (bug reported by Christopher Perkins) data = "asdf" * 123456 decompress = pylzma.decompressobj() infile = StringIO(pylzma.compress(data)) outfile = StringIO() while 1: tmp = infile.read(1) if not tmp: break outfile.write(decompress.decompress(tmp)) outfile.write(decompress.flush()) self.failUnless(data == outfile.getvalue())
def test_compress_large_stream(self): # decompress large block of repeating data, stream version (bug reported by Christopher Perkins) data = bytes("asdf", 'ascii')*123456 decompress = pylzma.decompressobj() infile = BytesIO(pylzma.compress(data)) outfile = BytesIO() while 1: tmp = infile.read(1) if not tmp: break outfile.write(decompress.decompress(tmp)) outfile.write(decompress.flush()) self.failUnless(data == outfile.getvalue())
def test_compress_large_stream_bigchunks(self): # decompress large block of repeating data, stream version with big chunks data = bytes("asdf", 'ascii')*123456 decompress = pylzma.decompressobj() infile = BytesIO(pylzma.compress(data)) outfile = BytesIO() while 1: tmp = infile.read(1024) if not tmp: break outfile.write(decompress.decompress(tmp)) outfile.write(decompress.flush()) self.failUnless(data == outfile.getvalue())
def test_compress_large_stream_bigchunks(self): # decompress large block of repeating data, stream version with big chunks data = "asdf" * 123456 decompress = pylzma.decompressobj() infile = StringIO(pylzma.compress(data)) outfile = StringIO() while 1: tmp = infile.read(1024) if not tmp: break outfile.write(decompress.decompress(tmp)) outfile.write(decompress.flush()) self.failUnless(data == outfile.getvalue())
def unjsllzma(fi, fi_close=True, parse_fn=None, handle_fail=None, bufs=655350): """ Make a generator for reading an lzma-compressed file with json(or something else) in lines. `parse_fn` is th function(v) to process lines with (defaults to `json.loads`) `handle_fail` if a fuction(value, exception) for handling a failure to parse the value; value is skipped if it raises _IgnoreTheError exception, otherwise its return value is yielded. default: skip all failures. """ if parse_fn is None: try: import simplejson as json except ImportError: print("Error importing (preferred) simplejson") import json parse_fn = json.loads if handle_fail is None: handle_fail = _handle_fail_default def try_loads(v): try: return parse_fn(v) except Exception as e: return handle_fail(v, e) if isinstance(fi, str): fi = open(fi, 'rb') tmp2 = '' # buffer for unfunushed lines in_bufs = int(bufs / 100) # XXX: see lzcat.py note around in_bufs s = pylzma.decompressobj() cont = True while cont: tmp = fi.read(in_bufs) if not tmp: # nothing more can be read tmp2 += s.flush() cont = False else: # XXX: TODO: use bytearray.extend (likely). tmp2 = tmp2 + s.decompress(tmp, bufs) tmp3 = tmp2.split('\n') # finished and unfinished lines for v in tmp3[:-1]: try: r = try_loads(v) except _IgnoreTheError: continue # no more handling requested, just skip it yield r tmp2 = tmp3[-1] if fi_close: fi.close()
def unpack_file_from_disk(packed_file, out_file): ''' unpack file ''' print 'Uncompressing file' with open(packed_file, 'rb') as packed, open(out_file, 'wb') as out: obj = pylzma.decompressobj() decompress = '' while True: b = packed.read(1) if not b: break dec = obj.decompress(b) out.write(dec) return True
def decomp_lzma(inputname, outputname): prev_time = datetime.now() comp_file = open(inputname, 'rb') ret_file = open(outputname, 'wb') obj = pylzma.decompressobj() while True: tmp = comp_file.read(8192) if not tmp: break ret_file.write(obj.decompress(tmp)) ret_file.write(obj.flush()) comp_file.close() ret_file.close() time_diff = str(datetime.now() - prev_time) return outputname, str(time_diff)
def decomp_lzma(inputname, outputname): prev_time = datetime.now() comp_file = open(inputname, 'rb') ret_file = open(outputname, 'wb') obj = pylzma.decompressobj() while True: tmp = comp_file.read(8192) if not tmp: break ret_file.write(obj.decompress(tmp)) ret_file.write(obj.flush()) comp_file.close() ret_file.close() time_diff = str(datetime.now()-prev_time) return outputname, str(time_diff)
def decomp_lzma(inputname, outputname): prev_time = datetime.now() comp_file = open(inputname, 'rb') ret_file = open(outputname, 'wb') obj = pylzma.decompressobj() while True: tmp = comp_file.read(8192) if not tmp: break ret_file.write(obj.decompress(tmp)) ret_file.write(obj.flush()) comp_file.close() ret_file.close() time_diff = (datetime.now()-prev_time) if time_diff.seconds == 0: return outputname, str(time_diff), '-1' else: return outputname, str(time_diff), str(os.path.getsize(inputname)/time_diff.seconds)
def decomp_lzma(inputname, outputname): prev_time = datetime.now() comp_file = open(inputname, 'rb') ret_file = open(outputname, 'wb') obj = pylzma.decompressobj() while True: tmp = comp_file.read(8192) if not tmp: break ret_file.write(obj.decompress(tmp)) ret_file.write(obj.flush()) comp_file.close() ret_file.close() time_diff = (datetime.now() - prev_time) if time_diff.seconds == 0: return outputname, str(time_diff), '-1' else: return outputname, str(time_diff), str( os.path.getsize(inputname) / time_diff.seconds)
def extractxz(infile, outfile): # unicode, unicode -> bool """ @param infile unicode @param outfile unicode @return bool """ import pylzma try: with open(infile, 'rb') as i: with open(outfile, 'wb') as o: z = pylzma.decompressobj() while True: data = i.read(1) if not data: break o.write(z.decompress(data)) o.write(z.flush()) except Exception, e: dwarn(e)
def do_decompress(self, comp_type, off, size): comp_success = True if comp_type == self.TYPE_LZMA: try: # 전체 압축한 경우인지 확인해 본다. obj = pylzma.decompressobj(maxlength=12) uncmp_data = obj.decompress(self.mm[0x1c:]) except TypeError: comp_success = False elif comp_type == self.TYPE_ZLIB: try: # 전체 압축한 경우인지 확인해 본다. uncmp_data = zlib.decompress(self.mm[off:off + size], -15) except zlib.error: comp_success = False else: uncmp_data = None comp_success = False if comp_success: return uncmp_data
def read(self): data = '' idx = 0 cnt = 0 dec = pylzma.decompressobj(maxlength=self._start+self.size) self._file.seek(self._src_start) dec.decompress(self._folder.coders[0]['properties']) total = self.compressed if total is None: remaining = self._start+self.size out = StringIO() while remaining > 0: data = self._file.read(1024) tmp = dec.decompress(data, remaining) out.write(tmp) remaining -= len(tmp) data = out.getvalue() else: data = dec.decompress(self._file.read(total), self._start+self.size) return data[self._start:self._start+self.size]
def decomp_worker(in_queue, out_queue, time_queue): start_time = datetime.now() data_size = 0 counter = 0 obj = pylzma.decompressobj() while True: chunk = in_queue.get() if chunk == END_OF_FILE: break data_size = data_size + len(chunk) decomp_chunk = obj.decompress(chunk) #print "in decomp : %d %d" % (data_size, len(decomp_chunk)) in_queue.task_done() out_queue.put(decomp_chunk) counter = counter + 1 out_queue.put(END_OF_FILE) end_time = datetime.now() time_queue.put({'start_time':start_time, 'end_time':end_time}) print "[Decomp] : (%s)-(%s)=(%s) (%d loop, %d bytes)" % (start_time.strftime('%X'), end_time.strftime('%X'), str(end_time-start_time), counter, data_size)
def read_size( self, toread ): data = '' idx = 0 #IGNORE:W0612 cnt = 0 #IGNORE:W0612 dec = pylzma.decompressobj( maxlength = self._start + toread ) self._file.seek( self._src_start ) dec.decompress( self._folder.coders[0]['properties'] ) total = self.compressed #IGNORE:E1101 if total is None: remaining = self._start + toread out = StringIO() while remaining > 0: data = self._file.read( 1024 ) tmp = dec.decompress( data, remaining ) out.write( tmp ) remaining -= len( tmp ) data = out.getvalue() else: data = dec.decompress( self._file.read( total ), self._start + toread ) return data[self._start:self._start + toread]
def read_size(self, toread): data = '' idx = 0 #IGNORE:W0612 cnt = 0 #IGNORE:W0612 dec = pylzma.decompressobj(maxlength=self._start + toread) self._file.seek(self._src_start) dec.decompress(self._folder.coders[0]['properties']) total = self.compressed #IGNORE:E1101 if total is None: remaining = self._start + toread out = StringIO() while remaining > 0: data = self._file.read(1024) tmp = dec.decompress(data, remaining) out.write(tmp) remaining -= len(tmp) data = out.getvalue() else: data = dec.decompress(self._file.read(total), self._start + toread) return data[self._start:self._start + toread]
def read(self): data = '' idx = 0 cnt = 0 dec = pylzma.decompressobj(maxlength=self._start + self.size) self._file.seek(self._src_start) dec.decompress(self._folder.coders[0]['properties']) total = self.compressed if total is None: remaining = self._start + self.size out = StringIO() while remaining > 0: data = self._file.read(1024) tmp = dec.decompress(data, remaining) out.write(tmp) remaining -= len(tmp) data = out.getvalue() else: data = dec.decompress(self._file.read(total), self._start + self.size) return data[self._start:self._start + self.size]
def setup_0_extracted(self): """Decompress setup-0 data to disk""" # decompress setup-0.bin data DecompressBuffer = 4096 DecompressCRCSize = 4 f = open(self.filename, 'rb') f.seek(self.TSetupLdrOffsetTable['Offset0'] + self.SetupIDSize + self.CRCCompressedBlockHeaderSize) decompress = pylzma.decompressobj() with open(self.setup_0_filename, 'wb') as o: read_count = 0 while read_count < self.TCompressedBlockHeader['StoredSize']: crc = f.read(DecompressCRCSize) data = f.read(DecompressBuffer) #assert(zlib.crc32(data) == struct.unpack('<l', crc)[0]) o.write(decompress.decompress(data, DecompressBuffer)) read_count += len(crc) + len(data) o.write(decompress.flush()) f.close() return True
def decomp_worker(in_path, out_path, time_queue): in_file = open(in_path, "rb") out_file = open(out_path, "w+b") start_time = datetime.now() data_size = 0 counter = 0 obj = pylzma.decompressobj() while True: chunk = in_file.read(CHUNK_SIZE) if not chunk: break data_size = data_size + len(chunk) decomp_chunk = obj.decompress(chunk) #print "in decomp : %d %d" % (data_size, len(decomp_chunk)) out_file.write(decomp_chunk) counter = counter + 1 in_file.close() out_file.close() end_time = datetime.now() time_queue.put({'start_time':start_time, 'end_time':end_time}) print "[Decomp] : (%s)-(%s)=(%s) (%d loop, %d bytes)" % (start_time.strftime('%X'), end_time.strftime('%X'), str(end_time-start_time), counter, data_size)
def extracttarxz(path, location): # unicode, unicode -> bool """ @param path unicode @param location unicode @return bool """ try: import pylzma with open(path, 'rb') as fp: z = pylzma.decompressobj() data = '' while True: trunk = fp.read(1) if not trunk: break data += z.decompress(trunk) data += z.flush() import tarfile from cStringIO import StringIO with tarfile.open(mode= "r:", fileobj=StringIO(data)) as t: t.extractall(location) return True except Exception, e: dwarn(e)
def read(self, filename): if filename in self.nsis_header.files: data = None (foff, ftime, extract_type) = self.nsis_header.files[filename] if self.case_type == 1: # case 1: 설치 파일 전부를 압축한 경우 # print '#Case 1' # print hex(foff) # print hex(kavutil.get_uint32(self.body_data, foff) & 0x7fffffff) fsize = kavutil.get_uint32(self.body_data, foff) & 0x7fffffff return self.body_data[foff + 4:foff + 4 + fsize] elif self.case_type == 2: # case 2: 개별로 압축한 경우 # print '#Case 2' # print hex(foff) # print hex(kavutil.get_uint32(self.body_data, foff) & 0x7fffffff) fsize = kavutil.get_uint32(self.body_data, foff) & 0x7fffffff fdata = self.body_data[foff + 4:foff + 4 + fsize] comp_type = self.__get_comp_type(kavutil.get_uint32(fdata, 0)) # print comp_type if comp_type == self.TYPE_LZMA: try: # 전체 압축한 경우인지 확인해 본다. obj = pylzma.decompressobj(maxlength=12) data = obj.decompress(fdata) except TypeError: pass elif comp_type == self.TYPE_ZLIB: if kavutil.get_uint32(self.body_data, foff) & 0x80000000 == 0x80000000: try: data = zlib.decompress(fdata, -15) except zlib.error: pass else: data = fdata # TYPE_COPY return data else: return None
#!/usr/bin/env python3 import pylzma import sys import struct if len(sys.argv) < 3: print("Usage: ./decompress-sc.py [input.sc] [output.scf]") exit(0) with open(sys.argv[1], 'rb') as fin, open(sys.argv[2], 'wb') as fout: header = fin.read(5) raw_size = fin.read(4) size = struct.unpack('<I', raw_size)[0] obj = pylzma.decompressobj(maxlength=size) fout.write(obj.decompress(header)) while True: chunk = fin.read(4096) if not chunk: break fout.write(obj.decompress(chunk)) # Sometimes the following line will crash with # 'Negative size passed to PyBytes_FromStringAndSize' try: fout.write(obj.flush()) except SystemError as e: print(e) # decompress-sc.py --- Decompress CoC SC files. # Copyright (C) 2013 kennytm
def _read_lzma(self, coder, input): dec = pylzma.decompressobj(maxlength=self._start + self.size) return self._read_from_decompressor(coder, dec, checkremaining=True)
def __init__(self, fp, length): FileEntry.__init__(self, fp, length) self.decomp = pylzma.decompressobj() self.buffer = ""
def test_decompression_stream(self): # test decompression object in one steps decompress = pylzma.decompressobj() data = decompress.decompress(self.plain_with_eos) data += decompress.flush() self.assertEqual(data, self.plain)
def _read_lzma(self, coder, input): dec = pylzma.decompressobj(maxlength=self._start+self.size) return self._read_from_decompressor(coder, dec, checkremaining=True)