def _read_lzma2(self, coder, input, level, num_coders):
size = self._uncompressed[level]
is_last_coder = (level + 1) == num_coders
if is_last_coder and not self._folder.solid:
dec = pylzma.decompressobj(maxlength=self._start+size, lzma2=True)
else:
dec = pylzma.decompressobj(lzma2=True)
try:
return self._read_from_decompressor(coder, dec, input, level, num_coders, with_cache=True)
except ValueError:
if self._is_encrypted():
raise WrongPasswordError('invalid password')
raise
def _read_lzma2(self, coder, input, level, num_coders):
size = self._uncompressed[level]
is_last_coder = (level + 1) == num_coders
if is_last_coder and not self._folder.solid:
dec = pylzma.decompressobj(maxlength=self._start+size, lzma2=True)
else:
dec = pylzma.decompressobj(lzma2=True)
try:
return self._read_from_decompressor(coder, dec, input, level, num_coders, with_cache=True)
except ValueError:
if self._is_encrypted():
raise WrongPasswordError('invalid password')
raise
def test_decompression_stream_two(self):
# test decompression in two steps
decompress = pylzma.decompressobj()
data = decompress.decompress(self.plain_with_eos[:10])
data += decompress.decompress(self.plain_with_eos[10:])
data += decompress.flush()
self.assertEqual(data, self.plain)
def test_decompression_stream_two(self):
# test decompression in two steps
decompress = pylzma.decompressobj()
data = decompress.decompress(self.plain_with_eos[:10])
data += decompress.decompress(self.plain_with_eos[10:])
data += decompress.flush()
self.assertEqual(data, self.plain)
def test_decompression_stream_props(self):
# test decompression with properties in separate step
decompress = pylzma.decompressobj()
data = decompress.decompress(self.plain_with_eos[:5])
data += decompress.decompress(self.plain_with_eos[5:])
data += decompress.flush()
self.assertEqual(data, self.plain)
def unlzma(fi, fo, fi_close=True, fo_close=True, bufs=6553500):
""" Decompress `fi` into `fo` (`file` or filename) """
if isinstance(fi, str):
fi, fi_n = open(fi, "rb"), fi
# fi_close = True
if isinstance(fo, str):
fo, fo_n = open(fo, "wb"), fo
# fo_close = True
# i.seek(0)
# XXXX: better way?
# * s.decompress *requires* an `output buffer size`, i.e. size of the
# unpacked data, otherwise packed data is stored in internal buffers
# and returned on flush (which gets problematic).
# * Suggested default is to read by 1 byte and use the default buffer
# size. Which gets slow.
# * Nearest hax to fix is to use output buffer over 30x (or something)
# the size of input buffer. Which is not a nice thing to do, but
# works... mostly.
# * ... anyway, symptoms: slowdown on decompression down to zero speed,
# high memory usage (up to almost size of the uncompressed file),
# after which all the decompressed data is written in one go.
in_bufs = int(bufs / 100)
s = pylzma.decompressobj()
while True:
tmp = fi.read(in_bufs)
if not tmp:
break
fo.write(s.decompress(tmp, bufs))
fo.write(s.flush())
if fo_close:
fo.close()
if fi_close:
fi.close()
return fi, fo
def test_decompression_stream_props(self):
# test decompression with properties in separate step
decompress = pylzma.decompressobj()
data = decompress.decompress(self.plain_with_eos[:5])
data += decompress.decompress(self.plain_with_eos[5:])
data += decompress.flush()
self.assertEqual(data, self.plain)
def decomp_worker(in_path, out_path, time_queue):
in_file = open(in_path, "rb")
out_file = open(out_path, "w+b")
start_time = datetime.now()
data_size = 0
counter = 0
obj = pylzma.decompressobj()
while True:
chunk = in_file.read(CHUNK_SIZE)
if not chunk:
break
data_size = data_size + len(chunk)
decomp_chunk = obj.decompress(chunk)
#print "in decomp : %d %d" % (data_size, len(decomp_chunk))
out_file.write(decomp_chunk)
counter = counter + 1
in_file.close()
out_file.close()
end_time = datetime.now()
time_queue.put({'start_time': start_time, 'end_time': end_time})
print "[Decomp] : (%s)-(%s)=(%s) (%d loop, %d bytes)" % (
start_time.strftime('%X'), end_time.strftime('%X'),
str(end_time - start_time), counter, data_size)
def test_decompression_stream_reset(self):
# test reset
decompress = pylzma.decompressobj()
data = decompress.decompress(self.plain_with_eos[:10])
decompress.reset()
data = decompress.decompress(self.plain_with_eos[:15])
data += decompress.decompress(self.plain_with_eos[15:])
data += decompress.flush()
self.assertEqual(data, self.plain)
def test_decompression_stream_reset(self):
# test reset
decompress = pylzma.decompressobj()
data = decompress.decompress(self.plain_with_eos[:10])
decompress.reset()
data = decompress.decompress(self.plain_with_eos[:15])
data += decompress.decompress(self.plain_with_eos[15:])
data += decompress.flush()
self.assertEqual(data, self.plain)
def _read_lzma(self, coder, input):
dec = pylzma.decompressobj(maxlength=self._start+self.size)
try:
return self._read_from_decompressor(coder, dec, input, checkremaining=True, with_cache=True)
except ValueError:
if self._is_encrypted():
raise WrongPasswordError('invalid password')
raise
def _read_lzma(self, coder, input, level):
size = self._uncompressed[level]
dec = pylzma.decompressobj(maxlength=self._start+size)
try:
return self._read_from_decompressor(coder, dec, input, level, checkremaining=True, with_cache=True)
except ValueError:
if self._is_encrypted():
raise WrongPasswordError('invalid password')
raise
def test_decompression_streaming_noeos(self):
# test decompressing with one byte at a time...
decompress = pylzma.decompressobj(maxlength=len(self.plain))
infile = StringIO(self.plain_without_eos)
outfile = StringIO()
while 1:
data = infile.read(1)
if not data: break
outfile.write(decompress.decompress(data, 1))
outfile.write(decompress.flush())
self.assertEqual(outfile.getvalue(), self.plain)
def test_decompression_streaming_noeos(self):
# test decompressing with one byte at a time...
decompress = pylzma.decompressobj(maxlength=len(self.plain))
infile = BytesIO(self.plain_without_eos)
outfile = BytesIO()
while 1:
data = infile.read(1)
if not data: break
outfile.write(decompress.decompress(data, 1))
outfile.write(decompress.flush())
self.assertEqual(outfile.getvalue(), self.plain)
def de7ZipIt(compressed_file,decompressed_file):
# Decomrpess the file (as a stream) to a file (as a stream)
i = open(compressed_file, 'rb')
o = open(decompressed_file, 'wb')
s = pylzma.decompressobj()
while True:
tmp = i.read(1)
if not tmp: break
o.write(s.decompress(tmp))
o.close()
i.close()
def test_compress_large_stream(self):
# decompress large block of repeating data, stream version (bug reported by Christopher Perkins)
data = "asdf" * 123456
decompress = pylzma.decompressobj()
infile = StringIO(pylzma.compress(data))
outfile = StringIO()
while 1:
tmp = infile.read(1)
if not tmp: break
outfile.write(decompress.decompress(tmp))
outfile.write(decompress.flush())
self.failUnless(data == outfile.getvalue())
def test_compress_large_stream(self):
# decompress large block of repeating data, stream version (bug reported by Christopher Perkins)
data = bytes("asdf", 'ascii')*123456
decompress = pylzma.decompressobj()
infile = BytesIO(pylzma.compress(data))
outfile = BytesIO()
while 1:
tmp = infile.read(1)
if not tmp: break
outfile.write(decompress.decompress(tmp))
outfile.write(decompress.flush())
self.failUnless(data == outfile.getvalue())
def test_compress_large_stream_bigchunks(self):
# decompress large block of repeating data, stream version with big chunks
data = bytes("asdf", 'ascii')*123456
decompress = pylzma.decompressobj()
infile = BytesIO(pylzma.compress(data))
outfile = BytesIO()
while 1:
tmp = infile.read(1024)
if not tmp: break
outfile.write(decompress.decompress(tmp))
outfile.write(decompress.flush())
self.failUnless(data == outfile.getvalue())
def test_compress_large_stream_bigchunks(self):
# decompress large block of repeating data, stream version with big chunks
data = "asdf" * 123456
decompress = pylzma.decompressobj()
infile = StringIO(pylzma.compress(data))
outfile = StringIO()
while 1:
tmp = infile.read(1024)
if not tmp: break
outfile.write(decompress.decompress(tmp))
outfile.write(decompress.flush())
self.failUnless(data == outfile.getvalue())
def unjsllzma(fi, fi_close=True, parse_fn=None, handle_fail=None, bufs=655350):
""" Make a generator for reading an lzma-compressed file with
json(or something else) in lines.
`parse_fn` is th function(v) to process lines with (defaults to
`json.loads`)
`handle_fail` if a fuction(value, exception) for handling a failure to
parse the value; value is skipped if it raises _IgnoreTheError
exception, otherwise its return value is yielded. default: skip all
failures.
"""
if parse_fn is None:
try:
import simplejson as json
except ImportError:
print("Error importing (preferred) simplejson")
import json
parse_fn = json.loads
if handle_fail is None:
handle_fail = _handle_fail_default
def try_loads(v):
try:
return parse_fn(v)
except Exception as e:
return handle_fail(v, e)
if isinstance(fi, str):
fi = open(fi, 'rb')
tmp2 = '' # buffer for unfunushed lines
in_bufs = int(bufs / 100) # XXX: see lzcat.py note around in_bufs
s = pylzma.decompressobj()
cont = True
while cont:
tmp = fi.read(in_bufs)
if not tmp: # nothing more can be read
tmp2 += s.flush()
cont = False
else:
# XXX: TODO: use bytearray.extend (likely).
tmp2 = tmp2 + s.decompress(tmp, bufs)
tmp3 = tmp2.split('\n') # finished and unfinished lines
for v in tmp3[:-1]:
try:
r = try_loads(v)
except _IgnoreTheError:
continue # no more handling requested, just skip it
yield r
tmp2 = tmp3[-1]
if fi_close:
fi.close()
def unpack_file_from_disk(packed_file, out_file):
''' unpack file '''
print 'Uncompressing file'
with open(packed_file, 'rb') as packed, open(out_file, 'wb') as out:
obj = pylzma.decompressobj()
decompress = ''
while True:
b = packed.read(1)
if not b:
break
dec = obj.decompress(b)
out.write(dec)
return True
def decomp_lzma(inputname, outputname):
prev_time = datetime.now()
comp_file = open(inputname, 'rb')
ret_file = open(outputname, 'wb')
obj = pylzma.decompressobj()
while True:
tmp = comp_file.read(8192)
if not tmp: break
ret_file.write(obj.decompress(tmp))
ret_file.write(obj.flush())
comp_file.close()
ret_file.close()
time_diff = str(datetime.now() - prev_time)
return outputname, str(time_diff)
def decomp_lzma(inputname, outputname):
prev_time = datetime.now()
comp_file = open(inputname, 'rb')
ret_file = open(outputname, 'wb')
obj = pylzma.decompressobj()
while True:
tmp = comp_file.read(8192)
if not tmp: break
ret_file.write(obj.decompress(tmp))
ret_file.write(obj.flush())
comp_file.close()
ret_file.close()
time_diff = str(datetime.now()-prev_time)
return outputname, str(time_diff)
def decomp_lzma(inputname, outputname):
prev_time = datetime.now()
comp_file = open(inputname, 'rb')
ret_file = open(outputname, 'wb')
obj = pylzma.decompressobj()
while True:
tmp = comp_file.read(8192)
if not tmp: break
ret_file.write(obj.decompress(tmp))
ret_file.write(obj.flush())
comp_file.close()
ret_file.close()
time_diff = (datetime.now()-prev_time)
if time_diff.seconds == 0:
return outputname, str(time_diff), '-1'
else:
return outputname, str(time_diff), str(os.path.getsize(inputname)/time_diff.seconds)
def decomp_lzma(inputname, outputname):
prev_time = datetime.now()
comp_file = open(inputname, 'rb')
ret_file = open(outputname, 'wb')
obj = pylzma.decompressobj()
while True:
tmp = comp_file.read(8192)
if not tmp: break
ret_file.write(obj.decompress(tmp))
ret_file.write(obj.flush())
comp_file.close()
ret_file.close()
time_diff = (datetime.now() - prev_time)
if time_diff.seconds == 0:
return outputname, str(time_diff), '-1'
else:
return outputname, str(time_diff), str(
os.path.getsize(inputname) / time_diff.seconds)
def extractxz(infile, outfile): # unicode, unicode -> bool
"""
@param infile unicode
@param outfile unicode
@return bool
"""
import pylzma
try:
with open(infile, 'rb') as i:
with open(outfile, 'wb') as o:
z = pylzma.decompressobj()
while True:
data = i.read(1)
if not data: break
o.write(z.decompress(data))
o.write(z.flush())
except Exception, e:
dwarn(e)
def do_decompress(self, comp_type, off, size):
comp_success = True
if comp_type == self.TYPE_LZMA:
try: # 전체 압축한 경우인지 확인해 본다.
obj = pylzma.decompressobj(maxlength=12)
uncmp_data = obj.decompress(self.mm[0x1c:])
except TypeError:
comp_success = False
elif comp_type == self.TYPE_ZLIB:
try: # 전체 압축한 경우인지 확인해 본다.
uncmp_data = zlib.decompress(self.mm[off:off + size], -15)
except zlib.error:
comp_success = False
else:
uncmp_data = None
comp_success = False
if comp_success:
return uncmp_data
def read(self):
data = ''
idx = 0
cnt = 0
dec = pylzma.decompressobj(maxlength=self._start+self.size)
self._file.seek(self._src_start)
dec.decompress(self._folder.coders[0]['properties'])
total = self.compressed
if total is None:
remaining = self._start+self.size
out = StringIO()
while remaining > 0:
data = self._file.read(1024)
tmp = dec.decompress(data, remaining)
out.write(tmp)
remaining -= len(tmp)
data = out.getvalue()
else:
data = dec.decompress(self._file.read(total), self._start+self.size)
return data[self._start:self._start+self.size]
def decomp_worker(in_queue, out_queue, time_queue):
start_time = datetime.now()
data_size = 0
counter = 0
obj = pylzma.decompressobj()
while True:
chunk = in_queue.get()
if chunk == END_OF_FILE:
break
data_size = data_size + len(chunk)
decomp_chunk = obj.decompress(chunk)
#print "in decomp : %d %d" % (data_size, len(decomp_chunk))
in_queue.task_done()
out_queue.put(decomp_chunk)
counter = counter + 1
out_queue.put(END_OF_FILE)
end_time = datetime.now()
time_queue.put({'start_time':start_time, 'end_time':end_time})
print "[Decomp] : (%s)-(%s)=(%s) (%d loop, %d bytes)" % (start_time.strftime('%X'), end_time.strftime('%X'), str(end_time-start_time), counter, data_size)
def decomp_worker(in_queue, out_queue, time_queue):
start_time = datetime.now()
data_size = 0
counter = 0
obj = pylzma.decompressobj()
while True:
chunk = in_queue.get()
if chunk == END_OF_FILE:
break
data_size = data_size + len(chunk)
decomp_chunk = obj.decompress(chunk)
#print "in decomp : %d %d" % (data_size, len(decomp_chunk))
in_queue.task_done()
out_queue.put(decomp_chunk)
counter = counter + 1
out_queue.put(END_OF_FILE)
end_time = datetime.now()
time_queue.put({'start_time':start_time, 'end_time':end_time})
print "[Decomp] : (%s)-(%s)=(%s) (%d loop, %d bytes)" % (start_time.strftime('%X'), end_time.strftime('%X'), str(end_time-start_time), counter, data_size)
def read_size( self, toread ):
data = ''
idx = 0 #IGNORE:W0612
cnt = 0 #IGNORE:W0612
dec = pylzma.decompressobj( maxlength = self._start + toread )
self._file.seek( self._src_start )
dec.decompress( self._folder.coders[0]['properties'] )
total = self.compressed #IGNORE:E1101
if total is None:
remaining = self._start + toread
out = StringIO()
while remaining > 0:
data = self._file.read( 1024 )
tmp = dec.decompress( data, remaining )
out.write( tmp )
remaining -= len( tmp )
data = out.getvalue()
else:
data = dec.decompress( self._file.read( total ), self._start + toread )
return data[self._start:self._start + toread]
def read_size(self, toread):
data = ''
idx = 0 #IGNORE:W0612
cnt = 0 #IGNORE:W0612
dec = pylzma.decompressobj(maxlength=self._start + toread)
self._file.seek(self._src_start)
dec.decompress(self._folder.coders[0]['properties'])
total = self.compressed #IGNORE:E1101
if total is None:
remaining = self._start + toread
out = StringIO()
while remaining > 0:
data = self._file.read(1024)
tmp = dec.decompress(data, remaining)
out.write(tmp)
remaining -= len(tmp)
data = out.getvalue()
else:
data = dec.decompress(self._file.read(total), self._start + toread)
return data[self._start:self._start + toread]
def read(self):
data = ''
idx = 0
cnt = 0
dec = pylzma.decompressobj(maxlength=self._start + self.size)
self._file.seek(self._src_start)
dec.decompress(self._folder.coders[0]['properties'])
total = self.compressed
if total is None:
remaining = self._start + self.size
out = StringIO()
while remaining > 0:
data = self._file.read(1024)
tmp = dec.decompress(data, remaining)
out.write(tmp)
remaining -= len(tmp)
data = out.getvalue()
else:
data = dec.decompress(self._file.read(total),
self._start + self.size)
return data[self._start:self._start + self.size]
def setup_0_extracted(self):
"""Decompress setup-0 data to disk"""
# decompress setup-0.bin data
DecompressBuffer = 4096
DecompressCRCSize = 4
f = open(self.filename, 'rb')
f.seek(self.TSetupLdrOffsetTable['Offset0'] + self.SetupIDSize + self.CRCCompressedBlockHeaderSize)
decompress = pylzma.decompressobj()
with open(self.setup_0_filename, 'wb') as o:
read_count = 0
while read_count < self.TCompressedBlockHeader['StoredSize']:
crc = f.read(DecompressCRCSize)
data = f.read(DecompressBuffer)
#assert(zlib.crc32(data) == struct.unpack('<l', crc)[0])
o.write(decompress.decompress(data, DecompressBuffer))
read_count += len(crc) + len(data)
o.write(decompress.flush())
f.close()
return True
def decomp_worker(in_path, out_path, time_queue):
in_file = open(in_path, "rb")
out_file = open(out_path, "w+b")
start_time = datetime.now()
data_size = 0
counter = 0
obj = pylzma.decompressobj()
while True:
chunk = in_file.read(CHUNK_SIZE)
if not chunk:
break
data_size = data_size + len(chunk)
decomp_chunk = obj.decompress(chunk)
#print "in decomp : %d %d" % (data_size, len(decomp_chunk))
out_file.write(decomp_chunk)
counter = counter + 1
in_file.close()
out_file.close()
end_time = datetime.now()
time_queue.put({'start_time':start_time, 'end_time':end_time})
print "[Decomp] : (%s)-(%s)=(%s) (%d loop, %d bytes)" % (start_time.strftime('%X'), end_time.strftime('%X'), str(end_time-start_time), counter, data_size)
def setup_0_extracted(self):
"""Decompress setup-0 data to disk"""
# decompress setup-0.bin data
DecompressBuffer = 4096
DecompressCRCSize = 4
f = open(self.filename, 'rb')
f.seek(self.TSetupLdrOffsetTable['Offset0'] + self.SetupIDSize +
self.CRCCompressedBlockHeaderSize)
decompress = pylzma.decompressobj()
with open(self.setup_0_filename, 'wb') as o:
read_count = 0
while read_count < self.TCompressedBlockHeader['StoredSize']:
crc = f.read(DecompressCRCSize)
data = f.read(DecompressBuffer)
#assert(zlib.crc32(data) == struct.unpack('<l', crc)[0])
o.write(decompress.decompress(data, DecompressBuffer))
read_count += len(crc) + len(data)
o.write(decompress.flush())
f.close()
return True
def extracttarxz(path, location): # unicode, unicode -> bool
"""
@param path unicode
@param location unicode
@return bool
"""
try:
import pylzma
with open(path, 'rb') as fp:
z = pylzma.decompressobj()
data = ''
while True:
trunk = fp.read(1)
if not trunk: break
data += z.decompress(trunk)
data += z.flush()
import tarfile
from cStringIO import StringIO
with tarfile.open(mode= "r:", fileobj=StringIO(data)) as t:
t.extractall(location)
return True
except Exception, e:
dwarn(e)
def read(self, filename):
if filename in self.nsis_header.files:
data = None
(foff, ftime, extract_type) = self.nsis_header.files[filename]
if self.case_type == 1: # case 1: 설치 파일 전부를 압축한 경우
# print '#Case 1'
# print hex(foff)
# print hex(kavutil.get_uint32(self.body_data, foff) & 0x7fffffff)
fsize = kavutil.get_uint32(self.body_data, foff) & 0x7fffffff
return self.body_data[foff + 4:foff + 4 + fsize]
elif self.case_type == 2: # case 2: 개별로 압축한 경우
# print '#Case 2'
# print hex(foff)
# print hex(kavutil.get_uint32(self.body_data, foff) & 0x7fffffff)
fsize = kavutil.get_uint32(self.body_data, foff) & 0x7fffffff
fdata = self.body_data[foff + 4:foff + 4 + fsize]
comp_type = self.__get_comp_type(kavutil.get_uint32(fdata, 0))
# print comp_type
if comp_type == self.TYPE_LZMA:
try: # 전체 압축한 경우인지 확인해 본다.
obj = pylzma.decompressobj(maxlength=12)
data = obj.decompress(fdata)
except TypeError:
pass
elif comp_type == self.TYPE_ZLIB:
if kavutil.get_uint32(self.body_data,
foff) & 0x80000000 == 0x80000000:
try:
data = zlib.decompress(fdata, -15)
except zlib.error:
pass
else:
data = fdata # TYPE_COPY
return data
else:
return None
#!/usr/bin/env python3
import pylzma
import sys
import struct
if len(sys.argv) < 3:
print("Usage: ./decompress-sc.py [input.sc] [output.scf]")
exit(0)
with open(sys.argv[1], 'rb') as fin, open(sys.argv[2], 'wb') as fout:
header = fin.read(5)
raw_size = fin.read(4)
size = struct.unpack('<I', raw_size)[0]
obj = pylzma.decompressobj(maxlength=size)
fout.write(obj.decompress(header))
while True:
chunk = fin.read(4096)
if not chunk:
break
fout.write(obj.decompress(chunk))
# Sometimes the following line will crash with
# 'Negative size passed to PyBytes_FromStringAndSize'
try:
fout.write(obj.flush())
except SystemError as e:
print(e)
# decompress-sc.py --- Decompress CoC SC files.
# Copyright (C) 2013 kennytm
def _read_lzma(self, coder, input):
dec = pylzma.decompressobj(maxlength=self._start + self.size)
return self._read_from_decompressor(coder, dec, checkremaining=True)
def __init__(self, fp, length):
FileEntry.__init__(self, fp, length)
self.decomp = pylzma.decompressobj()
self.buffer = ""
def test_decompression_stream(self):
# test decompression object in one steps
decompress = pylzma.decompressobj()
data = decompress.decompress(self.plain_with_eos)
data += decompress.flush()
self.assertEqual(data, self.plain)
def _read_lzma(self, coder, input):
dec = pylzma.decompressobj(maxlength=self._start+self.size)
return self._read_from_decompressor(coder, dec, checkremaining=True)
def test_decompression_stream(self):
# test decompression object in one steps
decompress = pylzma.decompressobj()
data = decompress.decompress(self.plain_with_eos)
data += decompress.flush()
self.assertEqual(data, self.plain)
