def LzmaCompress(data):
data = data.encode(u'utf-8')
c = pylzma.compressfile(BytesIO(data), eos=0)
result_bin = c.read(5)
result_bin += struct.pack(u'<Q', len(data))
compressed_data = result_bin + c.read()
return compressed_data
def test_bugzilla_13(self):
# prevent regression of bugzilla #13
if sys.version_info[:2] < (3, 0):
fp = pylzma.compressfile('/tmp/test')
self.failUnless(isinstance(fp, pylzma.compressfile))
else:
self.failUnlessRaises(TypeError, pylzma.compressfile, '/tmp/test')
def test_bugzilla_13(self):
# prevent regression of bugzilla #13
if sys.version_info[:2] < (3, 0):
fp = pylzma.compressfile('/tmp/test')
self.failUnless(isinstance(fp, pylzma.compressfile))
else:
self.failUnlessRaises(TypeError, pylzma.compressfile, '/tmp/test')
def LZMACompression(inputFileName, outputFileName):
import pylzma, struct
i = open(inputFileName, 'rb')
o = open(outputFileName, 'wb')
i.seek(0)
s = pylzma.compressfile(i, eos=1)
statinfo = os.stat(inputFileName)
result = s.read(5)
# size of uncompressed data
result += struct.pack('<Q', statinfo.st_size)
# compressed data
result += s.read()
o.write(result)
o.close()
i.close()
oSize = float(os.path.getsize(outputFileName))
iSize = float(os.path.getsize(inputFileName))
os.remove(inputFileName)
# print "%s %5.1f %s %5.1f %s %s%5.2f" % \
# ("LZMACompression():\n", iSize/1000, "KB -->", oSize/1000, "KB\n", "Compression Ratio = ", iSize/oSize)
# print 'Written to "', outputFileName, '" -- Removed according uncompressed file'
print outputFileName, "\n"
sys.stdout.flush()
return
def lzma_compress(data):
""" LZMA compression using pylzma """
c = pylzma.compressfile(StringIO(data), dictionary=8, fastBytes=128,
algorithm=0, eos=0, matchfinder='hc3')
result = c.read(5)
result += struct.pack('<Q', len(data))
return result + c.read()
def compress_compatible(data):
c = pylzma.compressfile(StringIO(data), algorithm = 0, dictionary = 16, fastBytes = 64)
# LZMA header
result = c.read(5)
# size of uncompressed data
result += struct.pack('<Q', len(data))
# compressed data
return result + c.read()
def compress_compatible(data):
c = pylzma.compressfile(StringIO(data))
# LZMA header
result = c.read(5)
# size of uncompressed data
result += struct.pack('<Q', len(data))
# compressed data
return result + c.read()
def compress_compatible(data):
#c = pylzma.compressfile(StringIO(data), algorithm = 0, dictionary = 16, fastBytes = 64)
c = pylzma.compressfile(StringIO(data), algorithm = 1, dictionary = 20, fastBytes = 64)
# LZMA header
result = c.read(5)
# size of uncompressed data
result += struct.pack('<Q', len(data))
# compressed data
return result + c.read()
def write(self,title,text):
if (len(self.curr_values)==self.max_inserts):
self.curs.executemany("INSERT INTO articles VALUES (NULL,?,?)",self.curr_values)
self.conn.commit()
self.curr_values=[]
else:
c=pylzma.compressfile(StringIO(text),dictionary=23)
result=c.read(5)
result+=struct.pack('<Q', len(text))
self.curr_values.append((title,buffer(result+c.read())))
def test_compression_file_python(self):
# test compressing from file-like object (Python class)
infile = PyStringIO(self.plain)
outfile = PyStringIO()
compress = pylzma.compressfile(infile, eos=1)
while 1:
data = compress.read(1)
if not data: break
outfile.write(data)
check = pylzma.decompress(outfile.getvalue())
self.assertEqual(check, self.plain)
def lzma_compress(data):
""" LZMA compression using pylzma """
c = pylzma.compressfile(StringIO(data),
dictionary=8,
fastBytes=128,
algorithm=0,
eos=0,
matchfinder='hc3')
result = c.read(5)
result += struct.pack('<Q', len(data))
return result + c.read()
def test_compression_file(self):
# test compressing from file-like object (C class)
infile = BytesIO(self.plain)
outfile = BytesIO()
compress = pylzma.compressfile(infile, eos=1)
while 1:
data = compress.read(1)
if not data: break
outfile.write(data)
check = pylzma.decompress(outfile.getvalue())
self.assertEqual(check, self.plain)
def compressLZMA(self):
i = open(self.txtFilePath, 'rb')
o = open('tra.%s.xz' % self.dateStr, 'wb')
i.seek(0)
s = pylzma.compressfile(i)
while True:
tmp = s.read(1)
if not tmp:
break
o.write(tmp)
o.close()
i.close()
def test_compression_file_python(self):
# test compressing from file-like object (Python class)
from StringIO import StringIO as PyStringIO
infile = PyStringIO(self.plain)
outfile = PyStringIO()
compress = pylzma.compressfile(infile, eos=1)
while 1:
data = compress.read(1)
if not data: break
outfile.write(data)
check = pylzma.decompress(outfile.getvalue())
self.assertEqual(check, self.plain)
def pack_file_to_disk(source_file):
''' takes the file and packs it '''
print 'Compressing file'
# TODO gen random temp here
target_file = 'tempfile.tmp'
with open(source_file, 'r') as source, open(target_file, 'wb') as target:
lz = pylzma.compressfile(source, eos=1)
while True:
b = lz.read(1)
if not b:
break
target.write(b)
target.close()
return target_file
def comp_lzma(inputname, outputname):
prev_time = datetime.now()
in_file = open(inputname, 'rb')
ret_file = open(outputname, 'wb')
c_fp = pylzma.compressfile(in_file, eos=1, algorithm=2, dictionary=28)
while True:
chunk = c_fp.read(8192)
if not chunk: break
ret_file.write(chunk)
in_file.close()
ret_file.close()
time_diff = str(datetime.now()-prev_time)
return outputname, str(time_diff)
def comp_lzma(inputname, outputname):
prev_time = datetime.now()
in_file = open(inputname, 'rb')
ret_file = open(outputname, 'wb')
c_fp = pylzma.compressfile(in_file, eos=1, algorithm=2, dictionary=28)
while True:
chunk = c_fp.read(8192)
if not chunk: break
ret_file.write(chunk)
in_file.close()
ret_file.close()
time_diff = str(datetime.now() - prev_time)
return outputname, str(time_diff)
def compress(f, rename_to=None, big_data=False, f_type='7z'):
"""
Compresses a file. Optimized for csv files.
:param f: Full file path
:param big_data: If True, directly reads data from file, compress it and
writes to new file. Uses less memory. Suitable for big
data.
:param f_type: File type: 7z | zip
:type f_type: str
:type big_data: bool
:type f: str
"""
f_types = ['7z', 'zip']
if f_type not in f_types:
raise ValueError("f_type must be one of %s" % f_types)
fn = f if rename_to is None else rename_to
fn = fn + '.' + f_type
if f_type == f_types[0]:
import struct
with open(f, "rb") as f1:
# pylint: disable=E1101
c = pylzma.compressfile(f1,
literalContextBits=4,
eos=0,
dictionary=24,
fastBytes=255)
result = c.read(5) + struct.pack('<Q', os.path.getsize(f))
with open(fn, 'wb') as f2:
f2.write(result)
with open(fn, 'ab') as f2:
if big_data:
while True:
tmp = c.read(1024)
if not tmp: break
f2.write(tmp)
else:
f2.write(c.read())
elif f_type == f_types[1]:
# http://stackoverflow.com/questions/14568647/create-zip-in-python?rq=1
with zipf.ZipFile(fn, 'w', zipf.ZIP_DEFLATED) as z:
f2 = os.path.splitext(fn)[0] + os.path.splitext(f)[1]
z.write(f, os.path.basename(f2))
return fn
def comp_lzma(inputname, outputname):
# comparable with 'xz -7 [filename]', which uses 200 MB Dictionary'
# original 723218356, this: 145385481, xz: 143296700
prev_time = datetime.now()
in_file = open(inputname, 'rb')
ret_file = open(outputname, 'wb')
c_fp = pylzma.compressfile(in_file, eos=1, algorithm=2, dictionary=28)
while True:
chunk = c_fp.read(8192)
if not chunk: break
ret_file.write(chunk)
in_file.close()
ret_file.close()
time_diff = (datetime.now()-prev_time)
if time_diff.seconds == 0:
return outputname, str(time_diff), '-1'
else:
return outputname, str(time_diff), str(os.path.getsize(inputname)/time_diff.seconds)
def post(self, request, format=None):
token = request.POST.get('token', '')
if Tokens.objects.filter(token=token).first() is not None:
file = request.FILES['file']
orignal_name = file.__str__()
org_size = file._size
if file._size > 26214400 or not isValidExtension(orignal_name):
return Response(
{"detail": "File Size Too Large Or Invalid Extension"},
status=413)
img = Images.objects.filter(orignal_name=orignal_name).first()
if img is None:
compressed_obj = pylzma.compressfile(file)
compressed = ''
while True:
tmp = compressed_obj.read(1)
if not tmp:
break
compressed += tmp
new_file_name = ''.join(
random.SystemRandom().choice(string.ascii_lowercase +
string.digits)
for _ in range(20)) + '.' + file.__str__().rsplit(
'.', 1)[1].lower()
if Images.objects.filter(
filename=new_file_name).first() is None:
namef = settings.IMAGES_ROOT + '/' + new_file_name + '.' + 'compr'
with open(namef, 'w') as cfile:
cfile.write(compressed)
ns = os.path.getsize(namef)
cr = (1 - (float(ns) / float(org_size))) * 100
image = Images(token=token,
orignal_name=orignal_name,
filename=new_file_name,
path=namef,
compression_percentage=cr)
image.save()
serializer = ImagesSerializer(image, many=False)
return Response(serializer.data, status=201)
else:
serializer = ImagesSerializer(img, many=False)
return Response(serializer.data, status=302)
else:
return Response({'detail': 'Authorization failed'}, status=401)
def comp_lzma(inputname, outputname):
# comparable with 'xz -7 [filename]', which uses 200 MB Dictionary'
# original 723218356, this: 145385481, xz: 143296700
prev_time = datetime.now()
in_file = open(inputname, 'rb')
ret_file = open(outputname, 'wb')
c_fp = pylzma.compressfile(in_file, eos=1, algorithm=2, dictionary=28)
while True:
chunk = c_fp.read(8192)
if not chunk: break
ret_file.write(chunk)
in_file.close()
ret_file.close()
time_diff = (datetime.now() - prev_time)
if time_diff.seconds == 0:
return outputname, str(time_diff), '-1'
else:
return outputname, str(time_diff), str(
os.path.getsize(inputname) / time_diff.seconds)
def compress(module, filename):
file = open(filename, "rb")
target = open(filename + ".lzma", "wb")
uncompressed_dat = file.read()
if module == "lzma":
import lzma, struct
compressed_dat = lzma.compress(uncompressed_dat, format=lzma.FORMAT_ALONE)
header = compressed_dat[0:5]
header += struct.pack('<Q', len(uncompressed_dat))
target.write(header + compressed_dat[13:])
elif module == "pylzma":
import pylzma, struct
raw_lzma = pylzma.compressfile(uncompressed_dat, eos=0)
header = raw_lzma.read(5)
header += struct.pack('<Q', len(uncompressed_dat))
target.write(header + raw_lzma.read())
else:
file.close(); target.close()
raise ValueError("Invalid module name: " + module)
file.close(); target.close()
def lzma_compress(fi, fo, fi_close=True, fo_close=True, bufs=65535):
""" Compress `fi` into `fo` (`file` or filename) """
if isinstance(fi, str):
fi, fi_n = open(fi, 'rb'), fi
#fi_close = True
if isinstance(fo, str):
fo, fo_n = open(fo, 'wb'), fo
#fo_close = True
#fi.seek(0)
s = pylzma.compressfile(fi)
while True:
tmp = s.read(bufs)
if not tmp:
break
fo.write(tmp)
if fo_close:
fo.close()
if fi_close:
fi.close()
return fi, fo
def compress(f, rename_to=None, big_data=False, f_type='7z'):
"""
Compresses a file. Optimized for csv files.
:param f: Full file path
:param big_data: If True, directly reads data from file, compress it and
writes to new file. Uses less memory. Suitable for big
data.
:param f_type: File type: 7z | zip
:type f_type: str
:type big_data: bool
:type f: str
"""
f_types = ['7z', 'zip']
if f_type not in f_types:
raise ValueError("f_type must be one of %s" % f_types)
fn = f if rename_to is None else rename_to
fn = fn + '.' + f_type
if f_type == f_types[0]:
import struct
with open(f, "rb") as f1:
# pylint: disable=E1101
c = pylzma.compressfile(f1, literalContextBits=4, eos=0,
dictionary=24, fastBytes=255)
result = c.read(5) + struct.pack('<Q', os.path.getsize(f))
with open(fn, 'wb') as f2: f2.write(result)
with open(fn, 'ab') as f2:
if big_data:
while True:
tmp = c.read(1024)
if not tmp: break
f2.write(tmp)
else:
f2.write(c.read())
elif f_type == f_types[1]:
# http://stackoverflow.com/questions/14568647/create-zip-in-python?rq=1
with zipf.ZipFile(fn, 'w', zipf.ZIP_DEFLATED) as z:
f2 = os.path.splitext(fn)[0] + os.path.splitext(f)[1]
z.write(f, os.path.basename(f2))
return fn
def patch(self, request, format=None):
token = request.data['token']
if Tokens.objects.filter(token=token).first() is not None:
image = Images.objects.filter(
filename=request.data['filename']).first()
if image is not None:
file = request.FILES['file']
orignal_name = file.__str__()
org_size = file._size
if file._size > 26214400 or not isValidExtension(orignal_name):
return Response(
{"detail": "File Size Too Large or Invalid Extension"},
status=413)
compressed_obj = pylzma.compressfile(file)
compressed = ''
while True:
tmp = compressed_obj.read(1)
if not tmp:
break
compressed += tmp
namef = settings.IMAGES_ROOT + '/' + request.data[
'filename'] + '.' + 'compr'
with open(namef, 'w') as cfile:
cfile.write(compressed)
ns = os.path.getsize(namef)
cr = (1 - (float(ns) / float(org_size))) * 100
image.delete()
image = Images(token=token,
orignal_name=orignal_name,
filename=request.data['filename'],
path=namef,
compression_percentage=cr)
image.save()
print "hello"
serializer = ImagesSerializer(image, many=False)
return Response(serializer.data, status=201)
else:
return Response({'detail': 'Filename not known'}, status=404)
else:
return Response({'detail': 'Authorization failed'}, status=401)
def compress(module, filename):
file = open(filename, "rb")
target = open(filename + ".lzma", "wb")
uncompressed_dat = file.read()
if module == "lzma":
import lzma, struct
compressed_dat = lzma.compress(uncompressed_dat,
format=lzma.FORMAT_ALONE)
header = compressed_dat[0:5]
header += struct.pack('<Q', len(uncompressed_dat))
target.write(header + compressed_dat[13:])
elif module == "pylzma":
import pylzma, struct
raw_lzma = pylzma.compressfile(uncompressed_dat, eos=0)
header = raw_lzma.read(5)
header += struct.pack('<Q', len(uncompressed_dat))
target.write(header + raw_lzma.read())
else:
file.close()
target.close()
raise ValueError("Invalid module name: " + module)
file.close()
target.close()
def test_bugzilla_13(self):
# prevent regression of bugzilla #13
fp = pylzma.compressfile('/tmp/test')
self.failUnless(isinstance(fp, pylzma.compressfile))
