def LzmaCompress(data): data = data.encode(u'utf-8') c = pylzma.compressfile(BytesIO(data), eos=0) result_bin = c.read(5) result_bin += struct.pack(u'<Q', len(data)) compressed_data = result_bin + c.read() return compressed_data
def test_bugzilla_13(self): # prevent regression of bugzilla #13 if sys.version_info[:2] < (3, 0): fp = pylzma.compressfile('/tmp/test') self.failUnless(isinstance(fp, pylzma.compressfile)) else: self.failUnlessRaises(TypeError, pylzma.compressfile, '/tmp/test')
def LZMACompression(inputFileName, outputFileName): import pylzma, struct i = open(inputFileName, 'rb') o = open(outputFileName, 'wb') i.seek(0) s = pylzma.compressfile(i, eos=1) statinfo = os.stat(inputFileName) result = s.read(5) # size of uncompressed data result += struct.pack('<Q', statinfo.st_size) # compressed data result += s.read() o.write(result) o.close() i.close() oSize = float(os.path.getsize(outputFileName)) iSize = float(os.path.getsize(inputFileName)) os.remove(inputFileName) # print "%s %5.1f %s %5.1f %s %s%5.2f" % \ # ("LZMACompression():\n", iSize/1000, "KB -->", oSize/1000, "KB\n", "Compression Ratio = ", iSize/oSize) # print 'Written to "', outputFileName, '" -- Removed according uncompressed file' print outputFileName, "\n" sys.stdout.flush() return
def lzma_compress(data): """ LZMA compression using pylzma """ c = pylzma.compressfile(StringIO(data), dictionary=8, fastBytes=128, algorithm=0, eos=0, matchfinder='hc3') result = c.read(5) result += struct.pack('<Q', len(data)) return result + c.read()
def compress_compatible(data): c = pylzma.compressfile(StringIO(data), algorithm = 0, dictionary = 16, fastBytes = 64) # LZMA header result = c.read(5) # size of uncompressed data result += struct.pack('<Q', len(data)) # compressed data return result + c.read()
def compress_compatible(data): c = pylzma.compressfile(StringIO(data)) # LZMA header result = c.read(5) # size of uncompressed data result += struct.pack('<Q', len(data)) # compressed data return result + c.read()
def compress_compatible(data): #c = pylzma.compressfile(StringIO(data), algorithm = 0, dictionary = 16, fastBytes = 64) c = pylzma.compressfile(StringIO(data), algorithm = 1, dictionary = 20, fastBytes = 64) # LZMA header result = c.read(5) # size of uncompressed data result += struct.pack('<Q', len(data)) # compressed data return result + c.read()
def write(self,title,text): if (len(self.curr_values)==self.max_inserts): self.curs.executemany("INSERT INTO articles VALUES (NULL,?,?)",self.curr_values) self.conn.commit() self.curr_values=[] else: c=pylzma.compressfile(StringIO(text),dictionary=23) result=c.read(5) result+=struct.pack('<Q', len(text)) self.curr_values.append((title,buffer(result+c.read())))
def test_compression_file_python(self): # test compressing from file-like object (Python class) infile = PyStringIO(self.plain) outfile = PyStringIO() compress = pylzma.compressfile(infile, eos=1) while 1: data = compress.read(1) if not data: break outfile.write(data) check = pylzma.decompress(outfile.getvalue()) self.assertEqual(check, self.plain)
def test_compression_file(self): # test compressing from file-like object (C class) infile = BytesIO(self.plain) outfile = BytesIO() compress = pylzma.compressfile(infile, eos=1) while 1: data = compress.read(1) if not data: break outfile.write(data) check = pylzma.decompress(outfile.getvalue()) self.assertEqual(check, self.plain)
def compressLZMA(self): i = open(self.txtFilePath, 'rb') o = open('tra.%s.xz' % self.dateStr, 'wb') i.seek(0) s = pylzma.compressfile(i) while True: tmp = s.read(1) if not tmp: break o.write(tmp) o.close() i.close()
def test_compression_file_python(self): # test compressing from file-like object (Python class) from StringIO import StringIO as PyStringIO infile = PyStringIO(self.plain) outfile = PyStringIO() compress = pylzma.compressfile(infile, eos=1) while 1: data = compress.read(1) if not data: break outfile.write(data) check = pylzma.decompress(outfile.getvalue()) self.assertEqual(check, self.plain)
def pack_file_to_disk(source_file): ''' takes the file and packs it ''' print 'Compressing file' # TODO gen random temp here target_file = 'tempfile.tmp' with open(source_file, 'r') as source, open(target_file, 'wb') as target: lz = pylzma.compressfile(source, eos=1) while True: b = lz.read(1) if not b: break target.write(b) target.close() return target_file
def comp_lzma(inputname, outputname): prev_time = datetime.now() in_file = open(inputname, 'rb') ret_file = open(outputname, 'wb') c_fp = pylzma.compressfile(in_file, eos=1, algorithm=2, dictionary=28) while True: chunk = c_fp.read(8192) if not chunk: break ret_file.write(chunk) in_file.close() ret_file.close() time_diff = str(datetime.now()-prev_time) return outputname, str(time_diff)
def comp_lzma(inputname, outputname): prev_time = datetime.now() in_file = open(inputname, 'rb') ret_file = open(outputname, 'wb') c_fp = pylzma.compressfile(in_file, eos=1, algorithm=2, dictionary=28) while True: chunk = c_fp.read(8192) if not chunk: break ret_file.write(chunk) in_file.close() ret_file.close() time_diff = str(datetime.now() - prev_time) return outputname, str(time_diff)
def compress(f, rename_to=None, big_data=False, f_type='7z'): """ Compresses a file. Optimized for csv files. :param f: Full file path :param big_data: If True, directly reads data from file, compress it and writes to new file. Uses less memory. Suitable for big data. :param f_type: File type: 7z | zip :type f_type: str :type big_data: bool :type f: str """ f_types = ['7z', 'zip'] if f_type not in f_types: raise ValueError("f_type must be one of %s" % f_types) fn = f if rename_to is None else rename_to fn = fn + '.' + f_type if f_type == f_types[0]: import struct with open(f, "rb") as f1: # pylint: disable=E1101 c = pylzma.compressfile(f1, literalContextBits=4, eos=0, dictionary=24, fastBytes=255) result = c.read(5) + struct.pack('<Q', os.path.getsize(f)) with open(fn, 'wb') as f2: f2.write(result) with open(fn, 'ab') as f2: if big_data: while True: tmp = c.read(1024) if not tmp: break f2.write(tmp) else: f2.write(c.read()) elif f_type == f_types[1]: # http://stackoverflow.com/questions/14568647/create-zip-in-python?rq=1 with zipf.ZipFile(fn, 'w', zipf.ZIP_DEFLATED) as z: f2 = os.path.splitext(fn)[0] + os.path.splitext(f)[1] z.write(f, os.path.basename(f2)) return fn
def comp_lzma(inputname, outputname): # comparable with 'xz -7 [filename]', which uses 200 MB Dictionary' # original 723218356, this: 145385481, xz: 143296700 prev_time = datetime.now() in_file = open(inputname, 'rb') ret_file = open(outputname, 'wb') c_fp = pylzma.compressfile(in_file, eos=1, algorithm=2, dictionary=28) while True: chunk = c_fp.read(8192) if not chunk: break ret_file.write(chunk) in_file.close() ret_file.close() time_diff = (datetime.now()-prev_time) if time_diff.seconds == 0: return outputname, str(time_diff), '-1' else: return outputname, str(time_diff), str(os.path.getsize(inputname)/time_diff.seconds)
def post(self, request, format=None): token = request.POST.get('token', '') if Tokens.objects.filter(token=token).first() is not None: file = request.FILES['file'] orignal_name = file.__str__() org_size = file._size if file._size > 26214400 or not isValidExtension(orignal_name): return Response( {"detail": "File Size Too Large Or Invalid Extension"}, status=413) img = Images.objects.filter(orignal_name=orignal_name).first() if img is None: compressed_obj = pylzma.compressfile(file) compressed = '' while True: tmp = compressed_obj.read(1) if not tmp: break compressed += tmp new_file_name = ''.join( random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(20)) + '.' + file.__str__().rsplit( '.', 1)[1].lower() if Images.objects.filter( filename=new_file_name).first() is None: namef = settings.IMAGES_ROOT + '/' + new_file_name + '.' + 'compr' with open(namef, 'w') as cfile: cfile.write(compressed) ns = os.path.getsize(namef) cr = (1 - (float(ns) / float(org_size))) * 100 image = Images(token=token, orignal_name=orignal_name, filename=new_file_name, path=namef, compression_percentage=cr) image.save() serializer = ImagesSerializer(image, many=False) return Response(serializer.data, status=201) else: serializer = ImagesSerializer(img, many=False) return Response(serializer.data, status=302) else: return Response({'detail': 'Authorization failed'}, status=401)
def comp_lzma(inputname, outputname): # comparable with 'xz -7 [filename]', which uses 200 MB Dictionary' # original 723218356, this: 145385481, xz: 143296700 prev_time = datetime.now() in_file = open(inputname, 'rb') ret_file = open(outputname, 'wb') c_fp = pylzma.compressfile(in_file, eos=1, algorithm=2, dictionary=28) while True: chunk = c_fp.read(8192) if not chunk: break ret_file.write(chunk) in_file.close() ret_file.close() time_diff = (datetime.now() - prev_time) if time_diff.seconds == 0: return outputname, str(time_diff), '-1' else: return outputname, str(time_diff), str( os.path.getsize(inputname) / time_diff.seconds)
def compress(module, filename): file = open(filename, "rb") target = open(filename + ".lzma", "wb") uncompressed_dat = file.read() if module == "lzma": import lzma, struct compressed_dat = lzma.compress(uncompressed_dat, format=lzma.FORMAT_ALONE) header = compressed_dat[0:5] header += struct.pack('<Q', len(uncompressed_dat)) target.write(header + compressed_dat[13:]) elif module == "pylzma": import pylzma, struct raw_lzma = pylzma.compressfile(uncompressed_dat, eos=0) header = raw_lzma.read(5) header += struct.pack('<Q', len(uncompressed_dat)) target.write(header + raw_lzma.read()) else: file.close(); target.close() raise ValueError("Invalid module name: " + module) file.close(); target.close()
def lzma_compress(fi, fo, fi_close=True, fo_close=True, bufs=65535): """ Compress `fi` into `fo` (`file` or filename) """ if isinstance(fi, str): fi, fi_n = open(fi, 'rb'), fi #fi_close = True if isinstance(fo, str): fo, fo_n = open(fo, 'wb'), fo #fo_close = True #fi.seek(0) s = pylzma.compressfile(fi) while True: tmp = s.read(bufs) if not tmp: break fo.write(tmp) if fo_close: fo.close() if fi_close: fi.close() return fi, fo
def patch(self, request, format=None): token = request.data['token'] if Tokens.objects.filter(token=token).first() is not None: image = Images.objects.filter( filename=request.data['filename']).first() if image is not None: file = request.FILES['file'] orignal_name = file.__str__() org_size = file._size if file._size > 26214400 or not isValidExtension(orignal_name): return Response( {"detail": "File Size Too Large or Invalid Extension"}, status=413) compressed_obj = pylzma.compressfile(file) compressed = '' while True: tmp = compressed_obj.read(1) if not tmp: break compressed += tmp namef = settings.IMAGES_ROOT + '/' + request.data[ 'filename'] + '.' + 'compr' with open(namef, 'w') as cfile: cfile.write(compressed) ns = os.path.getsize(namef) cr = (1 - (float(ns) / float(org_size))) * 100 image.delete() image = Images(token=token, orignal_name=orignal_name, filename=request.data['filename'], path=namef, compression_percentage=cr) image.save() print "hello" serializer = ImagesSerializer(image, many=False) return Response(serializer.data, status=201) else: return Response({'detail': 'Filename not known'}, status=404) else: return Response({'detail': 'Authorization failed'}, status=401)
def compress(module, filename): file = open(filename, "rb") target = open(filename + ".lzma", "wb") uncompressed_dat = file.read() if module == "lzma": import lzma, struct compressed_dat = lzma.compress(uncompressed_dat, format=lzma.FORMAT_ALONE) header = compressed_dat[0:5] header += struct.pack('<Q', len(uncompressed_dat)) target.write(header + compressed_dat[13:]) elif module == "pylzma": import pylzma, struct raw_lzma = pylzma.compressfile(uncompressed_dat, eos=0) header = raw_lzma.read(5) header += struct.pack('<Q', len(uncompressed_dat)) target.write(header + raw_lzma.read()) else: file.close() target.close() raise ValueError("Invalid module name: " + module) file.close() target.close()
def test_bugzilla_13(self): # prevent regression of bugzilla #13 fp = pylzma.compressfile('/tmp/test') self.failUnless(isinstance(fp, pylzma.compressfile))