def compress_files(): """Compresses downloaded data files.""" lcsv = [f for f in os.listdir(DIR.data) if f.endswith('.csv')] today_e_tag = _now().strftime('%Y%m%d') for f in lcsv: ff = os.path.join(DIR.data, f) if today_e_tag not in ff: c.compress(ff) log.info('%s compressed.', os.path.basename(ff)) os.remove(ff)
def test_compression(): """ Test compression and decompression functions """ f_sha = hashlib.sha256(open(_f, 'rb').read()).hexdigest() temp_name = os.path.join('tests', next(tempfile._get_candidate_names())) fc = c.compress(_f, temp_name) fd = c.decompress(fc) assert f_sha == hashlib.sha256(open(fd, 'rb').read()).hexdigest() os.remove(fc); os.remove(fd) fc = c.compress(_f, temp_name, f_type='zip') fd = c.decompress(fc) assert f_sha == hashlib.sha256(open(fd, 'rb').read()).hexdigest() os.remove(fc); os.remove(fd)
def test_compression(): """ Test compression and decompression functions """ f_sha = hashlib.sha256(open(_f, 'rb').read()).hexdigest() temp_name = os.path.join('tests', next(tempfile._get_candidate_names())) fc = c.compress(_f, temp_name) fd = c.decompress(fc) assert f_sha == hashlib.sha256(open(fd, 'rb').read()).hexdigest() os.remove(fc) os.remove(fd) fc = c.compress(_f, temp_name, f_type='zip') fd = c.decompress(fc) assert f_sha == hashlib.sha256(open(fd, 'rb').read()).hexdigest() os.remove(fc) os.remove(fd)
def test2(self): """ An arbitrarily long string :return: """ self.assertEqual(compress('abadbadsakdlfjieafnealfjiasfjaaaaanadddddddkkkjj'), \ "abadbadsakdlfjieafnealfjiasfja#5nad#7kkkjj")
def test3(self): """ A string where a lot has to be compressed :return: """ self.assertEqual(compress('kkkkkkjjjjjjjnnnnniiiiiiiooooook'), 'k#6j#7n#5i#7o#6k')
def resultat(): results = request.files filename = results['myFile'] fileCompressed, b = compression.compress(filename, 0, 2, 0, 1000, 1000, False, False) response = make_response(fileCompressed) response.headers["Content-type"] = "application/wav" response.headers[ "Content-Disposition"] = "attachment; filename=compressed.wav" return response
def convert_data_from_zip_file(f): """ Directly reads eoner's csv from zip file and converts it to ours. :param f: Full path to file name. :type f: str """ # f = '/Users/isezen/project/tkmpy/tkmdata/2015-07-31.zip' date = os.path.splitext(os.path.basename(f))[0] date = dt.strptime(date,'%Y-%m-%d') fnames = [_add_date_to_file_name(os.path.splitext(os.path.basename(url))[0], date.strftime('%Y%m%d')) +'.csv' for url in tkm.URL[:5]] text = comp.read_from_zip(f) k = 0 for l in [x for x in text.split('\n') if not re.match(r'^\s*$', x)]: d = l.replace(u'\ufeff','').split('#') if len(d)>6: for i in range(6,len(d)): d[5] = d[5] + '#' + d[i] d = d[0:6] d = [i.replace(';', '|').replace('|&', '&') for i in d] for i in range(len(d)): if d[i][len(d[i])-1] == u'&': d[i] = d[i][:(len(d[i])-1)] date = _oa_to_datetime(d[0]) date = date.replace(tzinfo=tz.tzutc()).astimezone(tz.tzlocal()) k+=1 for i in range(len(d[:-1])): f_name = _add_date_to_file_name(os.path.basename(tkm.URL[i]), date.strftime('%Y%m%d')) f_name = os.path.splitext(f_name)[0] + '.csv' if d[i+1] != 'error': tkmd = tkm.TKM_DATA(date=date, e_tag=None, filename=f_name, data=d[i+1]) tkm.save_instant_data(tkmd) for f in fnames: f = os.path.join(tkm.DIR.data, f) comp.compress(f) if os.path.exists(f): os.remove(f)
def encodeLayerData(self, tileLayer, format): if format in [Map.LayerDataFormat.XML, Map.LayerDataFormat.CSV]: raise tileData = QByteArray() for y in range(tileLayer.height()): for x in range(tileLayer.width()): gid = self.cellToGid(tileLayer.cellAt(x, y)) tileData.append(bytes([gid&0xff])) tileData.append(bytes([(gid >> 8)&0xff])) tileData.append(bytes([(gid >> 16)&0xff])) tileData.append(bytes([(gid >> 24)&0xff])) if len(tileData)%4 != 0: raise if (format == Map.LayerDataFormat.Base64Gzip): tileData = compress(tileData, CompressionMethod.Gzip) elif (format == Map.LayerDataFormat.Base64Zlib): tileData = compress(tileData, CompressionMethod.Zlib) return tileData.toBase64()
def getContents(self): fileslist = self.getFilesList() assert fileslist, "Must contain at least one resource." result = fileslist[0].getContents() content_type = result["content_type"] data = [result["data"]] for subres in fileslist[1:]: d = subres.getContents() # all elements must have the same content type. assert d["content_type"] == content_type data.append(d["data"]) result["data"] = "\n".join(data) result["compress_level"] = self.compress_level # Do compression on the result result["data"] = compress(**result) return result
def test_compression(): filename = "lena.bmp" image = io.imread(filename) filename = filename.split(".")[0] co.write_as_mcf(image, filename) vec = co.read_as_np(filename) com = co.compress(image) co.write_as_mcf(com, "comp_lena") img2 = co.read_as_np("comp_lena") print(img2.shape) image = co.decompress(img2, image.shape) plt.imshow(image, cmap="gray") plt.show()
def write(self, map, fileName): # Check layer count and type if (map.layerCount() != 1 or not map.layerAt(0).isTileLayer()): self.mError = self.tr( "The map needs to have exactly one tile layer!") return False mapLayer = map.layerAt(0).asTileLayer() # Check layer size if (mapLayer.width() != 48 or mapLayer.height() != 48): self.mError = self.tr( "The layer must have a size of 48 x 48 tiles!") return False # Create QByteArray and compress it uncompressed = QByteArray(48 * 48, b'\x00') width = mapLayer.width() height = mapLayer.height() for y in range(0, height): for x in range(0, width): tile = mapLayer.cellAt(x, y).tile if tile: # 'QByteArray' object does not support item assignment uncompressed.replace(y * width + x, 1, bytes([tile.id() & 0xff])) compressed = compress(uncompressed, CompressionMethod.Gzip) # Write QByteArray file = QSaveFile(fileName) if (not file.open(QIODevice.WriteOnly)): self.mError = self.tr("Could not open file for writing.") return False file.write(compressed) if not file.commit(): self.mError = file.errorString() return False return True
def write(self, map, fileName): # Check layer count and type if (map.layerCount() != 1 or not map.layerAt(0).isTileLayer()) : self.mError = self.tr("The map needs to have exactly one tile layer!") return False mapLayer = map.layerAt(0).asTileLayer() # Check layer size if (mapLayer.width() != 48 or mapLayer.height() != 48) : self.mError = self.tr("The layer must have a size of 48 x 48 tiles!") return False # Create QByteArray and compress it uncompressed = QByteArray(48 * 48, b'\x00') width = mapLayer.width() height = mapLayer.height() for y in range(0, height): for x in range(0, width): tile = mapLayer.cellAt(x, y).tile if tile: # 'QByteArray' object does not support item assignment uncompressed.replace(y * width + x, 1, bytes([tile.id()&0xff])) compressed = compress(uncompressed, CompressionMethod.Gzip) # Write QByteArray file = QSaveFile(fileName) if (not file.open(QIODevice.WriteOnly)) : self.mError = self.tr("Could not open file for writing.") return False file.write(compressed) if not file.commit(): self.mError = file.errorString() return False return True
def _compress2(): comp.compress(csv, True) _compress1()
for record in wfdb.get_record_list(db_name, records='all'): # record = 'Person_01/rec_10' # record = '203' # get data for current record data = wfdb.rdsamp(record, pb_dir=db_name + '/' + record.split('/')[0]) Fs = data[1]['fs'] ecg = data[0][:, 0] #[0:Fs*20] # zero-mean ecg = ecg - np.mean(ecg) # call compression function CR, ecg_compressed, wc_orig = compress(ecg, Fs) # call reconstruction function ecg_recon, wc_recon = reconstruct(ecg_compressed) # compute and store evaluation metrics PRD, R = calc_PRD(ecg, ecg_recon) ps = analysis(ecg, Fs) pr = analysis(ecg_recon, Fs) CR_arr.append(CR) PRD_arr.append(PRD) R_arr.append(R) SNRo_arr.append(ps['SNR']) SNRr_arr.append(pr['SNR']) d_SNR.append(pr['SNR'] - ps['SNR'])
def _compress1(): comp.compress(csv, f_type='.zip') # noinspection PyMissingOrEmptyDocstring def _compress2(): comp.compress(csv, True)
from sys import argv, exit from compression import compress, decompress, usage if len(argv) < 3: usage(argv[0]) if int(argv[1]) == 1: print "%r compressed is %r" % (argv[2], compress(argv[2])) elif int(argv[1]) == 0: print "%r decompressed is %r" % (argv[2], decompress(argv[2])) else: usage(argv[0])
def test_js(self): in_file = os.path.join('tests', 'resources', 'js', 'one-line.js') compression.compress(in_file, self.out_file) expected = 'var test;' result = open(self.out_file).read() self.assertEquals(result, expected)
def _save_manifest(self, data): data = crypto.encrypt_then_mac( self.config, compression.compress(self.config, msgpack.dumps(data))) self.transport.write_manifest(data, self.inittime)
def test_css(self): in_file = os.path.join('tests', 'resources', 'css', 'one-line.css') compression.compress(in_file, self.out_file, in_type='css') expected = 'body{background-color:red;}' result = open(self.out_file).read() self.assertEquals(result, expected)
if __name__ == "__main__": cli_input = sys.argv if invalid_input(cli_input): print("Please insert a valid command") raise Exception( "Commands should be like './main.py -(c/x) file.(txt/lz78) [-o result_file_name]'" ) f_name = "" if cli_input[1] == "-c": if len(cli_input) == 5: f_name = cli_input[4] + ".z78" else: f_name = cli_input[2].split(".")[0] + ".z78" lz78comp.compress(cli_input[2], f_name) print("File ", cli_input[2], " compressed to ", f_name, " !") elif cli_input[1] == "-x": if len(cli_input) == 5: f_name = cli_input[4] + ".txt" else: f_name = cli_input[2].split(".")[0] + ".txt" decoded_text = lz78comp.decompress(cli_input[2]) lz78comp.create_txt_file(f_name, decoded_text) print("File ", cli_input[2], " decompressed to ", f_name, " !")
#!/usr/bin/env python3 # Compress and store as (1x900) GIF import os import sys import imageio import cv2 as cv import numpy as np from compression import compress infilename = sys.argv[1] outdirectory = './dataset/compressed/' _, outfilename = os.path.split(infilename) # Extract first frame from GIF (only one frame) gif = imageio.mimread(infilename)[0] stacked_gif = compress(gif) # Save output to file imageio.mimwrite(outdirectory + outfilename, stacked_gif) print(infilename, '-->', outdirectory + outfilename)
graph = utils.read_file(sys.stdin) n = graph.number_of_nodes() m = graph.number_of_edges() eprint("n:", n, "m:", m) #eprint(utils.degree_hist(graph)) vc = main(graph, greedy_vc=True) eprint("final vc size: {}/{}".format(len(vc), n)) print("s vc {0} {1}".format(n, len(vc))) print(*vc, sep='\n') eprint("time:", time() - start) if __name__ == '__main__': eprint("compresing...") res, svc = compression.compress(graph, vc) timestamp("compression") if res: eprint("smaller vc found, size:", len(svc)) else: eprint("vc could not be compressed") import psutil import matplotlib.pyplot as plt # nx.draw_random(graph, with_labels=True) # plt.show() process = psutil.Process(os.getpid()) # graph = utils.read_instance(1) # graph = nx.random_regular_graph(3,6000,seed=1) # graph = nx.Graph([(u+1,v+1) for u,v in graph.edges()]) eprint("Memory usage: {} MB".format(process.memory_info().rss / 1e6))
## Simple Lossless String Compression ## import compression phrase = "to be or not to be, that is the question" compressed, key = compression.compress(phrase) def utf8len(s): return len(s.encode('utf-8')) def dictStringBytes(dictString): sum = 0 for v in dictString: sum += utf8len(v) return sum print("Uncompressed: " + phrase) print("Compressed: " + compressed) print("Bytes before compression: %s, Bytes after compression: %s"%(utf8len(phrase), utf8len(compressed)+dictStringBytes(key)))
import sys import cv2 import numpy as np from scipy.misc import imread, imsave from PIL import Image from comic import comic from compression import compress from stylize import render img = sys.argv[1] img1 = comic(img) img2 = compress('images/tmp/tmp1.jpg') img3 = render(imread('images/tmp/tmp2.jpg'), ratio=0.001, verbose=True) img3 = imsave('images/tmp/tmp3.jpg', img3) img4 = render(imread('images/tmp/tmp3.jpg'), depth=4, verbose=True) img4 = imsave('images/tmp/tmp4.jpg', img4) list_im = [ sys.argv[1], 'images/tmp/tmp2.jpg', 'images/tmp/tmp3.jpg', 'images/tmp/tmp4.jpg' ] imgs = [Image.open(i) for i in list_im] min_shape = sorted([(np.sum(i.size), i.size) for i in imgs])[0][1] imgs_comb = np.vstack(np.asarray(i.resize(min_shape)) for i in imgs) imgs_comb = Image.fromarray(imgs_comb) imgs_comb.save('images/tmp/VStack.jpg') def getSize(): return min_shape
def _enc(self, data): encrypted = crypto.encrypt(self.config, compression.compress(self.config, data)) return encrypted
from compression import compress im = [ # Image goes here :) ] tftable = [3, 2, 1, 0] res = [] for i in range(0, len(im), 4): n = tftable[im[i + 0]] << 6 | tftable[im[i + 1]] << 4 | tftable[im[ i + 2]] << 2 | tftable[im[i + 3]] res.append(n) res = compress(res) nbytes = len(res) res = ["%d," % n for n in res] res = [" ".join(res[i:i + 16]) for i in range(0, len(res), 16)] print 'const unsigned char varname[%d] __attribute__((section(".text"), used)) = {' % nbytes print "\n".join(res) print "};"
# filename = "bee_movie_script.txt" # compress(filename) # decompress(filename + ".compressed") # Main code while True: print("Choose an option:") print("1- Compress a file") print("2- Decompress a file") print("3- Compress a folder") print("4- Decompress a folder") print("0- Exit") choice = input("Choice > ") if choice == '1': filename = input("Enter filename: ") compress(filename) elif choice == '2': filename = input("Enter filename: ") decompress(filename) elif choice == '3': foldername = input("Enter folder name: ") compress_folder(foldername + '/') elif choice == '4': foldername = input("Enter folder name: ") decompress_folder(foldername) elif choice == '0': break else: print("Invalid choice") print("\n\n")
def _compress1(): comp.compress(csv, f_type='.zip')
from compression import compress from decompression import decompress def parseArguments(): if len(argv) == 4: return argv[1], argv[2], argv[3] else: print("ArgError - Script accepts 3 arguemnts: Mode, Source_Path, Destination_Path") exit() try: mode, srcPath, destPath = parseArguments() srcFile = open(srcPath, "r") destFile = open(destPath, "w") if mode == "-c": compress(srcFile, destFile) elif mode == "-d": decompress(srcFile, destFile) else: print("ModeError - First Arg must be -c for Compression or -d for Decompression") srcFile.close() destFile.close() except FileNotFoundError: print("IOError - File Not Found")
def _compress2(): comp.compress(csv, True)
def test1(self): """ A small standard string :return: """ self.assertEqual(compress('aaaaa'), 'a#5')
def saveObject(self,obj): file = open(self.path % obj.client.name ,'wb') file.write(compression.compress(obj)) file.close()
def test_compress_decompress(self): for testCase in self.toTestList: # test compress output self.assertEqual(compress(testCase[0]),testCase[1]) # test decompress output -- compressing then decompressing self.assertEqual(decompress(compress(testCase[0])),testCase[0])
def do_compress(self, output): compression.compress(self.image_entry.get(), self.network_entry.get(), output, int(self.bits_entry.get()), self.smoothing.get())
import os from compression import compress, decompress degree = 2 directory1 = 'mri_mini' directory2 = directory1 + '_compressed' directory3 = directory1 + '_decompressed' for file in os.listdir(directory1): compress(directory1, file, degree, directory2) for file in os.listdir(directory2): decompress(directory2, file, directory3) for file in os.listdir(directory1): f1 = open(directory1 + '/' + file, 'rb') f2 = open(directory3 + '/' + file, 'rb') f1 = f1.read() f2 = f2.read() if len(f1) != len(f2): print(False) print(file, 'len') for i in range(0, len(f1)): if f1[i] != f2[i]: print(False) print(file, 'val', i) break