def compress_files():
"""Compresses downloaded data files."""
lcsv = [f for f in os.listdir(DIR.data) if f.endswith('.csv')]
today_e_tag = _now().strftime('%Y%m%d')
for f in lcsv:
ff = os.path.join(DIR.data, f)
if today_e_tag not in ff:
c.compress(ff)
log.info('%s compressed.', os.path.basename(ff))
os.remove(ff)
def compress_files():
"""Compresses downloaded data files."""
lcsv = [f for f in os.listdir(DIR.data) if f.endswith('.csv')]
today_e_tag = _now().strftime('%Y%m%d')
for f in lcsv:
ff = os.path.join(DIR.data, f)
if today_e_tag not in ff:
c.compress(ff)
log.info('%s compressed.', os.path.basename(ff))
os.remove(ff)
def test_compression():
""" Test compression and decompression functions """
f_sha = hashlib.sha256(open(_f, 'rb').read()).hexdigest()
temp_name = os.path.join('tests', next(tempfile._get_candidate_names()))
fc = c.compress(_f, temp_name)
fd = c.decompress(fc)
assert f_sha == hashlib.sha256(open(fd, 'rb').read()).hexdigest()
os.remove(fc); os.remove(fd)
fc = c.compress(_f, temp_name, f_type='zip')
fd = c.decompress(fc)
assert f_sha == hashlib.sha256(open(fd, 'rb').read()).hexdigest()
os.remove(fc); os.remove(fd)
def test_compression():
""" Test compression and decompression functions """
f_sha = hashlib.sha256(open(_f, 'rb').read()).hexdigest()
temp_name = os.path.join('tests', next(tempfile._get_candidate_names()))
fc = c.compress(_f, temp_name)
fd = c.decompress(fc)
assert f_sha == hashlib.sha256(open(fd, 'rb').read()).hexdigest()
os.remove(fc)
os.remove(fd)
fc = c.compress(_f, temp_name, f_type='zip')
fd = c.decompress(fc)
assert f_sha == hashlib.sha256(open(fd, 'rb').read()).hexdigest()
os.remove(fc)
os.remove(fd)
def test2(self):
"""
An arbitrarily long string
:return:
"""
self.assertEqual(compress('abadbadsakdlfjieafnealfjiasfjaaaaanadddddddkkkjj'), \
"abadbadsakdlfjieafnealfjiasfja#5nad#7kkkjj")
def test3(self):
"""
A string where a lot has to be compressed
:return:
"""
self.assertEqual(compress('kkkkkkjjjjjjjnnnnniiiiiiiooooook'),
'k#6j#7n#5i#7o#6k')
def resultat():
results = request.files
filename = results['myFile']
fileCompressed, b = compression.compress(filename, 0, 2, 0, 1000, 1000,
False, False)
response = make_response(fileCompressed)
response.headers["Content-type"] = "application/wav"
response.headers[
"Content-Disposition"] = "attachment; filename=compressed.wav"
return response
def convert_data_from_zip_file(f):
"""
Directly reads eoner's csv from zip file and converts it to ours.
:param f: Full path to file name.
:type f: str
"""
# f = '/Users/isezen/project/tkmpy/tkmdata/2015-07-31.zip'
date = os.path.splitext(os.path.basename(f))[0]
date = dt.strptime(date,'%Y-%m-%d')
fnames = [_add_date_to_file_name(os.path.splitext(os.path.basename(url))[0],
date.strftime('%Y%m%d')) +'.csv'
for url in tkm.URL[:5]]
text = comp.read_from_zip(f)
k = 0
for l in [x for x in text.split('\n') if not re.match(r'^\s*$', x)]:
d = l.replace(u'\ufeff','').split('#')
if len(d)>6:
for i in range(6,len(d)): d[5] = d[5] + '#' + d[i]
d = d[0:6]
d = [i.replace(';', '|').replace('|&', '&') for i in d]
for i in range(len(d)):
if d[i][len(d[i])-1] == u'&': d[i] = d[i][:(len(d[i])-1)]
date = _oa_to_datetime(d[0])
date = date.replace(tzinfo=tz.tzutc()).astimezone(tz.tzlocal())
k+=1
for i in range(len(d[:-1])):
f_name = _add_date_to_file_name(os.path.basename(tkm.URL[i]),
date.strftime('%Y%m%d'))
f_name = os.path.splitext(f_name)[0] + '.csv'
if d[i+1] != 'error':
tkmd = tkm.TKM_DATA(date=date, e_tag=None,
filename=f_name, data=d[i+1])
tkm.save_instant_data(tkmd)
for f in fnames:
f = os.path.join(tkm.DIR.data, f)
comp.compress(f)
if os.path.exists(f): os.remove(f)
def encodeLayerData(self, tileLayer, format):
if format in [Map.LayerDataFormat.XML, Map.LayerDataFormat.CSV]:
raise
tileData = QByteArray()
for y in range(tileLayer.height()):
for x in range(tileLayer.width()):
gid = self.cellToGid(tileLayer.cellAt(x, y))
tileData.append(bytes([gid&0xff]))
tileData.append(bytes([(gid >> 8)&0xff]))
tileData.append(bytes([(gid >> 16)&0xff]))
tileData.append(bytes([(gid >> 24)&0xff]))
if len(tileData)%4 != 0:
raise
if (format == Map.LayerDataFormat.Base64Gzip):
tileData = compress(tileData, CompressionMethod.Gzip)
elif (format == Map.LayerDataFormat.Base64Zlib):
tileData = compress(tileData, CompressionMethod.Zlib)
return tileData.toBase64()
def getContents(self):
fileslist = self.getFilesList()
assert fileslist, "Must contain at least one resource."
result = fileslist[0].getContents()
content_type = result["content_type"]
data = [result["data"]]
for subres in fileslist[1:]:
d = subres.getContents()
# all elements must have the same content type.
assert d["content_type"] == content_type
data.append(d["data"])
result["data"] = "\n".join(data)
result["compress_level"] = self.compress_level
# Do compression on the result
result["data"] = compress(**result)
return result
def test_compression():
filename = "lena.bmp"
image = io.imread(filename)
filename = filename.split(".")[0]
co.write_as_mcf(image, filename)
vec = co.read_as_np(filename)
com = co.compress(image)
co.write_as_mcf(com, "comp_lena")
img2 = co.read_as_np("comp_lena")
print(img2.shape)
image = co.decompress(img2, image.shape)
plt.imshow(image, cmap="gray")
plt.show()
def write(self, map, fileName):
# Check layer count and type
if (map.layerCount() != 1 or not map.layerAt(0).isTileLayer()):
self.mError = self.tr(
"The map needs to have exactly one tile layer!")
return False
mapLayer = map.layerAt(0).asTileLayer()
# Check layer size
if (mapLayer.width() != 48 or mapLayer.height() != 48):
self.mError = self.tr(
"The layer must have a size of 48 x 48 tiles!")
return False
# Create QByteArray and compress it
uncompressed = QByteArray(48 * 48, b'\x00')
width = mapLayer.width()
height = mapLayer.height()
for y in range(0, height):
for x in range(0, width):
tile = mapLayer.cellAt(x, y).tile
if tile:
# 'QByteArray' object does not support item assignment
uncompressed.replace(y * width + x, 1,
bytes([tile.id() & 0xff]))
compressed = compress(uncompressed, CompressionMethod.Gzip)
# Write QByteArray
file = QSaveFile(fileName)
if (not file.open(QIODevice.WriteOnly)):
self.mError = self.tr("Could not open file for writing.")
return False
file.write(compressed)
if not file.commit():
self.mError = file.errorString()
return False
return True
def write(self, map, fileName):
# Check layer count and type
if (map.layerCount() != 1 or not map.layerAt(0).isTileLayer()) :
self.mError = self.tr("The map needs to have exactly one tile layer!")
return False
mapLayer = map.layerAt(0).asTileLayer()
# Check layer size
if (mapLayer.width() != 48 or mapLayer.height() != 48) :
self.mError = self.tr("The layer must have a size of 48 x 48 tiles!")
return False
# Create QByteArray and compress it
uncompressed = QByteArray(48 * 48, b'\x00')
width = mapLayer.width()
height = mapLayer.height()
for y in range(0, height):
for x in range(0, width):
tile = mapLayer.cellAt(x, y).tile
if tile:
# 'QByteArray' object does not support item assignment
uncompressed.replace(y * width + x, 1, bytes([tile.id()&0xff]))
compressed = compress(uncompressed, CompressionMethod.Gzip)
# Write QByteArray
file = QSaveFile(fileName)
if (not file.open(QIODevice.WriteOnly)) :
self.mError = self.tr("Could not open file for writing.")
return False
file.write(compressed)
if not file.commit():
self.mError = file.errorString()
return False
return True
def _compress2(): comp.compress(csv, True)
_compress1()
for record in wfdb.get_record_list(db_name, records='all'):
# record = 'Person_01/rec_10'
# record = '203'
# get data for current record
data = wfdb.rdsamp(record, pb_dir=db_name + '/' + record.split('/')[0])
Fs = data[1]['fs']
ecg = data[0][:, 0] #[0:Fs*20]
# zero-mean
ecg = ecg - np.mean(ecg)
# call compression function
CR, ecg_compressed, wc_orig = compress(ecg, Fs)
# call reconstruction function
ecg_recon, wc_recon = reconstruct(ecg_compressed)
# compute and store evaluation metrics
PRD, R = calc_PRD(ecg, ecg_recon)
ps = analysis(ecg, Fs)
pr = analysis(ecg_recon, Fs)
CR_arr.append(CR)
PRD_arr.append(PRD)
R_arr.append(R)
SNRo_arr.append(ps['SNR'])
SNRr_arr.append(pr['SNR'])
d_SNR.append(pr['SNR'] - ps['SNR'])
def _compress1(): comp.compress(csv, f_type='.zip')
# noinspection PyMissingOrEmptyDocstring
def _compress2(): comp.compress(csv, True)
from sys import argv, exit from compression import compress, decompress, usage if len(argv) < 3: usage(argv[0]) if int(argv[1]) == 1: print "%r compressed is %r" % (argv[2], compress(argv[2])) elif int(argv[1]) == 0: print "%r decompressed is %r" % (argv[2], decompress(argv[2])) else: usage(argv[0])
def test_js(self):
in_file = os.path.join('tests', 'resources', 'js', 'one-line.js')
compression.compress(in_file, self.out_file)
expected = 'var test;'
result = open(self.out_file).read()
self.assertEquals(result, expected)
def _save_manifest(self, data):
data = crypto.encrypt_then_mac(
self.config, compression.compress(self.config,
msgpack.dumps(data)))
self.transport.write_manifest(data, self.inittime)
def test_css(self):
in_file = os.path.join('tests', 'resources', 'css', 'one-line.css')
compression.compress(in_file, self.out_file, in_type='css')
expected = 'body{background-color:red;}'
result = open(self.out_file).read()
self.assertEquals(result, expected)
def test_js(self):
in_file = os.path.join('tests', 'resources', 'js', 'one-line.js')
compression.compress(in_file, self.out_file)
expected = 'var test;'
result = open(self.out_file).read()
self.assertEquals(result, expected)
if __name__ == "__main__":
cli_input = sys.argv
if invalid_input(cli_input):
print("Please insert a valid command")
raise Exception(
"Commands should be like './main.py -(c/x) file.(txt/lz78) [-o result_file_name]'"
)
f_name = ""
if cli_input[1] == "-c":
if len(cli_input) == 5:
f_name = cli_input[4] + ".z78"
else:
f_name = cli_input[2].split(".")[0] + ".z78"
lz78comp.compress(cli_input[2], f_name)
print("File ", cli_input[2], " compressed to ", f_name, " !")
elif cli_input[1] == "-x":
if len(cli_input) == 5:
f_name = cli_input[4] + ".txt"
else:
f_name = cli_input[2].split(".")[0] + ".txt"
decoded_text = lz78comp.decompress(cli_input[2])
lz78comp.create_txt_file(f_name, decoded_text)
print("File ", cli_input[2], " decompressed to ", f_name, " !")
#!/usr/bin/env python3 # Compress and store as (1x900) GIF import os import sys import imageio import cv2 as cv import numpy as np from compression import compress infilename = sys.argv[1] outdirectory = './dataset/compressed/' _, outfilename = os.path.split(infilename) # Extract first frame from GIF (only one frame) gif = imageio.mimread(infilename)[0] stacked_gif = compress(gif) # Save output to file imageio.mimwrite(outdirectory + outfilename, stacked_gif) print(infilename, '-->', outdirectory + outfilename)
graph = utils.read_file(sys.stdin)
n = graph.number_of_nodes()
m = graph.number_of_edges()
eprint("n:", n, "m:", m)
#eprint(utils.degree_hist(graph))
vc = main(graph, greedy_vc=True)
eprint("final vc size: {}/{}".format(len(vc), n))
print("s vc {0} {1}".format(n, len(vc)))
print(*vc, sep='\n')
eprint("time:", time() - start)
if __name__ == '__main__':
eprint("compresing...")
res, svc = compression.compress(graph, vc)
timestamp("compression")
if res:
eprint("smaller vc found, size:", len(svc))
else:
eprint("vc could not be compressed")
import psutil
import matplotlib.pyplot as plt
# nx.draw_random(graph, with_labels=True)
# plt.show()
process = psutil.Process(os.getpid())
# graph = utils.read_instance(1)
# graph = nx.random_regular_graph(3,6000,seed=1)
# graph = nx.Graph([(u+1,v+1) for u,v in graph.edges()])
eprint("Memory usage: {} MB".format(process.memory_info().rss / 1e6))
## Simple Lossless String Compression ##
import compression
phrase = "to be or not to be, that is the question"
compressed, key = compression.compress(phrase)
def utf8len(s):
return len(s.encode('utf-8'))
def dictStringBytes(dictString):
sum = 0
for v in dictString:
sum += utf8len(v)
return sum
print("Uncompressed: " + phrase)
print("Compressed: " + compressed)
print("Bytes before compression: %s, Bytes after compression: %s"%(utf8len(phrase), utf8len(compressed)+dictStringBytes(key)))
import sys
import cv2
import numpy as np
from scipy.misc import imread, imsave
from PIL import Image
from comic import comic
from compression import compress
from stylize import render
img = sys.argv[1]
img1 = comic(img)
img2 = compress('images/tmp/tmp1.jpg')
img3 = render(imread('images/tmp/tmp2.jpg'), ratio=0.001, verbose=True)
img3 = imsave('images/tmp/tmp3.jpg', img3)
img4 = render(imread('images/tmp/tmp3.jpg'), depth=4, verbose=True)
img4 = imsave('images/tmp/tmp4.jpg', img4)
list_im = [
sys.argv[1], 'images/tmp/tmp2.jpg', 'images/tmp/tmp3.jpg',
'images/tmp/tmp4.jpg'
]
imgs = [Image.open(i) for i in list_im]
min_shape = sorted([(np.sum(i.size), i.size) for i in imgs])[0][1]
imgs_comb = np.vstack(np.asarray(i.resize(min_shape)) for i in imgs)
imgs_comb = Image.fromarray(imgs_comb)
imgs_comb.save('images/tmp/VStack.jpg')
def getSize():
return min_shape
def _enc(self, data):
encrypted = crypto.encrypt(self.config,
compression.compress(self.config, data))
return encrypted
from compression import compress
im = [
# Image goes here :)
]
tftable = [3, 2, 1, 0]
res = []
for i in range(0, len(im), 4):
n = tftable[im[i + 0]] << 6 | tftable[im[i + 1]] << 4 | tftable[im[
i + 2]] << 2 | tftable[im[i + 3]]
res.append(n)
res = compress(res)
nbytes = len(res)
res = ["%d," % n for n in res]
res = [" ".join(res[i:i + 16]) for i in range(0, len(res), 16)]
print 'const unsigned char varname[%d] __attribute__((section(".text"), used)) = {' % nbytes
print "\n".join(res)
print "};"
# filename = "bee_movie_script.txt"
# compress(filename)
# decompress(filename + ".compressed")
# Main code
while True:
print("Choose an option:")
print("1- Compress a file")
print("2- Decompress a file")
print("3- Compress a folder")
print("4- Decompress a folder")
print("0- Exit")
choice = input("Choice > ")
if choice == '1':
filename = input("Enter filename: ")
compress(filename)
elif choice == '2':
filename = input("Enter filename: ")
decompress(filename)
elif choice == '3':
foldername = input("Enter folder name: ")
compress_folder(foldername + '/')
elif choice == '4':
foldername = input("Enter folder name: ")
decompress_folder(foldername)
elif choice == '0':
break
else:
print("Invalid choice")
print("\n\n")
def _compress1():
comp.compress(csv, f_type='.zip')
from compression import compress
from decompression import decompress
def parseArguments():
if len(argv) == 4:
return argv[1], argv[2], argv[3]
else:
print("ArgError - Script accepts 3 arguemnts: Mode, Source_Path, Destination_Path")
exit()
try:
mode, srcPath, destPath = parseArguments()
srcFile = open(srcPath, "r")
destFile = open(destPath, "w")
if mode == "-c":
compress(srcFile, destFile)
elif mode == "-d":
decompress(srcFile, destFile)
else:
print("ModeError - First Arg must be -c for Compression or -d for Decompression")
srcFile.close()
destFile.close()
except FileNotFoundError:
print("IOError - File Not Found")
def _compress2():
comp.compress(csv, True)
def test1(self):
"""
A small standard string
:return:
"""
self.assertEqual(compress('aaaaa'), 'a#5')
def saveObject(self,obj):
file = open(self.path % obj.client.name ,'wb')
file.write(compression.compress(obj))
file.close()
def test_css(self):
in_file = os.path.join('tests', 'resources', 'css', 'one-line.css')
compression.compress(in_file, self.out_file, in_type='css')
expected = 'body{background-color:red;}'
result = open(self.out_file).read()
self.assertEquals(result, expected)
def test_compress_decompress(self): for testCase in self.toTestList: # test compress output self.assertEqual(compress(testCase[0]),testCase[1]) # test decompress output -- compressing then decompressing self.assertEqual(decompress(compress(testCase[0])),testCase[0])
def do_compress(self, output):
compression.compress(self.image_entry.get(), self.network_entry.get(), output, int(self.bits_entry.get()), self.smoothing.get())
import os
from compression import compress, decompress
degree = 2
directory1 = 'mri_mini'
directory2 = directory1 + '_compressed'
directory3 = directory1 + '_decompressed'
for file in os.listdir(directory1):
compress(directory1, file, degree, directory2)
for file in os.listdir(directory2):
decompress(directory2, file, directory3)
for file in os.listdir(directory1):
f1 = open(directory1 + '/' + file, 'rb')
f2 = open(directory3 + '/' + file, 'rb')
f1 = f1.read()
f2 = f2.read()
if len(f1) != len(f2):
print(False)
print(file, 'len')
for i in range(0, len(f1)):
if f1[i] != f2[i]:
print(False)
print(file, 'val', i)
break
