Пример #1
0
def compress_files():
    """Compresses downloaded data files."""
    lcsv = [f for f in os.listdir(DIR.data) if f.endswith('.csv')]
    today_e_tag = _now().strftime('%Y%m%d')
    for f in lcsv:
        ff = os.path.join(DIR.data, f)
        if today_e_tag not in ff:
            c.compress(ff)
            log.info('%s compressed.', os.path.basename(ff))
            os.remove(ff)
Пример #2
0
def compress_files():
    """Compresses downloaded data files."""
    lcsv = [f for f in os.listdir(DIR.data) if f.endswith('.csv')]
    today_e_tag = _now().strftime('%Y%m%d')
    for f in lcsv:
        ff = os.path.join(DIR.data, f)
        if today_e_tag not in ff:
            c.compress(ff)
            log.info('%s compressed.', os.path.basename(ff))
            os.remove(ff)
Пример #3
0
def test_compression():
    """ Test compression and decompression functions """
    f_sha = hashlib.sha256(open(_f, 'rb').read()).hexdigest()
    temp_name = os.path.join('tests', next(tempfile._get_candidate_names()))

    fc = c.compress(_f, temp_name)
    fd = c.decompress(fc)
    assert f_sha == hashlib.sha256(open(fd, 'rb').read()).hexdigest()
    os.remove(fc); os.remove(fd)

    fc = c.compress(_f, temp_name, f_type='zip')
    fd = c.decompress(fc)
    assert f_sha == hashlib.sha256(open(fd, 'rb').read()).hexdigest()
    os.remove(fc); os.remove(fd)
Пример #4
0
def test_compression():
    """ Test compression and decompression functions """
    f_sha = hashlib.sha256(open(_f, 'rb').read()).hexdigest()
    temp_name = os.path.join('tests', next(tempfile._get_candidate_names()))

    fc = c.compress(_f, temp_name)
    fd = c.decompress(fc)
    assert f_sha == hashlib.sha256(open(fd, 'rb').read()).hexdigest()
    os.remove(fc)
    os.remove(fd)

    fc = c.compress(_f, temp_name, f_type='zip')
    fd = c.decompress(fc)
    assert f_sha == hashlib.sha256(open(fd, 'rb').read()).hexdigest()
    os.remove(fc)
    os.remove(fd)
Пример #5
0
 def test2(self):
     """
     An arbitrarily long string
     :return:
     """
     self.assertEqual(compress('abadbadsakdlfjieafnealfjiasfjaaaaanadddddddkkkjj'), \
         "abadbadsakdlfjieafnealfjiasfja#5nad#7kkkjj")
Пример #6
0
 def test3(self):
     """
     A string where a lot has to be compressed
     :return:
     """
     self.assertEqual(compress('kkkkkkjjjjjjjnnnnniiiiiiiooooook'),
                      'k#6j#7n#5i#7o#6k')
Пример #7
0
def resultat():
    results = request.files
    filename = results['myFile']
    fileCompressed, b = compression.compress(filename, 0, 2, 0, 1000, 1000,
                                             False, False)
    response = make_response(fileCompressed)
    response.headers["Content-type"] = "application/wav"
    response.headers[
        "Content-Disposition"] = "attachment; filename=compressed.wav"
    return response
Пример #8
0
def convert_data_from_zip_file(f):
    """
    Directly reads eoner's csv from zip file and converts it to ours.
    :param f: Full path to file name.
    :type f: str
    """

    # f = '/Users/isezen/project/tkmpy/tkmdata/2015-07-31.zip'
    date = os.path.splitext(os.path.basename(f))[0]
    date = dt.strptime(date,'%Y-%m-%d')
    fnames = [_add_date_to_file_name(os.path.splitext(os.path.basename(url))[0],
                                     date.strftime('%Y%m%d')) +'.csv'
              for url in tkm.URL[:5]]
    text = comp.read_from_zip(f)
    k = 0
    for l in [x for x in text.split('\n') if not re.match(r'^\s*$', x)]:
        d = l.replace(u'\ufeff','').split('#')
        if len(d)>6:
            for i in range(6,len(d)): d[5] = d[5] + '#' + d[i]
            d = d[0:6]

        d = [i.replace(';', '|').replace('|&', '&') for i in d]
        for i in range(len(d)):
            if d[i][len(d[i])-1] == u'&': d[i] = d[i][:(len(d[i])-1)]
        date = _oa_to_datetime(d[0])
        date = date.replace(tzinfo=tz.tzutc()).astimezone(tz.tzlocal())
        k+=1
        for i in range(len(d[:-1])):
            f_name = _add_date_to_file_name(os.path.basename(tkm.URL[i]),
                                            date.strftime('%Y%m%d'))
            f_name = os.path.splitext(f_name)[0] + '.csv'
            if d[i+1] != 'error':
                tkmd = tkm.TKM_DATA(date=date, e_tag=None,
                                    filename=f_name, data=d[i+1])
                tkm.save_instant_data(tkmd)

    for f in fnames:
        f = os.path.join(tkm.DIR.data, f)
        comp.compress(f)
        if os.path.exists(f): os.remove(f)
Пример #9
0
    def encodeLayerData(self, tileLayer, format):
        if format in [Map.LayerDataFormat.XML, Map.LayerDataFormat.CSV]:
            raise

        tileData = QByteArray()
        for y in range(tileLayer.height()):
            for x in range(tileLayer.width()):
                gid = self.cellToGid(tileLayer.cellAt(x, y))
                tileData.append(bytes([gid&0xff]))
                tileData.append(bytes([(gid >> 8)&0xff]))
                tileData.append(bytes([(gid >> 16)&0xff]))
                tileData.append(bytes([(gid >> 24)&0xff]))
                
        if len(tileData)%4 != 0:
            raise

        if (format == Map.LayerDataFormat.Base64Gzip):
            tileData = compress(tileData, CompressionMethod.Gzip)
        elif (format == Map.LayerDataFormat.Base64Zlib):
            tileData = compress(tileData, CompressionMethod.Zlib)

        return tileData.toBase64()
Пример #10
0
 def getContents(self):
     fileslist = self.getFilesList()
     assert fileslist, "Must contain at least one resource."
     result = fileslist[0].getContents()
     content_type = result["content_type"]
     data = [result["data"]]
     for subres in fileslist[1:]:
         d = subres.getContents()
         # all elements must have the same content type.
         assert d["content_type"] == content_type
         data.append(d["data"])
     result["data"] = "\n".join(data)
     result["compress_level"] = self.compress_level
     # Do compression on the result
     result["data"] = compress(**result)
     return result
Пример #11
0
def test_compression():

    filename = "lena.bmp"
    image = io.imread(filename)
    filename = filename.split(".")[0]

    co.write_as_mcf(image, filename)

    vec = co.read_as_np(filename)
    com = co.compress(image)

    co.write_as_mcf(com, "comp_lena")
    img2 = co.read_as_np("comp_lena")
    print(img2.shape)
    image = co.decompress(img2, image.shape)
    plt.imshow(image, cmap="gray")
    plt.show()
Пример #12
0
    def write(self, map, fileName):
        # Check layer count and type
        if (map.layerCount() != 1 or not map.layerAt(0).isTileLayer()):
            self.mError = self.tr(
                "The map needs to have exactly one tile layer!")
            return False

        mapLayer = map.layerAt(0).asTileLayer()
        # Check layer size
        if (mapLayer.width() != 48 or mapLayer.height() != 48):
            self.mError = self.tr(
                "The layer must have a size of 48 x 48 tiles!")
            return False

        # Create QByteArray and compress it
        uncompressed = QByteArray(48 * 48, b'\x00')
        width = mapLayer.width()
        height = mapLayer.height()
        for y in range(0, height):
            for x in range(0, width):
                tile = mapLayer.cellAt(x, y).tile
                if tile:
                    # 'QByteArray' object does not support item assignment
                    uncompressed.replace(y * width + x, 1,
                                         bytes([tile.id() & 0xff]))

        compressed = compress(uncompressed, CompressionMethod.Gzip)

        # Write QByteArray
        file = QSaveFile(fileName)
        if (not file.open(QIODevice.WriteOnly)):
            self.mError = self.tr("Could not open file for writing.")
            return False

        file.write(compressed)
        if not file.commit():
            self.mError = file.errorString()
            return False

        return True
Пример #13
0
    def write(self, map, fileName):
        # Check layer count and type
        if (map.layerCount() != 1 or not map.layerAt(0).isTileLayer()) :
            self.mError = self.tr("The map needs to have exactly one tile layer!")
            return False
        
        mapLayer = map.layerAt(0).asTileLayer()
        # Check layer size
        if (mapLayer.width() != 48 or mapLayer.height() != 48) :
            self.mError = self.tr("The layer must have a size of 48 x 48 tiles!")
            return False
        
        # Create QByteArray and compress it
        uncompressed = QByteArray(48 * 48, b'\x00')
        width = mapLayer.width()
        height = mapLayer.height()
        for y in range(0, height):
            for x in range(0, width):
                tile = mapLayer.cellAt(x, y).tile
                if tile:
                    # 'QByteArray' object does not support item assignment
                    uncompressed.replace(y * width + x, 1, bytes([tile.id()&0xff]))

        compressed = compress(uncompressed, CompressionMethod.Gzip)
        
        # Write QByteArray
        file = QSaveFile(fileName)
        if (not file.open(QIODevice.WriteOnly)) :
            self.mError = self.tr("Could not open file for writing.")
            return False
        
        file.write(compressed)
        if not file.commit():
            self.mError = file.errorString()
            return False

        return True
Пример #14
0
    def _compress2(): comp.compress(csv, True)

    _compress1()
Пример #15
0
    for record in wfdb.get_record_list(db_name, records='all'):

        # record = 'Person_01/rec_10'
        # record = '203'

        # get data for current record
        data = wfdb.rdsamp(record, pb_dir=db_name + '/' + record.split('/')[0])

        Fs = data[1]['fs']
        ecg = data[0][:, 0]  #[0:Fs*20]

        # zero-mean
        ecg = ecg - np.mean(ecg)

        # call compression function
        CR, ecg_compressed, wc_orig = compress(ecg, Fs)

        # call reconstruction function
        ecg_recon, wc_recon = reconstruct(ecg_compressed)

        # compute and store evaluation metrics
        PRD, R = calc_PRD(ecg, ecg_recon)
        ps = analysis(ecg, Fs)
        pr = analysis(ecg_recon, Fs)
        CR_arr.append(CR)
        PRD_arr.append(PRD)
        R_arr.append(R)
        SNRo_arr.append(ps['SNR'])
        SNRr_arr.append(pr['SNR'])
        d_SNR.append(pr['SNR'] - ps['SNR'])
Пример #16
0
    def _compress1(): comp.compress(csv, f_type='.zip')

    # noinspection PyMissingOrEmptyDocstring
    def _compress2(): comp.compress(csv, True)
Пример #17
0
from sys import argv, exit
from compression import compress, decompress, usage

if len(argv) < 3:
	usage(argv[0])

if int(argv[1]) == 1:
	print "%r compressed is %r" % (argv[2], compress(argv[2]))
elif int(argv[1]) == 0:
	print "%r decompressed is %r" % (argv[2], decompress(argv[2]))
else:
	usage(argv[0])
 def test_js(self):
     in_file = os.path.join('tests', 'resources', 'js', 'one-line.js')
     compression.compress(in_file, self.out_file)
     expected = 'var test;'
     result = open(self.out_file).read()
     self.assertEquals(result, expected)
Пример #19
0
 def _save_manifest(self, data):
     data = crypto.encrypt_then_mac(
         self.config, compression.compress(self.config,
                                           msgpack.dumps(data)))
     self.transport.write_manifest(data, self.inittime)
 def test_css(self):
     in_file = os.path.join('tests', 'resources', 'css', 'one-line.css')
     compression.compress(in_file, self.out_file, in_type='css')
     expected = 'body{background-color:red;}'
     result = open(self.out_file).read()
     self.assertEquals(result, expected)
Пример #21
0
 def test_js(self):
     in_file = os.path.join('tests', 'resources', 'js', 'one-line.js')
     compression.compress(in_file, self.out_file)
     expected = 'var test;'
     result = open(self.out_file).read()
     self.assertEquals(result, expected)
Пример #22
0

if __name__ == "__main__":
    cli_input = sys.argv

    if invalid_input(cli_input):
        print("Please insert a valid command")
        raise Exception(
            "Commands should be like './main.py -(c/x) file.(txt/lz78) [-o result_file_name]'"
        )

    f_name = ""

    if cli_input[1] == "-c":
        if len(cli_input) == 5:
            f_name = cli_input[4] + ".z78"
        else:
            f_name = cli_input[2].split(".")[0] + ".z78"

        lz78comp.compress(cli_input[2], f_name)
        print("File ", cli_input[2], " compressed to ", f_name, " !")

    elif cli_input[1] == "-x":
        if len(cli_input) == 5:
            f_name = cli_input[4] + ".txt"
        else:
            f_name = cli_input[2].split(".")[0] + ".txt"

        decoded_text = lz78comp.decompress(cli_input[2])
        lz78comp.create_txt_file(f_name, decoded_text)
        print("File ", cli_input[2], " decompressed to ", f_name, " !")
Пример #23
0
#!/usr/bin/env python3
# Compress and store as (1x900) GIF
import os
import sys
import imageio
import cv2 as cv
import numpy as np
from compression import compress
infilename = sys.argv[1]
outdirectory = './dataset/compressed/'
_, outfilename = os.path.split(infilename)
# Extract first frame from GIF (only one frame)
gif = imageio.mimread(infilename)[0]
stacked_gif = compress(gif)
# Save output to file
imageio.mimwrite(outdirectory + outfilename, stacked_gif)
print(infilename, '-->', outdirectory + outfilename)
Пример #24
0

graph = utils.read_file(sys.stdin)
n = graph.number_of_nodes()
m = graph.number_of_edges()
eprint("n:", n, "m:", m)
#eprint(utils.degree_hist(graph))
vc = main(graph, greedy_vc=True)
eprint("final vc size: {}/{}".format(len(vc), n))
print("s vc {0} {1}".format(n, len(vc)))
print(*vc, sep='\n')
eprint("time:", time() - start)

if __name__ == '__main__':
    eprint("compresing...")
    res, svc = compression.compress(graph, vc)
    timestamp("compression")
    if res:
        eprint("smaller vc found, size:", len(svc))
    else:
        eprint("vc could not be compressed")
    import psutil
    import matplotlib.pyplot as plt
    # nx.draw_random(graph, with_labels=True)
    # plt.show()
    process = psutil.Process(os.getpid())
    # graph = utils.read_instance(1)
    # graph = nx.random_regular_graph(3,6000,seed=1)
    # graph = nx.Graph([(u+1,v+1) for u,v in graph.edges()])
    eprint("Memory usage: {} MB".format(process.memory_info().rss / 1e6))
Пример #25
0
## Simple Lossless String Compression ##

import compression

phrase = "to be or not to be, that is the question"
compressed, key = compression.compress(phrase)

def utf8len(s):
  return len(s.encode('utf-8'))

def dictStringBytes(dictString):
  sum = 0 
  for v in dictString:
    sum += utf8len(v)
  return sum

print("Uncompressed: " + phrase)
print("Compressed: " + compressed)
print("Bytes before compression: %s, Bytes after compression: %s"%(utf8len(phrase), utf8len(compressed)+dictStringBytes(key)))
Пример #26
0
import sys
import cv2
import numpy as np
from scipy.misc import imread, imsave
from PIL import Image
from comic import comic
from compression import compress
from stylize import render

img = sys.argv[1]
img1 = comic(img)
img2 = compress('images/tmp/tmp1.jpg')
img3 = render(imread('images/tmp/tmp2.jpg'), ratio=0.001, verbose=True)
img3 = imsave('images/tmp/tmp3.jpg', img3)
img4 = render(imread('images/tmp/tmp3.jpg'), depth=4, verbose=True)
img4 = imsave('images/tmp/tmp4.jpg', img4)

list_im = [
    sys.argv[1], 'images/tmp/tmp2.jpg', 'images/tmp/tmp3.jpg',
    'images/tmp/tmp4.jpg'
]
imgs = [Image.open(i) for i in list_im]
min_shape = sorted([(np.sum(i.size), i.size) for i in imgs])[0][1]

imgs_comb = np.vstack(np.asarray(i.resize(min_shape)) for i in imgs)
imgs_comb = Image.fromarray(imgs_comb)
imgs_comb.save('images/tmp/VStack.jpg')


def getSize():
    return min_shape
Пример #27
0
 def _enc(self, data):
     encrypted = crypto.encrypt(self.config,
                                compression.compress(self.config, data))
     return encrypted
Пример #28
0
from compression import compress

im = [
    # Image goes here :)
]

tftable = [3, 2, 1, 0]

res = []
for i in range(0, len(im), 4):
    n = tftable[im[i + 0]] << 6 | tftable[im[i + 1]] << 4 | tftable[im[
        i + 2]] << 2 | tftable[im[i + 3]]

    res.append(n)

res = compress(res)
nbytes = len(res)

res = ["%d," % n for n in res]

res = [" ".join(res[i:i + 16]) for i in range(0, len(res), 16)]

print 'const unsigned char varname[%d] __attribute__((section(".text"), used)) = {' % nbytes
print "\n".join(res)
print "};"
Пример #29
0
# filename = "bee_movie_script.txt"
# compress(filename)
# decompress(filename + ".compressed")

# Main code

while True:
    print("Choose an option:")
    print("1- Compress a file")
    print("2- Decompress a file")
    print("3- Compress a folder")
    print("4- Decompress a folder")
    print("0- Exit")
    choice = input("Choice > ")
    if choice == '1':
        filename = input("Enter filename: ")
        compress(filename)
    elif choice == '2':
        filename = input("Enter filename: ")
        decompress(filename)
    elif choice == '3':
        foldername = input("Enter folder name: ")
        compress_folder(foldername + '/')
    elif choice == '4':
        foldername = input("Enter folder name: ")
        decompress_folder(foldername)
    elif choice == '0':
        break
    else:
        print("Invalid choice")
    print("\n\n")
Пример #30
0
 def _compress1():
     comp.compress(csv, f_type='.zip')
Пример #31
0
from compression import compress
from decompression import decompress


def parseArguments():
    if len(argv) == 4:
        return argv[1], argv[2], argv[3]
    else:
        print("ArgError - Script accepts 3 arguemnts: Mode, Source_Path, Destination_Path")
        exit()


try:
    mode, srcPath, destPath = parseArguments()

    srcFile = open(srcPath, "r")
    destFile = open(destPath, "w")

    if mode == "-c":
        compress(srcFile, destFile)
    elif mode == "-d":
        decompress(srcFile, destFile)
    else:
        print("ModeError - First Arg must be -c for Compression or -d for Decompression")

    srcFile.close()
    destFile.close()

except FileNotFoundError:
    print("IOError - File Not Found")
Пример #32
0
 def _compress2():
     comp.compress(csv, True)
Пример #33
0
 def test1(self):
     """
     A small standard string
     :return:
     """
     self.assertEqual(compress('aaaaa'), 'a#5')
Пример #34
0
 def saveObject(self,obj):
     file = open(self.path % obj.client.name ,'wb')
     file.write(compression.compress(obj))
     file.close()
Пример #35
0
 def test_css(self):
     in_file = os.path.join('tests', 'resources', 'css', 'one-line.css')
     compression.compress(in_file, self.out_file, in_type='css')
     expected = 'body{background-color:red;}'
     result = open(self.out_file).read()
     self.assertEquals(result, expected)
Пример #36
0
	def test_compress_decompress(self):
		for testCase in self.toTestList:
			# test compress output
			self.assertEqual(compress(testCase[0]),testCase[1])
			# test decompress output -- compressing then decompressing
			self.assertEqual(decompress(compress(testCase[0])),testCase[0])
Пример #37
0
 def do_compress(self, output):
     compression.compress(self.image_entry.get(), self.network_entry.get(), output, int(self.bits_entry.get()), self.smoothing.get())
Пример #38
0
import os
from compression import compress, decompress

degree = 2
directory1 = 'mri_mini'
directory2 = directory1 + '_compressed'
directory3 = directory1 + '_decompressed'

for file in os.listdir(directory1):
    compress(directory1, file, degree, directory2)

for file in os.listdir(directory2):
    decompress(directory2, file, directory3)

for file in os.listdir(directory1):
    f1 = open(directory1 + '/' + file, 'rb')
    f2 = open(directory3 + '/' + file, 'rb')
    f1 = f1.read()
    f2 = f2.read()

    if len(f1) != len(f2):
        print(False)
        print(file, 'len')

    for i in range(0, len(f1)):
        if f1[i] != f2[i]:
            print(False)
            print(file, 'val', i)
            break