Example #1
0
File: tkm.py Project: isezen/pytkm
def compress_files():
    """Compresses downloaded data files."""
    lcsv = [f for f in os.listdir(DIR.data) if f.endswith('.csv')]
    today_e_tag = _now().strftime('%Y%m%d')
    for f in lcsv:
        ff = os.path.join(DIR.data, f)
        if today_e_tag not in ff:
            c.compress(ff)
            log.info('%s compressed.', os.path.basename(ff))
            os.remove(ff)
Example #2
0
def test_compression():
    """ Test compression and decompression functions """
    f_sha = hashlib.sha256(open(_f, 'rb').read()).hexdigest()
    temp_name = os.path.join('tests', next(tempfile._get_candidate_names()))

    fc = c.compress(_f, temp_name)
    fd = c.decompress(fc)
    assert f_sha == hashlib.sha256(open(fd, 'rb').read()).hexdigest()
    os.remove(fc); os.remove(fd)

    fc = c.compress(_f, temp_name, f_type='zip')
    fd = c.decompress(fc)
    assert f_sha == hashlib.sha256(open(fd, 'rb').read()).hexdigest()
    os.remove(fc); os.remove(fd)
Example #3
0
def convert_data_from_zip_file(f):
    """
    Directly reads eoner's csv from zip file and converts it to ours.
    :param f: Full path to file name.
    :type f: str
    """

    # f = '/Users/isezen/project/tkmpy/tkmdata/2015-07-31.zip'
    date = os.path.splitext(os.path.basename(f))[0]
    date = dt.strptime(date,'%Y-%m-%d')
    fnames = [_add_date_to_file_name(os.path.splitext(os.path.basename(url))[0],
                                     date.strftime('%Y%m%d')) +'.csv'
              for url in tkm.URL[:5]]
    text = comp.read_from_zip(f)
    k = 0
    for l in [x for x in text.split('\n') if not re.match(r'^\s*$', x)]:
        d = l.replace(u'\ufeff','').split('#')
        if len(d)>6:
            for i in range(6,len(d)): d[5] = d[5] + '#' + d[i]
            d = d[0:6]

        d = [i.replace(';', '|').replace('|&', '&') for i in d]
        for i in range(len(d)):
            if d[i][len(d[i])-1] == u'&': d[i] = d[i][:(len(d[i])-1)]
        date = _oa_to_datetime(d[0])
        date = date.replace(tzinfo=tz.tzutc()).astimezone(tz.tzlocal())
        k+=1
        for i in range(len(d[:-1])):
            f_name = _add_date_to_file_name(os.path.basename(tkm.URL[i]),
                                            date.strftime('%Y%m%d'))
            f_name = os.path.splitext(f_name)[0] + '.csv'
            if d[i+1] != 'error':
                tkmd = tkm.TKM_DATA(date=date, e_tag=None,
                                    filename=f_name, data=d[i+1])
                tkm.save_instant_data(tkmd)

    for f in fnames:
        f = os.path.join(tkm.DIR.data, f)
        comp.compress(f)
        if os.path.exists(f): os.remove(f)
Example #4
0
    def encodeLayerData(self, tileLayer, format):
        if format in [Map.LayerDataFormat.XML, Map.LayerDataFormat.CSV]:
            raise

        tileData = QByteArray()
        for y in range(tileLayer.height()):
            for x in range(tileLayer.width()):
                gid = self.cellToGid(tileLayer.cellAt(x, y))
                tileData.append(bytes([gid&0xff]))
                tileData.append(bytes([(gid >> 8)&0xff]))
                tileData.append(bytes([(gid >> 16)&0xff]))
                tileData.append(bytes([(gid >> 24)&0xff]))
                
        if len(tileData)%4 != 0:
            raise

        if (format == Map.LayerDataFormat.Base64Gzip):
            tileData = compress(tileData, CompressionMethod.Gzip)
        elif (format == Map.LayerDataFormat.Base64Zlib):
            tileData = compress(tileData, CompressionMethod.Zlib)

        return tileData.toBase64()
Example #5
0
 def getContents(self):
     fileslist = self.getFilesList()
     assert fileslist, "Must contain at least one resource."
     result = fileslist[0].getContents()
     content_type = result["content_type"]
     data = [result["data"]]
     for subres in fileslist[1:]:
         d = subres.getContents()
         # all elements must have the same content type.
         assert d["content_type"] == content_type
         data.append(d["data"])
     result["data"] = "\n".join(data)
     result["compress_level"] = self.compress_level
     # Do compression on the result
     result["data"] = compress(**result)
     return result
Example #6
0
    def write(self, map, fileName):
        # Check layer count and type
        if (map.layerCount() != 1 or not map.layerAt(0).isTileLayer()) :
            self.mError = self.tr("The map needs to have exactly one tile layer!")
            return False
        
        mapLayer = map.layerAt(0).asTileLayer()
        # Check layer size
        if (mapLayer.width() != 48 or mapLayer.height() != 48) :
            self.mError = self.tr("The layer must have a size of 48 x 48 tiles!")
            return False
        
        # Create QByteArray and compress it
        uncompressed = QByteArray(48 * 48, b'\x00')
        width = mapLayer.width()
        height = mapLayer.height()
        for y in range(0, height):
            for x in range(0, width):
                tile = mapLayer.cellAt(x, y).tile
                if tile:
                    # 'QByteArray' object does not support item assignment
                    uncompressed.replace(y * width + x, 1, bytes([tile.id()&0xff]))

        compressed = compress(uncompressed, CompressionMethod.Gzip)
        
        # Write QByteArray
        file = QSaveFile(fileName)
        if (not file.open(QIODevice.WriteOnly)) :
            self.mError = self.tr("Could not open file for writing.")
            return False
        
        file.write(compressed)
        if not file.commit():
            self.mError = file.errorString()
            return False

        return True
Example #7
0
 def do_compress(self, output):
     compression.compress(self.image_entry.get(), self.network_entry.get(), output, int(self.bits_entry.get()), self.smoothing.get())
from compression import compress
from decompression import decompress


def parseArguments():
    if len(argv) == 4:
        return argv[1], argv[2], argv[3]
    else:
        print("ArgError - Script accepts 3 arguemnts: Mode, Source_Path, Destination_Path")
        exit()


try:
    mode, srcPath, destPath = parseArguments()

    srcFile = open(srcPath, "r")
    destFile = open(destPath, "w")

    if mode == "-c":
        compress(srcFile, destFile)
    elif mode == "-d":
        decompress(srcFile, destFile)
    else:
        print("ModeError - First Arg must be -c for Compression or -d for Decompression")

    srcFile.close()
    destFile.close()

except FileNotFoundError:
    print("IOError - File Not Found")
Example #9
0
from sys import argv, exit
from compression import compress, decompress, usage

if len(argv) < 3:
	usage(argv[0])

if int(argv[1]) == 1:
	print "%r compressed is %r" % (argv[2], compress(argv[2]))
elif int(argv[1]) == 0:
	print "%r decompressed is %r" % (argv[2], decompress(argv[2]))
else:
	usage(argv[0])
Example #10
0
 def saveObject(self,obj):
     file = open(self.path % obj.client.name ,'wb')
     file.write(compression.compress(obj))
     file.close()
Example #11
0
	def test_compress_decompress(self):
		for testCase in self.toTestList:
			# test compress output
			self.assertEqual(compress(testCase[0]),testCase[1])
			# test decompress output -- compressing then decompressing
			self.assertEqual(decompress(compress(testCase[0])),testCase[0])
 def test_css(self):
     in_file = os.path.join('tests', 'resources', 'css', 'one-line.css')
     compression.compress(in_file, self.out_file, in_type='css')
     expected = 'body{background-color:red;}'
     result = open(self.out_file).read()
     self.assertEquals(result, expected)
 def test_js(self):
     in_file = os.path.join('tests', 'resources', 'js', 'one-line.js')
     compression.compress(in_file, self.out_file)
     expected = 'var test;'
     result = open(self.out_file).read()
     self.assertEquals(result, expected)
Example #14
0
    def _compress2(): comp.compress(csv, True)

    _compress1()
Example #15
0
    def _compress1(): comp.compress(csv, f_type='.zip')

    # noinspection PyMissingOrEmptyDocstring
    def _compress2(): comp.compress(csv, True)