def compress_files():
"""Compresses downloaded data files."""
lcsv = [f for f in os.listdir(DIR.data) if f.endswith('.csv')]
today_e_tag = _now().strftime('%Y%m%d')
for f in lcsv:
ff = os.path.join(DIR.data, f)
if today_e_tag not in ff:
c.compress(ff)
log.info('%s compressed.', os.path.basename(ff))
os.remove(ff)
def test_compression():
""" Test compression and decompression functions """
f_sha = hashlib.sha256(open(_f, 'rb').read()).hexdigest()
temp_name = os.path.join('tests', next(tempfile._get_candidate_names()))
fc = c.compress(_f, temp_name)
fd = c.decompress(fc)
assert f_sha == hashlib.sha256(open(fd, 'rb').read()).hexdigest()
os.remove(fc); os.remove(fd)
fc = c.compress(_f, temp_name, f_type='zip')
fd = c.decompress(fc)
assert f_sha == hashlib.sha256(open(fd, 'rb').read()).hexdigest()
os.remove(fc); os.remove(fd)
def convert_data_from_zip_file(f):
"""
Directly reads eoner's csv from zip file and converts it to ours.
:param f: Full path to file name.
:type f: str
"""
# f = '/Users/isezen/project/tkmpy/tkmdata/2015-07-31.zip'
date = os.path.splitext(os.path.basename(f))[0]
date = dt.strptime(date,'%Y-%m-%d')
fnames = [_add_date_to_file_name(os.path.splitext(os.path.basename(url))[0],
date.strftime('%Y%m%d')) +'.csv'
for url in tkm.URL[:5]]
text = comp.read_from_zip(f)
k = 0
for l in [x for x in text.split('\n') if not re.match(r'^\s*$', x)]:
d = l.replace(u'\ufeff','').split('#')
if len(d)>6:
for i in range(6,len(d)): d[5] = d[5] + '#' + d[i]
d = d[0:6]
d = [i.replace(';', '|').replace('|&', '&') for i in d]
for i in range(len(d)):
if d[i][len(d[i])-1] == u'&': d[i] = d[i][:(len(d[i])-1)]
date = _oa_to_datetime(d[0])
date = date.replace(tzinfo=tz.tzutc()).astimezone(tz.tzlocal())
k+=1
for i in range(len(d[:-1])):
f_name = _add_date_to_file_name(os.path.basename(tkm.URL[i]),
date.strftime('%Y%m%d'))
f_name = os.path.splitext(f_name)[0] + '.csv'
if d[i+1] != 'error':
tkmd = tkm.TKM_DATA(date=date, e_tag=None,
filename=f_name, data=d[i+1])
tkm.save_instant_data(tkmd)
for f in fnames:
f = os.path.join(tkm.DIR.data, f)
comp.compress(f)
if os.path.exists(f): os.remove(f)
def encodeLayerData(self, tileLayer, format):
if format in [Map.LayerDataFormat.XML, Map.LayerDataFormat.CSV]:
raise
tileData = QByteArray()
for y in range(tileLayer.height()):
for x in range(tileLayer.width()):
gid = self.cellToGid(tileLayer.cellAt(x, y))
tileData.append(bytes([gid&0xff]))
tileData.append(bytes([(gid >> 8)&0xff]))
tileData.append(bytes([(gid >> 16)&0xff]))
tileData.append(bytes([(gid >> 24)&0xff]))
if len(tileData)%4 != 0:
raise
if (format == Map.LayerDataFormat.Base64Gzip):
tileData = compress(tileData, CompressionMethod.Gzip)
elif (format == Map.LayerDataFormat.Base64Zlib):
tileData = compress(tileData, CompressionMethod.Zlib)
return tileData.toBase64()
def getContents(self):
fileslist = self.getFilesList()
assert fileslist, "Must contain at least one resource."
result = fileslist[0].getContents()
content_type = result["content_type"]
data = [result["data"]]
for subres in fileslist[1:]:
d = subres.getContents()
# all elements must have the same content type.
assert d["content_type"] == content_type
data.append(d["data"])
result["data"] = "\n".join(data)
result["compress_level"] = self.compress_level
# Do compression on the result
result["data"] = compress(**result)
return result
def write(self, map, fileName):
# Check layer count and type
if (map.layerCount() != 1 or not map.layerAt(0).isTileLayer()) :
self.mError = self.tr("The map needs to have exactly one tile layer!")
return False
mapLayer = map.layerAt(0).asTileLayer()
# Check layer size
if (mapLayer.width() != 48 or mapLayer.height() != 48) :
self.mError = self.tr("The layer must have a size of 48 x 48 tiles!")
return False
# Create QByteArray and compress it
uncompressed = QByteArray(48 * 48, b'\x00')
width = mapLayer.width()
height = mapLayer.height()
for y in range(0, height):
for x in range(0, width):
tile = mapLayer.cellAt(x, y).tile
if tile:
# 'QByteArray' object does not support item assignment
uncompressed.replace(y * width + x, 1, bytes([tile.id()&0xff]))
compressed = compress(uncompressed, CompressionMethod.Gzip)
# Write QByteArray
file = QSaveFile(fileName)
if (not file.open(QIODevice.WriteOnly)) :
self.mError = self.tr("Could not open file for writing.")
return False
file.write(compressed)
if not file.commit():
self.mError = file.errorString()
return False
return True
def do_compress(self, output):
compression.compress(self.image_entry.get(), self.network_entry.get(), output, int(self.bits_entry.get()), self.smoothing.get())
from compression import compress
from decompression import decompress
def parseArguments():
if len(argv) == 4:
return argv[1], argv[2], argv[3]
else:
print("ArgError - Script accepts 3 arguemnts: Mode, Source_Path, Destination_Path")
exit()
try:
mode, srcPath, destPath = parseArguments()
srcFile = open(srcPath, "r")
destFile = open(destPath, "w")
if mode == "-c":
compress(srcFile, destFile)
elif mode == "-d":
decompress(srcFile, destFile)
else:
print("ModeError - First Arg must be -c for Compression or -d for Decompression")
srcFile.close()
destFile.close()
except FileNotFoundError:
print("IOError - File Not Found")
from sys import argv, exit from compression import compress, decompress, usage if len(argv) < 3: usage(argv[0]) if int(argv[1]) == 1: print "%r compressed is %r" % (argv[2], compress(argv[2])) elif int(argv[1]) == 0: print "%r decompressed is %r" % (argv[2], decompress(argv[2])) else: usage(argv[0])
def saveObject(self,obj):
file = open(self.path % obj.client.name ,'wb')
file.write(compression.compress(obj))
file.close()
def test_compress_decompress(self): for testCase in self.toTestList: # test compress output self.assertEqual(compress(testCase[0]),testCase[1]) # test decompress output -- compressing then decompressing self.assertEqual(decompress(compress(testCase[0])),testCase[0])
def test_css(self):
in_file = os.path.join('tests', 'resources', 'css', 'one-line.css')
compression.compress(in_file, self.out_file, in_type='css')
expected = 'body{background-color:red;}'
result = open(self.out_file).read()
self.assertEquals(result, expected)
def test_js(self):
in_file = os.path.join('tests', 'resources', 'js', 'one-line.js')
compression.compress(in_file, self.out_file)
expected = 'var test;'
result = open(self.out_file).read()
self.assertEquals(result, expected)
def _compress2(): comp.compress(csv, True)
_compress1()
def _compress1(): comp.compress(csv, f_type='.zip')
# noinspection PyMissingOrEmptyDocstring
def _compress2(): comp.compress(csv, True)
