def _save_reducer(ex, tile, axis, path=None, prefix=None, iszip=None):
if not os.path.exists(path + '/' + prefix):
os.makedirs(path + '/' + prefix)
sparse = True if sp.issparse(tile) else False
tile_dict = {
'ul': ex.ul,
'lr': ex.lr,
'shape': tile.shape,
'dtype': str(tile.dtype),
'type': "SPARSE" if sparse else "DENSITY"
}
cnt = "\x93NUMPY\x01\x00" # Magic Number & Version
# Dictionary
dict_cnt = str(tile_dict)
if (len(cnt) + 2 + len(dict_cnt)) % 16 != 0:
dict_cnt += str((16 - (len(cnt) + 2 + len(dict_cnt)) % 16) * ' ')
cnt += chr(len(dict_cnt) % 256) + chr(len(dict_cnt) / 256) + dict_cnt
# Now data
kw = {
'path': path,
'prefix': prefix,
'suffix': '',
'ul': ex.ul,
'lr': ex.lr,
'ispickle': False,
'isnp': False
}
try:
if iszip:
kw['iszip'] = True
fn = save_filename(**kw)
fp = bz2.BZ2File(fn, 'w', compresslevel=1)
else:
kw['iszip'] = False
fn = save_filename(**kw)
fp = open(fn, 'w')
fp.write(cnt)
if sparse:
tile = tile.tocoo()
save = np.savez_compressed if iszip else np.savez
kw['isnp'] = True
save(save_filename(**kw),
row=tile.row,
col=tile.col,
data=tile.data,
shape=tile.shape)
else:
fp.write(tile.data)
fp.close()
except Exception as e:
util.log_error('Save %s tile(%s, %s) failed : %s', prefix, ex.ul,
ex.lr, e)
raise
return np.asarray(1)
def _save_reducer(ex, tile, axis, path=None, prefix=None, iszip=None):
if not os.path.exists(path + '/' + prefix):
os.makedirs(path + '/' + prefix)
sparse = True if sp.issparse(tile) else False
tile_dict = {'ul': ex.ul, 'lr': ex.lr, 'shape': tile.shape,
'dtype': str(tile.dtype),
'type': "SPARSE" if sparse else "DENSITY"}
cnt = "\x93NUMPY\x01\x00" # Magic Number & Version
# Dictionary
dict_cnt = str(tile_dict)
if (len(cnt) + 2 + len(dict_cnt)) % 16 != 0:
dict_cnt += str((16 - (len(cnt) + 2 + len(dict_cnt)) % 16) * ' ')
cnt += chr(len(dict_cnt) % 256) + chr(len(dict_cnt) / 256) + dict_cnt
# Now data
kw = {'path': path, 'prefix': prefix, 'suffix': '',
'ul': ex.ul, 'lr': ex.lr, 'ispickle': False, 'isnp': False}
try:
if iszip:
kw['iszip'] = True
fn = save_filename(**kw)
fp = bz2.BZ2File(fn, 'w', compresslevel=1)
else:
kw['iszip'] = False
fn = save_filename(**kw)
fp = open(fn, 'w')
fp.write(cnt)
if sparse:
tile = tile.tocoo()
save = np.savez_compressed if iszip else np.savez
kw['isnp'] = True
save(save_filename(**kw), row=tile.row, col=tile.col,
data=tile.data, shape=tile.shape)
else:
fp.write(tile.data)
fp.close()
except Exception as e:
util.log_error('Save %s tile(%s, %s) failed : %s', prefix, ex.ul, ex.lr, e)
raise
return np.asarray(1)
def _pickle_reducer(ex, tile, axis, path=None, prefix=None, sparse=None, iszip=None):
if not os.path.exists(path + '/' + prefix):
os.makedirs(path + '/' + prefix)
kw = {'path': path, 'prefix': prefix, 'suffix': '',
'ul': ex.ul, 'lr': ex.lr, 'ispickle': True, 'isnp': False}
try:
if iszip:
kw['iszip'] = True
fn = save_filename(**kw)
with bz2.BZ2File(fn, 'w', compresslevel=1) as fp:
cpickle.dump(tile, fp, -1)
else:
kw['iszip'] = False
fn = save_filename(**kw)
with open(fn, "wb") as fp:
cpickle.dump(tile, fp, -1)
except Exception as e:
util.log_error('Save %s tile(%s, %s) failed : %s' % prefix, ex.ul, ex.lr, e)
raise
return np.asarray(1)
def _pickle_reducer(ex,
tile,
axis,
path=None,
prefix=None,
sparse=None,
iszip=None):
if not os.path.exists(path + '/' + prefix):
os.makedirs(path + '/' + prefix)
kw = {
'path': path,
'prefix': prefix,
'suffix': '',
'ul': ex.ul,
'lr': ex.lr,
'ispickle': True,
'isnp': False
}
try:
if iszip:
kw['iszip'] = True
fn = save_filename(**kw)
with bz2.BZ2File(fn, 'w', compresslevel=1) as fp:
cpickle.dump(tile, fp, -1)
else:
kw['iszip'] = False
fn = save_filename(**kw)
with open(fn, "wb") as fp:
cpickle.dump(tile, fp, -1)
except Exception as e:
util.log_error('Save %s tile(%s, %s) failed : %s' % prefix, ex.ul,
ex.lr, e)
raise
return np.asarray(1)
