def _save_reducer(ex, tile, axis, path=None, prefix=None, iszip=None): if not os.path.exists(path + '/' + prefix): os.makedirs(path + '/' + prefix) sparse = True if sp.issparse(tile) else False tile_dict = { 'ul': ex.ul, 'lr': ex.lr, 'shape': tile.shape, 'dtype': str(tile.dtype), 'type': "SPARSE" if sparse else "DENSITY" } cnt = "\x93NUMPY\x01\x00" # Magic Number & Version # Dictionary dict_cnt = str(tile_dict) if (len(cnt) + 2 + len(dict_cnt)) % 16 != 0: dict_cnt += str((16 - (len(cnt) + 2 + len(dict_cnt)) % 16) * ' ') cnt += chr(len(dict_cnt) % 256) + chr(len(dict_cnt) / 256) + dict_cnt # Now data kw = { 'path': path, 'prefix': prefix, 'suffix': '', 'ul': ex.ul, 'lr': ex.lr, 'ispickle': False, 'isnp': False } try: if iszip: kw['iszip'] = True fn = save_filename(**kw) fp = bz2.BZ2File(fn, 'w', compresslevel=1) else: kw['iszip'] = False fn = save_filename(**kw) fp = open(fn, 'w') fp.write(cnt) if sparse: tile = tile.tocoo() save = np.savez_compressed if iszip else np.savez kw['isnp'] = True save(save_filename(**kw), row=tile.row, col=tile.col, data=tile.data, shape=tile.shape) else: fp.write(tile.data) fp.close() except Exception as e: util.log_error('Save %s tile(%s, %s) failed : %s', prefix, ex.ul, ex.lr, e) raise return np.asarray(1)
def _save_reducer(ex, tile, axis, path=None, prefix=None, iszip=None): if not os.path.exists(path + '/' + prefix): os.makedirs(path + '/' + prefix) sparse = True if sp.issparse(tile) else False tile_dict = {'ul': ex.ul, 'lr': ex.lr, 'shape': tile.shape, 'dtype': str(tile.dtype), 'type': "SPARSE" if sparse else "DENSITY"} cnt = "\x93NUMPY\x01\x00" # Magic Number & Version # Dictionary dict_cnt = str(tile_dict) if (len(cnt) + 2 + len(dict_cnt)) % 16 != 0: dict_cnt += str((16 - (len(cnt) + 2 + len(dict_cnt)) % 16) * ' ') cnt += chr(len(dict_cnt) % 256) + chr(len(dict_cnt) / 256) + dict_cnt # Now data kw = {'path': path, 'prefix': prefix, 'suffix': '', 'ul': ex.ul, 'lr': ex.lr, 'ispickle': False, 'isnp': False} try: if iszip: kw['iszip'] = True fn = save_filename(**kw) fp = bz2.BZ2File(fn, 'w', compresslevel=1) else: kw['iszip'] = False fn = save_filename(**kw) fp = open(fn, 'w') fp.write(cnt) if sparse: tile = tile.tocoo() save = np.savez_compressed if iszip else np.savez kw['isnp'] = True save(save_filename(**kw), row=tile.row, col=tile.col, data=tile.data, shape=tile.shape) else: fp.write(tile.data) fp.close() except Exception as e: util.log_error('Save %s tile(%s, %s) failed : %s', prefix, ex.ul, ex.lr, e) raise return np.asarray(1)
def _pickle_reducer(ex, tile, axis, path=None, prefix=None, sparse=None, iszip=None): if not os.path.exists(path + '/' + prefix): os.makedirs(path + '/' + prefix) kw = {'path': path, 'prefix': prefix, 'suffix': '', 'ul': ex.ul, 'lr': ex.lr, 'ispickle': True, 'isnp': False} try: if iszip: kw['iszip'] = True fn = save_filename(**kw) with bz2.BZ2File(fn, 'w', compresslevel=1) as fp: cpickle.dump(tile, fp, -1) else: kw['iszip'] = False fn = save_filename(**kw) with open(fn, "wb") as fp: cpickle.dump(tile, fp, -1) except Exception as e: util.log_error('Save %s tile(%s, %s) failed : %s' % prefix, ex.ul, ex.lr, e) raise return np.asarray(1)
def _pickle_reducer(ex, tile, axis, path=None, prefix=None, sparse=None, iszip=None): if not os.path.exists(path + '/' + prefix): os.makedirs(path + '/' + prefix) kw = { 'path': path, 'prefix': prefix, 'suffix': '', 'ul': ex.ul, 'lr': ex.lr, 'ispickle': True, 'isnp': False } try: if iszip: kw['iszip'] = True fn = save_filename(**kw) with bz2.BZ2File(fn, 'w', compresslevel=1) as fp: cpickle.dump(tile, fp, -1) else: kw['iszip'] = False fn = save_filename(**kw) with open(fn, "wb") as fp: cpickle.dump(tile, fp, -1) except Exception as e: util.log_error('Save %s tile(%s, %s) failed : %s' % prefix, ex.ul, ex.lr, e) raise return np.asarray(1)