def save(self, distrib, name, dir_data='data'):
help.save_npy(help.join(dir_data, '{}_sample'.format(name)),
distrib.sample)
help.save_npy(help.join(dir_data, '{}_normal'.format(name)),
distrib.normal)
help.save_json_pretty(help.join(dir_data, '{}_meta.json'.format(name)),
distrib.meta())
return distrib
def ds_sym_clone(dir_src,
dir_destination,
dsfield,
add_cwd=True,
meta_suffix='.meta'):
file_data = help.join(dir_src, str(dsfield))
file_data_out = help.join(dir_destination, str(dsfield))
help.symlink(file_data, file_data_out, add_cwd)
file_meta = '{}{}'.format(file_data, meta_suffix)
file_meta_out = '{}{}'.format(file_data_out, meta_suffix)
help.symlink(file_meta, file_meta_out, add_cwd)
def __init__(self, name, dir='', meta_suffix='meta'):
self.name = name
self.dir = dir
self.meta_suffix = meta_suffix
self.file_data = help.join(self.dir, name)
self.file_meta = help.join(self.dir,
'{}.{}'.format(name, self.meta_suffix))
self.meta = None
self.data = None
self.shape = None
self.dtype = None
def ds_dirs(dir_base, ds_name, ds_version, ds_type):
if ds_name:
dir_name = help.join(dir_base, str(ds_name))
else:
dir_name = dir_base
if ds_version:
dir_version = help.join(dir_name, str(ds_version))
else:
dir_version = dir_name
if ds_type:
dir_type = help.join(dir_version, str(ds_type))
else:
dir_type = dir_version
return dir_base, dir_name, dir_version, dir_type
def exists(self, meta_suffix='.meta'):
dirs = self.dirs()
for d in dirs:
if not help.exists_file(d):
return False
file_data = help.join(dirs[-1], str(self.field))
file_meta = '{}{}'.format(file_data, meta_suffix)
return help.exists_file(file_meta)
def load(self, mode='r', meta_suffix='.meta'):
_, dir_name, dir_version, dir_type = self.dirs()
file_data = help.join(dir_type, str(self.field))
file_meta = '{}{}'.format(file_data, meta_suffix)
meta = help.load_json(file_meta)
shape = tuple(meta['shape'])
dtype = help.str2dtype(meta['type'])
data = np.memmap(file_data, dtype, shape=shape, mode=mode)
return Dataset(self.name, self.type, self.field, data, self.dir_base,
dir_name, dir_version, dir_type, file_data, file_meta)
def load(self, name, dir_data='data'):
sample = help.load_npy(help.join(dir_data, '{}_sample'.format(name)))
normal = help.load_npy(help.join(dir_data, '{}_normal'.format(name)))
meta = to_conf().load(help.join(dir_data, '{}_meta.json'.format(name)))
dist_type = meta['type']
if dist_type == '1d':
return Distrib1d(sample,
meta['pre_norm'],
normal=normal,
mean=meta['mean'],
std=meta['std'],
alpha=meta['alpha', 1.0])
elif dist_type == 'nd':
return DistribNd(sample,
meta['pre_norm'],
normal=normal,
mean=meta['mean'],
std=meta['std'],
alpha=meta['alpha', 1.0])
else:
raise Exception('Unknown distribution type: {}'.format(dist_type))
def __init__(self,
ds_name,
ds_version=None,
ds_type=None,
dir_base='data'):
self.name = ds_name
self.version = ds_version
self.type = ds_type
self.dir_base = dir_base
if self.name:
self.dir_name = help.join(self.dir_base, str(self.name))
else:
self.dir_name = self.dir_base
if self.version:
self.dir_version = help.join(self.dir_name, str(self.version))
else:
self.dir_version = self.dir_name
if self.type:
self.dir_type = help.join(self.dir_version, str(self.type))
else:
self.dir_type = self.dir_version
def create(self, shape, dtype=np.float64, mode='w+', meta_suffix='.meta'):
if isinstance(shape, int):
shape = (shape, )
else:
shape = tuple([int(v) for v in shape])
_, dir_name, dir_version, dir_type = self.dirs()
help.mkdirs(dir_type)
file_data = help.join(dir_type, str(self.field))
data = np.memmap(file_data, dtype, shape=tuple(shape), mode=mode)
meta = {'type': help.dtype2str(dtype), 'shape': shape}
file_meta = '{}{}'.format(file_data, meta_suffix)
help.save_json_pretty(file_meta, meta)
return Dataset(self.name, self.type, self.field, data, self.dir_base,
dir_name, dir_version, dir_type, file_data, file_meta)
def symlink(self, dir_destination, add_cwd=True, meta_suffix='.meta'):
dir_ds = help.join(self.dir_base, str(self.name), str(self.type))
ds_sym_clone(dir_ds, dir_destination, self.field, add_cwd, meta_suffix)
return self