def make_raw_feature_file(self, **get_feature_kwargs):
if self._dataset is None:
raise RuntimeError(
'Dataset was not initialized. Please call "make_dataset" before starting.'
)
if not any([
isinstance(t, torchvision.transforms.Normalize)
for t in self._dataset.transform.transforms
]):
raise RuntimeWarning(
'Dataset is not being normalized. Did you forget calling "make_dataset"?'
)
features = self.get_features(dataset=self._dataset,
verbose=self.verbose,
**get_feature_kwargs)
if not os.path.isdir(self.feature_dir):
os.makedirs(self.feature_dir)
img_id = list(map(lambda x: x.split('.')[0], self._dataset.filenames))
out_dict = {'raw': {'image_id': img_id, 'features': features}}
if os.path.isfile(self.feature_file):
data_dict = dd.io.load(self.feature_file)
data_dict.update(out_dict)
# hdfdict.dump(data_dict, self.feature_file)
dd.io.save(self.feature_file, data_dict)
if self.verbose:
print('Appended features to {}'.format(self.feature_file))
else:
hdfdict.dump(out_dict, self.feature_file)
if self.verbose:
print('Saved features to {}'.format(self.feature_file))
def test_dict_to_hdf_with_datetime():
d = {
'e': [datetime.datetime.now() for i in range(5)],
'f': datetime.datetime.utcnow(),
'g': [('Hello', 5), (6, 'No HDF but json'), {
'foo': True
}]
}
fname = 'test_hdfdict.h5'
if os.path.isfile(fname):
os.unlink(fname)
hf = h5py.File(fname)
hdfdict.dump(d, hf)
res = hdfdict.load(hf)
def equaldt(a, b):
d = a - b
return d.total_seconds() < 1e-3
assert all([equaldt(a, b) for (a, b) in zip(d['e'], res['e'])])
assert equaldt(d['f'], res['f'])
assert d['g'][0][0] == 'Hello'
assert d['g'][1][0] == 6
assert d.keys() == res.keys()
hf.close()
if os.path.isfile(fname):
os.unlink(fname)
def test_dict_to_hdf():
if os.path.isfile(fname):
os.unlink(fname)
hdfdict.dump(d, fname)
for lazy in [True, False]:
res = hdfdict.load(fname, lazy=lazy)
assume(np.all(d['a'] == res['a']))
assume(np.all(d['b'] == res['b']))
assume(np.all(d['c'] == res['c']))
assume(tuple(d.keys()) == tuple(res.keys()))
def _dump_tds3000_isf_to_hdf5(fn, f5grp):
print("Processing {} ...".format(fn))
hd = parse_tds3000_isf(fn)
wave = hd.pop('wave')
hdfdict.dump(hd, f5grp)
f5grp.create_dataset("waveform", (hd['NR_PT'], ),
dtype=wave_dtype,
data=wave,
compression=9,
chunks=True)
def save_model(self, fname):
"""
Save Model
Save the parameters and the architecture of the model for reuse once trained through hdfdict
:param fname: Directory where the model should be saved with filename with .h5 extension
"""
params = self.params_dict(print_params=False)
archi = self.arch
model_dict = {"Parameters": params, "Architecture": archi}
hdfdict.dump(model_dict, fname)
print("Model saved!")
def process_genres(dataset):
dataset_path = os.path.join(datasets_path, dataset)
folders = sorted(os.listdir(dataset_path))
folders = [os.path.join(dataset_path, folder) for folder in folders]
features_path = '../features/chromagram'
for i, folder in enumerate(tqdm(folders)):
data = process_genre(i, folder)
dst_path = folder.split('/')[-2]
folder = folder.split('/')[-1]
print(' ', folder)
dst_path = os.path.join(features_path, dst_path, folder + '.hdf5')
hdfdict.dump(data, dst_path)
def test_dict_to_hdf_with_datetime():
d = {
'e': [datetime.datetime.now() for i in range(5)],
'f': datetime.datetime.utcnow(),
'g': [('Hello', 5), (6, 'No HDF but json'), {
'foo': True
}],
'h': {
'test2': datetime.datetime.now(),
'again': ['a', 1],
(1, 2): (3, 4)
}
}
fname = 'test_hdfdict.h5'
if os.path.isfile(fname):
os.unlink(fname)
hf = h5py.File(fname)
hdfdict.dump(d, hf)
res = hdfdict.load(hf, lazy=False)
res = hdfdict.load(hf)
res.unlazy() # all lazy objects will be rolled out.
def equaldt(a, b):
d = a - b
return d.total_seconds() < 1e-3
assume(all([equaldt(a, b) for (a, b) in zip(d['e'], res['e'])]))
assume(equaldt(d['f'], res['f']))
assume(d['g'][0][0] == 'Hello')
assume(d['g'][1][0] == 6)
assume(d.keys() == res.keys())
assume(isinstance(res['h']['test2'], datetime.datetime))
assume(res['h']['again'][0] == 'a')
assume(res['h']['again'][1] == 1)
hf.close()
if os.path.isfile(fname):
os.unlink(fname)
def save(self, path='.', fname=''):
if not fname:
fname = '.'.join((self.uid, _cfg.H5FILEEXT))
fpath = str(_pathlib.Path(path) / fname)
_hdfdict.dump(_dataclasses.asdict(self), fpath)
return fpath