def make_raw_feature_file(self, **get_feature_kwargs): if self._dataset is None: raise RuntimeError( 'Dataset was not initialized. Please call "make_dataset" before starting.' ) if not any([ isinstance(t, torchvision.transforms.Normalize) for t in self._dataset.transform.transforms ]): raise RuntimeWarning( 'Dataset is not being normalized. Did you forget calling "make_dataset"?' ) features = self.get_features(dataset=self._dataset, verbose=self.verbose, **get_feature_kwargs) if not os.path.isdir(self.feature_dir): os.makedirs(self.feature_dir) img_id = list(map(lambda x: x.split('.')[0], self._dataset.filenames)) out_dict = {'raw': {'image_id': img_id, 'features': features}} if os.path.isfile(self.feature_file): data_dict = dd.io.load(self.feature_file) data_dict.update(out_dict) # hdfdict.dump(data_dict, self.feature_file) dd.io.save(self.feature_file, data_dict) if self.verbose: print('Appended features to {}'.format(self.feature_file)) else: hdfdict.dump(out_dict, self.feature_file) if self.verbose: print('Saved features to {}'.format(self.feature_file))
def test_dict_to_hdf_with_datetime(): d = { 'e': [datetime.datetime.now() for i in range(5)], 'f': datetime.datetime.utcnow(), 'g': [('Hello', 5), (6, 'No HDF but json'), { 'foo': True }] } fname = 'test_hdfdict.h5' if os.path.isfile(fname): os.unlink(fname) hf = h5py.File(fname) hdfdict.dump(d, hf) res = hdfdict.load(hf) def equaldt(a, b): d = a - b return d.total_seconds() < 1e-3 assert all([equaldt(a, b) for (a, b) in zip(d['e'], res['e'])]) assert equaldt(d['f'], res['f']) assert d['g'][0][0] == 'Hello' assert d['g'][1][0] == 6 assert d.keys() == res.keys() hf.close() if os.path.isfile(fname): os.unlink(fname)
def test_dict_to_hdf(): if os.path.isfile(fname): os.unlink(fname) hdfdict.dump(d, fname) for lazy in [True, False]: res = hdfdict.load(fname, lazy=lazy) assume(np.all(d['a'] == res['a'])) assume(np.all(d['b'] == res['b'])) assume(np.all(d['c'] == res['c'])) assume(tuple(d.keys()) == tuple(res.keys()))
def _dump_tds3000_isf_to_hdf5(fn, f5grp): print("Processing {} ...".format(fn)) hd = parse_tds3000_isf(fn) wave = hd.pop('wave') hdfdict.dump(hd, f5grp) f5grp.create_dataset("waveform", (hd['NR_PT'], ), dtype=wave_dtype, data=wave, compression=9, chunks=True)
def save_model(self, fname): """ Save Model Save the parameters and the architecture of the model for reuse once trained through hdfdict :param fname: Directory where the model should be saved with filename with .h5 extension """ params = self.params_dict(print_params=False) archi = self.arch model_dict = {"Parameters": params, "Architecture": archi} hdfdict.dump(model_dict, fname) print("Model saved!")
def process_genres(dataset): dataset_path = os.path.join(datasets_path, dataset) folders = sorted(os.listdir(dataset_path)) folders = [os.path.join(dataset_path, folder) for folder in folders] features_path = '../features/chromagram' for i, folder in enumerate(tqdm(folders)): data = process_genre(i, folder) dst_path = folder.split('/')[-2] folder = folder.split('/')[-1] print(' ', folder) dst_path = os.path.join(features_path, dst_path, folder + '.hdf5') hdfdict.dump(data, dst_path)
def test_dict_to_hdf_with_datetime(): d = { 'e': [datetime.datetime.now() for i in range(5)], 'f': datetime.datetime.utcnow(), 'g': [('Hello', 5), (6, 'No HDF but json'), { 'foo': True }], 'h': { 'test2': datetime.datetime.now(), 'again': ['a', 1], (1, 2): (3, 4) } } fname = 'test_hdfdict.h5' if os.path.isfile(fname): os.unlink(fname) hf = h5py.File(fname) hdfdict.dump(d, hf) res = hdfdict.load(hf, lazy=False) res = hdfdict.load(hf) res.unlazy() # all lazy objects will be rolled out. def equaldt(a, b): d = a - b return d.total_seconds() < 1e-3 assume(all([equaldt(a, b) for (a, b) in zip(d['e'], res['e'])])) assume(equaldt(d['f'], res['f'])) assume(d['g'][0][0] == 'Hello') assume(d['g'][1][0] == 6) assume(d.keys() == res.keys()) assume(isinstance(res['h']['test2'], datetime.datetime)) assume(res['h']['again'][0] == 'a') assume(res['h']['again'][1] == 1) hf.close() if os.path.isfile(fname): os.unlink(fname)
def save(self, path='.', fname=''): if not fname: fname = '.'.join((self.uid, _cfg.H5FILEEXT)) fpath = str(_pathlib.Path(path) / fname) _hdfdict.dump(_dataclasses.asdict(self), fpath) return fpath