def test_create(self): """ Enum datasets can be created and type correctly round-trips """ dt = h5py.enum_dtype(self.EDICT, basetype='i') ds = self.f.create_dataset('x', (100, 100), dtype=dt) dt2 = ds.dtype dict2 = h5py.check_enum_dtype(dt2) self.assertEqual(dict2, self.EDICT)
def test_vlen_enum(self): fname = self.mktemp() arr1 = [[1], [1, 2]] dt1 = h5py.vlen_dtype(h5py.enum_dtype(dict(foo=1, bar=2), 'i')) with h5py.File(fname, 'w') as f: df1 = f.create_dataset('test', (len(arr1), ), dtype=dt1) df1[:] = np.array(arr1) with h5py.File(fname, 'r') as f: df2 = f['test'] dt2 = df2.dtype arr2 = [e.tolist() for e in df2[:]] self.assertEqual(arr1, arr2) self.assertEqual(h5py.check_enum_dtype(h5py.check_vlen_dtype(dt1)), h5py.check_enum_dtype(h5py.check_vlen_dtype(dt2)))
def test_vlen_enum(self): fname = self.mktemp() arr1 = [[1],[1,2]] dt1 = h5py.vlen_dtype(h5py.enum_dtype(dict(foo=1, bar=2), 'i')) with h5py.File(fname,'w') as f: df1 = f.create_dataset('test', (len(arr1),), dtype=dt1) df1[:] = np.array(arr1) with h5py.File(fname,'r') as f: df2 = f['test'] dt2 = df2.dtype arr2 = [e.tolist() for e in df2[:]] self.assertEqual(arr1, arr2) self.assertEqual(h5py.check_enum_dtype(h5py.check_vlen_dtype(dt1)), h5py.check_enum_dtype(h5py.check_vlen_dtype(dt2)))
def run_analysis(input_file): # input_file = 'network_clamp.optimize.h5' opt_id = 'network_clamp.optimize' f = h5py.File(input_file, 'r') opt_grp = f[opt_id] parameter_enum_dict = h5py.check_enum_dtype( opt_grp['parameter_enum'].dtype) parameters_idx_dict = { parm: idx for parm, idx in parameter_enum_dict.items() } parameters_name_dict = { idx: parm for parm, idx in parameters_idx_dict.items() } problem_parameters = { parameters_name_dict[idx]: val for idx, val in opt_grp['problem_parameters'] } parameter_specs = [(parameters_name_dict[spec[0]], tuple(spec)[1:]) for spec in iter(opt_grp['parameter_spec'])] problem_ids = None if 'problem_ids' in opt_grp: problem_ids = set(opt_grp['problem_ids']) M = len(parameter_specs) raw_results = {} if problem_ids is not None: for problem_id in problem_ids: data = opt_grp['%d' % problem_id]['results'][:].reshape( (-1, M + 1)) # np.array of shape [N, M+1] raw_results[problem_id] = { 'x': data[:, 1:], 'y': data[:, 0], } else: data = opt_grp['%d' % 0]['results'][:].reshape( (-1, M + 1)) # np.array of shape [N, M+1] raw_results[0] = { 'x': data[:, 1:], 'y': data[:, 0], } f.close() pprint.pprint(raw_results)
def _column(self, k): cached_data = self._cache.get(k) if cached_data is not None: return cached_data dset = self._hf.get(k) if dset is not None: if h5py.check_string_dtype(dset.dtype): py_data = dset.asstr() else: enum_dict = h5py.check_enum_dtype(dset.dtype) if enum_dict: inv_enum_dict = dict((i, k) for k, i in enum_dict.items()) py_data = [inv_enum_dict[x] for x in np.array(dset)] else: py_data = np.array(dset) else: py_data = None self._cache[k] = py_data return py_data
import pprint import h5py input_file = 'network_clamp.optimize.h5' opt_id = 'network_clamp.optimize' f = h5py.File(input_file, 'r') opt_grp = f[opt_id] parameter_enum_dict = h5py.check_enum_dtype(opt_grp['parameter_enum'].dtype) parameters_idx_dict = {parm: idx for parm, idx in parameter_enum_dict.items()} parameters_name_dict = {idx: parm for parm, idx in parameters_idx_dict.items()} problem_parameters = { parameters_name_dict[idx]: val for idx, val in opt_grp['problem_parameters'] } parameter_specs = [(parameters_name_dict[spec[0]], tuple(spec)[1:]) for spec in iter(opt_grp['parameter_spec'])] problem_ids = None if 'problem_ids' in opt_grp: problem_ids = set(opt_grp['problem_ids']) M = len(parameter_specs) raw_results = {} if problem_ids is not None: for problem_id in problem_ids: data = opt_grp['%d' % 0]['results'][:].reshape( (-1, M + 1)) # np.array of shape [N, M+1] raw_results[problem_id] = {
def h5_load_raw(input_file, opt_id): ## N is number of trials ## M is number of hyperparameters f = h5py.File(input_file, 'r') opt_grp = h5_get_group(f, opt_id) solver_epsilon = opt_grp['solver_epsilon'][()] relative_noise_magnitude = opt_grp['relative_noise_magnitude'][()] n_features = 0 feature_names = None feature_types = None if 'feature_enum' in opt_grp: feature_enum_dict = h5py.check_enum_dtype( opt_grp['feature_enum'].dtype) feature_idx_dict = { parm: idx for parm, idx in feature_enum_dict.items() } feature_name_dict = { idx: parm for parm, idx in feature_idx_dict.items() } n_features = len(feature_enum_dict) feature_names = [ feature_name_dict[spec[0]] for spec in iter(opt_grp['feature_spec']) ] feature_dtype = opt_grp['feature_type'] feature_types = [feature_dtype.fields[x] for x in feature_dtype] n_constraints = 0 constraint_names = None if 'constraint_enum' in opt_grp: constraint_enum_dict = h5py.check_enum_dtype( opt_grp['constraint_enum'].dtype) constraint_idx_dict = { parm: idx for parm, idx in constraint_enum_dict.items() } constraint_name_dict = { idx: parm for parm, idx in constraint_idx_dict.items() } n_constraints = len(constraint_enum_dict) constraint_names = [ constraint_name_dict[spec[0]] for spec in iter(opt_grp['constraint_spec']) ] parameter_enum_dict = h5py.check_enum_dtype( opt_grp['parameter_enum'].dtype) parameters_idx_dict = { parm: idx for parm, idx in parameter_enum_dict.items() } parameters_name_dict = { idx: parm for parm, idx in parameters_idx_dict.items() } problem_parameters = { parameters_name_dict[idx]: val for idx, val in opt_grp['problem_parameters'] } parameter_specs = [(parameters_name_dict[spec[0]], tuple(spec)[1:]) for spec in iter(opt_grp['parameter_spec'])] problem_ids = None if 'problem_ids' in opt_grp: problem_ids = set(opt_grp['problem_ids']) raw_results = {} for problem_id in problem_ids if problem_ids is not None else [0]: if str(problem_id) in opt_grp: raw_results[problem_id] = { 'objectives': opt_grp[str(problem_id)]['objectives'][:], 'parameters': opt_grp[str(problem_id)]['parameters'][:] } if 'features' in opt_grp[str(problem_id)]: raw_results[problem_id]['features'] = opt_grp[str( problem_id)]['features'][:] if 'constraints' in opt_grp[str(problem_id)]: raw_results[problem_id]['constraints'] = opt_grp[str( problem_id)]['constraints'][:] f.close() param_names = [] is_integer = [] lower = [] upper = [] for parm, spec in parameter_specs: param_names.append(parm) is_int, lo, hi = spec is_integer.append(is_int) lower.append(lo) upper.append(hi) raw_spec = (is_integer, lower, upper) info = { 'features': feature_names, 'feature_types': feature_types, 'constraints': constraint_names, 'params': param_names, 'solver_epsilon': solver_epsilon, 'relative_noise_magnitude': relative_noise_magnitude, 'problem_parameters': problem_parameters, 'problem_ids': problem_ids } return raw_spec, raw_results, info