def test_create(self):
""" Enum datasets can be created and type correctly round-trips """
dt = h5py.enum_dtype(self.EDICT, basetype='i')
ds = self.f.create_dataset('x', (100, 100), dtype=dt)
dt2 = ds.dtype
dict2 = h5py.check_enum_dtype(dt2)
self.assertEqual(dict2, self.EDICT)
def test_vlen_enum(self):
fname = self.mktemp()
arr1 = [[1], [1, 2]]
dt1 = h5py.vlen_dtype(h5py.enum_dtype(dict(foo=1, bar=2), 'i'))
with h5py.File(fname, 'w') as f:
df1 = f.create_dataset('test', (len(arr1), ), dtype=dt1)
df1[:] = np.array(arr1)
with h5py.File(fname, 'r') as f:
df2 = f['test']
dt2 = df2.dtype
arr2 = [e.tolist() for e in df2[:]]
self.assertEqual(arr1, arr2)
self.assertEqual(h5py.check_enum_dtype(h5py.check_vlen_dtype(dt1)),
h5py.check_enum_dtype(h5py.check_vlen_dtype(dt2)))
def test_vlen_enum(self):
fname = self.mktemp()
arr1 = [[1],[1,2]]
dt1 = h5py.vlen_dtype(h5py.enum_dtype(dict(foo=1, bar=2), 'i'))
with h5py.File(fname,'w') as f:
df1 = f.create_dataset('test', (len(arr1),), dtype=dt1)
df1[:] = np.array(arr1)
with h5py.File(fname,'r') as f:
df2 = f['test']
dt2 = df2.dtype
arr2 = [e.tolist() for e in df2[:]]
self.assertEqual(arr1, arr2)
self.assertEqual(h5py.check_enum_dtype(h5py.check_vlen_dtype(dt1)),
h5py.check_enum_dtype(h5py.check_vlen_dtype(dt2)))
def run_analysis(input_file):
# input_file = 'network_clamp.optimize.h5'
opt_id = 'network_clamp.optimize'
f = h5py.File(input_file, 'r')
opt_grp = f[opt_id]
parameter_enum_dict = h5py.check_enum_dtype(
opt_grp['parameter_enum'].dtype)
parameters_idx_dict = {
parm: idx
for parm, idx in parameter_enum_dict.items()
}
parameters_name_dict = {
idx: parm
for parm, idx in parameters_idx_dict.items()
}
problem_parameters = {
parameters_name_dict[idx]: val
for idx, val in opt_grp['problem_parameters']
}
parameter_specs = [(parameters_name_dict[spec[0]], tuple(spec)[1:])
for spec in iter(opt_grp['parameter_spec'])]
problem_ids = None
if 'problem_ids' in opt_grp:
problem_ids = set(opt_grp['problem_ids'])
M = len(parameter_specs)
raw_results = {}
if problem_ids is not None:
for problem_id in problem_ids:
data = opt_grp['%d' % problem_id]['results'][:].reshape(
(-1, M + 1)) # np.array of shape [N, M+1]
raw_results[problem_id] = {
'x': data[:, 1:],
'y': data[:, 0],
}
else:
data = opt_grp['%d' % 0]['results'][:].reshape(
(-1, M + 1)) # np.array of shape [N, M+1]
raw_results[0] = {
'x': data[:, 1:],
'y': data[:, 0],
}
f.close()
pprint.pprint(raw_results)
def _column(self, k):
cached_data = self._cache.get(k)
if cached_data is not None:
return cached_data
dset = self._hf.get(k)
if dset is not None:
if h5py.check_string_dtype(dset.dtype):
py_data = dset.asstr()
else:
enum_dict = h5py.check_enum_dtype(dset.dtype)
if enum_dict:
inv_enum_dict = dict((i, k) for k, i in enum_dict.items())
py_data = [inv_enum_dict[x] for x in np.array(dset)]
else:
py_data = np.array(dset)
else:
py_data = None
self._cache[k] = py_data
return py_data
import pprint
import h5py
input_file = 'network_clamp.optimize.h5'
opt_id = 'network_clamp.optimize'
f = h5py.File(input_file, 'r')
opt_grp = f[opt_id]
parameter_enum_dict = h5py.check_enum_dtype(opt_grp['parameter_enum'].dtype)
parameters_idx_dict = {parm: idx for parm, idx in parameter_enum_dict.items()}
parameters_name_dict = {idx: parm for parm, idx in parameters_idx_dict.items()}
problem_parameters = {
parameters_name_dict[idx]: val
for idx, val in opt_grp['problem_parameters']
}
parameter_specs = [(parameters_name_dict[spec[0]], tuple(spec)[1:])
for spec in iter(opt_grp['parameter_spec'])]
problem_ids = None
if 'problem_ids' in opt_grp:
problem_ids = set(opt_grp['problem_ids'])
M = len(parameter_specs)
raw_results = {}
if problem_ids is not None:
for problem_id in problem_ids:
data = opt_grp['%d' % 0]['results'][:].reshape(
(-1, M + 1)) # np.array of shape [N, M+1]
raw_results[problem_id] = {
def h5_load_raw(input_file, opt_id):
## N is number of trials
## M is number of hyperparameters
f = h5py.File(input_file, 'r')
opt_grp = h5_get_group(f, opt_id)
solver_epsilon = opt_grp['solver_epsilon'][()]
relative_noise_magnitude = opt_grp['relative_noise_magnitude'][()]
n_features = 0
feature_names = None
feature_types = None
if 'feature_enum' in opt_grp:
feature_enum_dict = h5py.check_enum_dtype(
opt_grp['feature_enum'].dtype)
feature_idx_dict = {
parm: idx
for parm, idx in feature_enum_dict.items()
}
feature_name_dict = {
idx: parm
for parm, idx in feature_idx_dict.items()
}
n_features = len(feature_enum_dict)
feature_names = [
feature_name_dict[spec[0]]
for spec in iter(opt_grp['feature_spec'])
]
feature_dtype = opt_grp['feature_type']
feature_types = [feature_dtype.fields[x] for x in feature_dtype]
n_constraints = 0
constraint_names = None
if 'constraint_enum' in opt_grp:
constraint_enum_dict = h5py.check_enum_dtype(
opt_grp['constraint_enum'].dtype)
constraint_idx_dict = {
parm: idx
for parm, idx in constraint_enum_dict.items()
}
constraint_name_dict = {
idx: parm
for parm, idx in constraint_idx_dict.items()
}
n_constraints = len(constraint_enum_dict)
constraint_names = [
constraint_name_dict[spec[0]]
for spec in iter(opt_grp['constraint_spec'])
]
parameter_enum_dict = h5py.check_enum_dtype(
opt_grp['parameter_enum'].dtype)
parameters_idx_dict = {
parm: idx
for parm, idx in parameter_enum_dict.items()
}
parameters_name_dict = {
idx: parm
for parm, idx in parameters_idx_dict.items()
}
problem_parameters = {
parameters_name_dict[idx]: val
for idx, val in opt_grp['problem_parameters']
}
parameter_specs = [(parameters_name_dict[spec[0]], tuple(spec)[1:])
for spec in iter(opt_grp['parameter_spec'])]
problem_ids = None
if 'problem_ids' in opt_grp:
problem_ids = set(opt_grp['problem_ids'])
raw_results = {}
for problem_id in problem_ids if problem_ids is not None else [0]:
if str(problem_id) in opt_grp:
raw_results[problem_id] = {
'objectives': opt_grp[str(problem_id)]['objectives'][:],
'parameters': opt_grp[str(problem_id)]['parameters'][:]
}
if 'features' in opt_grp[str(problem_id)]:
raw_results[problem_id]['features'] = opt_grp[str(
problem_id)]['features'][:]
if 'constraints' in opt_grp[str(problem_id)]:
raw_results[problem_id]['constraints'] = opt_grp[str(
problem_id)]['constraints'][:]
f.close()
param_names = []
is_integer = []
lower = []
upper = []
for parm, spec in parameter_specs:
param_names.append(parm)
is_int, lo, hi = spec
is_integer.append(is_int)
lower.append(lo)
upper.append(hi)
raw_spec = (is_integer, lower, upper)
info = {
'features': feature_names,
'feature_types': feature_types,
'constraints': constraint_names,
'params': param_names,
'solver_epsilon': solver_epsilon,
'relative_noise_magnitude': relative_noise_magnitude,
'problem_parameters': problem_parameters,
'problem_ids': problem_ids
}
return raw_spec, raw_results, info
