def test_writing_and_reading_dicts_to_hdf5_int_keys(self):
test_dict = {
0: {
"name": "I",
"theta": 0,
"phi": 0,
"type": "ge"
},
1: {
"name": "rX180",
"theta": 180,
"phi": 0,
"type": "ge"
}
}
data_object = h5d.Data(name='test_object', datadir=self.datadir)
h5d.write_dict_to_hdf5(test_dict, data_object)
data_object.close()
filepath = data_object.filepath
new_dict = {}
opened_hdf5_file = h5py.File(filepath, 'r')
h5d.read_dict_from_hdf5(new_dict, opened_hdf5_file)
self.assertEqual(test_dict.keys(), new_dict.keys())
self.assertEqual(test_dict[0], new_dict[0])
def test_storing_and_loading_station_snapshot(self):
"""
Stores and writes a station (instrument) snapshot.
"""
self.mock_parabola_2.x(1)
self.mock_parabola_2.y(2.245)
self.mock_parabola_2.array_like(np.linspace(0, 11, 23))
snap = self.station.snapshot(update=True)
data_object = h5d.Data(name='test_object_snap', datadir=self.datadir)
h5d.write_dict_to_hdf5(snap, data_object)
data_object.close()
filepath = data_object.filepath
new_dict = {}
opened_hdf5_file = h5py.File(filepath, 'r')
h5d.read_dict_from_hdf5(new_dict, opened_hdf5_file)
self.assertEqual(snap.keys(), new_dict.keys())
self.assertEqual(snap['instruments'].keys(),
new_dict['instruments'].keys())
mock_parab_pars = snap['instruments']['mock_parabola_2']['parameters']
self.assertEqual(mock_parab_pars['x']['value'], 1)
self.assertEqual(mock_parab_pars['y']['value'], 2.245)
np.testing.assert_array_equal(mock_parab_pars['array_like']['value'],
np.linspace(0, 11, 23))
def test_save_exp_metadata(self):
metadata_dict = {
'intParam': 1,
'floatParam': 2.5e-3,
'strParam': 'spam',
'listParam': [1, 2, 3, 4],
'arrayParam': np.array([4e5, 5e5]),
'dictParam': {'a': 1, 'b': 2},
'tupleParam': (3, 'c')
}
old_a_tools_datadir = a_tools.datadir
a_tools.datadir = self.MC.datadir()
sweep_pts = np.linspace(0, 10, 30)
self.MC.set_sweep_function(None_Sweep())
self.MC.set_sweep_points(sweep_pts)
self.MC.set_detector_function(det.Dummy_Detector_Soft())
self.MC.run('test_exp_metadata', exp_metadata=metadata_dict)
a = ma.MeasurementAnalysis(label='test_exp_metadata', auto=False)
a_tools.datadir = old_a_tools_datadir
loaded_dict = read_dict_from_hdf5(
{}, a.data_file['Experimental Data']['Experimental Metadata'])
np.testing.assert_equal(metadata_dict, loaded_dict)
def test_save_exp_metadata(self):
metadata_dict = {
"intParam": 1,
"floatParam": 2.5e-3,
"strParam": "spam",
"listParam": [1, 2, 3, 4],
"arrayParam": np.array([4e5, 5e5]),
"dictParam": {
"a": 1,
"b": 2
},
"tupleParam": (3, "c"),
}
old_a_tools_datadir = a_tools.datadir
a_tools.datadir = self.MC.datadir()
sweep_pts = np.linspace(0, 10, 30)
self.MC.set_sweep_function(None_Sweep())
self.MC.set_sweep_points(sweep_pts)
self.MC.set_detector_function(det.Dummy_Detector_Soft())
self.MC.run("test_exp_metadata", exp_metadata=metadata_dict)
a = ma.MeasurementAnalysis(label="test_exp_metadata", auto=False)
a_tools.datadir = old_a_tools_datadir
loaded_dict = read_dict_from_hdf5(
{}, a.data_file["Experimental Data"]["Experimental Metadata"])
np.testing.assert_equal(metadata_dict, loaded_dict)
def test_wr_rd_hdf5_array():
datadir = os.path.join(pq.__path__[0], 'tests', 'test_data')
test_dict = {
'x': np.linspace(0, 1, 14),
'y': np.cos(np.linspace(0, 2*np.pi, 11))}
data_object = h5d.Data(name='test_object', datadir=datadir)
h5d.write_dict_to_hdf5(test_dict, data_object)
data_object.close()
filepath = data_object.filepath
new_dict = {}
opened_hdf5_file = h5py.File(filepath, 'r')
h5d.read_dict_from_hdf5(new_dict, opened_hdf5_file)
assert test_dict.keys() == new_dict.keys()
np.testing.assert_allclose(test_dict['x'], new_dict['x'])
np.testing.assert_allclose(test_dict['y'], new_dict['y'])
def get_param_from_metadata_group(timestamp=None, param_name=None, file_id=None,
data_file=None, close_file=True, mode='r'):
"""
Get a parameter with param_name from the Experimental Metadata group in
the HDF5 file specified by timestamp, or return the whole group if
param_name is None.
:param timestamp: (str) measurement timestamp of form YYYYMMDD_hhmmsss
:param param_name: (str) name of a key in Experimental Metadata group
:param data_file: (HDF file) opened HDF5 file
:param close_file: (bool) whether to close the HDF5 file
:return: the value of the param_name or the whole experimental metadata
dictionary
"""
if data_file is None:
if timestamp is None:
raise ValueError('Please provide either timestamp or data_file.')
folder = a_tools.get_folder(timestamp)
h5filepath = a_tools.measurement_filename(folder, file_id=file_id)
data_file = h5py.File(h5filepath, mode)
try:
if param_name is None:
group = data_file['Experimental Data']
return read_dict_from_hdf5({}, group['Experimental Metadata'])
group = data_file['Experimental Data']['Experimental Metadata']
if param_name in group:
group = group[param_name]
param_value = OrderedDict()
if isinstance(group, h5py._hl.dataset.Dataset):
param_value = list(np.array(group).flatten())
param_value = [x.decode('utf-8') if isinstance(x, bytes)
else x for x in param_value]
else:
param_value = read_dict_from_hdf5(param_value, group)
elif param_name in group.attrs:
param_value = get_hdf_param_value(group, param_name)
else:
raise KeyError(f'{param_name} was not found in metadata.')
if close_file:
data_file.close()
except Exception as e:
data_file.close()
raise e
return param_value
def load_settings_onto_instrument_v2(
instrument,
load_from_instr: str = None,
label: str = "",
filepath: str = None,
timestamp: str = None,
ignore_pars: set = None,
):
"""
Loads settings from an hdf5 file onto the instrument handed to the
function. By default uses the last hdf5 file in the datadirectory.
By giving a label or timestamp another file can be chosen as the
settings file.
Args:
instrument (instrument) : instrument onto which settings should be
loaded
load_from_instr (str) : optional name of another instrument from
which to load the settings.
label (str) : label used for finding the last datafile
filepath (str) : exact filepath of the hdf5 file to load.
if filepath is specified, this takes precedence over the file
locating options (label, timestamp etc.).
timestamp (str) : timestamp of file in the datadir
"""
older_than = None
folder = None
instrument_name = instrument.name
success = False
count = 0
# Will try multiple times in case the last measurements failed and
# created corrupt data files.
while success is False and count < 3:
if filepath is None:
folder = a_tools.get_folder(
timestamp=timestamp, label=label, older_than=older_than
)
filepath = a_tools.measurement_filename(folder)
try:
f = h5py.File(filepath, "r")
snapshot = {}
read_dict_from_hdf5(snapshot, h5_group=f["Snapshot"])
if load_from_instr is None:
ins_group = snapshot["instruments"][instrument_name]
else:
ins_group = snapshot["instruments"][load_from_instr]
success = True
except Exception as e:
logging.warning("Exception occured reading from {}".format(folder))
logging.warning(e)
# This check makes this snippet a bit more robust
if folder is not None:
older_than = (
os.path.split(folder)[0][-8:] + "_" + os.path.split(folder)[1][:6]
)
# important to set all to None, otherwise the try except loop
# will not look for an earlier data file
folder = None
filepath = None
success = False
count += 1
if not success:
logging.warning(
'Could not open settings for instrument "%s"' % (instrument_name)
)
return False
for parname, par in ins_group["parameters"].items():
try:
if hasattr(instrument.parameters[parname], "set") and (
par["value"] is not None
):
if ignore_pars is None or parname not in ignore_pars:
par_value = par["value"]
if type(par_value) == str:
try:
instrument.parameters[parname].validate(par_value)
except TypeError:
# This detects that in the hdf5 file the parameter
# was saved as string due to type incompatibility
par_value = eval(par_value)
instrument.set(parname, par_value)
except Exception as e:
print(
'Could not set parameter: "{}" to "{}" '
'for instrument "{}"'.format(parname, par["value"], instrument_name)
)
logging.warning(e)
f.close()
return True
def load_settings_onto_instrument_v2(instrument,
load_from_instr: str = None,
label: str = '',
filepath: str = None,
timestamp: str = None):
'''
Loads settings from an hdf5 file onto the instrument handed to the
function. By default uses the last hdf5 file in the datadirectory.
By giving a label or timestamp another file can be chosen as the
settings file.
Args:
instrument (instrument) : instrument onto which settings should be
loaded
load_from_instr (str) : optional name of another instrument from
which to load the settings.
label (str) : label used for finding the last datafile
filepath (str) : exact filepath of the hdf5 file to load.
if filepath is specified, this takes precedence over the file
locating options (label, timestamp etc.).
timestamp (str) : timestamp of file in the datadir
'''
older_than = None
folder = None
instrument_name = instrument.name
success = False
count = 0
# Will try multiple times in case the last measurements failed and
# created corrupt data files.
while success is False and count < 3:
if filepath is None:
folder = a_tools.get_folder(timestamp=timestamp,
label=label,
older_than=older_than)
filepath = a_tools.measurement_filename(folder)
try:
f = h5py.File(filepath, 'r')
snapshot = {}
h5d.read_dict_from_hdf5(snapshot, h5_group=f['Snapshot'])
if load_from_instr is None:
ins_group = snapshot['instruments'][instrument_name]
else:
ins_group = snapshot['instruments'][load_from_instr]
success = True
except Exception as e:
logging.warning('Exception occured reading from {}'.format(folder))
logging.warning(e)
# This check makes this snippet a bit more robust
if folder is not None:
older_than = os.path.split(folder)[0][-8:] \
+ '_' + os.path.split(folder)[1][:6]
# important to set all to None, otherwise the try except loop
# will not look for an earlier data file
folder = None
filepath = None
success = False
count += 1
if not success:
logging.warning('Could not open settings for instrument "%s"' %
(instrument_name))
return False
for parname, par in ins_group['parameters'].items():
try:
if hasattr(instrument.parameters[parname], 'set'):
instrument.set(parname, par['value'])
except Exception as e:
print('Could not set parameter: "{}" to "{}" '
'for instrument "{}"'.format(parname, par['value'],
instrument_name))
logging.warning(e)
f.close()
return True
def test_writing_and_reading_dicts_to_hdf5(self):
"""
Tests dumping some random dictionary to hdf5 and reading back the
stored values. The input dictionary contains:
- list of ints
- list of floats
- nested dict
- 1D array
- 2D array
"""
test_dict = {
'list_of_ints': list(np.arange(5)),
'list_of_floats': list(np.arange(5.1)),
'some_bool': True,
'weird_dict': {'a': 5},
'dataset1': np.linspace(0, 20, 31),
'dataset2': np.array([[2, 3, 4, 5],
[2, 3, 1, 2]]),
'list_of_mixed_type': ['hello', 4, 4.2, {'a': 5}, [4, 3]],
'tuple_of_mixed_type': tuple(['hello', 4, 4.2, {'a': 5}, [4, 3]]),
'a list of strings': ['my ', 'name ', 'is ', 'earl.'],
'some_np_bool': np.bool(True),
'list_of_dicts': [{'a': 5}, {'b': 3}],
'some_int': 3,
'some_float': 3.5,
'some_np_int': np.int(3),
'some_np_float': np.float(3.5)
}
data_object = h5d.Data(name='test_object', datadir=self.datadir)
h5d.write_dict_to_hdf5(test_dict, data_object)
data_object.close()
filepath = data_object.filepath
new_dict = {}
opened_hdf5_file = h5py.File(filepath, 'r')
try:
h5d.read_dict_from_hdf5(new_dict, opened_hdf5_file)
# objects are not identical but the string representation should be
self.assertEqual(test_dict.keys(), new_dict.keys())
self.assertEqual(test_dict['list_of_ints'], new_dict['list_of_ints'])
self.assertEqual(test_dict['list_of_floats'],
new_dict['list_of_floats'])
self.assertEqual(test_dict['weird_dict'], new_dict['weird_dict'])
self.assertEqual(test_dict['some_bool'], new_dict['some_bool'])
self.assertEqual(test_dict['list_of_dicts'],
new_dict['list_of_dicts'])
self.assertEqual(test_dict['list_of_mixed_type'],
new_dict['list_of_mixed_type'])
self.assertEqual(test_dict['list_of_mixed_type'][0],
new_dict['list_of_mixed_type'][0])
self.assertEqual(test_dict['list_of_mixed_type'][2],
new_dict['list_of_mixed_type'][2])
self.assertEqual(test_dict['tuple_of_mixed_type'],
new_dict['tuple_of_mixed_type'])
self.assertEqual(type(test_dict['tuple_of_mixed_type']),
type(new_dict['tuple_of_mixed_type']))
self.assertEqual(test_dict['tuple_of_mixed_type'][0],
new_dict['tuple_of_mixed_type'][0])
self.assertEqual(test_dict['tuple_of_mixed_type'][2],
new_dict['tuple_of_mixed_type'][2])
self.assertEqual(test_dict['some_np_bool'],
new_dict['some_np_bool'])
self.assertEqual(test_dict['some_int'], new_dict['some_int'])
self.assertEqual(test_dict['some_np_float'], new_dict['some_np_float'])
self.assertEqual(test_dict['a list of strings'],
new_dict['a list of strings'])
self.assertEqual(test_dict['a list of strings'][0],
new_dict['a list of strings'][0])
opened_hdf5_file.close()
except Exception as e:
opened_hdf5_file.close()
raise e
def get_params_from_hdf_file(data_dict, params_dict=None, numeric_params=None,
add_param_method=None, folder=None, **params):
"""
Extracts the parameter provided in params_dict from an HDF file
and saves them in data_dict.
:param data_dict: OrderedDict where parameters and their values are saved
:param params_dict: OrderedDict with key being the parameter name that will
be used as key in data_dict for this parameter, and value being a
parameter name or a path + parameter name indie the HDF file.
:param numeric_params: list of parameter names from amount the keys of
params_dict. This specifies that those parameters are numbers and will
be converted to floats.
:param folder: path to file from which data will be read
:param params: keyword arguments:
append_value (bool, default: True): whether to append an
already-existing key
update_value (bool, default: False): whether to replace an
already-existing key
h5mode (str, default: 'r+'): reading mode of the HDF file
close_file (bool, default: True): whether to close the HDF file(s)
"""
if params_dict is None:
params_dict = get_param('params_dict', data_dict, raise_error=True,
**params)
if numeric_params is None:
numeric_params = get_param('numeric_params', data_dict,
default_value=[], **params)
# if folder is not specified, will take the last folder in the list from
# data_dict['folders']
if folder is None:
folder = get_param('folders', data_dict, raise_error=True, **params)
if len(folder) > 0:
folder = folder[-1]
h5mode = get_param('h5mode', data_dict, default_value='r', **params)
h5filepath = a_tools.measurement_filename(folder, **params)
data_file = h5py.File(h5filepath, h5mode)
try:
for save_par, file_par in params_dict.items():
epd = data_dict
all_keys = save_par.split('.')
for i in range(len(all_keys)-1):
if all_keys[i] not in epd:
epd[all_keys[i]] = OrderedDict()
epd = epd[all_keys[i]]
if isinstance(epd, list):
epd = epd[-1]
if file_par == 'measurementstring':
add_param(all_keys[-1],
[os.path.split(folder)[1][7:]],
epd, add_param_method='append')
continue
group_name = '/'.join(file_par.split('.')[:-1])
par_name = file_par.split('.')[-1]
if group_name == '':
group = data_file
attrs = []
else:
group = data_file[group_name]
attrs = list(group.attrs)
if group_name in data_file or group_name == '':
if par_name in attrs:
add_param(all_keys[-1],
get_hdf_param_value(group,
par_name),
epd, add_param_method=add_param_method)
elif par_name in list(group.keys()) or file_par == '':
par = group[par_name] if par_name != '' else group
if isinstance(par,
h5py._hl.dataset.Dataset):
add_param(all_keys[-1],
np.array(par),
epd, add_param_method=add_param_method)
else:
add_param(all_keys[-1],
read_dict_from_hdf5(
{}, par),
epd, add_param_method=add_param_method)
if all_keys[-1] not in epd:
# search through the attributes of all groups
for group_name in data_file.keys():
if par_name in list(data_file[group_name].attrs):
add_param(all_keys[-1],
get_hdf_param_value(data_file[group_name],
par_name),
epd, add_param_method=add_param_method)
if all_keys[-1] not in epd:
log.warning(f'Parameter {file_par} was not found.')
data_file.close()
except Exception as e:
data_file.close()
raise e
for par_name in data_dict:
if par_name in numeric_params:
if hasattr(data_dict[par_name], '__iter__'):
data_dict[par_name] = [np.float(p) for p
in data_dict[par_name]]
data_dict[par_name] = np.asarray(data_dict[par_name])
else:
data_dict[par_name] = np.float(data_dict[par_name])
if get_param('close_file', data_dict, default_value=True, **params):
data_file.close()
else:
if 'data_files' in data_dict:
data_dict['data_files'] += [data_file]
else:
data_dict['data_files'] = [data_file]
return data_dict
