def test_get_spec_values_illegal(self):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
usi_main = USIDataset(h5_f['/Raw_Measurement/source_main'])
with self.assertRaises(KeyError):
_ = usi_main.get_spec_values('blah')
with self.assertRaises(TypeError):
_ = usi_main.get_spec_values(np.array(5))
def test_get_pos_spec_slices_pos_and_spec_sliced_list(self):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
pycro_main = USIDataset(h5_f['/Raw_Measurement/source_main'])
h5_pos_inds = pycro_main.h5_pos_inds
h5_spec_inds = pycro_main.h5_spec_inds
actual_pos, actual_spec = pycro_main._get_pos_spec_slices({
'X': [1, 2, 4],
'Bias':
slice(1, 7, 3)
})
# we want every fifth position starting from 3
positions = []
for col_ind in [1, 2, 4]:
positions += np.argwhere(h5_pos_inds[h5_pos_inds.attrs['X']] ==
col_ind)[:, 0].tolist()
specs = []
for bias_ind in range(1, 7, 3):
specs += np.argwhere(h5_spec_inds[h5_spec_inds.attrs['Bias']]
== bias_ind)[:, 1].tolist()
expected_pos = np.expand_dims(positions, axis=1)
expected_spec = np.expand_dims(specs, axis=1)
expected_pos.sort(axis=0)
expected_spec.sort(axis=0)
self.assertTrue(np.allclose(expected_spec, actual_spec))
self.assertTrue(np.allclose(expected_pos, actual_pos))
def test_get_n_dim_form_unsorted(self):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
h5_main = h5_f['/Raw_Measurement/source_main']
expected = np.reshape(h5_main, (3, 5, 7, 2))
expected = np.transpose(expected, (1, 0, 2, 3))
pycro_dset = USIDataset(h5_main)
self.assertTrue(np.allclose(expected, pycro_dset.get_n_dim_form()))
def setUp(self):
data_utils.make_beps_file()
self.orig_labels_order = ['X', 'Y', 'Cycle', 'Bias']
self.h5_file = h5py.File(data_utils.std_beps_path, mode='r')
h5_grp = self.h5_file['/Raw_Measurement/']
self.source_nd_s2f = h5_grp['n_dim_form'][()]
self.source_nd_f2s = self.source_nd_s2f.transpose(1, 0, 3, 2)
self.h5_source = USIDataset(h5_grp['source_main'])
self.pos_dims=[]
self.spec_dims=[]
for dim_name, dim_units in zip(self.h5_source.pos_dim_labels,
hdf_utils.get_attr(self.h5_source.h5_pos_inds, 'units')):
self.pos_dims.append(
Dimension(dim_name, dim_units, h5_grp[dim_name][()]))
for dim_name, dim_units in zip(self.h5_source.spec_dim_labels,
hdf_utils.get_attr(self.h5_source.h5_spec_inds, 'units')):
self.spec_dims.append(
Dimension(dim_name, dim_units, h5_grp[dim_name][()]))
res_grp_0 = h5_grp['source_main-Fitter_000']
self.results_0_nd_s2f = res_grp_0['n_dim_form'][()]
self.results_0_nd_f2s = self.results_0_nd_s2f.transpose(1, 0, 3, 2)
self.h5_compound = USIDataset(res_grp_0['results_main'])
res_grp_1 = h5_grp['source_main-Fitter_001']
self.results_1_nd_s2f = res_grp_1['n_dim_form'][()]
self.results_1_nd_f2s = self.results_1_nd_s2f.transpose(1, 0, 3, 2)
self.h5_complex = USIDataset(res_grp_1['results_main'])
def test_equality_incorrect_USIDataset(self):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
h5_main = h5_f['/Raw_Measurement/source_main']
expected = USIDataset(h5_main)
incorrect = USIDataset(
h5_f['/Raw_Measurement/source_main-Fitter_000/results_main'])
self.assertFalse(expected == incorrect)
def test_get_spec_values(self):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
usi_main = USIDataset(h5_f['/Raw_Measurement/source_main'])
for dim_name in ['Bias', 'Cycle']:
expected = h5_f['/Raw_Measurement/' + dim_name][()]
actual = usi_main.get_spec_values(dim_name)
self.assertTrue(np.allclose(expected, actual))
def test_sorted(self):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
h5_main = h5_f['/Raw_Measurement/source_main']
expected = np.reshape(h5_main, (3, 5, 7, 2))
expected = np.transpose(expected, (1, 0, 3, 2))
usi_dset = USIDataset(h5_main)
usi_dset.toggle_sorting()
self.assertTrue(np.allclose(expected, usi_dset.get_n_dim_form(lazy=False)))
def test_both_pos_removed(self):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
usi_main = USIDataset(h5_f['/Raw_Measurement/source_main'])
actual, success = usi_main.slice({'X': 3, 'Y': 1}, lazy=False)
n_dim_form = np.transpose(np.reshape(usi_main[()], (3, 5, 7, 2)), (1, 0, 2, 3))
expected = n_dim_form[3, 1, :, :]
self.assertTrue(np.allclose(expected, actual))
self.assertTrue(success)
def test_pos_and_spec_sliced_list(self):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
usi_main = USIDataset(h5_f['/Raw_Measurement/source_main'])
actual, success = usi_main.slice({'X': [1, 2, 4], 'Bias': slice(1, 7, 3)}, lazy=False)
n_dim_form = np.transpose(np.reshape(usi_main[()], (3, 5, 7, 2)), (1, 0, 2, 3))
expected = n_dim_form[[1, 2, 4], :, slice(1, 7, 3), :]
self.assertTrue(np.allclose(expected, actual))
self.assertTrue(success)
def test_get_pos_values(self):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
usi_main = USIDataset(h5_f['/Raw_Measurement/source_main'])
expected = usi_main.h5_pos_vals[:5, 0]
actual = usi_main.get_pos_values('X')
self.assertTrue(np.allclose(expected, actual))
expected = usi_main.h5_pos_vals[0:None:5, 1]
actual = usi_main.get_pos_values('Y')
self.assertTrue(np.allclose(expected, actual))
def test_get_spec_values(self):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
usi_main = USIDataset(h5_f['/Raw_Measurement/source_main'])
expected = usi_main.h5_spec_vals[0, ::2]
actual = usi_main.get_spec_values('Bias')
self.assertTrue(np.allclose(expected, actual))
expected = usi_main.h5_spec_vals[1, 0:None:7]
actual = usi_main.get_spec_values('Cycle')
self.assertTrue(np.allclose(expected, actual))
def test_slice_two_pos_dim_sliced_list(self):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
pycro_main = USIDataset(h5_f['/Raw_Measurement/source_main'])
actual, success = pycro_main.slice({'X': [1, 2, 4], 'Y': 1})
n_dim_form = np.transpose(np.reshape(pycro_main[()], (3, 5, 7, 2)),
(1, 0, 2, 3))
expected = n_dim_form[[1, 2, 4], 1, :, :]
self.assertTrue(np.allclose(expected, actual))
self.assertTrue(success)
def test_get_pos_spec_slices_non_existent_dim(self):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
pycro_main = USIDataset(h5_f['/Raw_Measurement/source_main'])
with self.assertRaises(KeyError):
_ = pycro_main._get_pos_spec_slices({
'blah': 4,
'X': 3,
'Y': 1
})
def test_empty_dict(self):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
usi_main = USIDataset(h5_f['/Raw_Measurement/source_main'])
actual_pos, actual_spec = usi_main._get_pos_spec_slices({})
self.assertTrue(
np.allclose(np.expand_dims(np.arange(14), axis=1),
actual_spec))
self.assertTrue(
np.allclose(np.expand_dims(np.arange(15), axis=1), actual_pos))
def test_both_pos_removed(self):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
usi_main = USIDataset(h5_f['/Raw_Measurement/source_main'])
actual_pos, actual_spec = usi_main._get_pos_spec_slices({'X': 3, 'Y': 1})
# we want every fifth position starting from 3
expected_pos = np.expand_dims([1 * 5 + 3], axis=1)
expected_spec = np.expand_dims(np.arange(14), axis=1)
self.assertTrue(np.allclose(expected_spec, actual_spec))
self.assertTrue(np.allclose(expected_pos, actual_pos))
def test_toggle_sorting(self):
# Need to change data file so that sorting actually does something
with h5py.File(test_h5_file_path, mode='r') as h5_f:
usi_main = USIDataset(h5_f['/Raw_Measurement/source_main'])
self.assertTrue(usi_main.n_dim_labels == ['X', 'Y', 'Bias', 'Cycle'])
usi_main.toggle_sorting()
self.assertTrue(usi_main.n_dim_labels==['X', 'Y', 'Cycle', 'Bias'])
def base(self, slice_dict, pos_exp, spec_exp, verbose=False):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
usi_main = USIDataset(h5_f['/Raw_Measurement/source_main'])
pos_act, spec_act = usi_main._get_dims_for_slice(
slice_dict=slice_dict, verbose=verbose)
if verbose:
print(pos_act)
print(spec_act)
self.__validate_dim_list(pos_act, pos_exp)
self.__validate_dim_list(spec_act, spec_exp)
def test_one_pos_dim_removed(self):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
usi_main = USIDataset(h5_f['/Raw_Measurement/source_main'])
# orig_pos = np.vstack([np.tile(np.arange(5), 3), np.repeat(np.arange(3), 5)]).T
# orig_spec = np.vstack([np.tile(np.arange(7), 2), np.repeat(np.arange(2), 7)])
actual_pos, actual_spec = usi_main._get_pos_spec_slices({'X': 3})
# we want every fifth position starting from 3
expected_pos = np.expand_dims(np.arange(3, 15, 5), axis=1)
expected_spec = np.expand_dims(np.arange(14), axis=1)
self.assertTrue(np.allclose(expected_spec, actual_spec))
self.assertTrue(np.allclose(expected_pos, actual_pos))
def test_two_pos_dim_sliced_list(self):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
usi_main = USIDataset(h5_f['/Raw_Measurement/source_main'])
actual_pos, actual_spec = usi_main._get_pos_spec_slices({'X': [1, 2, 4], 'Y': 1})
# we want every fifth position starting from 3
positions = []
for row_ind in range(1, 2):
for col_ind in [1, 2, 4]:
positions.append(5 * row_ind + col_ind)
expected_pos = np.expand_dims(positions, axis=1)
expected_spec = np.expand_dims(np.arange(14), axis=1)
self.assertTrue(np.allclose(expected_spec, actual_spec))
self.assertTrue(np.allclose(expected_pos, actual_pos))
def test_two_spec(self):
if skip_viz_tests: return
with h5py.File(test_h5_file_path, mode='r') as h5_f:
dset_path = '/Raw_Measurement/source_main'
usi_main = USIDataset(h5_f[dset_path])
slice_dict = {'X': 3, 'Y': 2}
exp_data, success = usi_main.slice(slice_dict=slice_dict)
self.assertTrue(success)
fig, axis = usi_main.visualize(slice_dict=slice_dict)
validate_imshow(self, axis, exp_data, title=dset_path,
x_vec=h5_f['/Raw_Measurement/' + usi_main.spec_dim_labels[1]],
y_vec=h5_f['/Raw_Measurement/' + usi_main.spec_dim_labels[0]],
x_label=usi_main.spec_dim_descriptors[1],
y_label=usi_main.spec_dim_descriptors[0])
def test_incorrect_h5_dataset(self):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
h5_main = h5_f['/Raw_Measurement/source_main']
expected = USIDataset(h5_main)
incorrect = h5_f[
'/Raw_Measurement/source_main-Fitter_000/Spectroscopic_Indices']
self.assertFalse(expected == incorrect)
def test_one_spec(self):
if skip_viz_tests: return
with h5py.File(test_h5_file_path, mode='r') as h5_f:
dset_path = '/Raw_Measurement/source_main'
usi_main = USIDataset(h5_f[dset_path])
slice_dict = {'Bias': 4, 'X': 1, 'Y': 2}
rem_dim_name = 'Cycle'
spec_ind = usi_main.spec_dim_labels.index(rem_dim_name)
exp_data, success = usi_main.slice(slice_dict=slice_dict)
self.assertTrue(success)
fig, axis = usi_main.visualize(slice_dict=slice_dict)
validate_single_curve(self, axis, h5_f['/Raw_Measurement/' + rem_dim_name],
exp_data,
title=dset_path,
x_label=usi_main.spec_dim_descriptors[spec_ind],
y_label=usi_main.data_descriptor)
def test_string_representation(self):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
h5_main = h5_f['/Raw_Measurement/source_main']
usi_dset = USIDataset(h5_main)
actual = usi_dset.__repr__()
actual = [line.strip() for line in actual.split("\n")]
actual = [actual[line_ind] for line_ind in [0, 2, 4, 7, 8, 10, 11]]
expected = list()
expected.append(h5_main.__repr__())
expected.append(h5_main.name)
expected.append(hdf_utils.get_attr(h5_main, "quantity") + " (" + hdf_utils.get_attr(h5_main, "units") + ")")
for h5_inds in [usi_dset.h5_pos_inds, usi_dset.h5_spec_inds]:
for dim_name, dim_size in zip(hdf_utils.get_attr(h5_inds, "labels"),
hdf_utils.get_dimensionality(h5_inds)):
expected.append(dim_name + ' - size: ' + str(dim_size))
self.assertTrue(np.all([x == y for x, y in zip(actual, expected)]))
def base(self,
slice_dict,
f2s_slice_list,
result_as_nd,
lazy_result,
verbose=False):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
usi_main = USIDataset(h5_f['/Raw_Measurement/source_main'])
actual, success = usi_main.slice(slice_dict,
ndim_form=result_as_nd,
lazy=lazy_result,
verbose=verbose)
if verbose:
print('Status: {}, actual.shape: {}, actual.dtype: {}, '
'type(actual): {}'.format(success, actual.shape,
actual.dtype, type(actual)))
self.assertTrue(success)
n_dim_s2f, n_dim_f2s = self.get_expected_n_dim(h5_f)
if result_as_nd:
expected = n_dim_f2s[tuple(f2s_slice_list)]
expected = expected.squeeze()
else:
s2f_slice_list = f2s_slice_list[:2][::-1] + \
f2s_slice_list[2:][::-1]
if verbose:
print('Slice list converted from: {} to {}'
''.format(f2s_slice_list, s2f_slice_list))
expected = n_dim_s2f[tuple(s2f_slice_list)]
if verbose:
print('Expected in N-dim form: {}'.format(expected.shape))
expected = expected.reshape(np.prod(expected.shape[:2]),
np.prod(expected.shape[2:]))
if verbose:
print('Expected after flattening of shape: {}'
''.format(expected.shape))
if lazy_result:
self.assertIsInstance(actual, da.core.Array)
actual = actual.compute()
self.assertTrue(np.allclose(expected, actual))
def get_all_dimensions():
pos_dims = []
spec_dims = []
with h5py.File(test_h5_file_path, mode='r') as h5_f:
h5_raw_grp = h5_f['Raw_Measurement']
usi_main = USIDataset(h5_raw_grp['source_main'])
for dim_name, dim_units in zip(usi_main.pos_dim_labels,
hdf_utils.get_attr(usi_main.h5_pos_inds, 'units')):
pos_dims.append(Dimension(dim_name, dim_units, h5_raw_grp[dim_name][()]))
for dim_name, dim_units in zip(usi_main.spec_dim_labels,
hdf_utils.get_attr(
usi_main.h5_spec_inds, 'units')):
spec_dims.append(Dimension(dim_name, dim_units, h5_raw_grp[dim_name][()]))
return pos_dims, spec_dims
def test_get_current_sorting(self):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
usi_main = USIDataset(h5_f['/Raw_Measurement/source_main'])
unsorted_str = 'Data dimensions are in the order they occur in the file.\n'
sorted_str = 'Data dimensions are sorted in order from fastest changing dimension to slowest.\n'
# Initial state should be unsorted
self.assertFalse(usi_main._USIDataset__sort_dims)
with data_utils.capture_stdout() as get_value:
usi_main.get_current_sorting()
test_str = get_value()
self.assertTrue(test_str == unsorted_str)
# Toggle sorting. Sorting should now be true.
usi_main.toggle_sorting()
self.assertTrue(usi_main._USIDataset__sort_dims)
with data_utils.capture_stdout() as get_value:
usi_main.get_current_sorting()
test_str = get_value()
self.assertTrue(test_str == sorted_str)
def test_sorted_and_unsorted(self):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
usi_dset = USIDataset(h5_f['/Raw_Measurement/source_main'])
nd_slow_to_fast, nd_fast_to_slow = self.get_expected_n_dim(h5_f)
actual_f2s = usi_dset.get_n_dim_form(lazy=False)
self.assertTrue(np.allclose(nd_fast_to_slow, actual_f2s))
nd_form, success = reshape_to_n_dims(usi_dset, sort_dims=True)
print(nd_form.shape)
usi_dset.toggle_sorting()
actual_s2f = usi_dset.get_n_dim_form(lazy=False)
self.assertTrue(np.allclose(nd_slow_to_fast, actual_s2f))
def basic_file_validation(self, h5_f):
self.assertEqual('ImageTranslator',
hdf_utils.get_attr(h5_f, 'translator'))
# First level should have absolutely nothing besides one group
self.assertEqual(len(h5_f.items()), 1)
self.assertTrue('Measurement_000' in h5_f.keys())
h5_meas_grp = h5_f['Measurement_000']
self.assertIsInstance(h5_meas_grp, h5py.Group)
# Again, this group should only have one group - Channel_000
self.assertEqual(len(h5_meas_grp.items()), 1)
self.assertTrue('Channel_000' in h5_meas_grp.keys())
h5_chan_grp = h5_meas_grp['Channel_000']
self.assertIsInstance(h5_chan_grp, h5py.Group)
# This channel group is not expected to have any (custom) attributes but it will contain the main dataset
self.assertEqual(len(h5_chan_grp.items()), 5)
for dset_name in [
'Raw_Data', 'Position_Indices', 'Position_Values',
'Spectroscopic_Indices', 'Spectroscopic_Values'
]:
self.assertTrue(dset_name in h5_chan_grp.keys())
h5_dset = h5_chan_grp[dset_name]
self.assertIsInstance(h5_dset, h5py.Dataset)
usid_main = USIDataset(h5_chan_grp['Raw_Data'])
self.assertIsInstance(usid_main, USIDataset)
self.assertEqual(usid_main.name.split('/')[-1], 'Raw_Data')
self.assertEqual(usid_main.parent, h5_chan_grp)
validate_aux_dset_pair(self,
h5_chan_grp,
usid_main.h5_spec_inds,
usid_main.h5_spec_vals, ['arb'], ['a.u.'],
np.atleast_2d([0]),
h5_main=usid_main,
is_spectral=True)
def test_negative_index_2d_numpy(self):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
usi_main = USIDataset(h5_f['/Raw_Measurement/source_main'])
with self.assertRaises(ValueError):
_ = usi_main.slice({'X': -2, 'Y': 1}, ndim_form=False)
def test_out_of_bounds(self):
with h5py.File(test_h5_file_path, mode='r') as h5_f:
usi_main = USIDataset(h5_f['/Raw_Measurement/source_main'])
with self.assertRaises(IndexError):
_ = usi_main.slice({'X': 15, 'Y': 1})
