def test_get_spec_values_illegal(self): with h5py.File(test_h5_file_path, mode='r') as h5_f: usi_main = USIDataset(h5_f['/Raw_Measurement/source_main']) with self.assertRaises(KeyError): _ = usi_main.get_spec_values('blah') with self.assertRaises(TypeError): _ = usi_main.get_spec_values(np.array(5))
def test_get_pos_spec_slices_pos_and_spec_sliced_list(self): with h5py.File(test_h5_file_path, mode='r') as h5_f: pycro_main = USIDataset(h5_f['/Raw_Measurement/source_main']) h5_pos_inds = pycro_main.h5_pos_inds h5_spec_inds = pycro_main.h5_spec_inds actual_pos, actual_spec = pycro_main._get_pos_spec_slices({ 'X': [1, 2, 4], 'Bias': slice(1, 7, 3) }) # we want every fifth position starting from 3 positions = [] for col_ind in [1, 2, 4]: positions += np.argwhere(h5_pos_inds[h5_pos_inds.attrs['X']] == col_ind)[:, 0].tolist() specs = [] for bias_ind in range(1, 7, 3): specs += np.argwhere(h5_spec_inds[h5_spec_inds.attrs['Bias']] == bias_ind)[:, 1].tolist() expected_pos = np.expand_dims(positions, axis=1) expected_spec = np.expand_dims(specs, axis=1) expected_pos.sort(axis=0) expected_spec.sort(axis=0) self.assertTrue(np.allclose(expected_spec, actual_spec)) self.assertTrue(np.allclose(expected_pos, actual_pos))
def test_get_n_dim_form_unsorted(self): with h5py.File(test_h5_file_path, mode='r') as h5_f: h5_main = h5_f['/Raw_Measurement/source_main'] expected = np.reshape(h5_main, (3, 5, 7, 2)) expected = np.transpose(expected, (1, 0, 2, 3)) pycro_dset = USIDataset(h5_main) self.assertTrue(np.allclose(expected, pycro_dset.get_n_dim_form()))
def setUp(self): data_utils.make_beps_file() self.orig_labels_order = ['X', 'Y', 'Cycle', 'Bias'] self.h5_file = h5py.File(data_utils.std_beps_path, mode='r') h5_grp = self.h5_file['/Raw_Measurement/'] self.source_nd_s2f = h5_grp['n_dim_form'][()] self.source_nd_f2s = self.source_nd_s2f.transpose(1, 0, 3, 2) self.h5_source = USIDataset(h5_grp['source_main']) self.pos_dims=[] self.spec_dims=[] for dim_name, dim_units in zip(self.h5_source.pos_dim_labels, hdf_utils.get_attr(self.h5_source.h5_pos_inds, 'units')): self.pos_dims.append( Dimension(dim_name, dim_units, h5_grp[dim_name][()])) for dim_name, dim_units in zip(self.h5_source.spec_dim_labels, hdf_utils.get_attr(self.h5_source.h5_spec_inds, 'units')): self.spec_dims.append( Dimension(dim_name, dim_units, h5_grp[dim_name][()])) res_grp_0 = h5_grp['source_main-Fitter_000'] self.results_0_nd_s2f = res_grp_0['n_dim_form'][()] self.results_0_nd_f2s = self.results_0_nd_s2f.transpose(1, 0, 3, 2) self.h5_compound = USIDataset(res_grp_0['results_main']) res_grp_1 = h5_grp['source_main-Fitter_001'] self.results_1_nd_s2f = res_grp_1['n_dim_form'][()] self.results_1_nd_f2s = self.results_1_nd_s2f.transpose(1, 0, 3, 2) self.h5_complex = USIDataset(res_grp_1['results_main'])
def test_equality_incorrect_USIDataset(self): with h5py.File(test_h5_file_path, mode='r') as h5_f: h5_main = h5_f['/Raw_Measurement/source_main'] expected = USIDataset(h5_main) incorrect = USIDataset( h5_f['/Raw_Measurement/source_main-Fitter_000/results_main']) self.assertFalse(expected == incorrect)
def test_get_spec_values(self): with h5py.File(test_h5_file_path, mode='r') as h5_f: usi_main = USIDataset(h5_f['/Raw_Measurement/source_main']) for dim_name in ['Bias', 'Cycle']: expected = h5_f['/Raw_Measurement/' + dim_name][()] actual = usi_main.get_spec_values(dim_name) self.assertTrue(np.allclose(expected, actual))
def test_sorted(self): with h5py.File(test_h5_file_path, mode='r') as h5_f: h5_main = h5_f['/Raw_Measurement/source_main'] expected = np.reshape(h5_main, (3, 5, 7, 2)) expected = np.transpose(expected, (1, 0, 3, 2)) usi_dset = USIDataset(h5_main) usi_dset.toggle_sorting() self.assertTrue(np.allclose(expected, usi_dset.get_n_dim_form(lazy=False)))
def test_both_pos_removed(self): with h5py.File(test_h5_file_path, mode='r') as h5_f: usi_main = USIDataset(h5_f['/Raw_Measurement/source_main']) actual, success = usi_main.slice({'X': 3, 'Y': 1}, lazy=False) n_dim_form = np.transpose(np.reshape(usi_main[()], (3, 5, 7, 2)), (1, 0, 2, 3)) expected = n_dim_form[3, 1, :, :] self.assertTrue(np.allclose(expected, actual)) self.assertTrue(success)
def test_pos_and_spec_sliced_list(self): with h5py.File(test_h5_file_path, mode='r') as h5_f: usi_main = USIDataset(h5_f['/Raw_Measurement/source_main']) actual, success = usi_main.slice({'X': [1, 2, 4], 'Bias': slice(1, 7, 3)}, lazy=False) n_dim_form = np.transpose(np.reshape(usi_main[()], (3, 5, 7, 2)), (1, 0, 2, 3)) expected = n_dim_form[[1, 2, 4], :, slice(1, 7, 3), :] self.assertTrue(np.allclose(expected, actual)) self.assertTrue(success)
def test_get_pos_values(self): with h5py.File(test_h5_file_path, mode='r') as h5_f: usi_main = USIDataset(h5_f['/Raw_Measurement/source_main']) expected = usi_main.h5_pos_vals[:5, 0] actual = usi_main.get_pos_values('X') self.assertTrue(np.allclose(expected, actual)) expected = usi_main.h5_pos_vals[0:None:5, 1] actual = usi_main.get_pos_values('Y') self.assertTrue(np.allclose(expected, actual))
def test_get_spec_values(self): with h5py.File(test_h5_file_path, mode='r') as h5_f: usi_main = USIDataset(h5_f['/Raw_Measurement/source_main']) expected = usi_main.h5_spec_vals[0, ::2] actual = usi_main.get_spec_values('Bias') self.assertTrue(np.allclose(expected, actual)) expected = usi_main.h5_spec_vals[1, 0:None:7] actual = usi_main.get_spec_values('Cycle') self.assertTrue(np.allclose(expected, actual))
def test_slice_two_pos_dim_sliced_list(self): with h5py.File(test_h5_file_path, mode='r') as h5_f: pycro_main = USIDataset(h5_f['/Raw_Measurement/source_main']) actual, success = pycro_main.slice({'X': [1, 2, 4], 'Y': 1}) n_dim_form = np.transpose(np.reshape(pycro_main[()], (3, 5, 7, 2)), (1, 0, 2, 3)) expected = n_dim_form[[1, 2, 4], 1, :, :] self.assertTrue(np.allclose(expected, actual)) self.assertTrue(success)
def test_get_pos_spec_slices_non_existent_dim(self): with h5py.File(test_h5_file_path, mode='r') as h5_f: pycro_main = USIDataset(h5_f['/Raw_Measurement/source_main']) with self.assertRaises(KeyError): _ = pycro_main._get_pos_spec_slices({ 'blah': 4, 'X': 3, 'Y': 1 })
def test_empty_dict(self): with h5py.File(test_h5_file_path, mode='r') as h5_f: usi_main = USIDataset(h5_f['/Raw_Measurement/source_main']) actual_pos, actual_spec = usi_main._get_pos_spec_slices({}) self.assertTrue( np.allclose(np.expand_dims(np.arange(14), axis=1), actual_spec)) self.assertTrue( np.allclose(np.expand_dims(np.arange(15), axis=1), actual_pos))
def test_both_pos_removed(self): with h5py.File(test_h5_file_path, mode='r') as h5_f: usi_main = USIDataset(h5_f['/Raw_Measurement/source_main']) actual_pos, actual_spec = usi_main._get_pos_spec_slices({'X': 3, 'Y': 1}) # we want every fifth position starting from 3 expected_pos = np.expand_dims([1 * 5 + 3], axis=1) expected_spec = np.expand_dims(np.arange(14), axis=1) self.assertTrue(np.allclose(expected_spec, actual_spec)) self.assertTrue(np.allclose(expected_pos, actual_pos))
def test_toggle_sorting(self): # Need to change data file so that sorting actually does something with h5py.File(test_h5_file_path, mode='r') as h5_f: usi_main = USIDataset(h5_f['/Raw_Measurement/source_main']) self.assertTrue(usi_main.n_dim_labels == ['X', 'Y', 'Bias', 'Cycle']) usi_main.toggle_sorting() self.assertTrue(usi_main.n_dim_labels==['X', 'Y', 'Cycle', 'Bias'])
def base(self, slice_dict, pos_exp, spec_exp, verbose=False): with h5py.File(test_h5_file_path, mode='r') as h5_f: usi_main = USIDataset(h5_f['/Raw_Measurement/source_main']) pos_act, spec_act = usi_main._get_dims_for_slice( slice_dict=slice_dict, verbose=verbose) if verbose: print(pos_act) print(spec_act) self.__validate_dim_list(pos_act, pos_exp) self.__validate_dim_list(spec_act, spec_exp)
def test_one_pos_dim_removed(self): with h5py.File(test_h5_file_path, mode='r') as h5_f: usi_main = USIDataset(h5_f['/Raw_Measurement/source_main']) # orig_pos = np.vstack([np.tile(np.arange(5), 3), np.repeat(np.arange(3), 5)]).T # orig_spec = np.vstack([np.tile(np.arange(7), 2), np.repeat(np.arange(2), 7)]) actual_pos, actual_spec = usi_main._get_pos_spec_slices({'X': 3}) # we want every fifth position starting from 3 expected_pos = np.expand_dims(np.arange(3, 15, 5), axis=1) expected_spec = np.expand_dims(np.arange(14), axis=1) self.assertTrue(np.allclose(expected_spec, actual_spec)) self.assertTrue(np.allclose(expected_pos, actual_pos))
def test_two_pos_dim_sliced_list(self): with h5py.File(test_h5_file_path, mode='r') as h5_f: usi_main = USIDataset(h5_f['/Raw_Measurement/source_main']) actual_pos, actual_spec = usi_main._get_pos_spec_slices({'X': [1, 2, 4], 'Y': 1}) # we want every fifth position starting from 3 positions = [] for row_ind in range(1, 2): for col_ind in [1, 2, 4]: positions.append(5 * row_ind + col_ind) expected_pos = np.expand_dims(positions, axis=1) expected_spec = np.expand_dims(np.arange(14), axis=1) self.assertTrue(np.allclose(expected_spec, actual_spec)) self.assertTrue(np.allclose(expected_pos, actual_pos))
def test_two_spec(self): if skip_viz_tests: return with h5py.File(test_h5_file_path, mode='r') as h5_f: dset_path = '/Raw_Measurement/source_main' usi_main = USIDataset(h5_f[dset_path]) slice_dict = {'X': 3, 'Y': 2} exp_data, success = usi_main.slice(slice_dict=slice_dict) self.assertTrue(success) fig, axis = usi_main.visualize(slice_dict=slice_dict) validate_imshow(self, axis, exp_data, title=dset_path, x_vec=h5_f['/Raw_Measurement/' + usi_main.spec_dim_labels[1]], y_vec=h5_f['/Raw_Measurement/' + usi_main.spec_dim_labels[0]], x_label=usi_main.spec_dim_descriptors[1], y_label=usi_main.spec_dim_descriptors[0])
def test_incorrect_h5_dataset(self): with h5py.File(test_h5_file_path, mode='r') as h5_f: h5_main = h5_f['/Raw_Measurement/source_main'] expected = USIDataset(h5_main) incorrect = h5_f[ '/Raw_Measurement/source_main-Fitter_000/Spectroscopic_Indices'] self.assertFalse(expected == incorrect)
def test_one_spec(self): if skip_viz_tests: return with h5py.File(test_h5_file_path, mode='r') as h5_f: dset_path = '/Raw_Measurement/source_main' usi_main = USIDataset(h5_f[dset_path]) slice_dict = {'Bias': 4, 'X': 1, 'Y': 2} rem_dim_name = 'Cycle' spec_ind = usi_main.spec_dim_labels.index(rem_dim_name) exp_data, success = usi_main.slice(slice_dict=slice_dict) self.assertTrue(success) fig, axis = usi_main.visualize(slice_dict=slice_dict) validate_single_curve(self, axis, h5_f['/Raw_Measurement/' + rem_dim_name], exp_data, title=dset_path, x_label=usi_main.spec_dim_descriptors[spec_ind], y_label=usi_main.data_descriptor)
def test_string_representation(self): with h5py.File(test_h5_file_path, mode='r') as h5_f: h5_main = h5_f['/Raw_Measurement/source_main'] usi_dset = USIDataset(h5_main) actual = usi_dset.__repr__() actual = [line.strip() for line in actual.split("\n")] actual = [actual[line_ind] for line_ind in [0, 2, 4, 7, 8, 10, 11]] expected = list() expected.append(h5_main.__repr__()) expected.append(h5_main.name) expected.append(hdf_utils.get_attr(h5_main, "quantity") + " (" + hdf_utils.get_attr(h5_main, "units") + ")") for h5_inds in [usi_dset.h5_pos_inds, usi_dset.h5_spec_inds]: for dim_name, dim_size in zip(hdf_utils.get_attr(h5_inds, "labels"), hdf_utils.get_dimensionality(h5_inds)): expected.append(dim_name + ' - size: ' + str(dim_size)) self.assertTrue(np.all([x == y for x, y in zip(actual, expected)]))
def base(self, slice_dict, f2s_slice_list, result_as_nd, lazy_result, verbose=False): with h5py.File(test_h5_file_path, mode='r') as h5_f: usi_main = USIDataset(h5_f['/Raw_Measurement/source_main']) actual, success = usi_main.slice(slice_dict, ndim_form=result_as_nd, lazy=lazy_result, verbose=verbose) if verbose: print('Status: {}, actual.shape: {}, actual.dtype: {}, ' 'type(actual): {}'.format(success, actual.shape, actual.dtype, type(actual))) self.assertTrue(success) n_dim_s2f, n_dim_f2s = self.get_expected_n_dim(h5_f) if result_as_nd: expected = n_dim_f2s[tuple(f2s_slice_list)] expected = expected.squeeze() else: s2f_slice_list = f2s_slice_list[:2][::-1] + \ f2s_slice_list[2:][::-1] if verbose: print('Slice list converted from: {} to {}' ''.format(f2s_slice_list, s2f_slice_list)) expected = n_dim_s2f[tuple(s2f_slice_list)] if verbose: print('Expected in N-dim form: {}'.format(expected.shape)) expected = expected.reshape(np.prod(expected.shape[:2]), np.prod(expected.shape[2:])) if verbose: print('Expected after flattening of shape: {}' ''.format(expected.shape)) if lazy_result: self.assertIsInstance(actual, da.core.Array) actual = actual.compute() self.assertTrue(np.allclose(expected, actual))
def get_all_dimensions(): pos_dims = [] spec_dims = [] with h5py.File(test_h5_file_path, mode='r') as h5_f: h5_raw_grp = h5_f['Raw_Measurement'] usi_main = USIDataset(h5_raw_grp['source_main']) for dim_name, dim_units in zip(usi_main.pos_dim_labels, hdf_utils.get_attr(usi_main.h5_pos_inds, 'units')): pos_dims.append(Dimension(dim_name, dim_units, h5_raw_grp[dim_name][()])) for dim_name, dim_units in zip(usi_main.spec_dim_labels, hdf_utils.get_attr( usi_main.h5_spec_inds, 'units')): spec_dims.append(Dimension(dim_name, dim_units, h5_raw_grp[dim_name][()])) return pos_dims, spec_dims
def test_get_current_sorting(self): with h5py.File(test_h5_file_path, mode='r') as h5_f: usi_main = USIDataset(h5_f['/Raw_Measurement/source_main']) unsorted_str = 'Data dimensions are in the order they occur in the file.\n' sorted_str = 'Data dimensions are sorted in order from fastest changing dimension to slowest.\n' # Initial state should be unsorted self.assertFalse(usi_main._USIDataset__sort_dims) with data_utils.capture_stdout() as get_value: usi_main.get_current_sorting() test_str = get_value() self.assertTrue(test_str == unsorted_str) # Toggle sorting. Sorting should now be true. usi_main.toggle_sorting() self.assertTrue(usi_main._USIDataset__sort_dims) with data_utils.capture_stdout() as get_value: usi_main.get_current_sorting() test_str = get_value() self.assertTrue(test_str == sorted_str)
def test_sorted_and_unsorted(self): with h5py.File(test_h5_file_path, mode='r') as h5_f: usi_dset = USIDataset(h5_f['/Raw_Measurement/source_main']) nd_slow_to_fast, nd_fast_to_slow = self.get_expected_n_dim(h5_f) actual_f2s = usi_dset.get_n_dim_form(lazy=False) self.assertTrue(np.allclose(nd_fast_to_slow, actual_f2s)) nd_form, success = reshape_to_n_dims(usi_dset, sort_dims=True) print(nd_form.shape) usi_dset.toggle_sorting() actual_s2f = usi_dset.get_n_dim_form(lazy=False) self.assertTrue(np.allclose(nd_slow_to_fast, actual_s2f))
def basic_file_validation(self, h5_f): self.assertEqual('ImageTranslator', hdf_utils.get_attr(h5_f, 'translator')) # First level should have absolutely nothing besides one group self.assertEqual(len(h5_f.items()), 1) self.assertTrue('Measurement_000' in h5_f.keys()) h5_meas_grp = h5_f['Measurement_000'] self.assertIsInstance(h5_meas_grp, h5py.Group) # Again, this group should only have one group - Channel_000 self.assertEqual(len(h5_meas_grp.items()), 1) self.assertTrue('Channel_000' in h5_meas_grp.keys()) h5_chan_grp = h5_meas_grp['Channel_000'] self.assertIsInstance(h5_chan_grp, h5py.Group) # This channel group is not expected to have any (custom) attributes but it will contain the main dataset self.assertEqual(len(h5_chan_grp.items()), 5) for dset_name in [ 'Raw_Data', 'Position_Indices', 'Position_Values', 'Spectroscopic_Indices', 'Spectroscopic_Values' ]: self.assertTrue(dset_name in h5_chan_grp.keys()) h5_dset = h5_chan_grp[dset_name] self.assertIsInstance(h5_dset, h5py.Dataset) usid_main = USIDataset(h5_chan_grp['Raw_Data']) self.assertIsInstance(usid_main, USIDataset) self.assertEqual(usid_main.name.split('/')[-1], 'Raw_Data') self.assertEqual(usid_main.parent, h5_chan_grp) validate_aux_dset_pair(self, h5_chan_grp, usid_main.h5_spec_inds, usid_main.h5_spec_vals, ['arb'], ['a.u.'], np.atleast_2d([0]), h5_main=usid_main, is_spectral=True)
def test_negative_index_2d_numpy(self): with h5py.File(test_h5_file_path, mode='r') as h5_f: usi_main = USIDataset(h5_f['/Raw_Measurement/source_main']) with self.assertRaises(ValueError): _ = usi_main.slice({'X': -2, 'Y': 1}, ndim_form=False)
def test_out_of_bounds(self): with h5py.File(test_h5_file_path, mode='r') as h5_f: usi_main = USIDataset(h5_f['/Raw_Measurement/source_main']) with self.assertRaises(IndexError): _ = usi_main.slice({'X': 15, 'Y': 1})