def test_inequality(self): name = 'Bias' units = 'V' values = [0, 1, 2, 3] left = write_utils.Dimension(name, units, values) right = write_utils.Dimension(name, units, [0, 1, 2, 4]) self.assertFalse(left == right) left = write_utils.Dimension(name, units, [0, 1, 2]) right = write_utils.Dimension(name, units, values) self.assertFalse(left == right) left = write_utils.Dimension('name', units, values) right = write_utils.Dimension(name, units, values) self.assertFalse(left == right) left = write_utils.Dimension(name, 'units', values) right = write_utils.Dimension(name, units, values) self.assertFalse(left == right) left = write_utils.Dimension(name, units, values, mode=write_utils.DimType.DEPENDENT) right = write_utils.Dimension(name, units, values) self.assertFalse(left == right)
def test_not_numpy_or_dask_array_main(self): translator = ArrayTranslator() with self.assertRaises(TypeError): delete_existing_file(file_path) _ = translator.translate(file_path, 'Blah', {'This is not a dataset': True}, 'quant', 'unit', write_utils.Dimension('Position_Dim', 'au', 5), write_utils.Dimension('Spec_Dim', 'au', 3))
def test_equality(self): name = 'Bias' units = 'V' dim_1 = write_utils.Dimension(name, units, [0, 1, 2, 3, 4]) dim_2 = write_utils.Dimension(name, units, np.arange(5, dtype=np.float32)) self.assertEqual(dim_1, dim_2)
def test_prod_sizes_mismatch(self): file_path = 'test.h5' data_utils.delete_existing_file(file_path) main_data = np.random.rand(15, 14) main_data_name = 'Test_Main' quantity = 'Current' dset_units = 'nA' pos_sizes = [5, 15] # too many steps in the Y direction pos_names = ['X', 'Y'] pos_units = ['nm', 'um'] pos_dims = [] for length, name, units in zip(pos_sizes, pos_names, pos_units): pos_dims.append( write_utils.Dimension(name, units, np.arange(length))) spec_sizes = [7, 2] spec_names = ['Bias', 'Cycle'] spec_units = ['V', ''] spec_dims = [] for length, name, units in zip(spec_sizes, spec_names, spec_units): spec_dims.append( write_utils.Dimension(name, units, np.arange(length))) with h5py.File(file_path) as h5_f: with self.assertRaises(ValueError): _ = hdf_utils.write_main_dataset(h5_f, main_data, main_data_name, quantity, dset_units, pos_dims, spec_dims) os.remove(file_path)
def test_objects(self): translator = ArrayTranslator() with self.assertRaises(TypeError): delete_existing_file(file_path) _ = translator.translate( file_path, 'Blah', np.random.rand(5, 13), 'quant', 'unit', write_utils.Dimension('Dim_1', 'au', 5), ['blah', write_utils.Dimension('Dim_2', 'au', 4)])
def test_object_single(self): translator = ArrayTranslator() with self.assertRaises(TypeError): delete_existing_file(file_path) _ = translator.translate(file_path, 'Blah', np.random.rand(5, 13), 'quant', 'unit', 'my_string_Dimension', [write_utils.Dimension('Spec_Dim', 'au', 3), write_utils.Dimension('Dim_2', 'au', 4)])
def test_position(self): translator = ArrayTranslator() with self.assertRaises(ValueError): delete_existing_file(file_path) _ = translator.translate(file_path, 'Blah', np.random.rand(15, 3), 'quant', 'unit', [write_utils.Dimension('Dim_1', 'au', 5), write_utils.Dimension('Dim_2', 'au', 4)], write_utils.Dimension('Spec_Dim', 'au', 3))
def test_empty_name(self): translator = ArrayTranslator() with self.assertRaises(ValueError): delete_existing_file(file_path) _ = translator.translate(file_path, 'Blah', np.random.rand(5, 3), 'quant', 'unit', write_utils.Dimension('Position_Dim', 'au', 5), write_utils.Dimension('Spec_Dim', 'au', 3), extra_dsets={' ': [1, 2, 3]})
def test_not_arrays(self): translator = ArrayTranslator() with self.assertRaises(TypeError): delete_existing_file(file_path) _ = translator.translate(file_path, 'Blah', np.random.rand(5, 3), 'quant', 'unit', write_utils.Dimension('Position_Dim', 'au', 5), write_utils.Dimension('Spec_Dim', 'au', 3), extra_dsets={'Blah_other': 'I am not an array'})
def test_reserved_names(self): translator = ArrayTranslator() with self.assertRaises(KeyError): delete_existing_file(file_path) _ = translator.translate(file_path, 'Blah', np.random.rand(5, 3), 'quant', 'unit', write_utils.Dimension('Position_Dim', 'au', 5), write_utils.Dimension('Spec_Dim', 'au', 3), extra_dsets={'Spectroscopic_Indices': np.arange(4), 'Blah_other': np.arange(15)})
def test_main_dset_1D(self): translator = ArrayTranslator() with self.assertRaises(ValueError): delete_existing_file(file_path) _ = translator.translate(file_path, 'Blah', np.arange(4), 'quant', 'unit', write_utils.Dimension('Position_Dim', 'au', 5), write_utils.Dimension('Spec_Dim', 'au', 3)) with self.assertRaises(ValueError): delete_existing_file(file_path) _ = translator.translate(file_path, 'Blah', da.from_array(np.arange(4), chunks=(4)), 'quant', 'unit', write_utils.Dimension('Position_Dim', 'au', 5), write_utils.Dimension('Spec_Dim', 'au', 3))
def test_aux_dset_descriptor_illegal(self): with self.assertRaises(TypeError): _ = write_utils.Dimension('Name', 14, np.arange(4)) with self.assertRaises(TypeError): _ = write_utils.Dimension(14, 'nm', np.arange(4)) with self.assertRaises(ValueError): _ = write_utils.Dimension('Name', 'unit', 0) with self.assertRaises(TypeError): _ = write_utils.Dimension('Name', 'unit', 'invalid')
def test_illegal_instantiation(self): with self.assertRaises(TypeError): _ = write_utils.Dimension('Name', 14, np.arange(4)) with self.assertRaises(TypeError): _ = write_utils.Dimension(14, 'nm', np.arange(4)) with self.assertRaises(ValueError): _ = write_utils.Dimension('Name', 'unit', 0) with self.assertRaises(TypeError): _ = write_utils.Dimension('Name', 'unit', 'invalid')
def test_values_as_array(self): name = 'Bias' units = 'V' values = np.random.rand(5) descriptor = write_utils.Dimension(name, units, values) for expected, actual in zip([name, units, values], [descriptor.name, descriptor.units, descriptor.values]): self.assertTrue(np.all([x == y for x, y in zip(expected, actual)]))
def test_repr(self): name = 'Bias' units = 'V' values = np.arange(5) descriptor = write_utils.Dimension(name, units, len(values)) actual = '{}'.format(descriptor) expected = '{} ({}) mode:{} : {}'.format(name, units, descriptor.mode, values) self.assertEqual(actual, expected)
def test_values_as_length(self): name = 'Bias' units = 'V' values = np.arange(5) descriptor = write_utils.Dimension(name, units, len(values)) for expected, actual in zip([name, units], [descriptor.name, descriptor.units]): self.assertTrue(np.all([x == y for x, y in zip(expected, actual)])) self.assertTrue(np.allclose(values, descriptor.values))
def test_repr(self): name = 'Bias' quantity = 'generic' units = 'V' values = np.arange(5, dtype=np.float) descriptor = write_utils.Dimension(name, units, len(values)) print(type(descriptor)) actual = '{}'.format(descriptor) expected = '{}: {} ({}) mode:{} : {}'.format(name, quantity, units, descriptor.mode, values) self.assertEqual(actual, expected)
def test_inequality(self): name = 'Bias' units = 'V' self.assertNotEqual(write_utils.Dimension(name, units, [0, 1, 2, 3]), write_utils.Dimension(name, units, [0, 1, 2, 4])) self.assertNotEqual(write_utils.Dimension('fdfd', units, [0, 1, 2, 3]), write_utils.Dimension(name, units, [0, 1, 2, 3])) self.assertNotEqual(write_utils.Dimension(name, 'fdfd', [0, 1, 2, 3]), write_utils.Dimension(name, units, [0, 1, 2, 3])) self.assertNotEqual( write_utils.Dimension(name, units, [0, 1, 2, 3], mode=write_utils.DimType.DEPENDENT), write_utils.Dimension(name, units, [0, 1, 2, 3], mode=write_utils.DimType.INCOMPLETE)) self.assertNotEqual(write_utils.Dimension(name, units, [0, 1, 2]), write_utils.Dimension(name, units, [0, 1, 2, 3]))
def test_not_strings(self): translator = ArrayTranslator() with self.assertRaises(TypeError): delete_existing_file(file_path) _ = translator.translate(file_path, 1.2345, np.random.rand(5, 3), 'quant', 'unit', write_utils.Dimension('Position_Dim', 'au', 5), write_utils.Dimension('Spec_Dim', 'au', 3)) with self.assertRaises(TypeError): delete_existing_file(file_path) _ = translator.translate(file_path, 'Blah', np.random.rand(5, 3), {'quant': 1}, 'unit', write_utils.Dimension('Position_Dim', 'au', 5), write_utils.Dimension('Spec_Dim', 'au', 3)) with self.assertRaises(TypeError): delete_existing_file(file_path) _ = translator.translate(file_path, 'Blah', np.random.rand(5, 3), 'quant', ['unit'], write_utils.Dimension('Position_Dim', 'au', 5), write_utils.Dimension('Spec_Dim', 'au', 3))
def test_existing_both_aux(self): file_path = 'test.h5' data_utils.delete_existing_file(file_path) main_data = np.random.rand(15, 14) main_data_name = 'Test_Main' quantity = 'Current' dset_units = 'nA' pos_sizes = [5, 3] pos_names = ['X', 'Y'] pos_units = ['nm', 'um'] pos_dims = [] for length, name, units in zip(pos_sizes, pos_names, pos_units): pos_dims.append( write_utils.Dimension(name, units, np.arange(length))) pos_data = np.vstack((np.tile(np.arange(5), 3), np.repeat(np.arange(3), 5))).T spec_sizes = [7, 2] spec_names = ['Bias', 'Cycle'] spec_units = ['V', ''] spec_dims = [] for length, name, units in zip(spec_sizes, spec_names, spec_units): spec_dims.append( write_utils.Dimension(name, units, np.arange(length))) spec_data = np.vstack((np.tile(np.arange(7), 2), np.repeat(np.arange(2), 7))) with h5py.File(file_path) as h5_f: h5_spec_inds, h5_spec_vals = hdf_utils.write_ind_val_dsets( h5_f, spec_dims, is_spectral=True) h5_pos_inds, h5_pos_vals = hdf_utils.write_ind_val_dsets( h5_f, pos_dims, is_spectral=False) usid_main = hdf_utils.write_main_dataset(h5_f, main_data, main_data_name, quantity, dset_units, None, None, h5_spec_inds=h5_spec_inds, h5_spec_vals=h5_spec_vals, h5_pos_vals=h5_pos_vals, h5_pos_inds=h5_pos_inds, main_dset_attrs=None) data_utils.validate_aux_dset_pair(self, h5_f, h5_pos_inds, h5_pos_vals, pos_names, pos_units, pos_data, h5_main=usid_main, is_spectral=False) data_utils.validate_aux_dset_pair(self, h5_f, h5_spec_inds, h5_spec_vals, spec_names, spec_units, spec_data, h5_main=usid_main, is_spectral=True) os.remove(file_path)
def test_empty(self): file_path = 'test.h5' data_utils.delete_existing_file(file_path) main_data = (15, 14) main_data_name = 'Test_Main' quantity = 'Current' dset_units = 'nA' pos_sizes = [5, 3] pos_names = ['X', 'Y'] pos_units = ['nm', 'um'] pos_dims = [] for length, name, units in zip(pos_sizes, pos_names, pos_units): pos_dims.append( write_utils.Dimension(name, units, np.arange(length))) pos_data = np.vstack((np.tile(np.arange(5), 3), np.repeat(np.arange(3), 5))).T spec_sizes = [7, 2] spec_names = ['Bias', 'Cycle'] spec_units = ['V', ''] spec_dims = [] for length, name, units in zip(spec_sizes, spec_names, spec_units): spec_dims.append( write_utils.Dimension(name, units, np.arange(length))) spec_data = np.vstack((np.tile(np.arange(7), 2), np.repeat(np.arange(2), 7))) with h5py.File(file_path) as h5_f: usid_main = hdf_utils.write_main_dataset(h5_f, main_data, main_data_name, quantity, dset_units, pos_dims, spec_dims, dtype=np.float16, main_dset_attrs=None) self.assertIsInstance(usid_main, USIDataset) self.assertEqual(usid_main.name.split('/')[-1], main_data_name) self.assertEqual(usid_main.parent, h5_f) self.assertEqual(main_data, usid_main.shape) data_utils.validate_aux_dset_pair(self, h5_f, usid_main.h5_pos_inds, usid_main.h5_pos_vals, pos_names, pos_units, pos_data, h5_main=usid_main, is_spectral=False) data_utils.validate_aux_dset_pair(self, h5_f, usid_main.h5_spec_inds, usid_main.h5_spec_vals, spec_names, spec_units, spec_data, h5_main=usid_main, is_spectral=True) os.remove(file_path)
def test_default_mode(self): dim = write_utils.Dimension('Name', 'units', 1) self.assertEqual(dim.mode, write_utils.DimType.DEFAULT)
def test_invalid_mode(self): with self.assertRaises(TypeError): _ = write_utils.Dimension('Name', 'units', 5, mode='Incomplete')
def test_legal_translation(self): data_name = 'TestDataType' attrs = { 'att_1': 'string_val', 'att_2': 1.2345, 'att_3': [1, 2, 3, 4], 'att_4': ['str_1', 'str_2', 'str_3'] } extra_dsets = {'dset_1': np.random.rand(5), 'dset_2': np.arange(25)} file_path = 'test_numpy_translator.h5' self.__delete_existing_file(file_path) main_data = np.random.rand(15, 14) main_data_name = 'Test_Main' quantity = 'Current' units = 'nA' pos_sizes = [5, 3] pos_names = ['X', 'Y'] pos_units = ['nm', 'um'] pos_dims = [] for name, unit, length in zip(pos_names, pos_units, pos_sizes): pos_dims.append( write_utils.Dimension(name, unit, np.arange(length))) pos_data = np.vstack((np.tile(np.arange(5), 3), np.repeat(np.arange(3), 5))).T spec_sizes = [7, 2] spec_names = ['Bias', 'Cycle'] spec_units = ['V', ''] spec_dims = [] for name, unit, length in zip(spec_names, spec_units, spec_sizes): spec_dims.append( write_utils.Dimension(name, unit, np.arange(length))) spec_data = np.vstack((np.tile(np.arange(7), 2), np.repeat(np.arange(2), 7))) translator = NumpyTranslator() _ = translator.translate(file_path, data_name, main_data, quantity, units, pos_dims, spec_dims, parm_dict=attrs, extra_dsets=extra_dsets) with h5py.File(file_path, mode='r') as h5_f: # we are not interested in most of the attributes under root besides two: self.assertEqual(data_name, hdf_utils.get_attr(h5_f, 'data_type')) self.assertEqual('NumpyTranslator', hdf_utils.get_attr(h5_f, 'translator')) # First level should have absolutely nothing besides one group self.assertEqual(len(h5_f.items()), 1) self.assertTrue('Measurement_000' in h5_f.keys()) h5_meas_grp = h5_f['Measurement_000'] self.assertIsInstance(h5_meas_grp, h5py.Group) # check the attributes under this group self.assertEqual(len(h5_meas_grp.attrs), len(attrs)) for key, expected_val in attrs.items(): self.assertTrue( np.all( hdf_utils.get_attr(h5_meas_grp, key) == expected_val)) # Again, this group should only have one group - Channel_000 self.assertEqual(len(h5_meas_grp.items()), 1) self.assertTrue('Channel_000' in h5_meas_grp.keys()) h5_chan_grp = h5_meas_grp['Channel_000'] self.assertIsInstance(h5_chan_grp, h5py.Group) # This channel group is not expected to have any attributes but it will contain the main dataset self.assertEqual(len(h5_chan_grp.items()), 5 + len(extra_dsets)) for dset_name in [ 'Raw_Data', 'Position_Indices', 'Position_Values', 'Spectroscopic_Indices', 'Spectroscopic_Values' ]: self.assertTrue(dset_name in h5_chan_grp.keys()) h5_dset = h5_chan_grp[dset_name] self.assertIsInstance(h5_dset, h5py.Dataset) pycro_main = USIDataset(h5_chan_grp['Raw_Data']) self.assertIsInstance(pycro_main, USIDataset) self.assertEqual(pycro_main.name.split('/')[-1], 'Raw_Data') self.assertEqual(pycro_main.parent, h5_chan_grp) self.assertTrue(np.allclose(main_data, pycro_main[()])) self.__validate_aux_dset_pair(h5_chan_grp, pycro_main.h5_pos_inds, pycro_main.h5_pos_vals, pos_names, pos_units, pos_data, h5_main=pycro_main, is_spectral=False) self.__validate_aux_dset_pair(h5_chan_grp, pycro_main.h5_spec_inds, pycro_main.h5_spec_vals, spec_names, spec_units, spec_data, h5_main=pycro_main, is_spectral=True) # Now validate each of the extra datasets: for key, val in extra_dsets.items(): self.assertTrue(key in h5_chan_grp.keys()) h5_dset = h5_chan_grp[key] self.assertIsInstance(h5_dset, h5py.Dataset) self.assertTrue(np.allclose(val, h5_dset[()])) os.remove(file_path)
def base_translation_tester(self, main_dset_as_dask=False, extra_dsets_type='numpy', use_parm_dict=True): data_name = 'My_Awesome_Measurement' if use_parm_dict: attrs = { 'att_1': 'string_val', 'att_2': 1.2345, 'att_3': [1, 2, 3, 4], 'att_4': ['str_1', 'str_2', 'str_3'] } else: attrs = None extra_dsets = {} if extra_dsets_type is not None: ref_dsets = {'dset_1': np.random.rand(5), 'dset_2': np.arange(25)} if extra_dsets_type == 'numpy': extra_dsets = ref_dsets elif extra_dsets_type == 'dask': for key, val in ref_dsets.items(): extra_dsets.update( {key: da.from_array(val, chunks=val.shape)}) else: extra_dsets_type = None delete_existing_file(file_path) main_data = np.random.rand(15, 14) if main_dset_as_dask: main_data = da.from_array(main_data, chunks=main_data.shape) quantity = 'Current' units = 'nA' pos_sizes = [5, 3] pos_names = ['X', 'Y'] pos_units = ['nm', 'um'] pos_dims = [] for name, unit, length in zip(pos_names, pos_units, pos_sizes): pos_dims.append( write_utils.Dimension(name, unit, np.arange(length))) pos_data = np.vstack((np.tile(np.arange(5), 3), np.repeat(np.arange(3), 5))).T spec_sizes = [7, 2] spec_names = ['Bias', 'Cycle'] spec_units = ['V', ''] spec_dims = [] for name, unit, length in zip(spec_names, spec_units, spec_sizes): spec_dims.append( write_utils.Dimension(name, unit, np.arange(length))) spec_data = np.vstack((np.tile(np.arange(7), 2), np.repeat(np.arange(2), 7))) translator = ArrayTranslator() _ = translator.translate(file_path, data_name, main_data, quantity, units, pos_dims, spec_dims, parm_dict=attrs, extra_dsets=extra_dsets) with h5py.File(file_path, mode='r') as h5_f: # we are not interested in most of the attributes under root besides two: self.assertEqual(data_name, hdf_utils.get_attr(h5_f, 'data_type')) # self.assertEqual('NumpyTranslator', hdf_utils.get_attr(h5_f, 'translator')) # First level should have absolutely nothing besides one group self.assertEqual(len(h5_f.items()), 1) self.assertTrue('Measurement_000' in h5_f.keys()) h5_meas_grp = h5_f['Measurement_000'] self.assertIsInstance(h5_meas_grp, h5py.Group) # check the attributes under this group # self.assertEqual(len(h5_meas_grp.attrs), len(attrs)) if use_parm_dict: for key, expected_val in attrs.items(): self.assertTrue( np.all( hdf_utils.get_attr(h5_meas_grp, key) == expected_val)) # Again, this group should only have one group - Channel_000 self.assertEqual(len(h5_meas_grp.items()), 1) self.assertTrue('Channel_000' in h5_meas_grp.keys()) h5_chan_grp = h5_meas_grp['Channel_000'] self.assertIsInstance(h5_chan_grp, h5py.Group) # This channel group is not expected to have any (custom) attributes but it will contain the main dataset self.assertEqual(len(h5_chan_grp.items()), 5 + len(extra_dsets)) for dset_name in [ 'Raw_Data', 'Position_Indices', 'Position_Values', 'Spectroscopic_Indices', 'Spectroscopic_Values' ]: self.assertTrue(dset_name in h5_chan_grp.keys()) h5_dset = h5_chan_grp[dset_name] self.assertIsInstance(h5_dset, h5py.Dataset) usid_main = USIDataset(h5_chan_grp['Raw_Data']) self.assertIsInstance(usid_main, USIDataset) self.assertEqual(usid_main.name.split('/')[-1], 'Raw_Data') self.assertEqual(usid_main.parent, h5_chan_grp) self.assertTrue(np.allclose(main_data, usid_main[()])) validate_aux_dset_pair(self, h5_chan_grp, usid_main.h5_pos_inds, usid_main.h5_pos_vals, pos_names, pos_units, pos_data, h5_main=usid_main, is_spectral=False) validate_aux_dset_pair(self, h5_chan_grp, usid_main.h5_spec_inds, usid_main.h5_spec_vals, spec_names, spec_units, spec_data, h5_main=usid_main, is_spectral=True) # Now validate each of the extra datasets: if extra_dsets_type is not None: for key, val in extra_dsets.items(): self.assertTrue(key in h5_chan_grp.keys()) h5_dset = h5_chan_grp[key] self.assertIsInstance(h5_dset, h5py.Dataset) if extra_dsets_type == 'dask': val = val.compute() self.assertTrue(np.allclose(val, h5_dset[()])) os.remove(file_path)