def basic_file_validation(self, h5_f):
self.assertEqual('ImageTranslator',
hdf_utils.get_attr(h5_f, 'translator'))
# First level should have absolutely nothing besides one group
self.assertEqual(len(h5_f.items()), 1)
self.assertTrue('Measurement_000' in h5_f.keys())
h5_meas_grp = h5_f['Measurement_000']
self.assertIsInstance(h5_meas_grp, h5py.Group)
# Again, this group should only have one group - Channel_000
self.assertEqual(len(h5_meas_grp.items()), 1)
self.assertTrue('Channel_000' in h5_meas_grp.keys())
h5_chan_grp = h5_meas_grp['Channel_000']
self.assertIsInstance(h5_chan_grp, h5py.Group)
# This channel group is not expected to have any (custom) attributes but it will contain the main dataset
self.assertEqual(len(h5_chan_grp.items()), 5)
for dset_name in [
'Raw_Data', 'Position_Indices', 'Position_Values',
'Spectroscopic_Indices', 'Spectroscopic_Values'
]:
self.assertTrue(dset_name in h5_chan_grp.keys())
h5_dset = h5_chan_grp[dset_name]
self.assertIsInstance(h5_dset, h5py.Dataset)
usid_main = USIDataset(h5_chan_grp['Raw_Data'])
self.assertIsInstance(usid_main, USIDataset)
self.assertEqual(usid_main.name.split('/')[-1], 'Raw_Data')
self.assertEqual(usid_main.parent, h5_chan_grp)
validate_aux_dset_pair(self,
h5_chan_grp,
usid_main.h5_spec_inds,
usid_main.h5_spec_vals, ['arb'], ['a.u.'],
np.atleast_2d([0]),
h5_main=usid_main,
is_spectral=True)
def test_existing_both_aux(self):
file_path = 'test.h5'
data_utils.delete_existing_file(file_path)
main_data = np.random.rand(15, 14)
main_data_name = 'Test_Main'
quantity = 'Current'
dset_units = 'nA'
pos_sizes = [5, 3]
pos_names = ['X', 'Y']
pos_units = ['nm', 'um']
pos_dims = []
for length, name, units in zip(pos_sizes, pos_names, pos_units):
pos_dims.append(
write_utils.Dimension(name, units, np.arange(length)))
pos_data = np.vstack((np.tile(np.arange(5),
3), np.repeat(np.arange(3), 5))).T
spec_sizes = [7, 2]
spec_names = ['Bias', 'Cycle']
spec_units = ['V', '']
spec_dims = []
for length, name, units in zip(spec_sizes, spec_names, spec_units):
spec_dims.append(
write_utils.Dimension(name, units, np.arange(length)))
spec_data = np.vstack((np.tile(np.arange(7),
2), np.repeat(np.arange(2), 7)))
with h5py.File(file_path) as h5_f:
h5_spec_inds, h5_spec_vals = hdf_utils.write_ind_val_dsets(
h5_f, spec_dims, is_spectral=True)
h5_pos_inds, h5_pos_vals = hdf_utils.write_ind_val_dsets(
h5_f, pos_dims, is_spectral=False)
usid_main = hdf_utils.write_main_dataset(h5_f,
main_data,
main_data_name,
quantity,
dset_units,
None,
None,
h5_spec_inds=h5_spec_inds,
h5_spec_vals=h5_spec_vals,
h5_pos_vals=h5_pos_vals,
h5_pos_inds=h5_pos_inds,
main_dset_attrs=None)
data_utils.validate_aux_dset_pair(self,
h5_f,
h5_pos_inds,
h5_pos_vals,
pos_names,
pos_units,
pos_data,
h5_main=usid_main,
is_spectral=False)
data_utils.validate_aux_dset_pair(self,
h5_f,
h5_spec_inds,
h5_spec_vals,
spec_names,
spec_units,
spec_data,
h5_main=usid_main,
is_spectral=True)
os.remove(file_path)
def test_empty(self):
file_path = 'test.h5'
data_utils.delete_existing_file(file_path)
main_data = (15, 14)
main_data_name = 'Test_Main'
quantity = 'Current'
dset_units = 'nA'
pos_sizes = [5, 3]
pos_names = ['X', 'Y']
pos_units = ['nm', 'um']
pos_dims = []
for length, name, units in zip(pos_sizes, pos_names, pos_units):
pos_dims.append(
write_utils.Dimension(name, units, np.arange(length)))
pos_data = np.vstack((np.tile(np.arange(5),
3), np.repeat(np.arange(3), 5))).T
spec_sizes = [7, 2]
spec_names = ['Bias', 'Cycle']
spec_units = ['V', '']
spec_dims = []
for length, name, units in zip(spec_sizes, spec_names, spec_units):
spec_dims.append(
write_utils.Dimension(name, units, np.arange(length)))
spec_data = np.vstack((np.tile(np.arange(7),
2), np.repeat(np.arange(2), 7)))
with h5py.File(file_path) as h5_f:
usid_main = hdf_utils.write_main_dataset(h5_f,
main_data,
main_data_name,
quantity,
dset_units,
pos_dims,
spec_dims,
dtype=np.float16,
main_dset_attrs=None)
self.assertIsInstance(usid_main, USIDataset)
self.assertEqual(usid_main.name.split('/')[-1], main_data_name)
self.assertEqual(usid_main.parent, h5_f)
self.assertEqual(main_data, usid_main.shape)
data_utils.validate_aux_dset_pair(self,
h5_f,
usid_main.h5_pos_inds,
usid_main.h5_pos_vals,
pos_names,
pos_units,
pos_data,
h5_main=usid_main,
is_spectral=False)
data_utils.validate_aux_dset_pair(self,
h5_f,
usid_main.h5_spec_inds,
usid_main.h5_spec_vals,
spec_names,
spec_units,
spec_data,
h5_main=usid_main,
is_spectral=True)
os.remove(file_path)
def main_translate(self, **kwargs):
h5_path = kwargs.pop('h5_path', image_path.replace('.png', '.h5'))
delete_existing_file(h5_path)
input_image = rand_image.copy()
usize, vsize = input_image.shape[:2]
translator = ImageTranslator()
h5_path = translator.translate(image_path, **kwargs)
image_parms = dict()
if 'bin_factor' in kwargs.keys():
bin_factor = kwargs.pop('bin_factor')
if bin_factor is None:
_ = kwargs.pop('interp_func', None)
else:
if isinstance(bin_factor, int):
bin_factor = (bin_factor, bin_factor)
interp_func = kwargs.pop('interp_func', Image.BICUBIC)
image_parms.update({
'image_binning_size': np.array(bin_factor),
'image_PIL_resample_mode': interp_func
})
img_obj = Image.fromarray(input_image)
img_obj = img_obj.convert(mode="L")
img_obj = img_obj.resize(
(int(vsize / bin_factor[1]), int(usize / bin_factor[0])),
resample=interp_func)
input_image = np.asarray(img_obj)
image_parms.update({'normalized': False})
input_image = input_image.copy()
if 'normalize' in kwargs.keys():
normalize = kwargs.pop('normalize')
if normalize:
input_image -= np.min(input_image)
input_image = input_image / np.float32(np.max(input_image))
image_parms.update({'normalized': True})
image_parms.update({
'image_min': np.min(input_image),
'image_max': np.max(input_image)
})
with h5py.File(h5_path, mode='r') as h5_f:
self.basic_file_validation(h5_f)
h5_meas_grp = h5_f['Measurement_000']
h5_chan_grp = h5_meas_grp['Channel_000']
usid_main = USIDataset(h5_chan_grp['Raw_Data'])
# check the attributes under this group
for key, expected_val in image_parms.items():
self.assertTrue(
np.all(
hdf_utils.get_attr(h5_meas_grp, key) == expected_val))
one_d_image = input_image.T.reshape(-1, 1)
self.assertTrue(np.allclose(one_d_image, usid_main[()]))
# self.assertTrue(np.allclose(rand_image, np.reshape(usid_main[()], rand_image.shape)))
pos_data = np.vstack((np.tile(np.arange(input_image.shape[0]),
input_image.shape[1]),
np.repeat(np.arange(input_image.shape[1]),
input_image.shape[0]))).T
validate_aux_dset_pair(self,
h5_chan_grp,
usid_main.h5_pos_inds,
usid_main.h5_pos_vals, ['Y', 'X'],
['a.u.', 'a.u.'],
pos_data,
h5_main=usid_main,
is_spectral=False)
delete_existing_file(h5_path)
def base_translation_tester(self,
main_dset_as_dask=False,
extra_dsets_type='numpy',
use_parm_dict=True):
data_name = 'My_Awesome_Measurement'
if use_parm_dict:
attrs = {
'att_1': 'string_val',
'att_2': 1.2345,
'att_3': [1, 2, 3, 4],
'att_4': ['str_1', 'str_2', 'str_3']
}
else:
attrs = None
extra_dsets = {}
if extra_dsets_type is not None:
ref_dsets = {'dset_1': np.random.rand(5), 'dset_2': np.arange(25)}
if extra_dsets_type == 'numpy':
extra_dsets = ref_dsets
elif extra_dsets_type == 'dask':
for key, val in ref_dsets.items():
extra_dsets.update(
{key: da.from_array(val, chunks=val.shape)})
else:
extra_dsets_type = None
delete_existing_file(file_path)
main_data = np.random.rand(15, 14)
if main_dset_as_dask:
main_data = da.from_array(main_data, chunks=main_data.shape)
quantity = 'Current'
units = 'nA'
pos_sizes = [5, 3]
pos_names = ['X', 'Y']
pos_units = ['nm', 'um']
pos_dims = []
for name, unit, length in zip(pos_names, pos_units, pos_sizes):
pos_dims.append(
write_utils.Dimension(name, unit, np.arange(length)))
pos_data = np.vstack((np.tile(np.arange(5),
3), np.repeat(np.arange(3), 5))).T
spec_sizes = [7, 2]
spec_names = ['Bias', 'Cycle']
spec_units = ['V', '']
spec_dims = []
for name, unit, length in zip(spec_names, spec_units, spec_sizes):
spec_dims.append(
write_utils.Dimension(name, unit, np.arange(length)))
spec_data = np.vstack((np.tile(np.arange(7),
2), np.repeat(np.arange(2), 7)))
translator = ArrayTranslator()
_ = translator.translate(file_path,
data_name,
main_data,
quantity,
units,
pos_dims,
spec_dims,
parm_dict=attrs,
extra_dsets=extra_dsets)
with h5py.File(file_path, mode='r') as h5_f:
# we are not interested in most of the attributes under root besides two:
self.assertEqual(data_name, hdf_utils.get_attr(h5_f, 'data_type'))
# self.assertEqual('NumpyTranslator', hdf_utils.get_attr(h5_f, 'translator'))
# First level should have absolutely nothing besides one group
self.assertEqual(len(h5_f.items()), 1)
self.assertTrue('Measurement_000' in h5_f.keys())
h5_meas_grp = h5_f['Measurement_000']
self.assertIsInstance(h5_meas_grp, h5py.Group)
# check the attributes under this group
# self.assertEqual(len(h5_meas_grp.attrs), len(attrs))
if use_parm_dict:
for key, expected_val in attrs.items():
self.assertTrue(
np.all(
hdf_utils.get_attr(h5_meas_grp, key) ==
expected_val))
# Again, this group should only have one group - Channel_000
self.assertEqual(len(h5_meas_grp.items()), 1)
self.assertTrue('Channel_000' in h5_meas_grp.keys())
h5_chan_grp = h5_meas_grp['Channel_000']
self.assertIsInstance(h5_chan_grp, h5py.Group)
# This channel group is not expected to have any (custom) attributes but it will contain the main dataset
self.assertEqual(len(h5_chan_grp.items()), 5 + len(extra_dsets))
for dset_name in [
'Raw_Data', 'Position_Indices', 'Position_Values',
'Spectroscopic_Indices', 'Spectroscopic_Values'
]:
self.assertTrue(dset_name in h5_chan_grp.keys())
h5_dset = h5_chan_grp[dset_name]
self.assertIsInstance(h5_dset, h5py.Dataset)
usid_main = USIDataset(h5_chan_grp['Raw_Data'])
self.assertIsInstance(usid_main, USIDataset)
self.assertEqual(usid_main.name.split('/')[-1], 'Raw_Data')
self.assertEqual(usid_main.parent, h5_chan_grp)
self.assertTrue(np.allclose(main_data, usid_main[()]))
validate_aux_dset_pair(self,
h5_chan_grp,
usid_main.h5_pos_inds,
usid_main.h5_pos_vals,
pos_names,
pos_units,
pos_data,
h5_main=usid_main,
is_spectral=False)
validate_aux_dset_pair(self,
h5_chan_grp,
usid_main.h5_spec_inds,
usid_main.h5_spec_vals,
spec_names,
spec_units,
spec_data,
h5_main=usid_main,
is_spectral=True)
# Now validate each of the extra datasets:
if extra_dsets_type is not None:
for key, val in extra_dsets.items():
self.assertTrue(key in h5_chan_grp.keys())
h5_dset = h5_chan_grp[key]
self.assertIsInstance(h5_dset, h5py.Dataset)
if extra_dsets_type == 'dask':
val = val.compute()
self.assertTrue(np.allclose(val, h5_dset[()]))
os.remove(file_path)