def test_file(self):
file_path = 'test.h5'
data_utils.delete_existing_file(file_path)
with h5py.File(file_path) as h5_f:
hdf_utils.write_book_keeping_attrs(h5_f)
data_utils.verify_book_keeping_attrs(self, h5_f)
os.remove(file_path)
def test_create_region_ref(self):
file_path = 'test.h5'
data_utils.delete_existing_file(file_path)
data = np.random.rand(5, 7)
with h5py.File(file_path) as h5_f:
h5_dset = h5_f.create_dataset('Source', data=data)
pos_inds = np.arange(0, h5_dset.shape[0], 2)
ref_inds = [((pos_start, 0), (pos_start, h5_dset.shape[1] - 1))
for pos_start in pos_inds]
ref_inds = np.array(ref_inds)
this_reg_ref = reg_ref.create_region_reference(h5_dset, ref_inds)
ref_slices = list()
for start, stop in ref_inds:
ref_slices.append(
[slice(start[0], stop[0] + 1),
slice(start[1], None)])
h5_reg = h5_dset[this_reg_ref]
h5_slice = np.vstack([
h5_dset[pos_slice, spec_slice]
for (pos_slice, spec_slice) in ref_slices
])
self.assertTrue(np.allclose(h5_reg, h5_slice))
os.remove(file_path)
def test_write_reg_ref_main_2nd_dim(self):
file_path = 'test.h5'
data_utils.delete_existing_file(file_path)
data = np.random.rand(5, 7)
with h5py.File(file_path) as h5_f:
h5_dset = h5_f.create_dataset('Main', data=data)
reg_refs = {
'even_rows': (slice(None), slice(0, None, 2)),
'odd_rows': (slice(None), slice(1, None, 2))
}
reg_ref.write_region_references(h5_dset,
reg_refs,
add_labels_attr=False)
self.assertEqual(len(h5_dset.attrs), len(reg_refs))
self.assertTrue('labels' not in h5_dset.attrs.keys())
expected_data = [data[:, 0:None:2], data[:, 1:None:2]]
written_data = [
h5_dset[h5_dset.attrs['even_rows']],
h5_dset[h5_dset.attrs['odd_rows']]
]
for exp, act in zip(expected_data, written_data):
self.assertTrue(np.allclose(exp, act))
os.remove(file_path)
def test_legal(self):
file_path = 'link_h5_objects_as_attrs.h5'
data_utils.delete_existing_file(file_path)
with h5py.File(file_path) as h5_f:
h5_main = h5_f.create_dataset('main', data=np.arange(5))
h5_anc = h5_f.create_dataset('Ancillary', data=np.arange(3))
h5_group = h5_f.create_group('Results')
hdf_utils.link_h5_objects_as_attrs(h5_f,
[h5_anc, h5_main, h5_group])
for exp, name in zip([h5_main, h5_anc, h5_group],
['main', 'Ancillary', 'Results']):
self.assertEqual(exp, h5_f[h5_f.attrs[name]])
# Single object
hdf_utils.link_h5_objects_as_attrs(h5_main, h5_anc)
self.assertEqual(h5_f[h5_main.attrs['Ancillary']], h5_anc)
# Linking to a group:
hdf_utils.link_h5_objects_as_attrs(h5_group, [h5_anc, h5_main])
for exp, name in zip([h5_main, h5_anc], ['main', 'Ancillary']):
self.assertEqual(exp, h5_group[h5_group.attrs[name]])
os.remove(file_path)
def test_legal(self):
file_path = 'link_as_alias.h5'
data_utils.delete_existing_file(file_path)
with h5py.File(file_path) as h5_f:
h5_main = h5_f.create_dataset('main', data=np.arange(5))
h5_anc = h5_f.create_dataset('Ancillary', data=np.arange(3))
h5_group = h5_f.create_group('Results')
# Linking to dataset:
hdf_utils.link_h5_obj_as_alias(h5_main, h5_anc, 'Blah')
hdf_utils.link_h5_obj_as_alias(h5_main, h5_group, 'Something')
self.assertEqual(h5_f[h5_main.attrs['Blah']], h5_anc)
self.assertEqual(h5_f[h5_main.attrs['Something']], h5_group)
# Linking ot Group:
hdf_utils.link_h5_obj_as_alias(h5_group, h5_main, 'Center')
hdf_utils.link_h5_obj_as_alias(h5_group, h5_anc, 'South')
self.assertEqual(h5_f[h5_group.attrs['Center']], h5_main)
self.assertEqual(h5_f[h5_group.attrs['South']], h5_anc)
# Linking to file:
hdf_utils.link_h5_obj_as_alias(h5_f, h5_main, 'Paris')
hdf_utils.link_h5_obj_as_alias(h5_f, h5_group, 'France')
self.assertEqual(h5_f[h5_f.attrs['Paris']], h5_main)
self.assertEqual(h5_f[h5_f.attrs['France']], h5_group)
def tearDown(self):
for file_path in [
data_utils.std_beps_path, data_utils.sparse_sampling_path,
data_utils.incomplete_measurement_path,
data_utils.relaxation_path
]:
data_utils.delete_existing_file(file_path)
def test_write_reg_ref_main_one_dim(self):
file_path = 'test.h5'
data_utils.delete_existing_file(file_path)
data = np.random.rand(7)
with h5py.File(file_path) as h5_f:
h5_dset = h5_f.create_dataset('Main', data=data)
reg_refs = {
'even_rows': (slice(0, None, 2)),
'odd_rows': (slice(1, None, 2))
}
reg_ref.write_region_references(h5_dset,
reg_refs,
add_labels_attr=True)
self.assertEqual(len(h5_dset.attrs), 1 + len(reg_refs))
actual = get_attr(h5_dset, 'labels')
self.assertTrue(
np.all([
x == y for x, y in zip(actual, ['even_rows', 'odd_rows'])
]))
expected_data = [data[0:None:2], data[1:None:2]]
written_data = [
h5_dset[h5_dset.attrs['even_rows']],
h5_dset[h5_dset.attrs['odd_rows']]
]
for exp, act in zip(expected_data, written_data):
self.assertTrue(np.allclose(exp, act))
os.remove(file_path)
def test_prod_sizes_mismatch(self):
file_path = 'test.h5'
data_utils.delete_existing_file(file_path)
main_data = np.random.rand(15, 14)
main_data_name = 'Test_Main'
quantity = 'Current'
dset_units = 'nA'
pos_sizes = [5, 15] # too many steps in the Y direction
pos_names = ['X', 'Y']
pos_units = ['nm', 'um']
pos_dims = []
for length, name, units in zip(pos_sizes, pos_names, pos_units):
pos_dims.append(
write_utils.Dimension(name, units, np.arange(length)))
spec_sizes = [7, 2]
spec_names = ['Bias', 'Cycle']
spec_units = ['V', '']
spec_dims = []
for length, name, units in zip(spec_sizes, spec_names, spec_units):
spec_dims.append(
write_utils.Dimension(name, units, np.arange(length)))
with h5py.File(file_path) as h5_f:
with self.assertRaises(ValueError):
_ = hdf_utils.write_main_dataset(h5_f, main_data,
main_data_name, quantity,
dset_units, pos_dims,
spec_dims)
os.remove(file_path)
def test_dset(self):
file_path = 'test.h5'
data_utils.delete_existing_file(file_path)
with h5py.File(file_path) as h5_f:
h5_dset = h5_f.create_dataset('dset', data=[1, 2, 3])
hdf_utils.write_book_keeping_attrs(h5_dset)
data_utils.verify_book_keeping_attrs(self, h5_dset)
os.remove(file_path)
def test_group(self):
file_path = 'test.h5'
data_utils.delete_existing_file(file_path)
with h5py.File(file_path) as h5_f:
h5_g = h5_f.create_group('group')
hdf_utils.write_book_keeping_attrs(h5_g)
data_utils.verify_book_keeping_attrs(self, h5_g)
os.remove(file_path)
def test_not_numpy_or_dask_array_main(self):
translator = ArrayTranslator()
with self.assertRaises(TypeError):
delete_existing_file(file_path)
_ = translator.translate(
file_path, 'Blah', {'This is not a dataset': True}, 'quant',
'unit', write_utils.Dimension('Position_Dim', 'au', 5),
write_utils.Dimension('Spec_Dim', 'au', 3))
def test_objects(self):
translator = ArrayTranslator()
with self.assertRaises(TypeError):
delete_existing_file(file_path)
_ = translator.translate(
file_path, 'Blah', np.random.rand(5, 13), 'quant', 'unit',
write_utils.Dimension('Dim_1', 'au', 5),
['blah', write_utils.Dimension('Dim_2', 'au', 4)])
def test_clean_reg_refs_2d(self):
file_path = 'test.h5'
data_utils.delete_existing_file(file_path)
with h5py.File(file_path) as h5_f:
h5_dset = h5_f.create_dataset('Test', data=np.random.rand(7, 5))
ref_in = (slice(0, None, 2), slice(None))
cleaned = reg_ref.clean_reg_ref(h5_dset, ref_in)
self.assertTrue(np.all([x == y for x, y in zip(ref_in, cleaned)]))
os.remove(file_path)
def test_clean_reg_refs_1d(self):
file_path = 'test.h5'
data_utils.delete_existing_file(file_path)
with h5py.File(file_path) as h5_f:
h5_dset = h5_f.create_dataset('Test', data=np.random.rand(7))
ref_in = (slice(0, None, 2))
cleaned = reg_ref.clean_reg_ref(h5_dset, ref_in)
self.assertEqual(ref_in, cleaned[0])
os.remove(file_path)
def test_simple_region_ref_copy(self):
# based on test_hdf_writer.test_write_legal_reg_ref_multi_dim_data()
file_path = 'test.h5'
data_utils.delete_existing_file(file_path)
with h5py.File(file_path) as h5_f:
data = np.random.rand(5, 7)
h5_orig_dset = h5_f.create_dataset('test', data=data)
self.assertIsInstance(h5_orig_dset, h5py.Dataset)
attrs = {
'labels': {
'even_rows': (slice(0, None, 2), slice(None)),
'odd_rows': (slice(1, None, 2), slice(None))
}
}
data_utils.write_main_reg_refs(h5_orig_dset, attrs['labels'])
h5_f.flush()
# two atts point to region references. one for labels
self.assertEqual(len(h5_orig_dset.attrs), 1 + len(attrs['labels']))
# check if the labels attribute was written:
self.assertTrue(
np.all([
x in list(attrs['labels'].keys())
for x in get_attr(h5_orig_dset, 'labels')
]))
expected_data = [data[:None:2], data[1:None:2]]
written_data = [
h5_orig_dset[h5_orig_dset.attrs['even_rows']],
h5_orig_dset[h5_orig_dset.attrs['odd_rows']]
]
for exp, act in zip(expected_data, written_data):
self.assertTrue(np.allclose(exp, act))
# Now write a new dataset without the region reference:
h5_new_dset = h5_f.create_dataset('other', data=data)
self.assertIsInstance(h5_orig_dset, h5py.Dataset)
h5_f.flush()
for key in attrs['labels'].keys():
reg_ref.simple_region_ref_copy(h5_orig_dset, h5_new_dset, key)
# now check to make sure that this dataset also has the same region references:
written_data = [
h5_new_dset[h5_new_dset.attrs['even_rows']],
h5_new_dset[h5_new_dset.attrs['odd_rows']]
]
for exp, act in zip(expected_data, written_data):
self.assertTrue(np.allclose(exp, act))
os.remove(file_path)
def test_invalid_attrs_dict(self):
file_path = 'test.h5'
data_utils.delete_existing_file(file_path)
with h5py.File(file_path) as h5_f:
h5_group = h5_f.create_group('Blah')
with self.assertRaises(TypeError):
hdf_utils.write_simple_attrs(
h5_group, ['attrs', 1.234, 'should be dict',
np.arange(3)])
def test_h5_dset_w_chunks(self):
h5_path = 'blah.h5'
delete_existing_file(h5_path)
with h5py.File(h5_path) as h5_f:
np_arr = np.random.rand(200, 30)
h5_dset = h5_f.create_dataset('Test', data=np_arr, chunks=(1, 30))
da_arr = da.from_array(np_arr, chunks=h5_dset.chunks)
self.assertTrue(np.allclose(da_arr, dtype_utils.lazy_load_array(h5_dset)))
os.remove(h5_path)
def test_position(self):
translator = ArrayTranslator()
with self.assertRaises(ValueError):
delete_existing_file(file_path)
_ = translator.translate(
file_path, 'Blah', np.random.rand(15, 3), 'quant', 'unit', [
write_utils.Dimension('Dim_1', 'au', 5),
write_utils.Dimension('Dim_2', 'au', 4)
], write_utils.Dimension('Spec_Dim', 'au', 3))
def test_text_to_numpy_simple(self):
img_data = rand_image.astype(np.uint8)
img_path = 'image_text.txt'
delete_existing_file(img_path)
np.savetxt(img_path, img_data)
np_data = read_image(image_path, as_numpy_array=True)
self.assertIsInstance(np_data, np.ndarray)
self.assertTrue(np.allclose(np_data, img_data))
delete_existing_file(img_path)
def test_object_single(self):
translator = ArrayTranslator()
with self.assertRaises(TypeError):
delete_existing_file(file_path)
_ = translator.translate(file_path, 'Blah', np.random.rand(
5, 13), 'quant', 'unit', 'my_string_Dimension', [
write_utils.Dimension('Spec_Dim', 'au', 3),
write_utils.Dimension('Dim_2', 'au', 4)
])
def test_attempt_reg_ref_build_pos_too_few_dims(self):
file_path = 'test.h5'
data_utils.delete_existing_file(file_path)
with h5py.File(file_path) as h5_f:
h5_dset = h5_f.create_dataset('Indices', data=np.random.rand(5, 2))
dim_names = ['Bias']
ret_val = reg_ref.attempt_reg_ref_build(h5_dset, dim_names)
self.assertEqual(ret_val, dict())
os.remove(file_path)
def test_clean_reg_refs_illegal_too_few_slices(self):
file_path = 'test.h5'
data_utils.delete_existing_file(file_path)
with h5py.File(file_path) as h5_f:
h5_dset = h5_f.create_dataset('Test', data=np.random.rand(7, 5))
ref_in = (slice(0, None, 2))
with self.assertRaises(ValueError):
_ = reg_ref.clean_reg_ref(h5_dset, ref_in)
os.remove(file_path)
def test_w(self):
file_path = 'test.h5'
data_utils.delete_existing_file(file_path)
with h5py.File(file_path) as h5_f:
h5_dset = h5_f.create_dataset('Test', data=np.arange(3))
h5_group = h5_f.create_group('blah')
self.assertTrue(hdf_utils.is_editable_h5(h5_group))
self.assertTrue(hdf_utils.is_editable_h5(h5_f))
self.assertTrue(hdf_utils.is_editable_h5(h5_dset))
os.remove(file_path)
def test_clean_reg_refs_out_of_bounds(self):
file_path = 'test.h5'
data_utils.delete_existing_file(file_path)
with h5py.File(file_path) as h5_f:
h5_dset = h5_f.create_dataset('Test', data=np.random.rand(7, 5))
ref_in = (slice(0, 13, 2), slice(None))
expected = (slice(0, 7, 2), slice(None))
cleaned = reg_ref.clean_reg_ref(h5_dset, ref_in, verbose=False)
self.assertTrue(np.all([x == y
for x, y in zip(expected, cleaned)]))
os.remove(file_path)
def test_none_ignored(self):
file_path = 'test.h5'
data_utils.delete_existing_file(file_path)
with h5py.File(file_path) as h5_f:
attrs = {'att_1': None}
hdf_utils.write_simple_attrs(h5_f, attrs)
self.assertTrue('att_1' not in h5_f.attrs.keys())
os.remove(file_path)
def test_empty_name(self):
translator = ArrayTranslator()
with self.assertRaises(ValueError):
delete_existing_file(file_path)
_ = translator.translate(
file_path,
'Blah',
np.random.rand(5, 3),
'quant',
'unit',
write_utils.Dimension('Position_Dim', 'au', 5),
write_utils.Dimension('Spec_Dim', 'au', 3),
extra_dsets={' ': [1, 2, 3]})
def test_not_arrays(self):
translator = ArrayTranslator()
with self.assertRaises(TypeError):
delete_existing_file(file_path)
_ = translator.translate(
file_path,
'Blah',
np.random.rand(5, 3),
'quant',
'unit',
write_utils.Dimension('Position_Dim', 'au', 5),
write_utils.Dimension('Spec_Dim', 'au', 3),
extra_dsets={'Blah_other': 'I am not an array'})
def test_wrong_type(self):
file_path = 'link_h5_objects_as_attrs.h5'
data_utils.delete_existing_file(file_path)
with h5py.File(file_path) as h5_f:
h5_main = h5_f.create_dataset('main', data=np.arange(5))
with self.assertRaises(TypeError):
hdf_utils.link_h5_objects_as_attrs(h5_main, np.arange(4))
with self.assertRaises(TypeError):
hdf_utils.link_h5_objects_as_attrs(np.arange(4), h5_main)
os.remove(file_path)
def test_np_array(self):
file_path = 'test.h5'
data_utils.delete_existing_file(file_path)
with h5py.File(file_path) as h5_f:
attrs = {'att_1': np.random.rand(4)}
hdf_utils.write_simple_attrs(h5_f, attrs)
for key, expected_val in attrs.items():
self.assertTrue(
np.all(hdf_utils.get_attr(h5_f, key) == expected_val))
os.remove(file_path)
def test_reserved_names(self):
translator = ArrayTranslator()
with self.assertRaises(KeyError):
delete_existing_file(file_path)
_ = translator.translate(
file_path,
'Blah',
np.random.rand(5, 3),
'quant',
'unit',
write_utils.Dimension('Position_Dim', 'au', 5),
write_utils.Dimension('Spec_Dim', 'au', 3),
extra_dsets={
'Spectroscopic_Indices': np.arange(4),
'Blah_other': np.arange(15)
})
