def test_write_dset_under_root(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: writer = HDFwriter(h5_f) data = np.random.rand(5, 7) attrs = {'att_1': 'string_val', 'att_2': 1.2345, 'att_3': [1, 2, 3, 4], 'att_4': ['str_1', 'str_2', 'str_3'], 'labels': {'even_rows': (slice(0, None, 2), slice(None)), 'odd_rows': (slice(1, None, 2), slice(None))} } micro_dset = VirtualDataset('test', data) micro_dset.attrs = attrs.copy() [h5_dset] = writer.write(micro_dset) self.assertIsInstance(h5_dset, h5py.Dataset) reg_ref = attrs.pop('labels') self.assertEqual(len(h5_dset.attrs), len(attrs) + 1 + len(reg_ref)) for key, expected_val in attrs.items(): self.assertTrue(np.all(get_attr(h5_dset, key) == expected_val)) self.assertTrue(np.all([x in list(reg_ref.keys()) for x in get_attr(h5_dset, 'labels')])) expected_data = [data[:None:2], data[1:None:2]] written_data = [h5_dset[h5_dset.attrs['even_rows']], h5_dset[h5_dset.attrs['odd_rows']]] for exp, act in zip(expected_data, written_data): self.assertTrue(np.allclose(exp, act)) os.remove(file_path)
def test_print(self): group_name = 'indexed_group_' group = VirtualGroup(group_name) dset_name_1 = 'dset_1' data_1 = np.arange(3) dset_1 = VirtualDataset(dset_name_1, data_1) dset_name_2 = 'dset_2' data_2 = np.random.rand(2, 3) dset_2 = VirtualDataset(dset_name_2, data_2) inner_grp_name = 'other_indexed_group_' inner_group = VirtualGroup(inner_grp_name) inner_group.add_children(dset_2) group.add_children([dset_1, inner_group]) # http://pragmaticpython.com/2017/03/23/unittesting-print-statements/ stdout_org = sys.stdout my_stdout = MyOutput() try: sys.stdout = my_stdout group.show_tree() finally: sys.stdout = stdout_org self.assertEqual(str(my_stdout), "/indexed_group_/dset_1\n/indexed_group_/other_indexed_group_\n/indexed_group_" "/other_indexed_group_/dset_2\n")
def test_add_children_legal_01(self): group_name = 'indexed_group_' group = VirtualGroup(group_name) dset_name_1 = 'dset_1' data_1 = np.arange(3) dset_1 = VirtualDataset(dset_name_1, data_1) dset_name_2 = 'dset_2' data_2 = np.random.rand(2, 3) dset_2 = VirtualDataset(dset_name_2, data_2) group.add_children([dset_1, dset_2]) self.assertEqual(len(group.children), 2)
def test_simple_dset_write_success_more_options_03(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: dset_name = 'test' data = np.random.rand(16, 1024) dtype = np.float16 compression = 'gzip' chunking = (1, 1024) microdset = VirtualDataset(dset_name, data, dtype=dtype, compression=compression, chunking=chunking) writer = HDFwriter(h5_f) h5_d = writer._create_simple_dset(h5_f, microdset) self.assertIsInstance(h5_d, h5py.Dataset) self.assertEqual(h5_d.parent, h5_f) self.assertEqual(h5_d.name, '/' + dset_name) self.assertEqual(h5_d.shape, data.shape) self.assertEqual(h5_d.dtype, dtype) self.assertEqual(h5_d.compression, compression) self.assertEqual(h5_d.chunks, chunking) self.assertTrue(np.all(h5_d[()] - data < 1E-3)) os.remove(file_path)
def test_incompatible_maxshape_chunking_01(self): name = 'test' chunking = (3, 128) maxshape = (1,129) data = np.random.rand(5, 128) with self.assertRaises(ValueError): _ = VirtualDataset(name, data, chunking=chunking, maxshape=maxshape)
def test_write_reg_ref_slice_dim_larger_than_data(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: writer = HDFwriter(h5_f) data = np.random.rand(5, 7) h5_dset = writer._create_simple_dset(h5_f, VirtualDataset('test', data)) self.assertIsInstance(h5_dset, h5py.Dataset) attrs = {'labels': {'even_rows': (slice(0, 15, 2), slice(None)), 'odd_rows': (slice(1, 15, 2), slice(None))}} writer._write_dset_attributes(h5_dset, attrs.copy()) h5_f.flush() # two atts point to region references. one for labels self.assertEqual(len(h5_dset.attrs), 1 + len(attrs['labels'])) # check if the labels attribute was written: self.assertTrue(np.all([x in list(attrs['labels'].keys()) for x in get_attr(h5_dset, 'labels')])) expected_data = [data[:None:2], data[1:None:2]] written_data = [h5_dset[h5_dset.attrs['even_rows']], h5_dset[h5_dset.attrs['odd_rows']]] for exp, act in zip(expected_data, written_data): self.assertTrue(np.allclose(exp, act)) os.remove(file_path)
def test_generate_and_write_reg_ref_legal(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: writer = HDFwriter(h5_f) data = np.random.rand(2, 7) h5_dset = writer._create_simple_dset(h5_f, VirtualDataset('test', data)) self.assertIsInstance(h5_dset, h5py.Dataset) attrs = {'labels': ['row_1', 'row_2']} if sys.version_info.major == 3: with self.assertWarns(UserWarning): writer._write_dset_attributes(h5_dset, attrs.copy()) else: writer._write_dset_attributes(h5_dset, attrs.copy()) h5_f.flush() # two atts point to region references. one for labels self.assertEqual(len(h5_dset.attrs), 1 + len(attrs['labels'])) # check if the labels attribute was written: self.assertTrue(np.all([x in list(attrs['labels']) for x in get_attr(h5_dset, 'labels')])) expected_data = [data[0], data[1]] written_data = [h5_dset[h5_dset.attrs['row_1']], h5_dset[h5_dset.attrs['row_2']]] for exp, act in zip(expected_data, written_data): self.assertTrue(np.allclose(np.squeeze(exp), np.squeeze(act))) os.remove(file_path)
def test_empty_dset_write_success_w_options_02(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: dset_name = 'test' maxshape = (16, 1024) chunking = (1, 1024) compression = 'gzip' dtype = np.float16 microdset = VirtualDataset(dset_name, None, maxshape=maxshape, dtype=dtype, compression=compression, chunking=chunking) writer = HDFwriter(h5_f) h5_d = writer._create_empty_dset(h5_f, microdset) self.assertIsInstance(h5_d, h5py.Dataset) self.assertEqual(h5_d.parent, h5_f) self.assertEqual(h5_d.name, '/' + dset_name) self.assertEqual(h5_d.dtype, dtype) self.assertEqual(h5_d.compression, compression) self.assertEqual(h5_d.chunks, chunking) self.assertEqual(h5_d.shape, maxshape) self.assertEqual(h5_d.maxshape, maxshape) os.remove(file_path)
def test_simple_correct_01(self): data = np.arange(5) name = 'test' dset = VirtualDataset(name, data) self.assertTrue(np.all(np.equal(dset.data, data))) self.assertEqual(dset.name, name) self.assertEqual(dset.resizable, False) self.assertEqual(dset.maxshape, None)
def test_large_empty_correct_01(self): data = None name = 'test' maxshape = (1024, 16384) dset = VirtualDataset(name, data, maxshape=maxshape) self.assertEqual(dset.data, data) self.assertEqual(dset.name, name) self.assertEqual(dset.resizable, False) self.assertEqual(dset.maxshape, maxshape)
def test_resizable_correct_01(self): dtype = np.uint16 data = np.zeros(shape=(1, 7), dtype=dtype) name = 'test' resizable = True dset = VirtualDataset(name, data, resizable=resizable) self.assertTrue(np.all(np.equal(dset.data, data))) self.assertEqual(dset.name, name) self.assertEqual(dset.resizable, resizable) self.assertEqual(dset.maxshape, None)
def test_simple_dset_str_dtype_valid(self): data = np.arange(5, dtype=np.complex64) name = 'test' dtype = 'uint8' dset = VirtualDataset(name, data, dtype=dtype) self.assertTrue(np.all(np.equal(dset.data, data))) self.assertEqual(dset.name, name) self.assertEqual(dset.resizable, False) self.assertEqual(dset.maxshape, None) self.assertEqual(dset.dtype, np.dtype(dtype))
def test_correct_compression_simple_dset(self): data = np.arange(5) name = 'test' compression = 'gzip' dset = VirtualDataset(name, data, compression=compression) self.assertTrue(np.all(np.equal(dset.data, data))) self.assertEqual(dset.name, name) self.assertEqual(dset.resizable, False) self.assertEqual(dset.maxshape, None) self.assertEqual(dset.compression, compression)
def test_write_dset_under_invalid_group(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: writer = HDFwriter(h5_f) with self.assertRaises(KeyError): _ = writer.write(VirtualDataset('test', np.random.rand(5, 7), parent='/does_not_exist')) os.remove(file_path)
def test_add_single_child_legal_01(self): dset_name_1 = 'dset_1' data_1 = np.arange(3) dset_1 = VirtualDataset(dset_name_1, data_1) group_name = 'indexed_group_' group = VirtualGroup(group_name) group.add_children(dset_1) group.show_tree() self.assertEqual(len(group.children), 1) in_dset = group.children[0] self.assertIsInstance(in_dset, VirtualDataset) self.assertEqual(in_dset.name, dset_name_1) self.assertTrue(np.all(np.equal(in_dset.data, data_1)))
def test_generate_and_write_reg_ref_illegal(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: writer = HDFwriter(h5_f) data = np.random.rand(2, 7) h5_dset = writer._create_simple_dset(h5_f, VirtualDataset('test', data)) self.assertIsInstance(h5_dset, h5py.Dataset) # with self.assertWarns(UserWarning): with self.assertRaises(TypeError): writer._write_dset_attributes(h5_dset, {'labels': [1, np.arange(3)]}) os.remove(file_path)
def test_write_illegal_reg_ref_not_slice_objs(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: writer = HDFwriter(h5_f) data = np.random.rand(5, 7) h5_dset = writer._create_simple_dset(h5_f, VirtualDataset('test', data)) self.assertIsInstance(h5_dset, h5py.Dataset) attrs = {'labels': {'even_rows': (slice(0, None, 2), 15), 'odd_rows': (slice(1, None, 2), 'hello')}} with self.assertRaises(TypeError): writer._write_dset_attributes(h5_dset, attrs.copy()) os.remove(file_path)
def test_generate_and_write_reg_ref_illegal(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: writer = HDFwriter(h5_f) data = np.random.rand(3, 7) h5_dset = writer._create_simple_dset(h5_f, VirtualDataset('test', data)) self.assertIsInstance(h5_dset, h5py.Dataset) # with self.assertWarns(UserWarning): writer._write_dset_attributes(h5_dset, {'labels': ['row_1', 'row_2']}) self.assertEqual(len(h5_dset.attrs), 0) h5_f.flush() os.remove(file_path)
def test_empty_dset_write_success_01(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: dset_name = 'test' maxshape = (16, 1024) microdset = VirtualDataset(dset_name, None, maxshape=maxshape) writer = HDFwriter(h5_f) h5_d = writer._create_empty_dset(h5_f, microdset) self.assertIsInstance(h5_d, h5py.Dataset) self.assertEqual(h5_d.parent, h5_f) self.assertEqual(h5_d.name, '/' + dset_name) self.assertEqual(h5_d.shape, maxshape) self.assertEqual(h5_d.maxshape, maxshape) # dtype is assigned automatically by h5py. Not to be tested here os.remove(file_path)
def test_simple_dset_write_success_01(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: dtype = np.uint16 dset_name = 'test' data = np.random.randint(0, high=15, size=5, dtype=dtype) microdset = VirtualDataset(dset_name, data) writer = HDFwriter(h5_f) h5_d = writer._create_simple_dset(h5_f, microdset) self.assertIsInstance(h5_d, h5py.Dataset) self.assertEqual(h5_d.parent, h5_f) self.assertEqual(h5_d.name, '/' + dset_name) self.assertEqual(h5_d.shape, data.shape) self.assertTrue(np.allclose(h5_d[()], data)) self.assertEqual(h5_d.dtype, dtype) os.remove(file_path)
def test_expandable_dset_write_success_01(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: dset_name = 'test' maxshape = (None, 1024) data = np.random.rand(1, 1024) microdset = VirtualDataset(dset_name, data, maxshape=maxshape) writer = HDFwriter(h5_f) h5_d = writer._create_resizeable_dset(h5_f, microdset) self.assertIsInstance(h5_d, h5py.Dataset) self.assertEqual(h5_d.parent, h5_f) self.assertEqual(h5_d.name, '/' + dset_name) self.assertEqual(h5_d.shape, data.shape) self.assertEqual(h5_d.maxshape, maxshape) self.assertTrue(np.allclose(h5_d[()], data)) # Now test to make sure that the dataset can be expanded: # TODO: add this to the example! expansion_axis = 0 h5_d.resize(h5_d.shape[expansion_axis] + 1, axis=expansion_axis) self.assertEqual(h5_d.shape, (data.shape[0]+1, data.shape[1])) self.assertEqual(h5_d.maxshape, maxshape) # Finally try checking to see if this new data is also present in the file new_data = np.random.rand(1024) h5_d[1] = new_data data = np.vstack((np.squeeze(data), new_data)) self.assertTrue(np.allclose(h5_d[()], data)) os.remove(file_path)
def test_invalid_chunking_argument_02(self): name = 'test' chunking = ('a', range(5)) data = np.random.rand(2, 128) with self.assertRaises(ValueError): _ = VirtualDataset(name, data, chunking=chunking)
def test_write_simple_tree(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: inner_dset_data = np.random.rand(5, 7) inner_dset_attrs = {'att_1': 'string_val', 'att_2': 1.2345, 'att_3': [1, 2, 3, 4], 'att_4': ['str_1', 'str_2', 'str_3'], 'labels': {'even_rows': (slice(0, None, 2), slice(None)), 'odd_rows': (slice(1, None, 2), slice(None))} } inner_dset = VirtualDataset('inner_dset', inner_dset_data) inner_dset.attrs = inner_dset_attrs.copy() attrs_inner_grp = {'att_1': 'string_val', 'att_2': 1.2345, 'att_3': [1, 2, 3, 4], 'att_4': ['str_1', 'str_2', 'str_3']} inner_group = VirtualGroup('indexed_inner_group_') inner_group.attrs = attrs_inner_grp inner_group.add_children(inner_dset) outer_dset_data = np.random.rand(5, 7) outer_dset_attrs = {'att_1': 'string_val', 'att_2': 1.2345, 'att_3': [1, 2, 3, 4], 'att_4': ['str_1', 'str_2', 'str_3'], 'labels': {'even_rows': (slice(0, None, 2), slice(None)), 'odd_rows': (slice(1, None, 2), slice(None))} } outer_dset = VirtualDataset('test', outer_dset_data, parent='/test_group') outer_dset.attrs = outer_dset_attrs.copy() attrs_outer_grp = {'att_1': 'string_val', 'att_2': 1.2345, 'att_3': [1, 2, 3, 4], 'att_4': ['str_1', 'str_2', 'str_3']} outer_group = VirtualGroup('unindexed_outer_group') outer_group.attrs = attrs_outer_grp outer_group.add_children([inner_group, outer_dset]) writer = HDFwriter(h5_f) h5_refs_list = writer.write(outer_group) # I don't know of a more elegant way to do this: [h5_outer_dset] = get_h5_obj_refs([outer_dset.name], h5_refs_list) [h5_inner_dset] = get_h5_obj_refs([inner_dset.name], h5_refs_list) [h5_outer_group] = get_h5_obj_refs([outer_group.name], h5_refs_list) [h5_inner_group] = get_h5_obj_refs(['indexed_inner_group_000'], h5_refs_list) self.assertIsInstance(h5_outer_dset, h5py.Dataset) self.assertIsInstance(h5_inner_dset, h5py.Dataset) self.assertIsInstance(h5_outer_group, h5py.Group) self.assertIsInstance(h5_inner_group, h5py.Group) # check assertions for the inner dataset first self.assertEqual(h5_inner_dset.parent, h5_inner_group) reg_ref = inner_dset_attrs.pop('labels') self.assertEqual(len(h5_inner_dset.attrs), len(inner_dset_attrs) + 1 + len(reg_ref)) for key, expected_val in inner_dset_attrs.items(): self.assertTrue(np.all(get_attr(h5_inner_dset, key) == expected_val)) self.assertTrue(np.all([x in list(reg_ref.keys()) for x in get_attr(h5_inner_dset, 'labels')])) expected_data = [inner_dset_data[:None:2], inner_dset_data[1:None:2]] written_data = [h5_inner_dset[h5_inner_dset.attrs['even_rows']], h5_inner_dset[h5_inner_dset.attrs['odd_rows']]] for exp, act in zip(expected_data, written_data): self.assertTrue(np.allclose(exp, act)) # check assertions for the inner data group next: self.assertEqual(h5_inner_group.parent, h5_outer_group) for key, expected_val in attrs_inner_grp.items(): self.assertTrue(np.all(get_attr(h5_inner_group, key) == expected_val)) # check the outer dataset next: self.assertEqual(h5_outer_dset.parent, h5_outer_group) reg_ref = outer_dset_attrs.pop('labels') self.assertEqual(len(h5_outer_dset.attrs), len(outer_dset_attrs) + 1 + len(reg_ref)) for key, expected_val in outer_dset_attrs.items(): self.assertTrue(np.all(get_attr(h5_outer_dset, key) == expected_val)) self.assertTrue(np.all([x in list(reg_ref.keys()) for x in get_attr(h5_outer_dset, 'labels')])) expected_data = [outer_dset_data[:None:2], outer_dset_data[1:None:2]] written_data = [h5_outer_dset[h5_outer_dset.attrs['even_rows']], h5_outer_dset[h5_outer_dset.attrs['odd_rows']]] for exp, act in zip(expected_data, written_data): self.assertTrue(np.allclose(exp, act)) # Finally check the outer group: self.assertEqual(h5_outer_group.parent, h5_f) for key, expected_val in attrs_outer_grp.items(): self.assertTrue(np.all(get_attr(h5_outer_group, key) == expected_val)) os.remove(file_path)
def test_chunking_w_none(self): name = 'test' chunking = (None, 128) data = np.random.rand(2, 128) with self.assertRaises(ValueError): _ = VirtualDataset(name, data, chunking=chunking)
def test_incompatible_maxshape_data_shapes(self): name = 'test' maxshape = 128 data = np.random.rand(2, 128) with self.assertRaises(ValueError): _ = VirtualDataset(name, data, maxshape=maxshape)
def test_incorrect_compression_simple_dset(self): data = np.arange(5) name = 'test' compression = 'blah' with self.assertRaises(ValueError): _ = VirtualDataset(name, data, compression=compression)
def test_empty_incorrect_01(self): data = None name = 'test' maxshape = (None, 16384) with self.assertRaises(ValueError): _ = VirtualDataset(name, data, maxshape=maxshape)
def test_insufficient_inputs(self): name = 'test' with self.assertRaises(ValueError): _ = VirtualDataset(name, data=None)
def test_incompatible_chunking_data(self): name = 'test' chunking = (4, 128) data = np.random.rand(2, 128) with self.assertRaises(ValueError): _ = VirtualDataset(name, data, chunking=chunking)
def test_incompatible_chunking_data_03(self): name = 'test' chunking = (4, 128) maxshape = (None, 128) with self.assertRaises(ValueError): _ = VirtualDataset(name, None, maxshape=maxshape, chunking=chunking)