def test_write_reg_ref_slice_dim_larger_than_data(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: writer = HDFwriter(h5_f) data = np.random.rand(5, 7) h5_dset = writer._create_simple_dset(h5_f, VirtualDataset('test', data)) self.assertIsInstance(h5_dset, h5py.Dataset) attrs = {'labels': {'even_rows': (slice(0, 15, 2), slice(None)), 'odd_rows': (slice(1, 15, 2), slice(None))}} writer._write_dset_attributes(h5_dset, attrs.copy()) h5_f.flush() # two atts point to region references. one for labels self.assertEqual(len(h5_dset.attrs), 1 + len(attrs['labels'])) # check if the labels attribute was written: self.assertTrue(np.all([x in list(attrs['labels'].keys()) for x in get_attr(h5_dset, 'labels')])) expected_data = [data[:None:2], data[1:None:2]] written_data = [h5_dset[h5_dset.attrs['even_rows']], h5_dset[h5_dset.attrs['odd_rows']]] for exp, act in zip(expected_data, written_data): self.assertTrue(np.allclose(exp, act)) os.remove(file_path)
def test_write_reg_ref_slice_dim_larger_than_data(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: writer = HDFwriter(h5_f) data = np.random.rand(5, 7) h5_dset = writer._create_simple_dset(h5_f, VirtualDataset('test', data)) self.assertIsInstance(h5_dset, h5py.Dataset) attrs = {'labels': {'even_rows': (slice(0, 15, 2), slice(None)), 'odd_rows': (slice(1, 15, 2), slice(None))}} writer._write_dset_attributes(h5_dset, attrs.copy()) h5_f.flush() # two atts point to region references. one for labels self.assertEqual(len(h5_dset.attrs), 1 + len(attrs['labels'])) # check if the labels attribute was written: self.assertTrue(np.all([x in list(attrs['labels'].keys()) for x in get_attr(h5_dset, 'labels')])) expected_data = [data[:None:2], data[1:None:2]] written_data = [h5_dset[h5_dset.attrs['even_rows']], h5_dset[h5_dset.attrs['odd_rows']]] for exp, act in zip(expected_data, written_data): self.assertTrue(np.allclose(exp, act)) os.remove(file_path)
def test_generate_and_write_reg_ref_legal(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: writer = HDFwriter(h5_f) data = np.random.rand(2, 7) h5_dset = writer._create_simple_dset(h5_f, VirtualDataset('test', data)) self.assertIsInstance(h5_dset, h5py.Dataset) attrs = {'labels': ['row_1', 'row_2']} if sys.version_info.major == 3: with self.assertWarns(UserWarning): writer._write_dset_attributes(h5_dset, attrs.copy()) else: writer._write_dset_attributes(h5_dset, attrs.copy()) h5_f.flush() # two atts point to region references. one for labels self.assertEqual(len(h5_dset.attrs), 1 + len(attrs['labels'])) # check if the labels attribute was written: self.assertTrue(np.all([x in list(attrs['labels']) for x in get_attr(h5_dset, 'labels')])) expected_data = [data[0], data[1]] written_data = [h5_dset[h5_dset.attrs['row_1']], h5_dset[h5_dset.attrs['row_2']]] for exp, act in zip(expected_data, written_data): self.assertTrue(np.allclose(np.squeeze(exp), np.squeeze(act))) os.remove(file_path)
def test_simple_dset_write_success_more_options_03(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: dset_name = 'test' data = np.random.rand(16, 1024) dtype = np.float16 compression = 'gzip' chunking = (1, 1024) microdset = VirtualDataset(dset_name, data, dtype=dtype, compression=compression, chunking=chunking) writer = HDFwriter(h5_f) h5_d = writer._create_simple_dset(h5_f, microdset) self.assertIsInstance(h5_d, h5py.Dataset) self.assertEqual(h5_d.parent, h5_f) self.assertEqual(h5_d.name, '/' + dset_name) self.assertEqual(h5_d.shape, data.shape) self.assertEqual(h5_d.dtype, dtype) self.assertEqual(h5_d.compression, compression) self.assertEqual(h5_d.chunks, chunking) self.assertTrue(np.all(h5_d[()] - data < 1E-3)) os.remove(file_path)
def test_generate_and_write_reg_ref_legal(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: writer = HDFwriter(h5_f) data = np.random.rand(2, 7) h5_dset = writer._create_simple_dset(h5_f, VirtualDataset('test', data)) self.assertIsInstance(h5_dset, h5py.Dataset) attrs = {'labels': ['row_1', 'row_2']} if sys.version_info.major == 3: with self.assertWarns(UserWarning): writer._write_dset_attributes(h5_dset, attrs.copy()) else: writer._write_dset_attributes(h5_dset, attrs.copy()) h5_f.flush() # two atts point to region references. one for labels self.assertEqual(len(h5_dset.attrs), 1 + len(attrs['labels'])) # check if the labels attribute was written: self.assertTrue(np.all([x in list(attrs['labels']) for x in get_attr(h5_dset, 'labels')])) expected_data = [data[0], data[1]] written_data = [h5_dset[h5_dset.attrs['row_1']], h5_dset[h5_dset.attrs['row_2']]] for exp, act in zip(expected_data, written_data): self.assertTrue(np.allclose(np.squeeze(exp), np.squeeze(act))) os.remove(file_path)
def test_generate_and_write_reg_ref_illegal(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: writer = HDFwriter(h5_f) data = np.random.rand(2, 7) h5_dset = writer._create_simple_dset(h5_f, VirtualDataset('test', data)) self.assertIsInstance(h5_dset, h5py.Dataset) # with self.assertWarns(UserWarning): with self.assertRaises(TypeError): writer._write_dset_attributes(h5_dset, {'labels': [1, np.arange(3)]}) os.remove(file_path)
def test_generate_and_write_reg_ref_illegal(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: writer = HDFwriter(h5_f) data = np.random.rand(2, 7) h5_dset = writer._create_simple_dset(h5_f, VirtualDataset('test', data)) self.assertIsInstance(h5_dset, h5py.Dataset) # with self.assertWarns(UserWarning): with self.assertRaises(TypeError): writer._write_dset_attributes(h5_dset, {'labels': [1, np.arange(3)]}) os.remove(file_path)
def test_write_illegal_reg_ref_not_slice_objs(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: writer = HDFwriter(h5_f) data = np.random.rand(5, 7) h5_dset = writer._create_simple_dset(h5_f, VirtualDataset('test', data)) self.assertIsInstance(h5_dset, h5py.Dataset) attrs = {'labels': {'even_rows': (slice(0, None, 2), 15), 'odd_rows': (slice(1, None, 2), 'hello')}} with self.assertRaises(TypeError): writer._write_dset_attributes(h5_dset, attrs.copy()) os.remove(file_path)
def test_write_illegal_reg_ref_not_slice_objs(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: writer = HDFwriter(h5_f) data = np.random.rand(5, 7) h5_dset = writer._create_simple_dset(h5_f, VirtualDataset('test', data)) self.assertIsInstance(h5_dset, h5py.Dataset) attrs = {'labels': {'even_rows': (slice(0, None, 2), 15), 'odd_rows': (slice(1, None, 2), 'hello')}} with self.assertRaises(TypeError): writer._write_dset_attributes(h5_dset, attrs.copy()) os.remove(file_path)
def test_write_simple_atts_reg_ref_to_dset(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: writer = HDFwriter(h5_f) data = np.random.rand(5, 7) h5_dset = writer._create_simple_dset(h5_f, VirtualDataset('test', data)) self.assertIsInstance(h5_dset, h5py.Dataset) attrs = { 'att_1': 'string_val', 'att_2': 1.2345, 'att_3': [1, 2, 3, 4], 'att_4': ['str_1', 'str_2', 'str_3'], 'labels': { 'even_rows': (slice(0, None, 2), slice(None)), 'odd_rows': (slice(1, None, 2), slice(None)) } } writer._write_dset_attributes(h5_dset, attrs.copy()) reg_ref = attrs.pop('labels') self.assertEqual(len(h5_dset.attrs), len(attrs) + 1 + len(reg_ref)) for key, expected_val in attrs.items(): self.assertTrue(np.all(get_attr(h5_dset, key) == expected_val)) self.assertTrue( np.all([ x in list(reg_ref.keys()) for x in get_attr(h5_dset, 'labels') ])) expected_data = [data[:None:2], data[1:None:2]] written_data = [ h5_dset[h5_dset.attrs['even_rows']], h5_dset[h5_dset.attrs['odd_rows']] ] for exp, act in zip(expected_data, written_data): self.assertTrue(np.allclose(exp, act)) os.remove(file_path)
def test_generate_and_write_reg_ref_illegal(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: writer = HDFwriter(h5_f) data = np.random.rand(3, 7) h5_dset = writer._create_simple_dset(h5_f, VirtualDataset('test', data)) self.assertIsInstance(h5_dset, h5py.Dataset) # with self.assertWarns(UserWarning): writer._write_dset_attributes(h5_dset, {'labels': ['row_1', 'row_2']}) self.assertEqual(len(h5_dset.attrs), 0) h5_f.flush() os.remove(file_path)
def test_generate_and_write_reg_ref_illegal(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: writer = HDFwriter(h5_f) data = np.random.rand(3, 7) h5_dset = writer._create_simple_dset(h5_f, VirtualDataset('test', data)) self.assertIsInstance(h5_dset, h5py.Dataset) # with self.assertWarns(UserWarning): writer._write_dset_attributes(h5_dset, {'labels': ['row_1', 'row_2']}) self.assertEqual(len(h5_dset.attrs), 0) h5_f.flush() os.remove(file_path)
def test_simple_dset_write_success_01(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: dtype = np.uint16 dset_name = 'test' data = np.random.randint(0, high=15, size=5, dtype=dtype) microdset = VirtualDataset(dset_name, data) writer = HDFwriter(h5_f) h5_d = writer._create_simple_dset(h5_f, microdset) self.assertIsInstance(h5_d, h5py.Dataset) self.assertEqual(h5_d.parent, h5_f) self.assertEqual(h5_d.name, '/' + dset_name) self.assertEqual(h5_d.shape, data.shape) self.assertTrue(np.allclose(h5_d[()], data)) self.assertEqual(h5_d.dtype, dtype) os.remove(file_path)
def test_simple_dset_write_success_01(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: dtype = np.uint16 dset_name = 'test' data = np.random.randint(0, high=15, size=5, dtype=dtype) microdset = VirtualDataset(dset_name, data) writer = HDFwriter(h5_f) h5_d = writer._create_simple_dset(h5_f, microdset) self.assertIsInstance(h5_d, h5py.Dataset) self.assertEqual(h5_d.parent, h5_f) self.assertEqual(h5_d.name, '/' + dset_name) self.assertEqual(h5_d.shape, data.shape) self.assertTrue(np.allclose(h5_d[()], data)) self.assertEqual(h5_d.dtype, dtype) os.remove(file_path)
def test_write_simple_atts_reg_ref_to_dset(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: writer = HDFwriter(h5_f) data = np.random.rand(5, 7) h5_dset = writer._create_simple_dset(h5_f, VirtualDataset('test', data)) self.assertIsInstance(h5_dset, h5py.Dataset) attrs = {'att_1': 'string_val', 'att_2': 1.2345, 'att_3': [1, 2, 3, 4], 'att_4': ['str_1', 'str_2', 'str_3'], 'labels': {'even_rows': (slice(0, None, 2), slice(None)), 'odd_rows': (slice(1, None, 2), slice(None))} } writer._write_dset_attributes(h5_dset, attrs.copy()) reg_ref = attrs.pop('labels') self.assertEqual(len(h5_dset.attrs), len(attrs) + 1 + len(reg_ref)) for key, expected_val in attrs.items(): self.assertTrue(np.all(get_attr(h5_dset, key) == expected_val)) self.assertTrue(np.all([x in list(reg_ref.keys()) for x in get_attr(h5_dset, 'labels')])) expected_data = [data[:None:2], data[1:None:2]] written_data = [h5_dset[h5_dset.attrs['even_rows']], h5_dset[h5_dset.attrs['odd_rows']]] for exp, act in zip(expected_data, written_data): self.assertTrue(np.allclose(exp, act)) os.remove(file_path)
def test_simple_dset_write_success_more_options_03(self): file_path = 'test.h5' self.__delete_existing_file(file_path) with h5py.File(file_path) as h5_f: dset_name = 'test' data = np.random.rand(16, 1024) dtype = np.float16 compression = 'gzip' chunking=(1, 1024) microdset = VirtualDataset(dset_name, data, dtype=dtype, compression=compression, chunking=chunking) writer = HDFwriter(h5_f) h5_d = writer._create_simple_dset(h5_f, microdset) self.assertIsInstance(h5_d, h5py.Dataset) self.assertEqual(h5_d.parent, h5_f) self.assertEqual(h5_d.name, '/' + dset_name) self.assertEqual(h5_d.shape, data.shape) self.assertEqual(h5_d.dtype, dtype) self.assertEqual(h5_d.compression, compression) self.assertEqual(h5_d.chunks, chunking) self.assertTrue(np.all(h5_d[()] - data < 1E-3)) os.remove(file_path)