def test_store_hdf5_execution(setup):
raw = np.random.RandomState(0).rand(10, 20)
group_name = 'test_group'
dataset_name = 'test_dataset'
t1 = tensor(raw, chunk_size=20)
t2 = tensor(raw, chunk_size=9)
with pytest.raises(TypeError):
tohdf5(object(), t2)
with tempfile.TemporaryDirectory() as d:
filename = os.path.join(d, f'test_store_{int(time.time())}.hdf5')
# test 1 chunk
r = tohdf5(filename, t1, group=group_name, dataset=dataset_name)
r.execute()
with h5py.File(filename, 'r') as f:
result = np.asarray(f[f'{group_name}/{dataset_name}'])
np.testing.assert_array_equal(result, raw)
# test filename
r = tohdf5(filename, t2, group=group_name, dataset=dataset_name)
r.execute()
with h5py.File(filename, 'r') as f:
result = np.asarray(f[f'{group_name}/{dataset_name}'])
np.testing.assert_array_equal(result, raw)
with pytest.raises(ValueError):
tohdf5(filename, t2)
with h5py.File(filename, 'r') as f:
# test file
r = tohdf5(f, t2, group=group_name, dataset=dataset_name)
r.execute()
with h5py.File(filename, 'r') as f:
result = np.asarray(f[f'{group_name}/{dataset_name}'])
np.testing.assert_array_equal(result, raw)
with pytest.raises(ValueError):
with h5py.File(filename, 'r') as f:
tohdf5(f, t2)
with h5py.File(filename, 'r') as f:
# test dataset
ds = f[f'{group_name}/{dataset_name}']
# test file
r = tohdf5(ds, t2)
r.execute()
with h5py.File(filename, 'r') as f:
result = np.asarray(f[f'{group_name}/{dataset_name}'])
np.testing.assert_array_equal(result, raw)
def testStoreHDF5ForLocalCluster(self):
with new_cluster(worker_n_process=2,
shared_memory='20M', web=True) as cluster:
session = cluster.session
raw = np.random.RandomState(0).rand(10, 20)
t = mt.tensor(raw, chunk_size=11)
dataset = 'test_dataset'
with tempfile.TemporaryDirectory() as d:
filename = os.path.join(d, 'test_read_{}.hdf5'.format(int(time.time())))
r = mt.tohdf5(filename, t, dataset=dataset)
session.run(r, timeout=_exec_timeout)
with h5py.File(filename, 'r') as f:
result = np.asarray(f[dataset])
np.testing.assert_array_equal(result, raw)
def testStoreHDF5Execution(self):
raw = np.random.RandomState(0).rand(10, 20)
group_name = 'test_group'
dataset_name = 'test_dataset'
t1 = tensor(raw, chunk_size=20)
t2 = tensor(raw, chunk_size=9)
with self.assertRaises(TypeError):
tohdf5(object(), t2)
ctx, executor = self._create_test_context(self.executor)
with ctx:
with tempfile.TemporaryDirectory() as d:
filename = os.path.join(d, 'test_store_{}.hdf5'.format(int(time.time())))
# test 1 chunk
r = tohdf5(filename, t1, group=group_name, dataset=dataset_name)
executor.execute_tensor(r)
with h5py.File(filename, 'r') as f:
result = np.asarray(f['{}/{}'.format(group_name, dataset_name)])
np.testing.assert_array_equal(result, raw)
# test filename
r = tohdf5(filename, t2, group=group_name, dataset=dataset_name)
executor.execute_tensor(r)
rt = get_tiled(r)
self.assertEqual(type(rt.chunks[0].inputs[1].op).__name__, 'SuccessorsExclusive')
self.assertEqual(len(rt.chunks[0].inputs[1].inputs), 0)
with h5py.File(filename, 'r') as f:
result = np.asarray(f['{}/{}'.format(group_name, dataset_name)])
np.testing.assert_array_equal(result, raw)
with self.assertRaises(ValueError):
tohdf5(filename, t2)
with h5py.File(filename, 'r') as f:
# test file
r = tohdf5(f, t2, group=group_name, dataset=dataset_name)
executor.execute_tensor(r)
with h5py.File(filename, 'r') as f:
result = np.asarray(f['{}/{}'.format(group_name, dataset_name)])
np.testing.assert_array_equal(result, raw)
with self.assertRaises(ValueError):
with h5py.File(filename, 'r') as f:
tohdf5(f, t2)
with h5py.File(filename, 'r') as f:
# test dataset
ds = f['{}/{}'.format(group_name, dataset_name)]
# test file
r = tohdf5(ds, t2)
executor.execute_tensor(r)
with h5py.File(filename, 'r') as f:
result = np.asarray(f['{}/{}'.format(group_name, dataset_name)])
np.testing.assert_array_equal(result, raw)
