def test_store_hdf5_execution(setup): raw = np.random.RandomState(0).rand(10, 20) group_name = 'test_group' dataset_name = 'test_dataset' t1 = tensor(raw, chunk_size=20) t2 = tensor(raw, chunk_size=9) with pytest.raises(TypeError): tohdf5(object(), t2) with tempfile.TemporaryDirectory() as d: filename = os.path.join(d, f'test_store_{int(time.time())}.hdf5') # test 1 chunk r = tohdf5(filename, t1, group=group_name, dataset=dataset_name) r.execute() with h5py.File(filename, 'r') as f: result = np.asarray(f[f'{group_name}/{dataset_name}']) np.testing.assert_array_equal(result, raw) # test filename r = tohdf5(filename, t2, group=group_name, dataset=dataset_name) r.execute() with h5py.File(filename, 'r') as f: result = np.asarray(f[f'{group_name}/{dataset_name}']) np.testing.assert_array_equal(result, raw) with pytest.raises(ValueError): tohdf5(filename, t2) with h5py.File(filename, 'r') as f: # test file r = tohdf5(f, t2, group=group_name, dataset=dataset_name) r.execute() with h5py.File(filename, 'r') as f: result = np.asarray(f[f'{group_name}/{dataset_name}']) np.testing.assert_array_equal(result, raw) with pytest.raises(ValueError): with h5py.File(filename, 'r') as f: tohdf5(f, t2) with h5py.File(filename, 'r') as f: # test dataset ds = f[f'{group_name}/{dataset_name}'] # test file r = tohdf5(ds, t2) r.execute() with h5py.File(filename, 'r') as f: result = np.asarray(f[f'{group_name}/{dataset_name}']) np.testing.assert_array_equal(result, raw)
def testStoreHDF5ForLocalCluster(self): with new_cluster(worker_n_process=2, shared_memory='20M', web=True) as cluster: session = cluster.session raw = np.random.RandomState(0).rand(10, 20) t = mt.tensor(raw, chunk_size=11) dataset = 'test_dataset' with tempfile.TemporaryDirectory() as d: filename = os.path.join(d, 'test_read_{}.hdf5'.format(int(time.time()))) r = mt.tohdf5(filename, t, dataset=dataset) session.run(r, timeout=_exec_timeout) with h5py.File(filename, 'r') as f: result = np.asarray(f[dataset]) np.testing.assert_array_equal(result, raw)
def testStoreHDF5Execution(self): raw = np.random.RandomState(0).rand(10, 20) group_name = 'test_group' dataset_name = 'test_dataset' t1 = tensor(raw, chunk_size=20) t2 = tensor(raw, chunk_size=9) with self.assertRaises(TypeError): tohdf5(object(), t2) ctx, executor = self._create_test_context(self.executor) with ctx: with tempfile.TemporaryDirectory() as d: filename = os.path.join(d, 'test_store_{}.hdf5'.format(int(time.time()))) # test 1 chunk r = tohdf5(filename, t1, group=group_name, dataset=dataset_name) executor.execute_tensor(r) with h5py.File(filename, 'r') as f: result = np.asarray(f['{}/{}'.format(group_name, dataset_name)]) np.testing.assert_array_equal(result, raw) # test filename r = tohdf5(filename, t2, group=group_name, dataset=dataset_name) executor.execute_tensor(r) rt = get_tiled(r) self.assertEqual(type(rt.chunks[0].inputs[1].op).__name__, 'SuccessorsExclusive') self.assertEqual(len(rt.chunks[0].inputs[1].inputs), 0) with h5py.File(filename, 'r') as f: result = np.asarray(f['{}/{}'.format(group_name, dataset_name)]) np.testing.assert_array_equal(result, raw) with self.assertRaises(ValueError): tohdf5(filename, t2) with h5py.File(filename, 'r') as f: # test file r = tohdf5(f, t2, group=group_name, dataset=dataset_name) executor.execute_tensor(r) with h5py.File(filename, 'r') as f: result = np.asarray(f['{}/{}'.format(group_name, dataset_name)]) np.testing.assert_array_equal(result, raw) with self.assertRaises(ValueError): with h5py.File(filename, 'r') as f: tohdf5(f, t2) with h5py.File(filename, 'r') as f: # test dataset ds = f['{}/{}'.format(group_name, dataset_name)] # test file r = tohdf5(ds, t2) executor.execute_tensor(r) with h5py.File(filename, 'r') as f: result = np.asarray(f['{}/{}'.format(group_name, dataset_name)]) np.testing.assert_array_equal(result, raw)