def test_clone_and_load(self): clone_dataset = self.dataset.clone("{}_clone".format( self.dataset.name)) clone_dataset.load(self.dataset[...]) if mpi_is_root(): np.testing.assert_array_equal(clone_dataset[...], self.data) mpi_barrier() clone_dataset.delete()
def test_non_overlapping_get(self): chunks_per_process = self.dataset.chunk_grid[0] // mpi_size() x_start = mpi_rank() * chunks_per_process * DATA_CHUNK_SIZE[0] if mpi_rank() == mpi_size() - 1: x_stop = self.dataset.shape[0] else: x_stop = (mpi_rank() + 1) * chunks_per_process * DATA_CHUNK_SIZE[0] if x_start < self.dataset.shape[0]: np.testing.assert_array_equal(self.dataset[x_start:x_stop, ...], self.data[x_start:x_stop, ...]) mpi_barrier()
def test_non_overlapping_set(self): chunks_per_process = self.dataset.chunk_grid[0] // mpi_size() x_start = mpi_rank() * chunks_per_process * DATA_CHUNK_SIZE[0] if mpi_rank() == mpi_size() - 1: x_stop = self.dataset.shape[0] else: x_stop = (mpi_rank() + 1) * chunks_per_process * DATA_CHUNK_SIZE[0] if x_start < self.dataset.shape[0]: self.dataset[x_start:x_stop, ...] = \ self.data[x_start:x_stop, ...] * 3 + 5 mpi_barrier() if mpi_is_root(): expected = self.data * 3 + 5 result = self.dataset[...] np.testing.assert_array_equal(result, expected) mpi_barrier()
def convolve1(ds, sigma): mpi_barrier() pprint('Convolving in 3D', rank=0) with MpiTimer('Separable 3D convolution') as T: ds3d_ = ds.clone('gaussian_3d') for z in range(mpi_rank(), ds.chunk_grid[0], mpi_size()): zS = z * ds.chunk_size[0] zE = min(zS + ds.chunk_size[0], ds.shape[0]) for y in range(ds.chunk_grid[1]): yS = y * ds.chunk_size[1] yE = min(yS + ds.chunk_size[1], ds.shape[1]) for x in range(ds.chunk_grid[2]): xS = x * ds.chunk_size[2] xE = min(xS + ds.chunk_size[2], ds.shape[2]) out = gaussian_filter(ds[zS:zE, yS:yE, xS:xE], sigma=sigma) ds3d_[zS:zE, yS:yE, xS:xE] = out mpi_barrier() if mpi_root(): try: imsave( join( OUT_PATH, 'conv3d_{}-{}-{}.png'.format(mpi_size(), ds.shape[0], ds.chunk_size[0])), ds3d_[ds.shape[0] // 2]) except NameError: pass mpi_barrier() ds3d_.delete() return T.time
def test_apply(self): self.dataset.apply(lambda x: x + 1) if mpi_is_root(): np.testing.assert_array_equal(self.dataset[...], self.data + 1) mpi_barrier()
def test_map(self): dataset2 = self.dataset.map(lambda x: x + 1, self.fake_dataset + '2') if mpi_is_root(): np.testing.assert_array_equal(dataset2[...], self.dataset[...] + 1) mpi_barrier() dataset2.delete()
def test_dataset_clear(self): self.dataset.clear() if mpi_is_root(): np.testing.assert_array_equal(self.dataset[...], self.dataset.fillvalue) mpi_barrier()
def test_load_function(self): np.testing.assert_array_equal(self.dataset[...], self.data) mpi_barrier()
def convolve2(ds, sigma): mpi_barrier() with MpiTimer('Three separable 1D convolutions') as T: pprint('Convolving axis Z', rank=0) ds1_ = ds.clone('gaussian_z') for z in range(mpi_rank(), ds.chunk_grid[0], mpi_size()): zS = z * ds.chunk_size[0] zE = min(zS + ds.chunk_size[0], ds.shape[0]) out = gaussian_filter1d(ds[zS:zE], sigma=sigma, axis=0) ds1_[zS:zE] = out mpi_barrier() # Gaussian second axis pprint('Convolving axis Y', rank=0) ds2_ = ds.clone('gaussian_y') for y in range(mpi_rank(), ds.chunk_grid[1], mpi_size()): yS = y * ds.chunk_size[1] yE = min(yS + ds.chunk_size[1], ds.shape[1]) out = gaussian_filter1d(ds1_[:, yS:yE], sigma=sigma, axis=1) ds2_[:, yS:yE] = out mpi_barrier() # Gaussian second axis pprint('Convolving axis X', rank=0) ds3_ = ds.clone('gaussian_x') for x in range(mpi_rank(), ds.chunk_grid[2], mpi_size()): xS = x * ds.chunk_size[2] xE = min(xS + ds.chunk_size[2], ds.shape[2]) out = gaussian_filter1d(ds2_[..., xS:xE], sigma=sigma, axis=2) ds3_[..., xS:xE] = out mpi_barrier() if mpi_root(): try: imsave( join( OUT_PATH, 'conv3x1d_{}-{}-{}.png'.format(mpi_size(), ds.shape[0], ds.chunk_size[0])), ds3_[ds.shape[0] // 2]) except NameError: pass mpi_barrier() ds1_.delete() ds2_.delete() ds3_.delete() return T.time
pprint("Connecting", rank=0) with dn.Cluster('Cluster', **params) as C: pprint("Connected", C.connected) with C.create_pool('test_dosna') as P: pprint("Pool Created", rank=0) with MpiTimer('Dataset Created'): data = P.create_dataset('test_data', data=rdata, chunks=(CS, CS, CS)) if mpi_root(): pprint('Asserting the quality') for i, j in [(0, 30), (10, 50), (5, 25), (37, 91)]: np.testing.assert_allclose(data[i:j, i:j, i:j], rdata[i:j, i:j, i:j]) print(data[25, 25, 25:35]) data[25:75, 25:75, 25:75] = np.random.rand(50, 50, 50) print(data[25, 25, 25:35]) data[25:75, 25:75, 25:75] = rdata[25:75, 25:75, 25:75] np.testing.assert_allclose(data[...], rdata[...]) mpi_barrier() with C['test_dosna'] as P: P.del_dataset('test_data') pprint('Dataset Destroyed', rank=0) C.del_pool('test_dosna') pprint("Pool Destroyed", rank=0)