def test_clone_and_load(self):
clone_dataset = self.dataset.clone("{}_clone".format(
self.dataset.name))
clone_dataset.load(self.dataset[...])
if mpi_is_root():
np.testing.assert_array_equal(clone_dataset[...], self.data)
mpi_barrier()
clone_dataset.delete()
def test_non_overlapping_get(self):
chunks_per_process = self.dataset.chunk_grid[0] // mpi_size()
x_start = mpi_rank() * chunks_per_process * DATA_CHUNK_SIZE[0]
if mpi_rank() == mpi_size() - 1:
x_stop = self.dataset.shape[0]
else:
x_stop = (mpi_rank() + 1) * chunks_per_process * DATA_CHUNK_SIZE[0]
if x_start < self.dataset.shape[0]:
np.testing.assert_array_equal(self.dataset[x_start:x_stop, ...],
self.data[x_start:x_stop, ...])
mpi_barrier()
def test_non_overlapping_set(self):
chunks_per_process = self.dataset.chunk_grid[0] // mpi_size()
x_start = mpi_rank() * chunks_per_process * DATA_CHUNK_SIZE[0]
if mpi_rank() == mpi_size() - 1:
x_stop = self.dataset.shape[0]
else:
x_stop = (mpi_rank() + 1) * chunks_per_process * DATA_CHUNK_SIZE[0]
if x_start < self.dataset.shape[0]:
self.dataset[x_start:x_stop, ...] = \
self.data[x_start:x_stop, ...] * 3 + 5
mpi_barrier()
if mpi_is_root():
expected = self.data * 3 + 5
result = self.dataset[...]
np.testing.assert_array_equal(result, expected)
mpi_barrier()
def convolve1(ds, sigma):
mpi_barrier()
pprint('Convolving in 3D', rank=0)
with MpiTimer('Separable 3D convolution') as T:
ds3d_ = ds.clone('gaussian_3d')
for z in range(mpi_rank(), ds.chunk_grid[0], mpi_size()):
zS = z * ds.chunk_size[0]
zE = min(zS + ds.chunk_size[0], ds.shape[0])
for y in range(ds.chunk_grid[1]):
yS = y * ds.chunk_size[1]
yE = min(yS + ds.chunk_size[1], ds.shape[1])
for x in range(ds.chunk_grid[2]):
xS = x * ds.chunk_size[2]
xE = min(xS + ds.chunk_size[2], ds.shape[2])
out = gaussian_filter(ds[zS:zE, yS:yE, xS:xE], sigma=sigma)
ds3d_[zS:zE, yS:yE, xS:xE] = out
mpi_barrier()
if mpi_root():
try:
imsave(
join(
OUT_PATH,
'conv3d_{}-{}-{}.png'.format(mpi_size(), ds.shape[0],
ds.chunk_size[0])),
ds3d_[ds.shape[0] // 2])
except NameError:
pass
mpi_barrier()
ds3d_.delete()
return T.time
def test_apply(self):
self.dataset.apply(lambda x: x + 1)
if mpi_is_root():
np.testing.assert_array_equal(self.dataset[...], self.data + 1)
mpi_barrier()
def test_map(self):
dataset2 = self.dataset.map(lambda x: x + 1, self.fake_dataset + '2')
if mpi_is_root():
np.testing.assert_array_equal(dataset2[...], self.dataset[...] + 1)
mpi_barrier()
dataset2.delete()
def test_dataset_clear(self):
self.dataset.clear()
if mpi_is_root():
np.testing.assert_array_equal(self.dataset[...],
self.dataset.fillvalue)
mpi_barrier()
def test_load_function(self):
np.testing.assert_array_equal(self.dataset[...], self.data)
mpi_barrier()
def convolve2(ds, sigma):
mpi_barrier()
with MpiTimer('Three separable 1D convolutions') as T:
pprint('Convolving axis Z', rank=0)
ds1_ = ds.clone('gaussian_z')
for z in range(mpi_rank(), ds.chunk_grid[0], mpi_size()):
zS = z * ds.chunk_size[0]
zE = min(zS + ds.chunk_size[0], ds.shape[0])
out = gaussian_filter1d(ds[zS:zE], sigma=sigma, axis=0)
ds1_[zS:zE] = out
mpi_barrier()
# Gaussian second axis
pprint('Convolving axis Y', rank=0)
ds2_ = ds.clone('gaussian_y')
for y in range(mpi_rank(), ds.chunk_grid[1], mpi_size()):
yS = y * ds.chunk_size[1]
yE = min(yS + ds.chunk_size[1], ds.shape[1])
out = gaussian_filter1d(ds1_[:, yS:yE], sigma=sigma, axis=1)
ds2_[:, yS:yE] = out
mpi_barrier()
# Gaussian second axis
pprint('Convolving axis X', rank=0)
ds3_ = ds.clone('gaussian_x')
for x in range(mpi_rank(), ds.chunk_grid[2], mpi_size()):
xS = x * ds.chunk_size[2]
xE = min(xS + ds.chunk_size[2], ds.shape[2])
out = gaussian_filter1d(ds2_[..., xS:xE], sigma=sigma, axis=2)
ds3_[..., xS:xE] = out
mpi_barrier()
if mpi_root():
try:
imsave(
join(
OUT_PATH,
'conv3x1d_{}-{}-{}.png'.format(mpi_size(), ds.shape[0],
ds.chunk_size[0])),
ds3_[ds.shape[0] // 2])
except NameError:
pass
mpi_barrier()
ds1_.delete()
ds2_.delete()
ds3_.delete()
return T.time
pprint("Connecting", rank=0)
with dn.Cluster('Cluster', **params) as C:
pprint("Connected", C.connected)
with C.create_pool('test_dosna') as P:
pprint("Pool Created", rank=0)
with MpiTimer('Dataset Created'):
data = P.create_dataset('test_data',
data=rdata,
chunks=(CS, CS, CS))
if mpi_root():
pprint('Asserting the quality')
for i, j in [(0, 30), (10, 50), (5, 25), (37, 91)]:
np.testing.assert_allclose(data[i:j, i:j, i:j],
rdata[i:j, i:j, i:j])
print(data[25, 25, 25:35])
data[25:75, 25:75, 25:75] = np.random.rand(50, 50, 50)
print(data[25, 25, 25:35])
data[25:75, 25:75, 25:75] = rdata[25:75, 25:75, 25:75]
np.testing.assert_allclose(data[...], rdata[...])
mpi_barrier()
with C['test_dosna'] as P:
P.del_dataset('test_data')
pprint('Dataset Destroyed', rank=0)
C.del_pool('test_dosna')
pprint("Pool Destroyed", rank=0)
