class TestDistArrayCreation(unittest.TestCase):
"""Test distarray creation methods"""
def setUp(self):
self.context = Context()
def tearDown(self):
self.context.close()
def test___init__(self):
shape = (100, 100)
distribution = Distribution.from_shape(self.context, shape, ('b', 'c'))
da = DistArray(distribution, dtype=int)
da.fill(42)
nda = numpy.empty(shape, dtype=int)
nda.fill(42)
assert_array_equal(da.tondarray(), nda)
def test_zeros(self):
shape = (16, 16)
zero_distarray = self.context.zeros(shape)
zero_ndarray = numpy.zeros(shape)
assert_array_equal(zero_distarray.tondarray(), zero_ndarray)
def test_ones(self):
shape = (16, 16)
one_distarray = self.context.ones(shape)
one_ndarray = numpy.ones(shape)
assert_array_equal(one_distarray.tondarray(), one_ndarray)
def test_empty(self):
shape = (16, 16)
empty_distarray = self.context.empty(shape)
self.assertEqual(empty_distarray.shape, shape)
def test_fromndarray(self):
ndarr = numpy.arange(16).reshape(4, 4)
distarr = self.context.fromndarray(ndarr)
for (i, j), val in numpy.ndenumerate(ndarr):
self.assertEqual(distarr[i, j], ndarr[i, j])
def test_grid_rank(self):
# regression test for issue #235
a = self.context.empty((4, 4, 4), dist=('b', 'n', 'b'),
grid_shape=(1, 1, 4))
self.assertEqual(a.grid_shape, (1, 1, 4))
def test_fromfunction(self):
fn = lambda i, j: i + j
shape = (7, 9)
expected = numpy.fromfunction(fn, shape, dtype=int)
result = self.context.fromfunction(fn, shape, dtype=int)
assert_array_equal(expected, result.tondarray())
def test_writing_two_datasets(self):
h5py = import_or_skip('h5py')
datalen = 33
dac = Context(self.client)
da = dac.empty((datalen,), dist={0: 'b'})
for i in range(datalen):
da[i] = i
output_path = temp_filepath('.hdf5')
try:
# make a file, and write to dataset 'foo'
with h5py.File(output_path, 'w') as fp:
fp['foo'] = np.arange(10)
# try saving to a different dataset
dac.save_hdf5(output_path, da, key='bar', mode='a')
with h5py.File(output_path, 'r') as fp:
self.assertTrue("foo" in fp)
self.assertTrue("bar" in fp)
finally:
if os.path.exists(output_path):
os.remove(output_path)
def test_write_3d(self):
h5py = import_or_skip('h5py')
shape = (4, 5, 3)
source = np.random.random(shape)
dac = Context(self.client)
dist = {0: 'b', 1: 'c', 2: 'n'}
da = dac.empty(shape, dist=dist)
for i in range(shape[0]):
for j in range(shape[1]):
for k in range(shape[2]):
da[i, j, k] = source[i, j, k]
output_path = temp_filepath('.hdf5')
try:
dac.save_hdf5(output_path, da, mode='w')
self.assertTrue(os.path.exists(output_path))
with h5py.File(output_path, 'r') as fp:
self.assertTrue("buffer" in fp)
assert_allclose(source, fp["buffer"])
finally:
if os.path.exists(output_path):
os.remove(output_path)
class TestDistArrayCreation(IpclusterTestCase):
"""Test distarray creation methods"""
def setUp(self):
self.context = Context(self.client)
def test_zeros(self):
shape = (16, 16)
zero_distarray = self.context.zeros(shape)
zero_ndarray = numpy.zeros(shape)
assert_array_equal(zero_distarray.tondarray(), zero_ndarray)
def test_ones(self):
shape = (16, 16)
one_distarray = self.context.ones(shape)
one_ndarray = numpy.ones(shape)
assert_array_equal(one_distarray.tondarray(), one_ndarray)
def test_empty(self):
shape = (16, 16)
empty_distarray = self.context.empty(shape)
self.assertEqual(empty_distarray.shape, shape)
def test_fromndarray(self):
ndarr = numpy.arange(16).reshape(4, 4)
distarr = self.context.fromndarray(ndarr)
for (i, j), val in numpy.ndenumerate(ndarr):
self.assertEqual(distarr[i, j], ndarr[i, j])
def test_save_load_with_filenames(self):
dac = Context(self.client)
da = dac.empty((100,), dist={0: 'b'})
output_paths = [temp_filepath() for target in dac.targets]
try:
dac.save(output_paths, da)
db = dac.load(output_paths)
self.assertTrue(isinstance(db, DistArray))
self.assertEqual(da, db)
finally:
for filepath in output_paths:
if os.path.exists(filepath):
os.remove(filepath)
def test_save_load_with_prefix(self):
dac = Context(self.client)
da = dac.empty((100,), dist={0: 'b'})
output_path = temp_filepath()
try:
dac.save(output_path, da)
db = dac.load(output_path)
self.assertTrue(isinstance(db, DistArray))
self.assertEqual(da, db)
finally:
for rank in dac.targets:
filepath = output_path + "_" + str(rank) + ".dnpy"
if os.path.exists(filepath):
os.remove(filepath)
def test_local(self):
"""Test the @local decorator"""
context = Context()
da = context.empty((4, 4))
a = numpy.empty((4, 4))
def fill_a(a):
for (i, j), _ in numpy.ndenumerate(a):
a[i, j] = i + j
return a
@local
def fill_da(da):
for i in da.distribution[0].global_iter:
for j in da.distribution[1].global_iter:
da.global_index[i, j] = i + j
return da
da = fill_da(da)
a = fill_a(a)
assert_array_equal(da.toarray(), a)
def test_write_block(self):
h5py = import_or_skip('h5py')
datalen = 33
dac = Context(self.client)
da = dac.empty((datalen,), dist={0: 'b'})
for i in range(datalen):
da[i] = i
output_path = temp_filepath('.hdf5')
try:
dac.save_hdf5(output_path, da, mode='w')
self.assertTrue(os.path.exists(output_path))
with h5py.File(output_path, 'r') as fp:
self.assertTrue("buffer" in fp)
expected = np.arange(datalen)
assert_equal(expected, fp["buffer"])
finally:
if os.path.exists(output_path):
os.remove(output_path)
class TestLocalDecorator(TestCase):
# Functions for @local decorator tests. These are here so we can
# guarantee they are pushed to the engines before we try to use them.
@local
def local_add50(da):
return da + 50
@local
def local_add_num(da, num):
return da + num
@local
def assert_allclose(da, db):
assert numpy.allclose(da, db), "Arrays not equal within tolerance."
@local
def local_sin(da):
return numpy.sin(da)
@local
def local_sum(da):
return numpy.sum(da.get_localarray())
@local
def call_barrier(da):
from mpi4py import MPI
MPI.COMM_WORLD.Barrier()
return da
@local
def local_add_nums(da, num1, num2, num3):
return da + num1 + num2 + num3
@local
def local_add_distarrayproxies(da, dg):
return da + dg
@local
def local_add_mixed(da, num1, dg, num2):
return da + num1 + dg + num2
@local
def local_add_ndarray(da, num, ndarr):
return da + num + ndarr
@local
def local_add_kwargs(da, num1, num2=55):
return da + num1 + num2
@local
def local_add_supermix(da, num1, db, num2, dc, num3=99, num4=66):
return da + num1 + db + num2 + dc + num3 + num4
@local
def local_none(da):
return None
@local
def parameterless():
"""This is a parameterless function."""
return None
@classmethod
def setUpClass(self):
self.context = Context()
self.da = self.context.empty((5, 5))
self.da.fill(2 * numpy.pi)
def test_local(self):
context = Context()
"""Test the @local decorator"""
da = context.empty((4, 4))
a = numpy.empty((4, 4))
def fill_a(a):
for (i, j), _ in numpy.ndenumerate(a):
a[i, j] = i + j
return a
@local
def fill_da(da):
for i in da.maps[0].global_index:
for j in da.maps[1].global_index:
da[i, j] = i + j
return da
da = fill_da(da)
a = fill_a(a)
assert_array_equal(da.toarray(), a)
def test_local_sin(self):
db = self.local_sin(self.da)
self.assert_allclose(db, 0)
def test_local_add50(self):
dc = self.local_add50(self.da)
self.assert_allclose(dc, 2 * numpy.pi + 50)
def test_local_sum(self):
dd = self.local_sum(self.da)
lshapes = self.da.get_localshapes()
expected = []
for lshape in lshapes:
expected.append(lshape[0] * lshape[1] * (2 * numpy.pi))
for (v, e) in zip(dd, expected):
self.assertAlmostEqual(v, e, places=5)
def test_local_add_num(self):
de = self.local_add_num(self.da, 11)
self.assert_allclose(de, 2 * numpy.pi + 11)
def test_local_add_nums(self):
df = self.local_add_nums(self.da, 11, 12, 13)
self.assert_allclose(df, 2 * numpy.pi + 11 + 12 + 13)
def test_local_add_distarrayproxies(self):
dg = self.context.empty((5, 5))
dg.fill(33)
dh = self.local_add_distarrayproxies(self.da, dg)
self.assert_allclose(dh, 33 + 2 * numpy.pi)
def test_local_add_mixed(self):
di = self.context.empty((5, 5))
di.fill(33)
dj = self.local_add_mixed(self.da, 11, di, 12)
self.assert_allclose(dj, 2 * numpy.pi + 11 + 33 + 12)
@unittest.skip('Locally adding ndarrays not supported.')
def test_local_add_ndarray(self):
shp = self.da.get_localshapes()[0]
ndarr = numpy.empty(shp)
ndarr.fill(33)
dk = self.local_add_ndarray(self.da, 11, ndarr)
self.assert_allclose(dk, 2 * numpy.pi + 11 + 33)
def test_local_add_kwargs(self):
dl = self.local_add_kwargs(self.da, 11, num2=12)
self.assert_allclose(dl, 2 * numpy.pi + 11 + 12)
def test_local_add_supermix(self):
dm = self.context.empty((5, 5))
dm.fill(22)
dn = self.context.empty((5, 5))
dn.fill(44)
do = self.local_add_supermix(self.da, 11, dm, 33, dc=dn, num3=55)
expected = 2 * numpy.pi + 11 + 22 + 33 + 44 + 55 + 66
self.assert_allclose(do, expected)
def test_local_none(self):
dp = self.local_none(self.da)
self.assertTrue(dp is None)
def test_barrier(self):
self.call_barrier(self.da)
def test_parameterless(self):
self.assertRaises(TypeError, self.parameterless)
def test_function_metadata(self):
name = "parameterless"
docstring = """This is a parameterless function."""
self.assertEqual(self.parameterless.__name__, name)
self.assertEqual(self.parameterless.__doc__, docstring)
class TestDistArray(unittest.TestCase):
def setUp(self):
self.dac = Context()
def tearDown(self):
self.dac.close()
def test_set_and_getitem_block_dist(self):
size = 10
dap = self.dac.empty((size,), dist={0: 'b'})
for val in range(size):
dap[val] = val
for val in range(size):
self.assertEqual(dap[val], val)
for i in range(1, size + 1):
dap[-i] = i
self.assertEqual(dap[-i], i)
def test_set_and_getitem_nd_block_dist(self):
size = 5
dap = self.dac.empty((size, size), dist={0: 'b', 1: 'b'})
for row in range(size):
for col in range(size):
val = size*row + col
dap[row, col] = val
self.assertEqual(dap[row, col], val)
for row in range(1 ,size + 1):
for col in range(1, size + 1):
dap[-row, -col] = row + col
self.assertEqual(dap[-row, -col], row + col)
def test_set_and_getitem_cyclic_dist(self):
size = 10
dap = self.dac.empty((size,), dist={0: 'c'})
for val in range(size):
dap[val] = val
self.assertEqual(dap[val], val)
for i in range(1, size + 1):
dap[-i] = i
self.assertEqual(dap[-i], i)
def test_get_index_error(self):
dap = self.dac.empty((10,), dist={0: 'c'})
with self.assertRaises(IndexError):
dap[11]
with self.assertRaises(IndexError):
dap[-11]
def test_set_index_error(self):
dap = self.dac.empty((10,), dist={0: 'c'})
with self.assertRaises(IndexError):
dap[11] = 55
with self.assertRaises(IndexError):
dap[-11] = 55
def test_iteration(self):
size = 10
dap = self.dac.empty((size,), dist={0: 'c'})
dap.fill(10)
for val in dap:
self.assertEqual(val, 10)
def test_tondarray(self):
dap = self.dac.empty((3, 3))
ndarr = numpy.arange(9).reshape(3, 3)
for (i, j), val in numpy.ndenumerate(ndarr):
dap[i, j] = ndarr[i, j]
numpy.testing.assert_array_equal(dap.tondarray(), ndarr)
def test_global_tolocal_bug(self):
# gh-issue #154
dap = self.dac.zeros((3, 3), dist=('n', 'b'))
ndarr = numpy.zeros((3, 3))
numpy.testing.assert_array_equal(dap.tondarray(), ndarr)
class TestDistArray(IpclusterTestCase):
def setUp(self):
self.dac = Context(self.client)
def test_set_and_getitem_block_dist(self):
size = 10
dap = self.dac.empty((size,), dist={0: 'b'})
for val in range(size):
dap[val] = val
for val in range(size):
self.assertEqual(dap[val], val)
def test_set_and_getitem_nd_block_dist(self):
size = 5
dap = self.dac.empty((size, size), dist={0: 'b', 1: 'b'})
for row in range(size):
for col in range(size):
val = size*row + col
dap[row, col] = val
for row in range(size):
for col in range(size):
val = size*row + col
self.assertEqual(dap[row, col], val)
def test_set_and_getitem_cyclic_dist(self):
size = 10
dap = self.dac.empty((size,), dist={0: 'c'})
for val in range(size):
dap[val] = val
for val in range(size):
self.assertEqual(dap[val], val)
@unittest.skip("Slicing not yet implemented.")
def test_slice_in_getitem_block_dist(self):
dap = self.dac.empty((100,), dist={0: 'b'})
self.assertIsInstance(dap[20:40], DistArray)
@unittest.skip("Slicing not yet implemented.")
def test_slice_in_setitem_raises_valueerror(self):
dap = self.dac.empty((100,), dist={0: 'b'})
vals = numpy.random.random(20)
with self.assertRaises(NotImplementedError):
dap[20:40] = vals
@unittest.skip('Slice assignment not yet implemented.')
def test_slice_size_error(self):
dap = self.dac.empty((100,), dist={0: 'c'})
with self.assertRaises(NotImplementedError):
dap[20:40] = (11, 12)
def test_get_index_error(self):
dap = self.dac.empty((10,), dist={0: 'c'})
with self.assertRaises(IndexError):
dap[11]
def test_set_index_error(self):
dap = self.dac.empty((10,), dist={0: 'c'})
with self.assertRaises(IndexError):
dap[11] = 55
def test_iteration(self):
size = 10
dap = self.dac.empty((size,), dist={0: 'c'})
dap.fill(10)
for val in dap:
self.assertEqual(val, 10)
def test_tondarray(self):
dap = self.dac.empty((3, 3))
ndarr = numpy.arange(9).reshape(3, 3)
for (i, j), val in numpy.ndenumerate(ndarr):
dap[i, j] = ndarr[i, j]
numpy.testing.assert_array_equal(dap.tondarray(), ndarr)
def test_global_tolocal_bug(self):
# gh-issue #154
dap = self.dac.zeros((3, 3), dist=('n', 'b'))
ndarr = numpy.zeros((3, 3))
numpy.testing.assert_array_equal(dap.tondarray(), ndarr)
