def test_is_compatible_degenerate_block_cyclic(self):
size = 19937
gdd_block_cyclic = ({
'dist_type': 'c',
'proc_grid_size': 1,
'block_size': 7,
'size': size,
}, )
gdd_block = ({
'dist_type': 'b',
'proc_grid_size': 1,
'bounds': [0, size],
}, )
gdd_cyclic = ({
'dist_type': 'c',
'proc_grid_size': 1,
'size': size,
}, )
dist_block_cyclic = Distribution.from_global_dim_data(
self.context, gdd_block_cyclic)
dist_block = Distribution.from_global_dim_data(self.context, gdd_block)
dist_cyclic = Distribution.from_global_dim_data(
self.context, gdd_cyclic)
self.assertTrue(dist_block_cyclic.is_compatible(dist_block))
self.assertTrue(dist_block_cyclic.is_compatible(dist_cyclic))
self.assertTrue(dist_block.is_compatible(dist_block_cyclic))
self.assertTrue(dist_cyclic.is_compatible(dist_block_cyclic))
def test_is_compatible_degenerate_block_cyclic(self):
size = 19937
gdd_block_cyclic = ({"dist_type": "c", "proc_grid_size": 1, "block_size": 7, "size": size},)
gdd_block = ({"dist_type": "b", "proc_grid_size": 1, "bounds": [0, size]},)
gdd_cyclic = ({"dist_type": "c", "proc_grid_size": 1, "size": size},)
dist_block_cyclic = Distribution.from_global_dim_data(self.context, gdd_block_cyclic)
dist_block = Distribution.from_global_dim_data(self.context, gdd_block)
dist_cyclic = Distribution.from_global_dim_data(self.context, gdd_cyclic)
self.assertTrue(dist_block_cyclic.is_compatible(dist_block))
self.assertTrue(dist_block_cyclic.is_compatible(dist_cyclic))
self.assertTrue(dist_block.is_compatible(dist_block_cyclic))
self.assertTrue(dist_cyclic.is_compatible(dist_block_cyclic))
def test_global_dim_data_local_dim_data_equivalence(self):
rows, cols = 5, 9
glb_dim_data = (
{"dist_type": "c", "block_size": 2, "size": rows, "proc_grid_size": 2},
{"dist_type": "c", "block_size": 2, "proc_grid_size": 2, "size": cols},
)
distribution = Distribution.from_global_dim_data(self.context, glb_dim_data)
actual = distribution.get_dim_data_per_rank()
expected = [
(
{"block_size": 2, "dist_type": "c", "proc_grid_rank": 0, "proc_grid_size": 2, "size": rows, "start": 0},
{"block_size": 2, "dist_type": "c", "proc_grid_rank": 0, "proc_grid_size": 2, "size": cols, "start": 0},
),
(
{"block_size": 2, "dist_type": "c", "proc_grid_rank": 0, "proc_grid_size": 2, "size": rows, "start": 0},
{"block_size": 2, "dist_type": "c", "proc_grid_rank": 1, "proc_grid_size": 2, "size": cols, "start": 2},
),
(
{"block_size": 2, "dist_type": "c", "proc_grid_rank": 1, "proc_grid_size": 2, "size": rows, "start": 2},
{"block_size": 2, "dist_type": "c", "proc_grid_rank": 0, "proc_grid_size": 2, "size": cols, "start": 0},
),
(
{"block_size": 2, "dist_type": "c", "proc_grid_rank": 1, "proc_grid_size": 2, "size": rows, "start": 2},
{"block_size": 2, "dist_type": "c", "proc_grid_rank": 1, "proc_grid_size": 2, "size": cols, "start": 2},
),
]
self.assertSequenceEqual(actual, expected)
def test_from_global_dim_data_irregular_block(self):
bounds = (0, 2, 3, 4, 10)
glb_dim_data = ({"dist_type": "b", "bounds": bounds},)
distribution = Distribution.from_global_dim_data(self.context, glb_dim_data)
distarr = DistArray(distribution, dtype=int)
distarr.toarray()
def test_not_compatible(self):
dist_b1 = Distribution(self.context, (10, ), ('b', ), (1, ),
targets=[0])
dist_b2 = Distribution(self.context, (9, ), ('b', ), (1, ),
targets=[0])
self.assertFalse(dist_b1.is_compatible(dist_b2))
self.assertFalse(dist_b2.is_compatible(dist_b1))
dist_b3 = Distribution(self.context, (10, ), ('b', ), (2, ),
targets=[0, 1])
self.assertFalse(dist_b1.is_compatible(dist_b3))
self.assertFalse(dist_b3.is_compatible(dist_b1))
dist_b4 = Distribution(self.context, (10, ), ('c', ), (2, ),
targets=[0, 1])
self.assertFalse(dist_b4.is_compatible(dist_b3))
self.assertFalse(dist_b3.is_compatible(dist_b4))
gdd_unstructured = ({
'dist_type': 'u',
'indices': [range(10)],
}, )
dist_u = Distribution.from_global_dim_data(self.context,
gdd_unstructured)
self.assertFalse(dist_u.is_compatible(dist_b1))
self.assertFalse(dist_b1.is_compatible(dist_u))
def test_from_global_dim_data_bc(self):
""" Test creation of a block-cyclic array. """
rows, cols = 5, 9
global_dim_data = (
# dim 0
{
'dist_type': 'c',
'proc_grid_size': 2,
'size': rows,
'block_size': 2,
},
# dim 1
{
'dist_type': 'c',
'proc_grid_size': 2,
'size': cols,
'block_size': 2,
},
)
distribution = Distribution.from_global_dim_data(
self.context, global_dim_data)
distarr = DistArray(distribution, dtype=int)
distarr.toarray()
las = distarr.get_localarrays()
local_shapes = [la.local_shape for la in las]
self.assertSequenceEqual(local_shapes, [(3, 5), (3, 4), (2, 5),
(2, 4)])
def test_from_global_dim_data_irregular_block(self):
bounds = (0, 2, 3, 4, 10)
glb_dim_data = ({'dist_type': 'b', 'bounds': bounds}, )
distribution = Distribution.from_global_dim_data(
self.context, glb_dim_data)
distarr = DistArray(distribution, dtype=int)
distarr.toarray()
def test_harder_distribution(self):
a = numpy.asarray(numpy.arange(30), dtype=numpy.int16)
gdd = (dict(dist_type="b", bounds=[0, 3, 12, 21, 30]),)
dist = Distribution.from_global_dim_data(context=self.context, global_dim_data=gdd)
da = self.context.fromndarray(a, distribution=dist)
dtype = numpy.int32
with self.assertRaises(ValueError):
da.view(dtype=dtype)
def test_harder_distribution(self):
a = numpy.asarray(numpy.arange(30), dtype=numpy.int16)
gdd = (dict(dist_type='b', bounds=[0, 3, 12, 21, 30]), )
dist = Distribution.from_global_dim_data(context=self.context,
global_dim_data=gdd)
da = self.context.fromndarray(a, distribution=dist)
dtype = numpy.int32
with self.assertRaises(ValueError):
da.view(dtype=dtype)
def test_from_global_dim_data_uu(self):
rows = 6
cols = 20
row_ixs = numpy.random.permutation(range(rows))
col_ixs = numpy.random.permutation(range(cols))
row_indices = [row_ixs[: rows // 2], row_ixs[rows // 2 :]]
col_indices = [col_ixs[: cols // 4], col_ixs[cols // 4 :]]
glb_dim_data = ({"dist_type": "u", "indices": row_indices}, {"dist_type": "u", "indices": col_indices})
distribution = Distribution.from_global_dim_data(self.context, glb_dim_data)
distarr = DistArray(distribution, dtype=int)
distarr.toarray()
def test_irregular_block_assignment(self):
global_dim_data = ({
'dist_type': 'b',
'bounds': (0, 5),
}, {
'dist_type': 'b',
'bounds': (0, 2, 6, 7, 9),
})
distribution = Distribution.from_global_dim_data(
self.context, global_dim_data)
distarr = DistArray(distribution, dtype=int)
distarr.toarray()
def test_redist_2D(self):
nrows, ncols = 7, 13
source_dist = Distribution(self.context, (nrows, ncols), ("b", "b"), (2, 2), targets=range(4))
dest_gdd = (
{"dist_type": "b", "bounds": [0, nrows // 3, nrows]},
{"dist_type": "b", "bounds": [0, ncols // 3, ncols]},
)
dest_dist = Distribution.from_global_dim_data(self.context, dest_gdd, targets=range(4))
source_da = self.context.empty(source_dist, dtype=numpy.int32)
source_da.fill(-42)
dest_da = source_da.distribute_as(dest_dist)
assert_array_equal(source_da.tondarray(), dest_da.tondarray())
def test_from_global_dim_data_bu(self):
rows = 9
row_break_point = rows // 2
cols = 10
col_indices = numpy.random.permutation(range(cols))
col_break_point = len(col_indices) // 3
indices = [col_indices[:col_break_point], col_indices[col_break_point:]]
glb_dim_data = (
{"dist_type": "b", "bounds": (0, row_break_point, rows)},
{"dist_type": "u", "indices": indices},
)
distribution = Distribution.from_global_dim_data(self.context, glb_dim_data)
distarr = DistArray(distribution, dtype=int)
distarr.toarray()
def test_from_global_dim_data_bc(self):
""" Test creation of a block-cyclic array. """
rows, cols = 5, 9
global_dim_data = (
# dim 0
{"dist_type": "c", "proc_grid_size": 2, "size": rows, "block_size": 2},
# dim 1
{"dist_type": "c", "proc_grid_size": 2, "size": cols, "block_size": 2},
)
distribution = Distribution.from_global_dim_data(self.context, global_dim_data)
distarr = DistArray(distribution, dtype=int)
distarr.toarray()
las = distarr.get_localarrays()
local_shapes = [la.local_shape for la in las]
self.assertSequenceEqual(local_shapes, [(3, 5), (3, 4), (2, 5), (2, 4)])
def test_from_global_dim_data_1d(self):
total_size = 40
list_of_indices = [
[29, 38, 18, 19, 11, 33, 10, 1, 22, 25],
[5, 15, 34, 12, 16, 24, 23, 39, 6, 36],
[0, 7, 27, 4, 32, 37, 21, 26, 9, 17],
[35, 14, 20, 13, 3, 30, 2, 8, 28, 31],
]
glb_dim_data = ({"dist_type": "u", "indices": list_of_indices},)
distribution = Distribution.from_global_dim_data(self.context, glb_dim_data)
distarr = DistArray(distribution, dtype=int)
for i in range(total_size):
distarr[i] = i
localarrays = distarr.get_localarrays()
for i, arr in enumerate(localarrays):
assert_allclose(arr, list_of_indices[i])
def test_from_global_dim_data_1d(self):
total_size = 40
list_of_indices = [
[29, 38, 18, 19, 11, 33, 10, 1, 22, 25],
[5, 15, 34, 12, 16, 24, 23, 39, 6, 36],
[0, 7, 27, 4, 32, 37, 21, 26, 9, 17],
[35, 14, 20, 13, 3, 30, 2, 8, 28, 31],
]
glb_dim_data = ({
'dist_type': 'u',
'indices': list_of_indices,
}, )
distribution = Distribution.from_global_dim_data(
self.context, glb_dim_data)
distarr = DistArray(distribution, dtype=int)
for i in range(total_size):
distarr[i] = i
localarrays = distarr.get_localarrays()
for i, arr in enumerate(localarrays):
assert_allclose(arr, list_of_indices[i])
def test_redist_2D(self):
nrows, ncols = 7, 13
source_dist = Distribution(self.context, (nrows, ncols), ('b', 'b'),
(2, 2),
targets=range(4))
dest_gdd = ({
'dist_type': 'b',
'bounds': [0, nrows // 3, nrows],
}, {
'dist_type': 'b',
'bounds': [0, ncols // 3, ncols],
})
dest_dist = Distribution.from_global_dim_data(self.context,
dest_gdd,
targets=range(4))
source_da = self.context.empty(source_dist, dtype=numpy.int32)
source_da.fill(-42)
dest_da = source_da.distribute_as(dest_dist)
assert_array_equal(source_da.tondarray(), dest_da.tondarray())
def test_from_global_dim_data_uu(self):
rows = 6
cols = 20
row_ixs = numpy.random.permutation(range(rows))
col_ixs = numpy.random.permutation(range(cols))
row_indices = [row_ixs[:rows // 2], row_ixs[rows // 2:]]
col_indices = [col_ixs[:cols // 4], col_ixs[cols // 4:]]
glb_dim_data = (
{
'dist_type': 'u',
'indices': row_indices
},
{
'dist_type': 'u',
'indices': col_indices
},
)
distribution = Distribution.from_global_dim_data(
self.context, glb_dim_data)
distarr = DistArray(distribution, dtype=int)
distarr.toarray()
def test_not_compatible(self):
dist_b1 = Distribution(self.context, (10,), ("b",), (1,), targets=[0])
dist_b2 = Distribution(self.context, (9,), ("b",), (1,), targets=[0])
self.assertFalse(dist_b1.is_compatible(dist_b2))
self.assertFalse(dist_b2.is_compatible(dist_b1))
dist_b3 = Distribution(self.context, (10,), ("b",), (2,), targets=[0, 1])
self.assertFalse(dist_b1.is_compatible(dist_b3))
self.assertFalse(dist_b3.is_compatible(dist_b1))
dist_b4 = Distribution(self.context, (10,), ("c",), (2,), targets=[0, 1])
self.assertFalse(dist_b4.is_compatible(dist_b3))
self.assertFalse(dist_b3.is_compatible(dist_b4))
gdd_unstructured = ({"dist_type": "u", "indices": [range(10)]},)
dist_u = Distribution.from_global_dim_data(self.context, gdd_unstructured)
self.assertFalse(dist_u.is_compatible(dist_b1))
self.assertFalse(dist_b1.is_compatible(dist_u))
def test_from_global_dim_data_bu(self):
rows = 9
row_break_point = rows // 2
cols = 10
col_indices = numpy.random.permutation(range(cols))
col_break_point = len(col_indices) // 3
indices = [
col_indices[:col_break_point], col_indices[col_break_point:]
]
glb_dim_data = (
{
'dist_type': 'b',
'bounds': (0, row_break_point, rows)
},
{
'dist_type': 'u',
'indices': indices
},
)
distribution = Distribution.from_global_dim_data(
self.context, glb_dim_data)
distarr = DistArray(distribution, dtype=int)
distarr.toarray()
def test_global_dim_data_local_dim_data_equivalence(self):
rows, cols = 5, 9
glb_dim_data = (
{
'dist_type': 'c',
'block_size': 2,
'size': rows,
'proc_grid_size': 2,
},
{
'dist_type': 'c',
'block_size': 2,
'proc_grid_size': 2,
'size': cols,
},
)
distribution = Distribution.from_global_dim_data(
self.context, glb_dim_data)
actual = distribution.get_dim_data_per_rank()
expected = [
({
'block_size': 2,
'dist_type': 'c',
'proc_grid_rank': 0,
'proc_grid_size': 2,
'size': rows,
'start': 0
}, {
'block_size': 2,
'dist_type': 'c',
'proc_grid_rank': 0,
'proc_grid_size': 2,
'size': cols,
'start': 0
}),
({
'block_size': 2,
'dist_type': 'c',
'proc_grid_rank': 0,
'proc_grid_size': 2,
'size': rows,
'start': 0
}, {
'block_size': 2,
'dist_type': 'c',
'proc_grid_rank': 1,
'proc_grid_size': 2,
'size': cols,
'start': 2
}),
({
'block_size': 2,
'dist_type': 'c',
'proc_grid_rank': 1,
'proc_grid_size': 2,
'size': rows,
'start': 2
}, {
'block_size': 2,
'dist_type': 'c',
'proc_grid_rank': 0,
'proc_grid_size': 2,
'size': cols,
'start': 0
}),
({
'block_size': 2,
'dist_type': 'c',
'proc_grid_rank': 1,
'proc_grid_size': 2,
'size': rows,
'start': 2
}, {
'block_size': 2,
'dist_type': 'c',
'proc_grid_rank': 1,
'proc_grid_size': 2,
'size': cols,
'start': 2
}),
]
self.assertSequenceEqual(actual, expected)
def test_irregular_block_assignment(self):
global_dim_data = ({"dist_type": "b", "bounds": (0, 5)}, {"dist_type": "b", "bounds": (0, 2, 6, 7, 9)})
distribution = Distribution.from_global_dim_data(self.context, global_dim_data)
distarr = DistArray(distribution, dtype=int)
distarr.toarray()
