def test_both_methods(self):
"""
Do the two methods of computing the multiplicative partitions agree?
"""
for s in [2,3]:
for n in range(2, 512):
self.assertEquals(utils.mult_partitions(n,s), utils.create_factors(n,s))
def test_mult_partitions(self):
self.assertEqual(utils.mult_partitions(1, 2), [(1, 1)])
self.assertEqual(utils.mult_partitions(2, 2), [(1, 2)])
self.assertEqual(utils.mult_partitions(1, 7), [(1,) * 7])
self.assertEqual(utils.mult_partitions(7, 2), [(1, 7)])
self.assertEqual(utils.mult_partitions(7, 3), [(1, 1, 7)])
self.assertEqual(utils.mult_partitions(16, 4), [(1, 1, 1, 16),
(1, 1, 2, 8),
(1, 1, 4, 4),
(1, 2, 2, 4),
(2, 2, 2, 2)])
self.assertEqual(utils.mult_partitions(6, 3), [(1, 1, 6), (1, 2, 3)])
def test_mult_partitions(self):
self.assertEqual(utils.mult_partitions(1, 2), [(1, 1)])
self.assertEqual(utils.mult_partitions(2, 2), [(1, 2)])
self.assertEqual(utils.mult_partitions(1, 7), [(1, ) * 7])
self.assertEqual(utils.mult_partitions(7, 2), [(1, 7)])
self.assertEqual(utils.mult_partitions(7, 3), [(1, 1, 7)])
self.assertEqual(utils.mult_partitions(16, 4), [(1, 1, 1, 16),
(1, 1, 2, 8),
(1, 1, 4, 4),
(1, 2, 2, 4),
(2, 2, 2, 2)])
self.assertEqual(utils.mult_partitions(6, 3), [(1, 1, 6), (1, 2, 3)])
def optimize_grid_shape(shape, grid_shape, distdims, comm_size):
ndistdim = len(distdims)
if ndistdim == 1:
grid_shape = (comm_size, )
else:
factors = utils.mult_partitions(comm_size, ndistdim)
if factors != []:
reduced_shape = [shape[i] for i in distdims]
factors = [utils.mirror_sort(f, reduced_shape) for f in factors]
rs_ratio = _compute_grid_ratios(reduced_shape)
f_ratios = [_compute_grid_ratios(f) for f in factors]
distances = [rs_ratio - f_ratio for f_ratio in f_ratios]
norms = np.array([np.linalg.norm(d, 2) for d in distances])
index = norms.argmin()
grid_shape = tuple(factors[index])
else:
raise GridShapeError("Cannot distribute array over processors")
return grid_shape
def optimize_grid_shape(shape, distdims, comm_size):
ndistdim = len(distdims)
if ndistdim==1:
grid_shape = (comm_size,)
else:
factors = utils.mult_partitions(comm_size, ndistdim)
if factors != []:
reduced_shape = [shape[i] for i in distdims]
factors = [utils.mirror_sort(f, reduced_shape) for f in factors]
rs_ratio = _compute_grid_ratios(reduced_shape)
f_ratios = [_compute_grid_ratios(f) for f in factors]
distances = [rs_ratio-f_ratio for f_ratio in f_ratios]
norms = np.array([np.linalg.norm(d,2) for d in distances])
index = norms.argmin()
grid_shape = tuple(factors[index])
else:
raise GridShapeError("Cannot distribute array over processors.")
return grid_shape
def make_grid_shape(shape, dist, comm_size):
""" Generate a `grid_shape` from `shape` tuple and `dist` tuple.
Does not assume that `dim_data` has `proc_grid_size` set for each
dimension.
Attempts to allocate processes optimally for distributed dimensions.
Parameters
----------
shape : tuple of int
The global shape of the array.
dist: tuple of str
dist_type character per dimension.
comm_size : int
Total number of processes to distribute.
Returns
-------
dist_grid_shape : tuple of int
Raises
------
GridShapeError
if not possible to distribute `comm_size` processes over number of
dimensions.
"""
check_grid_shape_preconditions(shape, dist, comm_size)
distdims = tuple(i for (i, v) in enumerate(dist) if v != 'n')
ndistdim = len(distdims)
if ndistdim == 0:
dist_grid_shape = ()
elif ndistdim == 1:
# Trivial case: all processes used for the one distributed dimension.
if comm_size >= shape[distdims[0]]:
dist_grid_shape = (shape[distdims[0]],)
else:
dist_grid_shape = (comm_size,)
elif comm_size == 1:
# Trivial case: only one process to distribute over!
dist_grid_shape = (1,) * ndistdim
else: # Main case: comm_size > 1, ndistdim > 1.
factors = utils.mult_partitions(comm_size, ndistdim)
if not factors: # Can't factorize appropriately.
raise GridShapeError("Cannot distribute array over processors.")
reduced_shape = [shape[i] for i in distdims]
# Reorder factors so they match the relative ordering in reduced_shape
factors = [utils.mirror_sort(f, reduced_shape) for f in factors]
# Pick the "best" factoring from `factors` according to which matches
# the ratios among the dimensions in `shape`.
rs_ratio = _compute_grid_ratios(reduced_shape)
f_ratios = [_compute_grid_ratios(f) for f in factors]
distances = [rs_ratio - f_ratio for f_ratio in f_ratios]
norms = numpy.array([numpy.linalg.norm(d, 2) for d in distances])
index = norms.argmin()
# we now have the grid shape for the distributed dimensions.
dist_grid_shape = tuple(int(i) for i in factors[index])
# Create the grid_shape, all 1's for now.
grid_shape = [1] * len(shape)
# Fill grid_shape in the distdim slots using dist_grid_shape
it = iter(dist_grid_shape)
for distdim in distdims:
grid_shape[distdim] = next(it)
out_grid_shape = tuple(grid_shape)
check_grid_shape_postconditions(out_grid_shape, shape, dist, comm_size)
return out_grid_shape
def make_grid_shape(shape, dist, comm_size):
""" Generate a `grid_shape` from `shape` tuple and `dist` tuple.
Does not assume that `dim_data` has `proc_grid_size` set for each
dimension.
Attempts to allocate processes optimally for distributed dimensions.
Parameters
----------
shape : tuple of int
The global shape of the array.
dist: tuple of str
dist_type character per dimension.
comm_size : int
Total number of processes to distribute.
Returns
-------
dist_grid_shape : tuple of int
Raises
------
GridShapeError
if not possible to distribute `comm_size` processes over number of
dimensions.
"""
check_grid_shape_preconditions(shape, dist, comm_size)
distdims = tuple(i for (i, v) in enumerate(dist) if v != 'n')
ndistdim = len(distdims)
if ndistdim == 0:
dist_grid_shape = ()
elif ndistdim == 1:
# Trivial case: all processes used for the one distributed dimension.
if comm_size >= shape[distdims[0]]:
dist_grid_shape = (shape[distdims[0]], )
else:
dist_grid_shape = (comm_size, )
elif comm_size == 1:
# Trivial case: only one process to distribute over!
dist_grid_shape = (1, ) * ndistdim
else: # Main case: comm_size > 1, ndistdim > 1.
factors = utils.mult_partitions(comm_size, ndistdim)
if not factors: # Can't factorize appropriately.
raise GridShapeError("Cannot distribute array over processors.")
reduced_shape = [shape[i] for i in distdims]
# Reorder factors so they match the relative ordering in reduced_shape
factors = [utils.mirror_sort(f, reduced_shape) for f in factors]
# Pick the "best" factoring from `factors` according to which matches
# the ratios among the dimensions in `shape`.
rs_ratio = _compute_grid_ratios(reduced_shape)
f_ratios = [_compute_grid_ratios(f) for f in factors]
distances = [rs_ratio - f_ratio for f_ratio in f_ratios]
norms = numpy.array([numpy.linalg.norm(d, 2) for d in distances])
index = norms.argmin()
# we now have the grid shape for the distributed dimensions.
dist_grid_shape = tuple(int(i) for i in factors[index])
# Create the grid_shape, all 1's for now.
grid_shape = [1] * len(shape)
# Fill grid_shape in the distdim slots using dist_grid_shape
it = iter(dist_grid_shape)
for distdim in distdims:
grid_shape[distdim] = next(it)
out_grid_shape = tuple(grid_shape)
check_grid_shape_postconditions(out_grid_shape, shape, dist, comm_size)
return out_grid_shape