def check_grid_shape_postconditions(grid_shape, shape, dist, comm_size):
""" Check grid_shape for reasonableness after creating it. """
if not (len(grid_shape) == len(shape) == len(dist)):
raise ValueError("len(gird_shape) == len(shape) == len(dist) not "
"satisfied, len(grid_shape) = %s and len(shape) = %s "
"and len(dist) = %s" % (len(grid_shape), len(shape),
len(dist)))
if any(gs < 1 for gs in grid_shape):
raise ValueError("all(gs >= 1 for gs in grid_shape) not satisfied, "
"grid_shape = %s" % (grid_shape,))
if any(gs != 1 for (d, gs) in zip(dist, grid_shape) if d == 'n'):
raise ValueError("all(gs == 1 for (d, gs) in zip(dist, grid_shape) if "
"d == 'n', not satified dist = %s and grid_shape = "
"%s" % (dist, grid_shape))
if any(gs > s for (s, gs) in zip(shape, grid_shape) if s > 0):
raise ValueError("all(gs <= s for (s, gs) in zip(shape, grid_shape) "
"if s > 0) not satisfied, shape = %s and grid_shape "
"= %s" % (shape, grid_shape))
if reduce(operator.mul, grid_shape, 1) > comm_size:
raise ValueError("reduce(operator.mul, grid_shape, 1) <= comm_size not"
" satisfied, grid_shape = %s product = %s and "
"comm_size = %s" % (
grid_shape,
reduce(operator.mul, grid_shape, 1),
comm_size))
def from_shape(cls, comm, shape, dist=None, grid_shape=None):
"""Create a Distribution from a `shape` and optional arguments."""
dist = {0: 'b'} if dist is None else dist
ndim = len(shape)
dist_tuple = normalize_dist(dist, ndim)
base_comm = construct.init_base_comm(comm)
comm_size = base_comm.Get_size()
if grid_shape is None: # Make a new grid_shape if not provided.
grid_shape = make_grid_shape(shape, dist_tuple, comm_size)
grid_shape = normalize_grid_shape(grid_shape, ndim,
dist_tuple, comm_size)
comm = construct.init_comm(base_comm, grid_shape)
grid_coords = comm.Get_coords(comm.Get_rank())
dim_data = []
for dist, size, grid_rank, grid_size in zip(dist_tuple, shape,
grid_coords, grid_shape):
dim_dict = dict(dist_type=dist,
size=size,
proc_grid_rank=grid_rank,
proc_grid_size=grid_size)
distribute_indices(dim_dict)
dim_data.append(dim_dict)
return cls(comm=base_comm, dim_data=dim_data)
def from_shape(cls, comm, shape, dist=None, grid_shape=None):
"""Create a Distribution from a `shape` and optional arguments."""
dist = {0: 'b'} if dist is None else dist
ndim = len(shape)
dist_tuple = normalize_dist(dist, ndim)
base_comm = construct.init_base_comm(comm)
comm_size = base_comm.Get_size()
if grid_shape is None: # Make a new grid_shape if not provided.
grid_shape = make_grid_shape(shape, dist_tuple, comm_size)
grid_shape = normalize_grid_shape(grid_shape, shape, dist_tuple,
comm_size)
comm = construct.init_comm(base_comm, grid_shape)
grid_coords = comm.Get_coords(comm.Get_rank())
dim_data = []
for dist, size, grid_rank, grid_size in zip(dist_tuple, shape,
grid_coords, grid_shape):
dim_dict = dict(dist_type=dist,
size=size,
proc_grid_rank=grid_rank,
proc_grid_size=grid_size)
distribute_indices(dim_dict)
dim_data.append(dim_dict)
return cls(comm=base_comm, dim_data=dim_data)
def __init__(self, global_size, grid_size, grid_rank, indices):
self.global_size = global_size
self.grid_size = grid_size
self.grid_rank = grid_rank
self.indices = np.asarray(indices)
self.local_size = len(self.indices)
local_indices = range(self.local_size)
self._local_index = dict(zip(self.indices, local_indices))
def __init__(self, global_size, grid_size, grid_rank, indices):
self.global_size = global_size
self.grid_size = grid_size
self.grid_rank = grid_rank
self.indices = np.asarray(indices)
self.local_size = len(self.indices)
local_indices = range(self.local_size)
self._local_index = dict(zip(self.indices, local_indices))
def global_from_local(self, local_ind):
""" Given `local_ind` indices, translate into global indices."""
global_idxs = []
for m, idx in zip(self._maps, local_ind):
if isinstance(idx, Integral):
global_idxs.append(m.global_from_local_index(idx))
elif isinstance(idx, slice):
global_idxs.append(m.global_from_local_slice(idx))
else:
raise TypeError("Index must be Integral or slice.")
return tuple(global_idxs)
def global_from_local(self, local_ind):
""" Given `local_ind` indices, translate into global indices."""
global_idxs = []
for m, idx in zip(self._maps, local_ind):
if isinstance(idx, Integral):
global_idxs.append(m.global_from_local_index(idx))
elif isinstance(idx, slice):
global_idxs.append(m.global_from_local_slice(idx))
else:
raise TypeError("Index must be Integral or slice.")
return tuple(global_idxs)
def local_from_global(self, global_ind):
""" Given `global_ind` indices, translate into local indices."""
_, idxs = sanitize_indices(global_ind, self.ndim, self.global_shape)
local_idxs = []
for m, idx in zip(self._maps, global_ind):
if isinstance(idx, Integral):
local_idxs.append(m.local_from_global_index(idx))
elif isinstance(idx, slice):
local_idxs.append(m.local_from_global_slice(idx))
else:
raise TypeError("Index must be Integral or slice.")
return tuple(local_idxs)
def __init__(self, global_indices):
"""Make an IndexMap from a local_index and global_index.
Parameters
----------
global_indices: list of int or range object
Each position contains the corresponding global index for a
local index (position).
"""
self.global_index = global_indices
local_indices = range(len(global_indices))
self.local_index = dict(zip(global_indices, local_indices))
def local_from_global(self, global_ind):
""" Given `global_ind` indices, translate into local indices."""
_, idxs = sanitize_indices(global_ind, self.ndim, self.global_shape)
local_idxs = []
for m, idx in zip(self._maps, global_ind):
if isinstance(idx, Integral):
local_idxs.append(m.local_from_global_index(idx))
elif isinstance(idx, slice):
local_idxs.append(m.local_from_global_slice(idx))
else:
raise TypeError("Index must be Integral or slice.")
return tuple(local_idxs)
def check_grid_shape_postconditions(grid_shape, shape, dist, comm_size):
""" Check grid_shape for reasonableness after creating it. """
if not (len(grid_shape) == len(shape) == len(dist)):
raise ValueError("len(gird_shape) == len(shape) == len(dist) not "
"satisfied, len(grid_shape) = %s and len(shape) = %s "
"and len(dist) = %s" %
(len(grid_shape), len(shape), len(dist)))
if any(gs < 1 for gs in grid_shape):
raise ValueError("all(gs >= 1 for gs in grid_shape) not satisfied, "
"grid_shape = %s" % (grid_shape, ))
if any(gs != 1 for (d, gs) in zip(dist, grid_shape) if d == 'n'):
raise ValueError("all(gs == 1 for (d, gs) in zip(dist, grid_shape) if "
"d == 'n', not satified dist = %s and grid_shape = "
"%s" % (dist, grid_shape))
if any(gs > s for (s, gs) in zip(shape, grid_shape) if s > 0):
raise ValueError("all(gs <= s for (s, gs) in zip(shape, grid_shape) "
"if s > 0) not satisfied, shape = %s and grid_shape "
"= %s" % (shape, grid_shape))
if reduce(operator.mul, grid_shape, 1) > comm_size:
raise ValueError(
"reduce(operator.mul, grid_shape, 1) <= comm_size not"
" satisfied, grid_shape = %s product = %s and "
"comm_size = %s" %
(grid_shape, reduce(operator.mul, grid_shape, 1), comm_size))
def make_partial_dim_data(shape, dist=None, grid_shape=None):
"""Create an (incomplete) dim_data structure from simple parameters.
Parameters
----------
shape : tuple of int
Number of elements in each dimension.
dist : dict mapping int -> str, default is {0: 'b'}
Keys are dimension number, values are dist_type, e.g 'b', 'c', or 'n'.
grid_shape : tuple of int, optional
Size of process grid in each dimension
Returns
-------
dim_data : tuple of dict
Partial dim_data structure as outlined in the Distributed Array
Protocol.
"""
supported_dist_types = ('n', 'b', 'c')
if dist is None:
dist = {0: 'b'}
dist_tuple = construct.init_dist(dist, len(shape))
if grid_shape: # if None, BaseLocalArray will initialize
grid_gen = iter(grid_shape)
dim_data = []
for size, dist_type in zip(shape, dist_tuple):
if dist_type not in supported_dist_types:
msg = "dist_type {} not supported. Try `from_dim_data`."
raise TypeError(msg.format(dist_type))
dimdict = dict(dist_type=dist_type, size=size)
if grid_shape is not None and dist_type != 'n':
dimdict["proc_grid_size"] = next(grid_gen)
dim_data.append(dimdict)
return tuple(dim_data)
def sanitize_indices(indices, ndim=None, shape=None):
"""Classify and sanitize `indices`.
* Wrap naked Integral, slice, or Ellipsis indices into tuples
* Classify result as 'value' or 'view'
* Expand `Ellipsis` objects to slices
* If the length of the tuple-ized `indices` is < ndim (and it's
provided), add slice(None)'s to indices until `indices` is ndim long
* If `shape` is provided, call `positivify` on the indices
Raises
------
TypeError
If `indices` is other than Integral, slice or a Sequence of these
IndexError
If len(indices) > ndim
Returns
-------
2-tuple of (str, n-tuple of slices and Integral values)
"""
if isinstance(indices, Integral):
rtype, sanitized = 'value', (indices,)
elif isinstance(indices, slice) or indices is Ellipsis:
rtype, sanitized = 'view', (indices,)
elif all(isinstance(i, Integral) for i in indices):
rtype, sanitized = 'value', indices
elif all(isinstance(i, Integral)
or isinstance(i, slice)
or i is Ellipsis for i in indices):
rtype, sanitized = 'view', indices
else:
msg = ("Index must be an Integral, a slice, or a sequence of "
"Integrals and slices.")
raise IndexError(msg)
if Ellipsis in sanitized:
if ndim is None:
raise RuntimeError("Can't call `sanitize_indices` on Ellipsis "
"without providing `ndim`.")
# expand first Ellipsis
diff = ndim - (len(sanitized) - 1)
filler = (slice(None),) * diff
epos = sanitized.index(Ellipsis)
sanitized = sanitized[:epos] + filler + sanitized[epos + 1:]
# remaining Ellipsis objects are just converted to slices
def replace_ellipsis(idx):
if idx is Ellipsis:
return slice(None)
else:
return idx
sanitized = tuple(replace_ellipsis(i) for i in sanitized)
if ndim is not None:
diff = ndim - len(sanitized)
if diff < 0:
raise IndexError("Too many indices.")
if diff > 0:
# allow incomplete indexing
rtype = 'view'
sanitized = sanitized + (slice(None),) * diff
if shape is not None:
sanitized = tuple(positivify(i, size) for (i, size) in zip(sanitized,
shape))
return (rtype, sanitized)
def _are_shapes_bcast(shape, target_shape):
for si, tsi in zip(shape, target_shape):
if not si == 1 and not si == tsi:
return False
return True
def _return_shape(*args):
return tuple([max(i) for i in zip(*args)])
def _return_shape(*args):
return tuple([max(i) for i in zip(*args)])
def flatten(idx, strides):
return sum(a * b for (a, b) in zip(idx, strides))
def flatten(idx, strides):
return sum(a * b for (a, b) in zip(idx, strides))
def _are_shapes_bcast(shape, target_shape):
for si, tsi in zip(shape, target_shape):
if not si == 1 and not si == tsi:
return False
return True
def sanitize_indices(indices, ndim=None, shape=None):
"""Classify and sanitize `indices`.
* Wrap naked Integral, slice, or Ellipsis indices into tuples
* Classify result as 'value' or 'view'
* Expand `Ellipsis` objects to slices
* If the length of the tuple-ized `indices` is < ndim (and it's
provided), add slice(None)'s to indices until `indices` is ndim long
* If `shape` is provided, call `positivify` on the indices
Raises
------
TypeError
If `indices` is other than Integral, slice or a Sequence of these
IndexError
If len(indices) > ndim
Returns
-------
2-tuple of (str, n-tuple of slices and Integral values)
"""
if isinstance(indices, Integral):
rtype, sanitized = 'value', (indices, )
elif isinstance(indices, slice) or indices is Ellipsis:
rtype, sanitized = 'view', (indices, )
elif all(isinstance(i, Integral) for i in indices):
rtype, sanitized = 'value', indices
elif all(
isinstance(i, Integral) or isinstance(i, slice) or i is Ellipsis
for i in indices):
rtype, sanitized = 'view', indices
else:
msg = ("Index must be an Integral, a slice, or a sequence of "
"Integrals and slices.")
raise IndexError(msg)
if Ellipsis in sanitized:
if ndim is None:
raise RuntimeError("Can't call `sanitize_indices` on Ellipsis "
"without providing `ndim`.")
# expand first Ellipsis
diff = ndim - (len(sanitized) - 1)
filler = (slice(None), ) * diff
epos = sanitized.index(Ellipsis)
sanitized = sanitized[:epos] + filler + sanitized[epos + 1:]
# remaining Ellipsis objects are just converted to slices
def replace_ellipsis(idx):
if idx is Ellipsis:
return slice(None)
else:
return idx
sanitized = tuple(replace_ellipsis(i) for i in sanitized)
if ndim is not None:
diff = ndim - len(sanitized)
if diff < 0:
raise IndexError("Too many indices.")
if diff > 0:
# allow incomplete indexing
rtype = 'view'
sanitized = sanitized + (slice(None), ) * diff
if shape is not None:
sanitized = tuple(
positivify(i, size) for (i, size) in zip(sanitized, shape))
return (rtype, sanitized)
