def check_grid_shape_postconditions(grid_shape, shape, dist, comm_size): """ Check grid_shape for reasonableness after creating it. """ if not (len(grid_shape) == len(shape) == len(dist)): raise ValueError("len(gird_shape) == len(shape) == len(dist) not " "satisfied, len(grid_shape) = %s and len(shape) = %s " "and len(dist) = %s" % (len(grid_shape), len(shape), len(dist))) if any(gs < 1 for gs in grid_shape): raise ValueError("all(gs >= 1 for gs in grid_shape) not satisfied, " "grid_shape = %s" % (grid_shape,)) if any(gs != 1 for (d, gs) in zip(dist, grid_shape) if d == 'n'): raise ValueError("all(gs == 1 for (d, gs) in zip(dist, grid_shape) if " "d == 'n', not satified dist = %s and grid_shape = " "%s" % (dist, grid_shape)) if any(gs > s for (s, gs) in zip(shape, grid_shape) if s > 0): raise ValueError("all(gs <= s for (s, gs) in zip(shape, grid_shape) " "if s > 0) not satisfied, shape = %s and grid_shape " "= %s" % (shape, grid_shape)) if reduce(operator.mul, grid_shape, 1) > comm_size: raise ValueError("reduce(operator.mul, grid_shape, 1) <= comm_size not" " satisfied, grid_shape = %s product = %s and " "comm_size = %s" % ( grid_shape, reduce(operator.mul, grid_shape, 1), comm_size))
def from_shape(cls, comm, shape, dist=None, grid_shape=None): """Create a Distribution from a `shape` and optional arguments.""" dist = {0: 'b'} if dist is None else dist ndim = len(shape) dist_tuple = normalize_dist(dist, ndim) base_comm = construct.init_base_comm(comm) comm_size = base_comm.Get_size() if grid_shape is None: # Make a new grid_shape if not provided. grid_shape = make_grid_shape(shape, dist_tuple, comm_size) grid_shape = normalize_grid_shape(grid_shape, ndim, dist_tuple, comm_size) comm = construct.init_comm(base_comm, grid_shape) grid_coords = comm.Get_coords(comm.Get_rank()) dim_data = [] for dist, size, grid_rank, grid_size in zip(dist_tuple, shape, grid_coords, grid_shape): dim_dict = dict(dist_type=dist, size=size, proc_grid_rank=grid_rank, proc_grid_size=grid_size) distribute_indices(dim_dict) dim_data.append(dim_dict) return cls(comm=base_comm, dim_data=dim_data)
def from_shape(cls, comm, shape, dist=None, grid_shape=None): """Create a Distribution from a `shape` and optional arguments.""" dist = {0: 'b'} if dist is None else dist ndim = len(shape) dist_tuple = normalize_dist(dist, ndim) base_comm = construct.init_base_comm(comm) comm_size = base_comm.Get_size() if grid_shape is None: # Make a new grid_shape if not provided. grid_shape = make_grid_shape(shape, dist_tuple, comm_size) grid_shape = normalize_grid_shape(grid_shape, shape, dist_tuple, comm_size) comm = construct.init_comm(base_comm, grid_shape) grid_coords = comm.Get_coords(comm.Get_rank()) dim_data = [] for dist, size, grid_rank, grid_size in zip(dist_tuple, shape, grid_coords, grid_shape): dim_dict = dict(dist_type=dist, size=size, proc_grid_rank=grid_rank, proc_grid_size=grid_size) distribute_indices(dim_dict) dim_data.append(dim_dict) return cls(comm=base_comm, dim_data=dim_data)
def __init__(self, global_size, grid_size, grid_rank, indices): self.global_size = global_size self.grid_size = grid_size self.grid_rank = grid_rank self.indices = np.asarray(indices) self.local_size = len(self.indices) local_indices = range(self.local_size) self._local_index = dict(zip(self.indices, local_indices))
def __init__(self, global_size, grid_size, grid_rank, indices): self.global_size = global_size self.grid_size = grid_size self.grid_rank = grid_rank self.indices = np.asarray(indices) self.local_size = len(self.indices) local_indices = range(self.local_size) self._local_index = dict(zip(self.indices, local_indices))
def global_from_local(self, local_ind): """ Given `local_ind` indices, translate into global indices.""" global_idxs = [] for m, idx in zip(self._maps, local_ind): if isinstance(idx, Integral): global_idxs.append(m.global_from_local_index(idx)) elif isinstance(idx, slice): global_idxs.append(m.global_from_local_slice(idx)) else: raise TypeError("Index must be Integral or slice.") return tuple(global_idxs)
def global_from_local(self, local_ind): """ Given `local_ind` indices, translate into global indices.""" global_idxs = [] for m, idx in zip(self._maps, local_ind): if isinstance(idx, Integral): global_idxs.append(m.global_from_local_index(idx)) elif isinstance(idx, slice): global_idxs.append(m.global_from_local_slice(idx)) else: raise TypeError("Index must be Integral or slice.") return tuple(global_idxs)
def local_from_global(self, global_ind): """ Given `global_ind` indices, translate into local indices.""" _, idxs = sanitize_indices(global_ind, self.ndim, self.global_shape) local_idxs = [] for m, idx in zip(self._maps, global_ind): if isinstance(idx, Integral): local_idxs.append(m.local_from_global_index(idx)) elif isinstance(idx, slice): local_idxs.append(m.local_from_global_slice(idx)) else: raise TypeError("Index must be Integral or slice.") return tuple(local_idxs)
def __init__(self, global_indices): """Make an IndexMap from a local_index and global_index. Parameters ---------- global_indices: list of int or range object Each position contains the corresponding global index for a local index (position). """ self.global_index = global_indices local_indices = range(len(global_indices)) self.local_index = dict(zip(global_indices, local_indices))
def local_from_global(self, global_ind): """ Given `global_ind` indices, translate into local indices.""" _, idxs = sanitize_indices(global_ind, self.ndim, self.global_shape) local_idxs = [] for m, idx in zip(self._maps, global_ind): if isinstance(idx, Integral): local_idxs.append(m.local_from_global_index(idx)) elif isinstance(idx, slice): local_idxs.append(m.local_from_global_slice(idx)) else: raise TypeError("Index must be Integral or slice.") return tuple(local_idxs)
def check_grid_shape_postconditions(grid_shape, shape, dist, comm_size): """ Check grid_shape for reasonableness after creating it. """ if not (len(grid_shape) == len(shape) == len(dist)): raise ValueError("len(gird_shape) == len(shape) == len(dist) not " "satisfied, len(grid_shape) = %s and len(shape) = %s " "and len(dist) = %s" % (len(grid_shape), len(shape), len(dist))) if any(gs < 1 for gs in grid_shape): raise ValueError("all(gs >= 1 for gs in grid_shape) not satisfied, " "grid_shape = %s" % (grid_shape, )) if any(gs != 1 for (d, gs) in zip(dist, grid_shape) if d == 'n'): raise ValueError("all(gs == 1 for (d, gs) in zip(dist, grid_shape) if " "d == 'n', not satified dist = %s and grid_shape = " "%s" % (dist, grid_shape)) if any(gs > s for (s, gs) in zip(shape, grid_shape) if s > 0): raise ValueError("all(gs <= s for (s, gs) in zip(shape, grid_shape) " "if s > 0) not satisfied, shape = %s and grid_shape " "= %s" % (shape, grid_shape)) if reduce(operator.mul, grid_shape, 1) > comm_size: raise ValueError( "reduce(operator.mul, grid_shape, 1) <= comm_size not" " satisfied, grid_shape = %s product = %s and " "comm_size = %s" % (grid_shape, reduce(operator.mul, grid_shape, 1), comm_size))
def make_partial_dim_data(shape, dist=None, grid_shape=None): """Create an (incomplete) dim_data structure from simple parameters. Parameters ---------- shape : tuple of int Number of elements in each dimension. dist : dict mapping int -> str, default is {0: 'b'} Keys are dimension number, values are dist_type, e.g 'b', 'c', or 'n'. grid_shape : tuple of int, optional Size of process grid in each dimension Returns ------- dim_data : tuple of dict Partial dim_data structure as outlined in the Distributed Array Protocol. """ supported_dist_types = ('n', 'b', 'c') if dist is None: dist = {0: 'b'} dist_tuple = construct.init_dist(dist, len(shape)) if grid_shape: # if None, BaseLocalArray will initialize grid_gen = iter(grid_shape) dim_data = [] for size, dist_type in zip(shape, dist_tuple): if dist_type not in supported_dist_types: msg = "dist_type {} not supported. Try `from_dim_data`." raise TypeError(msg.format(dist_type)) dimdict = dict(dist_type=dist_type, size=size) if grid_shape is not None and dist_type != 'n': dimdict["proc_grid_size"] = next(grid_gen) dim_data.append(dimdict) return tuple(dim_data)
def sanitize_indices(indices, ndim=None, shape=None): """Classify and sanitize `indices`. * Wrap naked Integral, slice, or Ellipsis indices into tuples * Classify result as 'value' or 'view' * Expand `Ellipsis` objects to slices * If the length of the tuple-ized `indices` is < ndim (and it's provided), add slice(None)'s to indices until `indices` is ndim long * If `shape` is provided, call `positivify` on the indices Raises ------ TypeError If `indices` is other than Integral, slice or a Sequence of these IndexError If len(indices) > ndim Returns ------- 2-tuple of (str, n-tuple of slices and Integral values) """ if isinstance(indices, Integral): rtype, sanitized = 'value', (indices,) elif isinstance(indices, slice) or indices is Ellipsis: rtype, sanitized = 'view', (indices,) elif all(isinstance(i, Integral) for i in indices): rtype, sanitized = 'value', indices elif all(isinstance(i, Integral) or isinstance(i, slice) or i is Ellipsis for i in indices): rtype, sanitized = 'view', indices else: msg = ("Index must be an Integral, a slice, or a sequence of " "Integrals and slices.") raise IndexError(msg) if Ellipsis in sanitized: if ndim is None: raise RuntimeError("Can't call `sanitize_indices` on Ellipsis " "without providing `ndim`.") # expand first Ellipsis diff = ndim - (len(sanitized) - 1) filler = (slice(None),) * diff epos = sanitized.index(Ellipsis) sanitized = sanitized[:epos] + filler + sanitized[epos + 1:] # remaining Ellipsis objects are just converted to slices def replace_ellipsis(idx): if idx is Ellipsis: return slice(None) else: return idx sanitized = tuple(replace_ellipsis(i) for i in sanitized) if ndim is not None: diff = ndim - len(sanitized) if diff < 0: raise IndexError("Too many indices.") if diff > 0: # allow incomplete indexing rtype = 'view' sanitized = sanitized + (slice(None),) * diff if shape is not None: sanitized = tuple(positivify(i, size) for (i, size) in zip(sanitized, shape)) return (rtype, sanitized)
def _are_shapes_bcast(shape, target_shape): for si, tsi in zip(shape, target_shape): if not si == 1 and not si == tsi: return False return True
def _return_shape(*args): return tuple([max(i) for i in zip(*args)])
def _return_shape(*args): return tuple([max(i) for i in zip(*args)])
def flatten(idx, strides): return sum(a * b for (a, b) in zip(idx, strides))
def flatten(idx, strides): return sum(a * b for (a, b) in zip(idx, strides))
def _are_shapes_bcast(shape, target_shape): for si, tsi in zip(shape, target_shape): if not si == 1 and not si == tsi: return False return True
def sanitize_indices(indices, ndim=None, shape=None): """Classify and sanitize `indices`. * Wrap naked Integral, slice, or Ellipsis indices into tuples * Classify result as 'value' or 'view' * Expand `Ellipsis` objects to slices * If the length of the tuple-ized `indices` is < ndim (and it's provided), add slice(None)'s to indices until `indices` is ndim long * If `shape` is provided, call `positivify` on the indices Raises ------ TypeError If `indices` is other than Integral, slice or a Sequence of these IndexError If len(indices) > ndim Returns ------- 2-tuple of (str, n-tuple of slices and Integral values) """ if isinstance(indices, Integral): rtype, sanitized = 'value', (indices, ) elif isinstance(indices, slice) or indices is Ellipsis: rtype, sanitized = 'view', (indices, ) elif all(isinstance(i, Integral) for i in indices): rtype, sanitized = 'value', indices elif all( isinstance(i, Integral) or isinstance(i, slice) or i is Ellipsis for i in indices): rtype, sanitized = 'view', indices else: msg = ("Index must be an Integral, a slice, or a sequence of " "Integrals and slices.") raise IndexError(msg) if Ellipsis in sanitized: if ndim is None: raise RuntimeError("Can't call `sanitize_indices` on Ellipsis " "without providing `ndim`.") # expand first Ellipsis diff = ndim - (len(sanitized) - 1) filler = (slice(None), ) * diff epos = sanitized.index(Ellipsis) sanitized = sanitized[:epos] + filler + sanitized[epos + 1:] # remaining Ellipsis objects are just converted to slices def replace_ellipsis(idx): if idx is Ellipsis: return slice(None) else: return idx sanitized = tuple(replace_ellipsis(i) for i in sanitized) if ndim is not None: diff = ndim - len(sanitized) if diff < 0: raise IndexError("Too many indices.") if diff > 0: # allow incomplete indexing rtype = 'view' sanitized = sanitized + (slice(None), ) * diff if shape is not None: sanitized = tuple( positivify(i, size) for (i, size) in zip(sanitized, shape)) return (rtype, sanitized)