def testCartcomm(self): comm = self.COMM size = comm.Get_size() rank = comm.Get_rank() for ndim in (1, 2, 3, 4, 5): dims = [0] * ndim dims = MPI.Compute_dims(size, [0] * ndim) periods = [True] * len(dims) topo = comm.Create_cart(dims, periods=periods) self.assertTrue(topo.is_topo) self.assertTrue(topo.topology, MPI.CART) self.checkFortran(topo) self.assertEqual(topo.dim, len(dims)) self.assertEqual(topo.ndim, len(dims)) coordinates = topo.coords self.assertEqual(coordinates, topo.Get_coords(topo.rank)) neighbors = [] for i in range(ndim): for d in (-1, +1): coord = list(coordinates) coord[i] = (coord[i] + d) % dims[i] neigh = topo.Get_cart_rank(coord) self.assertEqual(coord, topo.Get_coords(neigh)) source, dest = topo.Shift(i, d) self.assertEqual(neigh, dest) neighbors.append(neigh) self.assertEqual(topo.indegree, len(neighbors)) self.assertEqual(topo.outdegree, len(neighbors)) self.assertEqual(topo.inedges, neighbors) self.assertEqual(topo.outedges, neighbors) inedges, outedges = topo.inoutedges self.assertEqual(inedges, neighbors) self.assertEqual(outedges, neighbors) if ndim == 1: topo.Free() continue for i in range(ndim): rem_dims = [1] * ndim rem_dims[i] = 0 sub = topo.Sub(rem_dims) if sub != MPI.COMM_NULL: self.assertEqual(sub.dim, ndim - 1) dims = topo.dims del dims[i] self.assertEqual(sub.dims, dims) sub.Free() topo.Free() if size > 1: return if MPI.VERSION < 2: return topo = comm.Create_cart([]) self.assertEqual(topo.dim, 0) self.assertEqual(topo.dims, []) self.assertEqual(topo.periods, []) self.assertEqual(topo.coords, []) rank = topo.Get_cart_rank([]) self.assertEqual(rank, 0) inedges, outedges = topo.inoutedges self.assertEqual(inedges, []) self.assertEqual(outedges, []) topo.Free()
def test1(): dims = MPI.Compute_dims(MPISIZE, 3) if MPIRANK == 0 and DEBUG: print("DIM", dims) cc = MPI.COMM_WORLD.Create_cart(dims, (1, 1, 1), True) top = cc.Get_topo()[2] N = 64 dtype = ctypes.c_double grid_size = (N, N, N) rng = np.random.RandomState(seed=3857293) orig = np.array(rng.uniform(0., 100., size=grid_size), dtype=dtype) np0 = int(math.floor(grid_size[0] / dims[0])) np1 = int(math.floor(grid_size[1] / dims[1])) np2 = int(math.floor(grid_size[2] / dims[2])) s0 = top[0] * np0 s1 = top[1] * np1 s2 = top[2] * np2 e0 = (top[0] + 1) * np0 if top[0] < dims[0] - 1 else N e1 = (top[1] + 1) * np1 if top[1] < dims[1] - 1 else N e2 = (top[2] + 1) * np2 if top[2] < dims[2] - 1 else N works_size = (e0 - s0 + 2, e1 - s1 + 2, e2 - s2 + 2) he = pygcl.HaloExchange3D(cc, works_size) a = np.zeros(shape=list(works_size) + [1], dtype=dtype) a[1:-1:, 1:-1:, 1:-1, 0] = orig[s0:e0:, s1:e1:, s2:e2:] # shell should be zero and interior should be the copied data for iz in range(works_size[0]): for iy in range(works_size[1]): for ix in range(works_size[2]): # if shell if (iz == 0 or iz == works_size[2]-1) and \ (iy == 0 or iy == works_size[1]-1) and \ (ix == 0 or ix == works_size[0]-1): assert a[iz, iy, ix, 0] == 0 for iz in range(1, works_size[0] - 1): for iy in range(1, works_size[1] - 1): for ix in range(1, works_size[2] - 1): assert a[iz, iy, ix, 0] == orig[iz + s0 - 1, iy + s1 - 1, ix + s2 - 1] # exchange he.exchange(a) # interior should be unchanged, outer shell should be halo for iz in range(works_size[0]): for iy in range(works_size[1]): for ix in range(works_size[2]): assert a[iz, iy, ix, 0] == orig[(iz+s0-1) % grid_size[0], (iy+s1-1) % grid_size[1], (ix+s2-1) % grid_size[2]], \ "{} {} {}".format(iz, iy, ix)
def __init__(self, shape, dimensions, input_comm=None): self._glb_shape = shape self._dimensions = dimensions self._input_comm = (input_comm or MPI.COMM_WORLD).Clone() # `Compute_dims` sets the dimension sizes to be as close to each other # as possible, using an appropriate divisibility algorithm. Thus, in 3D: # * topology[0] >= topology[1] >= topology[2] # * topology[0] * topology[1] * topology[2] == self._input_comm.size topology = MPI.Compute_dims(self._input_comm.size, len(shape)) # At this point MPI's dimension 0 corresponds to the rightmost element # in `topology`. This is in reverse to `shape`'s ordering. Hence, we # now restore consistency self._topology = tuple(reversed(topology)) if self._input_comm is not input_comm: # By default, Devito arranges processes into a cartesian topology. # MPI works with numbered dimensions and follows the C row-major # numbering of the ranks, i.e. in a 2x3 Cartesian topology (0,0) # maps to rank 0, (0,1) maps to rank 1, (0,2) maps to rank 2, (1,0) # maps to rank 3, and so on. self._comm = self._input_comm.Create_cart(self._topology) else: self._comm = input_comm # Perform domain decomposition self._glb_numbs = [ np.array_split(range(i), j) for i, j in zip(shape, self._topology) ]
def __init__(self, rankinfo): '''Find out the best distribution of the lattice amonst the ranks of the communicator passed inside "rankinfo", create the topology, and save information about the neighbours of present rank. Parameters ---------- rankinfo : a rankinfo instance describing current communicator Attributes ---------- dims : dimensions of the cartesian MPI rank grid (not lattice!) topology : the new cartesian topology communicator shifts : a dict of dicts of which rank to talk to when going along "X", "Y", or "Z" direcion in "up" or "down" direction ''' self.me=rankinfo self.dims=MPI.Compute_dims(self.me.size, self.me.ndim) self.topology=MPI.COMM_WORLD.Create_cart(self.dims, periods=self.me.periods, reorder=True) left,right = self.topology.Shift(0,1) front,back = self.topology.Shift(1,1) up,down = self.topology.Shift(2,1) self.shifts={"X": {"up": up, "down": down}, "Y": {"up": back, "down": front}, "Z": {"up": right, "down": left}}
def __new__(cls, comm, dims=None, reorder=True): assert not comm.Is_inter() if comm.Get_topology() == MPI.CART: assert comm.Get_dim() > 0 assert dims is None cartcomm = comm else: if dims is None: dims = [0] elif np.ndim(dims) > 0: assert len(dims) > 0 dims = [max(0, d) for d in dims] else: assert dims > 0 dims = [0] * dims dims = MPI.Compute_dims(comm.Get_size(), dims) cartcomm = comm.Create_cart(dims, reorder=reorder) dim = cartcomm.Get_dim() subcomm = [None] * dim remdims = [False] * dim for i in range(dim): remdims[i] = True subcomm[i] = cartcomm.Sub(remdims) remdims[i] = False if cartcomm != comm: cartcomm.Free() return super(Subcomm, cls).__new__(cls, subcomm)
def testCartMap(self): comm = self.COMM size = comm.Get_size() for ndim in (1,2,3,4,5): for periods in (None, True, False): dims = MPI.Compute_dims(size, [0]*ndim) topo = comm.Create_cart(dims, periods, reorder=True) rank = comm.Cart_map(dims, periods) self.assertEqual(topo.Get_rank(), rank) topo.Free()
def compute_dims(nprocs, ndim): # We don't do anything clever here. In fact, we do something very basic -- # we just try to distribute `nprocs` evenly over the number of dimensions, # and if we can't we fallback to whatever MPI.Compute_dims gives... v = pow(nprocs, 1 / ndim) if not v.is_integer(): # Since pow(64, 1/3) == 3.999..4 v = int(ceil(v)) if not v**ndim == nprocs: # Fallback return MPI.Compute_dims(nprocs, ndim) else: v = int(v) return tuple(v for _ in range(ndim))
def createCart(parent): """Create a 2-d Cartesian communicator from parent communicator""" dims = MPI.Compute_dims(parent.size, 2) comm = parent.Create_cart(dims, periods=[True, False]) rank = comm.Get_rank() coords = comm.Get_coords(rank) upx = comm.Shift(0, 1) upy = comm.Shift(1, 1) out = "Rank{:2d} coords{:2d} {:2d} upx(src,dst) {} upy(src,dst) {}\n"\ .format(rank, coords[0], coords[1], upx, upy) sys.stdout.write(out)
def testCreateDarray(self): for dtype in datatypes: for ndim in range(1, 3+1): for size in (4, 8, 9, 27): for rank in (0, size-1): for dist in [MPI.DISTRIBUTE_BLOCK, MPI.DISTRIBUTE_CYCLIC]: for order in [MPI.ORDER_C, MPI.ORDER_F]: gsizes = [size]*ndim distribs = [dist]*ndim dargs = [MPI.DISTRIBUTE_DFLT_DARG]*ndim psizes = MPI.Compute_dims(size, [0]*ndim) factory = MPI.Datatype.Create_darray args = size, rank, gsizes, distribs, dargs, psizes, order self.check_datatype(dtype, factory, *args)
def main(): dim = 3 proc_sizes = tuple(MPI.Compute_dims(SIZE, dim)) # see lists.mcs.anl.gov/pipermail/petsc-dev/2016-April/018903.html # manually distribute the unkowns to the processors # mpiexec -n 3 if SIZE > 1: if 1: ownership_ranges = ( np.array([3, 4, 5], dtype='i4'), np.array([1], dtype='i4'), np.array([1], dtype='i4') ) else: ownership_ranges = ( # proc 0 (3, 4, 5), # proc 1 (1,), # proc 2 (1,) ) else: ownership_ranges = ((3,), (4,), (5,)) sizes = np.array(( sum(ownership_ranges[0]), sum(ownership_ranges[1]), sum(ownership_ranges[2])), dtype='i4' ) dm = PETSc.DMDA().create(dim=dim, sizes=sizes, proc_sizes=proc_sizes, ownership_ranges=ownership_ranges, boundary_type=(PETSc.DMDA.BoundaryType.PERIODIC, PETSc.DMDA.BoundaryType.PERIODIC, PETSc.DMDA.BoundaryType.GHOSTED), stencil_type=PETSc.DMDA.StencilType.BOX, stencil_width=1, dof=1, comm=PETSc.COMM_WORLD, setup=True) assert dm.getComm().Get_rank() == RANK assert dm.getComm().Get_size() == SIZE print(f'create problem with {sizes.prod()} unknowns split into dim={dim} (sizes={sizes}) split accross {SIZE} procs with distribution {proc_sizes}') vec_l = dm.createLocalVector() vec_g = dm.createGlobalVector() initialise_values(dm, vec_l, vec_g) result_l, result_g = find_max_grad(dm, vec_l, vec_g) PETSc.Sys.syncPrint(f'Rank {RANK} had max gradient {result_l} while the global was {result_g}.') if RANK == 0: if testme(result_g, dm.getComm()): print('Result is correct.') else: print('Result is incorrect!')
def __init__(self, dim, period, comm=mpi.COMM_WORLD): self.dim = dim self.set_options() self.comm = comm # if npx, npy and npz are all set to the default value (1) # then Compute_dims performs the splitting of the domain if self.npx == self.npy == self.npz == 1: size = comm.Get_size() split = mpi.Compute_dims(size, self.dim) else: split = (self.npx, self.npy, self.npz) self.split = np.asarray(split[:self.dim]) self.cartcomm = comm.Create_cart(self.split, period)
def __init__(self, comm, discretization): # set comm self._comm = comm # get size & rank self._size = comm.Get_size() self._rank = comm.Get_rank() # compute a distribution of processes per coordinate direction self._dims = [0, 0] self._dims = MPI.Compute_dims(self._size, self._dims) # create two-dimensional non-periodic Cartesian topology self._periods = [False, False] self._comm_cart = self._comm.Create_cart(self._dims, periods=self._periods) # get rank's coordinates in the topology self._coords = self._comm_cart.Get_coords(self._rank) # get rank's south/north/west/east neighbors self._neigh_west, self._neigh_east = self._comm_cart.Shift(0, 1) self._neigh_south, self._neigh_north = self._comm_cart.Shift(1, 1) # global domain discretization size self._global_nx = discretization.nx self._global_ny = discretization.ny # global start/end index in (sub)domain discretization chunkx = discretization.nx // self._dims[0] self._global_startx = chunkx * self._coords[0] if self._coords[0] == self._dims[0] - 1: self._global_endx = discretization.nx else: self._global_endx = self._global_startx + chunkx chunky = discretization.ny // self._dims[1] self._global_starty = chunky * self._coords[1] if self._coords[1] == self._dims[1] - 1: self._global_endy = discretization.ny else: self._global_endy = self._global_starty + chunky # local (sub)domain discretization size self._local_nx = self._global_endx - self._global_startx self._local_ny = self._global_endy - self._global_starty
def __init__(self, comm, idir, odir, N, ndims, decomp, periodic): self.idir = idir # users should be able to change directories at will self.odir = odir # users should be able to change directories at will self._ndims = ndims self._subarrays = dict() init_comm = comm if np.iterable(N): if len(N) == 1: self._nx = np.array(list(N)*ndims, dtype=int) elif len(N) == ndims: self._nx = np.array(N, dtype=int) else: raise IndexError("The length of N must be either 1 or ndims") else: self._nx = np.array([N]*ndims, dtype=int) if decomp is None: self._decomp = 1 elif decomp in [1, 2, 3] and decomp <= ndims: self._decomp = decomp else: raise IndexError("decomp must be 1, 2, 3 or None") if periodic is None: self._periodic = tuple([True]*ndims) elif len(periodic) == ndims: self._periodic = tuple(periodic) else: raise IndexError("Either len(periodic) must be ndims or " "periodic must be None") dims = MPI.Compute_dims(init_comm.size, self._decomp) dims.extend([1]*(ndims-self._decomp)) assert np.all(np.mod(self._nx, dims) == 0) self._comm = init_comm.Create_cart(dims, self._periodic) self._nnx = self._nx//dims self._ixs = self._nnx*self.comm.coords self._ixe = self._ixs+self._nnx
def main(): dims=tuple(MPI.Compute_dims(MPI.COMM_WORLD.Get_size(),3)) dm = PETSc.DMDA().create(dim=len(dims), ownership_ranges=(numpy.array([3]), numpy.array([4]), numpy.array([5])), proc_sizes=dims, boundary_type=(PETSc.DMDA.BoundaryType.PERIODIC, PETSc.DMDA.BoundaryType.PERIODIC, PETSc.DMDA.BoundaryType.GHOSTED), stencil_type=PETSc.DMDA.StencilType.BOX, stencil_width = 1, dof = 1, comm = PETSc.COMM_WORLD, setup = True) vec_l=dm.createLocalVector() vec_g=dm.createGlobalVector() initialise_values(dm, vec_l, vec_g) result_l, result_g = find_max_grad(dm, vec_l, vec_g) PETSc.Sys.syncPrint("Rank {rank} had max gradient {maxgrad_l} while the global was {maxgrad_g}." .format( rank=dm.getComm().getRank(), maxgrad_l=result_l, maxgrad_g=result_g)) if (dm.getComm().getRank() == 0): if (testme(result_g, dm.getComm())): print("Result is correct.") else: print("Result is incorrect!")
def get_comm_dims(procs, dist): """ Get dimensions of cartesian grid as determined by specified distribution. Parameters ---------- procs: int Size/number of processes in communicator dist : str Specified distribution of data among processes. Default value 'b' : Block Supported types: 'b' : Block 'r' : Replicated Returns ------- comm_dims : list, None Dimensions of cartesian grid """ if is_Replicated(dist): return None return MPI.Compute_dims(procs, distribution_to_dimensions(dist, procs))
def test2(): width = 2 dims = MPI.Compute_dims(MPISIZE, 3) if MPIRANK == 0 and DEBUG: print("DIM", dims) cc = MPI.COMM_WORLD.Create_cart(dims, (1, 1, 1), True) top = cc.Get_topo()[2] N = 64 M = 2 dtype = ctypes.c_double grid_size = (2 * N, N, N) rng = np.random.RandomState(seed=3857293) orig = np.array(rng.uniform(0., 100., size=list(grid_size) + [M]), dtype=dtype) np0 = int(math.floor(grid_size[0] / dims[0])) np1 = int(math.floor(grid_size[1] / dims[1])) np2 = int(math.floor(grid_size[2] / dims[2])) s0 = top[0] * np0 s1 = top[1] * np1 s2 = top[2] * np2 e0 = (top[0] + 1) * np0 if top[0] < dims[0] - 1 else grid_size[0] e1 = (top[1] + 1) * np1 if top[1] < dims[1] - 1 else grid_size[1] e2 = (top[2] + 1) * np2 if top[2] < dims[2] - 1 else grid_size[2] works_size = (e0 - s0 + 2 * width, e1 - s1 + 2 * width, e2 - s2 + 2 * width) he = pygcl.HaloExchange3D(cc, works_size, (width, width, width)) a = np.zeros(shape=list(works_size) + [M], dtype=dtype) a[width:-1*width:, width:-1*width:, width:-1*width, :] = \ orig[s0:e0:, s1:e1:, s2:e2:, :] # shell should be zero and interior should be the copied data for iz in range(width, works_size[0] - width): for iy in range(width, works_size[1] - width): for ix in range(width, works_size[2] - width): assert np.all(a[iz, iy, ix, :] == orig[iz + s0 - width, iy + s1 - width, ix + s2 - width, :]) if DEBUG: for rx in range(MPISIZE): if MPIRANK == rx: print("--", MPIRANK) print(orig) print(a[:, :, :, 0]) MPI.COMM_WORLD.Barrier() # exchange he.exchange(a) if DEBUG: for rx in range(MPISIZE): if MPIRANK == rx: print("--", MPIRANK) print('boundary', he.boundary_cells) print('halo', he.halo_cells) print(s0, e0, s1, e1, s2, e2) print(works_size) print(orig) print(a[:, :, :, 0]) MPI.COMM_WORLD.Barrier() # interior should be unchanged, outer shell should be halo for iz in range(works_size[0]): for iy in range(works_size[1]): for ix in range(works_size[2]): assert np.all(a[iz, iy, ix, :] == orig[(iz+s0-width) % grid_size[0], (iy+s1-width) % grid_size[1], (ix+s2-width) % grid_size[2], :]),\ "{} {} {}".format(iz, iy, ix)
import mpi4py.MPI as mpi comm = mpi.COMM_WORLD size = comm.Get_size() rank = comm.Get_rank() ndim = 2 dims = mpi.Compute_dims(size, [0] * ndim) topo = comm.Create_cart(dims, periods=False) print( f'Process {rank}: coordinate = {topo.coords}, neighbor = {topo.outedges}') comm.Barrier() recvbuf = topo.neighbor_alltoall([rank] * 4) print(rank, "->", recvbuf)
def test_mpifft(): from itertools import product comm = MPI.COMM_WORLD dims = (2, 3, 4,) sizes = (12, 13) assert MPI.COMM_WORLD.Get_size() < 8, "due to sizes" types = '' for t in 'fdg': if fftw.get_fftw_lib(t): types += t+t.upper() grids = {2: (None,), 3: ((-1,), None), 4: ((-1,), None)} for typecode in types: for dim in dims: for shape in product(*([sizes]*dim)): if dim < 3: n = min(shape) if typecode in 'fdg': n //= 2 n += 1 if n < comm.size: continue for grid in grids[dim]: padding = False for collapse in (True, False): for backend in backends: transforms = None if dim < 3: allaxes = [None, (-1,), (-2,), (-1, -2,), (-2, -1), (-1, 0), (0, -1), ((0,), (1,))] elif dim < 4: allaxes = [None, ((0,), (1, 2)), ((0,), (-2, -1))] elif dim > 3: allaxes = [None, ((0,), (1,), (2,), (3,)), ((0,), (1, 2, 3)), ((0,), (1,), (2, 3))] dctn = functools.partial(fftw.dctn, type=3) idctn = functools.partial(fftw.idctn, type=3) if not typecode in 'FDG': if backend == 'pyfftw': transforms = {(3,): (pyfftw.builders.rfftn, pyfftw.builders.irfftn), (2, 3): (pyfftw.builders.rfftn, pyfftw.builders.irfftn), (1, 2, 3): (pyfftw.builders.rfftn, pyfftw.builders.irfftn), (0, 1, 2, 3): (pyfftw.builders.rfftn, pyfftw.builders.irfftn)} else: transforms = {(3,): (dctn, idctn), (2, 3): (dctn, idctn), (1, 2, 3): (dctn, idctn), (0, 1, 2, 3): (dctn, idctn)} for axes in allaxes: # Test also the slab is number interface _grid = grid if grid is not None: ax = -1 if axes is not None: ax = axes[-1] if isinstance(axes[-1], int) else axes[-1][-1] _slab = (ax+1) % len(shape) _grid = [1]*(_slab+1) _grid[_slab] = 0 _comm = comm # Test also the comm is Subcomm interfaces # For PFFT the Subcomm needs to be as long as shape if len(shape) > 2 and axes is None and grid is None: _dims = [0] * len(shape) _dims[-1] = 1 # distribute all but last axis (axes is None) _comm = comm if random_true_or_false(comm) == 1: # then test Subcomm with a MPI.CART argument _dims = MPI.Compute_dims(comm.Get_size(), _dims) _comm = comm.Create_cart(_dims) _dims = None _comm = Subcomm(_comm, _dims) #print(typecode, shape, axes, collapse, _grid) fft = PFFT(_comm, shape, axes=axes, dtype=typecode, padding=padding, grid=_grid, collapse=collapse, backend=backend, transforms=transforms) #if comm.rank == 0: # grid_ = [c.size for c in fft.subcomm] # print('grid:{} shape:{} typecode:{} backend:{} axes:{}' # .format(grid_, shape, typecode, backend, axes)) assert fft.dtype(True) == fft.forward.output_array.dtype assert fft.dtype(False) == fft.forward.input_array.dtype assert len(fft.axes) == len(fft.xfftn) assert len(fft.axes) == len(fft.transfer) + 1 assert (fft.forward.input_pencil.subshape == fft.forward.input_array.shape) assert (fft.forward.output_pencil.subshape == fft.forward.output_array.shape) assert (fft.backward.input_pencil.subshape == fft.backward.input_array.shape) assert (fft.backward.output_pencil.subshape == fft.backward.output_array.shape) assert np.alltrue(np.array(fft.global_shape(True)) == np.array(fft.forward.output_pencil.shape)) assert np.alltrue(np.array(fft.global_shape(False)) == np.array(fft.forward.input_pencil.shape)) ax = -1 if axes is None else axes[-1] if isinstance(axes[-1], int) else axes[-1][-1] assert fft.forward.input_pencil.substart[ax] == 0 assert fft.backward.output_pencil.substart[ax] == 0 ax = 0 if axes is None else axes[0] if isinstance(axes[0], int) else axes[0][0] assert fft.forward.output_pencil.substart[ax] == 0 assert fft.backward.input_pencil.substart[ax] == 0 assert fft.dimensions == len(shape) U = random_like(fft.forward.input_array) if random_true_or_false(comm) == 1: F = fft.forward(U) V = fft.backward(F) assert allclose(V, U) else: fft.forward.input_array[...] = U fft.forward() fft.backward() V = fft.backward.output_array assert allclose(V, U) fft.destroy() padding = [1.5]*len(shape) for backend in backends: if dim < 3: allaxes = [None, (-1,), (-2,), (-1, -2,), (-2, -1), (-1, 0), (0, -1), ((0,), (1,))] elif dim < 4: allaxes = [None, ((0,), (1,), (2,)), ((0,), (-2,), (-1,))] elif dim > 3: allaxes = [None, (0, 1, -2, -1), ((0,), (1,), (2,), (3,))] for axes in allaxes: _grid = grid if grid is not None: ax = -1 if axes is not None: ax = axes[-1] if isinstance(axes[-1], int) else axes[-1][-1] _slab = (ax+1) % len(shape) _grid = [1]*(_slab+1) _grid[_slab] = 0 fft = PFFT(comm, shape, axes=axes, dtype=typecode, padding=padding, grid=_grid, backend=backend) #if comm.rank == 0: # grid = [c.size for c in fft.subcomm] # print('grid:{} shape:{} typecode:{} backend:{} axes:{}' # .format(grid, shape, typecode, backend, axes)) assert len(fft.axes) == len(fft.xfftn) assert len(fft.axes) == len(fft.transfer) + 1 assert (fft.forward.input_pencil.subshape == fft.forward.input_array.shape) assert (fft.forward.output_pencil.subshape == fft.forward.output_array.shape) assert (fft.backward.input_pencil.subshape == fft.backward.input_array.shape) assert (fft.backward.output_pencil.subshape == fft.backward.output_array.shape) ax = -1 if axes is None else axes[-1] if isinstance(axes[-1], int) else axes[-1][-1] assert fft.forward.input_pencil.substart[ax] == 0 assert fft.backward.output_pencil.substart[ax] == 0 ax = 0 if axes is None else axes[0] if isinstance(axes[0], int) else axes[0][0] assert fft.forward.output_pencil.substart[ax] == 0 assert fft.backward.input_pencil.substart[ax] == 0 U = random_like(fft.forward.input_array) F = fft.forward(U) if random_true_or_false(comm) == 1: Fc = F.copy() V = fft.backward(F) F = fft.forward(V) assert allclose(F, Fc) else: fft.backward.input_array[...] = F fft.backward() fft.forward() V = fft.forward.output_array assert allclose(F, V) # Test normalization on backward transform instead of default fft.backward.input_array[...] = F fft.backward(normalize=True) fft.forward(normalize=False) V = fft.forward.output_array assert allclose(F, V) fft.destroy()
# Timers timers = {"projection": 0, "loading": 0, "writing": 0, "total": 0} timers["total"] = time.time() # Setup xmin = 0 xmax = 2 * np.pi L = xmax - xmin # MPI setup minimize_communication = False comm = MPI.COMM_WORLD rank = comm.Get_rank() nprocs = comm.Get_size() if minimize_communication: dimensions = MPI.Compute_dims(nprocs, 3) else: # you won't have to sort data on merge dimensions = [1, 1, nprocs] periodicity = (False, False, False) grid = comm.Create_cart(dimensions, periodicity, reorder=True) coords = grid.Get_coords(rank) # Logging information if rank == 0: pfx = "fv_{0:d}".format(args.res) logname = pfx + ".log" logging.basicConfig( level=logging.INFO, format="%(asctime)s %(name)-12s %(levelname)-8s %(message)s", datefmt="%m-%d %H:%M", filename=logname,
from mpi4py import MPI import numpy as np # mostly adapted from the unit tests of mpi4py github repo comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() ndims = 2 dims = [0] * ndims dims = MPI.Compute_dims(size, dims) periods = [True] * ndims topo = comm.Create_cart(dims, periods=periods) coords = topo.Get_coords(rank) nw, ne = topo.Shift(0, 1) ns, nn = topo.Shift(1, 1) localgrid = np.array([[rank] * 8] * 8, dtype='i') req = [] if rank == 1: print("BEFORE\n ---------------------") print(localgrid) bN = localgrid[:, -1].flatten() bS = localgrid[:, 0].flatten() bE = localgrid[-1, :].flatten() bW = localgrid[0, :].flatten() bufN = np.zeros_like(bN) bufS = np.zeros_like(bS) bufE = np.zeros_like(bE) bufW = np.zeros_like(bW)
format='%(filename)s:%(lineno)d - %(levelname)s:%(message)s') log = logging.getLogger("main") console = logging.StreamHandler() console.setLevel(level=console_log_level) log.addHandler(console) else: logging.basicConfig(level=console_log_level) log = logging.getLogger("main") log.info("Starting configuration.") ############################################################################### # MPI set up ############################################################################### log.info("Initialising mpi.cart_comm") dims_list = [0, 0] dims = MPI.Compute_dims(MPI.COMM_WORLD.size, dims_list) periods = [1, 1] reorder = True mpi_comm = MPI.COMM_WORLD.Create_cart(dims, periods=periods, reorder=reorder) log.info("Initialisation of mpi.cart_comm complete") ################################################################################ # System, IBVP and Grid settings ################################################################################ log.info("Starting configuration.") # How many systems? num_of_grids = args.nsim # How many grid points? Nx = 50
def __init__(self, comm, odir, pid, ndims, decomp, periodic, L, N, method): # -------------------------------------------------------------- # Unprocessed user-input instance properties self._pid = pid self._ndims = ndims self._config = "Unknown (Base Configuration)" self._odir = odir init_comm = comm # -------------------------------------------------------------- # Make analysis output directory if init_comm.rank == 0: try: os.makedirs(odir) except OSError as e: if not os.path.isdir(odir): raise e else: status = e finally: if os.path.isdir(odir): status = 0 else: status = None status = init_comm.bcast(status) if status != 0: MPI.Finalize() sys.exit(999) # -------------------------------------------------------------- # Processed user-input instance properties if np.iterable(N): if len(N) == 1: self._nx = np.array(list(N) * ndims, dtype=np.int) elif len(N) == ndims: self._nx = np.array(N, dtype=np.int) else: raise IndexError("The length of N must be either 1 or ndims") else: self._nx = np.array([N] * ndims, dtype=np.int) if np.iterable(L): if len(L) == 1: self._L = np.array(list(L) * ndims) elif len(L) == ndims: self._L = np.array(L) else: raise IndexError("The length of L must be either 1 or ndims") else: self._L = np.array([L] * ndims, dtype=np.float) if decomp is None: self._decomp = 1 elif decomp in [1, 2, 3] and decomp <= ndims: self._decomp = decomp else: raise IndexError("decomp must be 1, 2, 3 or None") if periodic is None: self._periodic = tuple([False] * ndims) elif len(periodic) == ndims: self._periodic = tuple(periodic) else: raise IndexError("Either len(periodic) must be ndims or " "periodic must be None") if method == 'central_diff': self.deriv = self._centdiff_deriv elif method == 'spline_flux_diff': self.deriv = self._akima_deriv elif method == 'ignore': self.deriv = None else: if init_comm.rank == 0: print("mpiAnalyzer._baseAnalyzer.__init__(): " "'method' argument not recognized!\n" "Defaulting to Akima spline flux differencing.") self.deriv = self._akima_deriv self._dx = self._L / self._nx self._Nx = self._nx.prod() # -------------------------------------------------------------- # MPI domain decomposition dims = MPI.Compute_dims(init_comm.size, self._decomp) dims.extend([1] * (ndims - self._decomp)) assert np.all(np.mod(self._nx, dims) == 0) self._comm = init_comm.Create_cart(dims, self._periodic) self._nnx = self._nx // dims self._ixs = self._nnx * self.comm.coords self._ixe = self._ixs + self._nnx # -------------------------------------------------------------- # other stuff (DO NOT count on these being permanent!) self.tol = 1.0e-6 self.prefix = pid + '-' self.moments_file = '%s%s.moments' % (self.odir, self.prefix)
# Configuration of System CFLs = [0.5 for i in range(num_of_grids)] tau = 1 # Select diffop #raxis_1D_diffop = fd.FD12() raxis_1D_diffop = fd.FD14() # Configuration of IBVP maxIteration = 1000000 ############################################################################### # MPI set up ############################################################################### log.info("Initialising mpi.cart_comm") dims = MPI.Compute_dims(MPI.COMM_WORLD.size, [0]) periods = [0] reorder = True mpi_comm = MPI.COMM_WORLD.Create_cart(dims, periods=periods, reorder=reorder) log.info("Initialisation complete") ################################################################################ # Grid construction ################################################################################ # Grid point data raxis_gdp = [N*2**i for i in range(num_of_grids)] # Determine the boundary data ghost_points = (raxis_1D_diffop.ghost_points(),)
#!/usr/bin/env python from mpi4py import MPI # Example for creating a cartesian topology # and identifying your neighbors comm = MPI.COMM_WORLD world_rank = comm.Get_rank() size = comm.Get_size() ndim = 2 dims = MPI.Compute_dims(size, [0] * ndim) cart_comm = comm.Create_cart(dims, periods=[True, True], reorder=True) new_rank = cart_comm.Get_rank() if new_rank == 0: print("Cart dim: %s" % (dims)) for i in range(ndim): for d in (-1, +1): source, dest = cart_comm.Shift(i, d) if new_rank == 0: print("Dir %d, disp %d - Src %d - Dest %d" % (i, d, source, dest)) cart_comm.Free()
def __init__(self, comm, shape, dtype=float, **options): self.comm = MPI.COMM_NULL self._comm1 = MPI.COMM_NULL self._comm2 = MPI.COMM_NULL self._subarrays1A = [] self._subarrays1B = [] self._subarrays2A = [] self._subarrays2B = [] self._fft1 = self._ifft1 = None self._fft2 = self._ifft2 = None self._fft3 = self._ifft3 = None assert isinstance(comm, MPI.Intracomm) self.comm = comm size = comm.Get_size() rank = comm.Get_rank() size1, size2 = MPI.Compute_dims(size, 2) self._comm1 = comm.Split(rank % size2) self._comm2 = comm.Split(rank // size2) rank1 = self._comm1.Get_rank() rank2 = self._comm2.Get_rank() M, N, P = shape if np.issubdtype(dtype, np.floating): assert P % 2 == 0 Q = P // 2 + 1 else: Q = P assert M >= size1 assert N >= size1 assert N >= size2 assert Q >= size2 opts = dict( avoid_copy=True, overwrite_input=True, auto_align_input=True, auto_contiguous=True, threads=1, ) opts.update(options) dtype = np.dtype(dtype) empty = pyfftw.empty_aligned fft1 = pyfftw.builders.fft ifft1 = pyfftw.builders.ifft fft2 = pyfftw.builders.fft ifft2 = pyfftw.builders.ifft if np.issubdtype(dtype, np.floating): fft3 = pyfftw.builders.rfft ifft3 = pyfftw.builders.irfft else: fft3 = pyfftw.builders.fft ifft3 = pyfftw.builders.ifft m = _subsize(M, size1, rank1) n = _subsize(N, size2, rank2) dtype = np.dtype(dtype) U = empty([m, n, P], dtype=dtype) self._fft3 = fft3(U, axis=2, **opts) ctype = self._fft3.output_array.dtype F = self._fft3.output_array self._ifft3 = ifft3(F, axis=2, **opts) self._ifft3.update_arrays(F, U) m = _subsize(M, size1, rank1) q = _subsize(Q, size2, rank2) U = empty([m, N, q], dtype=ctype) self._fft2 = fft2(U, axis=1, **opts) F = self._fft2.output_array self._ifft2 = ifft2(F, axis=1, **opts) self._ifft2.update_arrays(F, U) n = _subsize(N, size1, rank1) q = _subsize(Q, size2, rank2) U = empty([M, n, q], dtype=ctype) self._fft1 = fft1(U, axis=0, **opts) F = self._fft1.output_array self._ifft1 = ifft1(F, axis=0, **opts) self._ifft1.update_arrays(F, U) datatype = MPI._typedict[ctype.char] m = _subsize(M, size1, rank1) n = _subsize(N, size1, rank1) q = _subsize(Q, size2, rank2) self._subarrays1A = [ datatype.Create_subarray([M, n, q], [l, n, q], [s, 0, 0]).Commit() for l, s in _distribution(M, size1) ] self._subarrays1B = [ datatype.Create_subarray([m, N, q], [m, l, q], [0, s, 0]).Commit() for l, s in _distribution(N, size1) ] self._counts_displs1 = ([1] * size1, [0] * size1) m = _subsize(M, size1, rank1) n = _subsize(N, size2, rank2) q = _subsize(Q, size2, rank2) self._subarrays2A = [ datatype.Create_subarray([m, N, q], [m, l, q], [0, s, 0]).Commit() for l, s in _distribution(N, size2) ] self._subarrays2B = [ datatype.Create_subarray([m, n, Q], [m, n, l], [0, 0, s]).Commit() for l, s in _distribution(Q, size2) ] self._counts_displs2 = ([1] * size2, [0] * size2)
sendbuf = np.copy(A[:, 1]).flatten() recvbuf = np.zeros(A.shape[0]) comm.Send(sendbuf, neighbors_y[0], 2) comm.Recv(recvbuf, neighbors_y[0], 3) A[:, 0] = recvbuf if neighbors_y[1] >= 0: # MPI_PROC_NULL? sendbuf = np.copy(A[:, -2]).flatten() recvbuf = np.zeros(A.shape[0]) comm.Send(sendbuf, neighbors_y[1], 3) comm.Recv(recvbuf, neighbors_y[1], 2) A[:, -1] = recvbuf # MPI nprocs = MPI.COMM_WORLD.Get_size() dims = MPI.Compute_dims(nprocs, [0, 0]) comm = MPI.COMM_WORLD.Create_cart(dims) me = comm.Get_rank() coords = comm.Get_coords(me) neighbors_x = comm.Shift(direction=0, disp=1) neighbors_y = comm.Shift(direction=1, disp=1) # Physics lam = 1.0 # Thermal conductivity cp_min = 1.0 # Minimal heat capacity lx, ly = 10.0, 10.0 # Length of computational domain in dimension x and y # Numerics nx, ny = 128, 128 # Number of gridpoints in dimensions x and y nt = 10000 # Number of time steps nx_g = dims[0] * (
from mpi4py import MPI import numpy as np # setup communicator and get # procs comm = MPI.COMM_WORLD nprocs = comm.Get_size() # We are solving the Inhomogeneous 1D groundwater flow problem # 0 = d/dx[k(x)dh/dx] # from x = 0 to x = L, i.e. we are solving along a 1 dim grid # To simplify the parallelization, we can use the Cartesian Conv. Function # from MPI. For this example, we could figure this out ourselves pretty easily # but when scaling up, this is really convenient # get dimensions dims = MPI.Compute_dims(nprocs, 1) # setup grid grid = comm.Create_cart(dims) # now we want the gridded rank rank = grid.Get_rank() # the jacobi algorithm is needs the previous and next values # grid.Shift(direction (dim), displacement) # So shift along the zeroth axis, 1 unit in both directions # Shift return -1 for values less than 0 or greater than nprocs left, right = grid.Shift(0,1) # grab the status to be used in sendrecv calls later status = MPI.Status() # setup problem parameters h0 = 1.0 # GW head at x = 0