def testCartcomm(self):
comm = self.COMM
size = comm.Get_size()
rank = comm.Get_rank()
for ndim in (1, 2, 3, 4, 5):
dims = [0] * ndim
dims = MPI.Compute_dims(size, [0] * ndim)
periods = [True] * len(dims)
topo = comm.Create_cart(dims, periods=periods)
self.assertTrue(topo.is_topo)
self.assertTrue(topo.topology, MPI.CART)
self.checkFortran(topo)
self.assertEqual(topo.dim, len(dims))
self.assertEqual(topo.ndim, len(dims))
coordinates = topo.coords
self.assertEqual(coordinates, topo.Get_coords(topo.rank))
neighbors = []
for i in range(ndim):
for d in (-1, +1):
coord = list(coordinates)
coord[i] = (coord[i] + d) % dims[i]
neigh = topo.Get_cart_rank(coord)
self.assertEqual(coord, topo.Get_coords(neigh))
source, dest = topo.Shift(i, d)
self.assertEqual(neigh, dest)
neighbors.append(neigh)
self.assertEqual(topo.indegree, len(neighbors))
self.assertEqual(topo.outdegree, len(neighbors))
self.assertEqual(topo.inedges, neighbors)
self.assertEqual(topo.outedges, neighbors)
inedges, outedges = topo.inoutedges
self.assertEqual(inedges, neighbors)
self.assertEqual(outedges, neighbors)
if ndim == 1:
topo.Free()
continue
for i in range(ndim):
rem_dims = [1] * ndim
rem_dims[i] = 0
sub = topo.Sub(rem_dims)
if sub != MPI.COMM_NULL:
self.assertEqual(sub.dim, ndim - 1)
dims = topo.dims
del dims[i]
self.assertEqual(sub.dims, dims)
sub.Free()
topo.Free()
if size > 1: return
if MPI.VERSION < 2: return
topo = comm.Create_cart([])
self.assertEqual(topo.dim, 0)
self.assertEqual(topo.dims, [])
self.assertEqual(topo.periods, [])
self.assertEqual(topo.coords, [])
rank = topo.Get_cart_rank([])
self.assertEqual(rank, 0)
inedges, outedges = topo.inoutedges
self.assertEqual(inedges, [])
self.assertEqual(outedges, [])
topo.Free()
def test1():
dims = MPI.Compute_dims(MPISIZE, 3)
if MPIRANK == 0 and DEBUG:
print("DIM", dims)
cc = MPI.COMM_WORLD.Create_cart(dims, (1, 1, 1), True)
top = cc.Get_topo()[2]
N = 64
dtype = ctypes.c_double
grid_size = (N, N, N)
rng = np.random.RandomState(seed=3857293)
orig = np.array(rng.uniform(0., 100., size=grid_size), dtype=dtype)
np0 = int(math.floor(grid_size[0] / dims[0]))
np1 = int(math.floor(grid_size[1] / dims[1]))
np2 = int(math.floor(grid_size[2] / dims[2]))
s0 = top[0] * np0
s1 = top[1] * np1
s2 = top[2] * np2
e0 = (top[0] + 1) * np0 if top[0] < dims[0] - 1 else N
e1 = (top[1] + 1) * np1 if top[1] < dims[1] - 1 else N
e2 = (top[2] + 1) * np2 if top[2] < dims[2] - 1 else N
works_size = (e0 - s0 + 2, e1 - s1 + 2, e2 - s2 + 2)
he = pygcl.HaloExchange3D(cc, works_size)
a = np.zeros(shape=list(works_size) + [1], dtype=dtype)
a[1:-1:, 1:-1:, 1:-1, 0] = orig[s0:e0:, s1:e1:, s2:e2:]
# shell should be zero and interior should be the copied data
for iz in range(works_size[0]):
for iy in range(works_size[1]):
for ix in range(works_size[2]):
# if shell
if (iz == 0 or iz == works_size[2]-1) and \
(iy == 0 or iy == works_size[1]-1) and \
(ix == 0 or ix == works_size[0]-1):
assert a[iz, iy, ix, 0] == 0
for iz in range(1, works_size[0] - 1):
for iy in range(1, works_size[1] - 1):
for ix in range(1, works_size[2] - 1):
assert a[iz, iy, ix, 0] == orig[iz + s0 - 1, iy + s1 - 1,
ix + s2 - 1]
# exchange
he.exchange(a)
# interior should be unchanged, outer shell should be halo
for iz in range(works_size[0]):
for iy in range(works_size[1]):
for ix in range(works_size[2]):
assert a[iz, iy, ix, 0] == orig[(iz+s0-1) % grid_size[0],
(iy+s1-1) % grid_size[1],
(ix+s2-1) % grid_size[2]], \
"{} {} {}".format(iz, iy, ix)
def __init__(self, shape, dimensions, input_comm=None):
self._glb_shape = shape
self._dimensions = dimensions
self._input_comm = (input_comm or MPI.COMM_WORLD).Clone()
# `Compute_dims` sets the dimension sizes to be as close to each other
# as possible, using an appropriate divisibility algorithm. Thus, in 3D:
# * topology[0] >= topology[1] >= topology[2]
# * topology[0] * topology[1] * topology[2] == self._input_comm.size
topology = MPI.Compute_dims(self._input_comm.size, len(shape))
# At this point MPI's dimension 0 corresponds to the rightmost element
# in `topology`. This is in reverse to `shape`'s ordering. Hence, we
# now restore consistency
self._topology = tuple(reversed(topology))
if self._input_comm is not input_comm:
# By default, Devito arranges processes into a cartesian topology.
# MPI works with numbered dimensions and follows the C row-major
# numbering of the ranks, i.e. in a 2x3 Cartesian topology (0,0)
# maps to rank 0, (0,1) maps to rank 1, (0,2) maps to rank 2, (1,0)
# maps to rank 3, and so on.
self._comm = self._input_comm.Create_cart(self._topology)
else:
self._comm = input_comm
# Perform domain decomposition
self._glb_numbs = [
np.array_split(range(i), j) for i, j in zip(shape, self._topology)
]
def __init__(self, rankinfo):
'''Find out the best distribution of the lattice amonst the ranks of the
communicator passed inside "rankinfo", create the topology, and
save information about the neighbours of present rank.
Parameters
----------
rankinfo : a rankinfo instance describing current communicator
Attributes
----------
dims : dimensions of the cartesian MPI rank grid (not lattice!)
topology : the new cartesian topology communicator
shifts : a dict of dicts of which rank to talk to when going
along "X", "Y", or "Z" direcion in "up" or "down"
direction
'''
self.me=rankinfo
self.dims=MPI.Compute_dims(self.me.size, self.me.ndim)
self.topology=MPI.COMM_WORLD.Create_cart(self.dims,
periods=self.me.periods, reorder=True)
left,right = self.topology.Shift(0,1)
front,back = self.topology.Shift(1,1)
up,down = self.topology.Shift(2,1)
self.shifts={"X": {"up": up, "down": down},
"Y": {"up": back, "down": front},
"Z": {"up": right, "down": left}}
def __new__(cls, comm, dims=None, reorder=True):
assert not comm.Is_inter()
if comm.Get_topology() == MPI.CART:
assert comm.Get_dim() > 0
assert dims is None
cartcomm = comm
else:
if dims is None:
dims = [0]
elif np.ndim(dims) > 0:
assert len(dims) > 0
dims = [max(0, d) for d in dims]
else:
assert dims > 0
dims = [0] * dims
dims = MPI.Compute_dims(comm.Get_size(), dims)
cartcomm = comm.Create_cart(dims, reorder=reorder)
dim = cartcomm.Get_dim()
subcomm = [None] * dim
remdims = [False] * dim
for i in range(dim):
remdims[i] = True
subcomm[i] = cartcomm.Sub(remdims)
remdims[i] = False
if cartcomm != comm:
cartcomm.Free()
return super(Subcomm, cls).__new__(cls, subcomm)
def testCartMap(self):
comm = self.COMM
size = comm.Get_size()
for ndim in (1,2,3,4,5):
for periods in (None, True, False):
dims = MPI.Compute_dims(size, [0]*ndim)
topo = comm.Create_cart(dims, periods, reorder=True)
rank = comm.Cart_map(dims, periods)
self.assertEqual(topo.Get_rank(), rank)
topo.Free()
def compute_dims(nprocs, ndim):
# We don't do anything clever here. In fact, we do something very basic --
# we just try to distribute `nprocs` evenly over the number of dimensions,
# and if we can't we fallback to whatever MPI.Compute_dims gives...
v = pow(nprocs, 1 / ndim)
if not v.is_integer():
# Since pow(64, 1/3) == 3.999..4
v = int(ceil(v))
if not v**ndim == nprocs:
# Fallback
return MPI.Compute_dims(nprocs, ndim)
else:
v = int(v)
return tuple(v for _ in range(ndim))
def createCart(parent):
"""Create a 2-d Cartesian communicator from parent communicator"""
dims = MPI.Compute_dims(parent.size, 2)
comm = parent.Create_cart(dims, periods=[True, False])
rank = comm.Get_rank()
coords = comm.Get_coords(rank)
upx = comm.Shift(0, 1)
upy = comm.Shift(1, 1)
out = "Rank{:2d} coords{:2d} {:2d} upx(src,dst) {} upy(src,dst) {}\n"\
.format(rank, coords[0], coords[1], upx, upy)
sys.stdout.write(out)
def testCreateDarray(self):
for dtype in datatypes:
for ndim in range(1, 3+1):
for size in (4, 8, 9, 27):
for rank in (0, size-1):
for dist in [MPI.DISTRIBUTE_BLOCK, MPI.DISTRIBUTE_CYCLIC]:
for order in [MPI.ORDER_C, MPI.ORDER_F]:
gsizes = [size]*ndim
distribs = [dist]*ndim
dargs = [MPI.DISTRIBUTE_DFLT_DARG]*ndim
psizes = MPI.Compute_dims(size, [0]*ndim)
factory = MPI.Datatype.Create_darray
args = size, rank, gsizes, distribs, dargs, psizes, order
self.check_datatype(dtype, factory, *args)
def main():
dim = 3
proc_sizes = tuple(MPI.Compute_dims(SIZE, dim))
# see lists.mcs.anl.gov/pipermail/petsc-dev/2016-April/018903.html
# manually distribute the unkowns to the processors
# mpiexec -n 3
if SIZE > 1:
if 1:
ownership_ranges = (
np.array([3, 4, 5], dtype='i4'),
np.array([1], dtype='i4'),
np.array([1], dtype='i4')
)
else:
ownership_ranges = (
# proc 0
(3, 4, 5),
# proc 1
(1,),
# proc 2
(1,)
)
else:
ownership_ranges = ((3,), (4,), (5,))
sizes = np.array((
sum(ownership_ranges[0]),
sum(ownership_ranges[1]),
sum(ownership_ranges[2])), dtype='i4'
)
dm = PETSc.DMDA().create(dim=dim, sizes=sizes, proc_sizes=proc_sizes,
ownership_ranges=ownership_ranges,
boundary_type=(PETSc.DMDA.BoundaryType.PERIODIC,
PETSc.DMDA.BoundaryType.PERIODIC,
PETSc.DMDA.BoundaryType.GHOSTED),
stencil_type=PETSc.DMDA.StencilType.BOX,
stencil_width=1, dof=1, comm=PETSc.COMM_WORLD, setup=True)
assert dm.getComm().Get_rank() == RANK
assert dm.getComm().Get_size() == SIZE
print(f'create problem with {sizes.prod()} unknowns split into dim={dim} (sizes={sizes}) split accross {SIZE} procs with distribution {proc_sizes}')
vec_l = dm.createLocalVector()
vec_g = dm.createGlobalVector()
initialise_values(dm, vec_l, vec_g)
result_l, result_g = find_max_grad(dm, vec_l, vec_g)
PETSc.Sys.syncPrint(f'Rank {RANK} had max gradient {result_l} while the global was {result_g}.')
if RANK == 0:
if testme(result_g, dm.getComm()):
print('Result is correct.')
else:
print('Result is incorrect!')
def __init__(self, dim, period, comm=mpi.COMM_WORLD):
self.dim = dim
self.set_options()
self.comm = comm
# if npx, npy and npz are all set to the default value (1)
# then Compute_dims performs the splitting of the domain
if self.npx == self.npy == self.npz == 1:
size = comm.Get_size()
split = mpi.Compute_dims(size, self.dim)
else:
split = (self.npx, self.npy, self.npz)
self.split = np.asarray(split[:self.dim])
self.cartcomm = comm.Create_cart(self.split, period)
def __init__(self, comm, discretization):
# set comm
self._comm = comm
# get size & rank
self._size = comm.Get_size()
self._rank = comm.Get_rank()
# compute a distribution of processes per coordinate direction
self._dims = [0, 0]
self._dims = MPI.Compute_dims(self._size, self._dims)
# create two-dimensional non-periodic Cartesian topology
self._periods = [False, False]
self._comm_cart = self._comm.Create_cart(self._dims,
periods=self._periods)
# get rank's coordinates in the topology
self._coords = self._comm_cart.Get_coords(self._rank)
# get rank's south/north/west/east neighbors
self._neigh_west, self._neigh_east = self._comm_cart.Shift(0, 1)
self._neigh_south, self._neigh_north = self._comm_cart.Shift(1, 1)
# global domain discretization size
self._global_nx = discretization.nx
self._global_ny = discretization.ny
# global start/end index in (sub)domain discretization
chunkx = discretization.nx // self._dims[0]
self._global_startx = chunkx * self._coords[0]
if self._coords[0] == self._dims[0] - 1:
self._global_endx = discretization.nx
else:
self._global_endx = self._global_startx + chunkx
chunky = discretization.ny // self._dims[1]
self._global_starty = chunky * self._coords[1]
if self._coords[1] == self._dims[1] - 1:
self._global_endy = discretization.ny
else:
self._global_endy = self._global_starty + chunky
# local (sub)domain discretization size
self._local_nx = self._global_endx - self._global_startx
self._local_ny = self._global_endy - self._global_starty
def __init__(self, comm, idir, odir, N, ndims, decomp, periodic):
self.idir = idir # users should be able to change directories at will
self.odir = odir # users should be able to change directories at will
self._ndims = ndims
self._subarrays = dict()
init_comm = comm
if np.iterable(N):
if len(N) == 1:
self._nx = np.array(list(N)*ndims, dtype=int)
elif len(N) == ndims:
self._nx = np.array(N, dtype=int)
else:
raise IndexError("The length of N must be either 1 or ndims")
else:
self._nx = np.array([N]*ndims, dtype=int)
if decomp is None:
self._decomp = 1
elif decomp in [1, 2, 3] and decomp <= ndims:
self._decomp = decomp
else:
raise IndexError("decomp must be 1, 2, 3 or None")
if periodic is None:
self._periodic = tuple([True]*ndims)
elif len(periodic) == ndims:
self._periodic = tuple(periodic)
else:
raise IndexError("Either len(periodic) must be ndims or "
"periodic must be None")
dims = MPI.Compute_dims(init_comm.size, self._decomp)
dims.extend([1]*(ndims-self._decomp))
assert np.all(np.mod(self._nx, dims) == 0)
self._comm = init_comm.Create_cart(dims, self._periodic)
self._nnx = self._nx//dims
self._ixs = self._nnx*self.comm.coords
self._ixe = self._ixs+self._nnx
def main():
dims=tuple(MPI.Compute_dims(MPI.COMM_WORLD.Get_size(),3))
dm = PETSc.DMDA().create(dim=len(dims), ownership_ranges=(numpy.array([3]), numpy.array([4]), numpy.array([5])),
proc_sizes=dims,
boundary_type=(PETSc.DMDA.BoundaryType.PERIODIC,
PETSc.DMDA.BoundaryType.PERIODIC,
PETSc.DMDA.BoundaryType.GHOSTED),
stencil_type=PETSc.DMDA.StencilType.BOX,
stencil_width = 1, dof = 1, comm = PETSc.COMM_WORLD, setup = True)
vec_l=dm.createLocalVector()
vec_g=dm.createGlobalVector()
initialise_values(dm, vec_l, vec_g)
result_l, result_g = find_max_grad(dm, vec_l, vec_g)
PETSc.Sys.syncPrint("Rank {rank} had max gradient {maxgrad_l} while the global was {maxgrad_g}."
.format(
rank=dm.getComm().getRank(),
maxgrad_l=result_l,
maxgrad_g=result_g))
if (dm.getComm().getRank() == 0):
if (testme(result_g, dm.getComm())):
print("Result is correct.")
else:
print("Result is incorrect!")
def get_comm_dims(procs, dist):
""" Get dimensions of cartesian grid as determined by specified
distribution.
Parameters
----------
procs: int
Size/number of processes in communicator
dist : str
Specified distribution of data among processes.
Default value 'b' : Block
Supported types:
'b' : Block
'r' : Replicated
Returns
-------
comm_dims : list, None
Dimensions of cartesian grid
"""
if is_Replicated(dist):
return None
return MPI.Compute_dims(procs, distribution_to_dimensions(dist, procs))
def test2():
width = 2
dims = MPI.Compute_dims(MPISIZE, 3)
if MPIRANK == 0 and DEBUG:
print("DIM", dims)
cc = MPI.COMM_WORLD.Create_cart(dims, (1, 1, 1), True)
top = cc.Get_topo()[2]
N = 64
M = 2
dtype = ctypes.c_double
grid_size = (2 * N, N, N)
rng = np.random.RandomState(seed=3857293)
orig = np.array(rng.uniform(0., 100., size=list(grid_size) + [M]),
dtype=dtype)
np0 = int(math.floor(grid_size[0] / dims[0]))
np1 = int(math.floor(grid_size[1] / dims[1]))
np2 = int(math.floor(grid_size[2] / dims[2]))
s0 = top[0] * np0
s1 = top[1] * np1
s2 = top[2] * np2
e0 = (top[0] + 1) * np0 if top[0] < dims[0] - 1 else grid_size[0]
e1 = (top[1] + 1) * np1 if top[1] < dims[1] - 1 else grid_size[1]
e2 = (top[2] + 1) * np2 if top[2] < dims[2] - 1 else grid_size[2]
works_size = (e0 - s0 + 2 * width, e1 - s1 + 2 * width,
e2 - s2 + 2 * width)
he = pygcl.HaloExchange3D(cc, works_size, (width, width, width))
a = np.zeros(shape=list(works_size) + [M], dtype=dtype)
a[width:-1*width:, width:-1*width:, width:-1*width, :] = \
orig[s0:e0:, s1:e1:, s2:e2:, :]
# shell should be zero and interior should be the copied data
for iz in range(width, works_size[0] - width):
for iy in range(width, works_size[1] - width):
for ix in range(width, works_size[2] - width):
assert np.all(a[iz, iy, ix, :] == orig[iz + s0 - width,
iy + s1 - width,
ix + s2 - width, :])
if DEBUG:
for rx in range(MPISIZE):
if MPIRANK == rx:
print("--", MPIRANK)
print(orig)
print(a[:, :, :, 0])
MPI.COMM_WORLD.Barrier()
# exchange
he.exchange(a)
if DEBUG:
for rx in range(MPISIZE):
if MPIRANK == rx:
print("--", MPIRANK)
print('boundary', he.boundary_cells)
print('halo', he.halo_cells)
print(s0, e0, s1, e1, s2, e2)
print(works_size)
print(orig)
print(a[:, :, :, 0])
MPI.COMM_WORLD.Barrier()
# interior should be unchanged, outer shell should be halo
for iz in range(works_size[0]):
for iy in range(works_size[1]):
for ix in range(works_size[2]):
assert np.all(a[iz, iy, ix, :] == orig[(iz+s0-width) % grid_size[0],
(iy+s1-width) % grid_size[1],
(ix+s2-width) % grid_size[2], :]),\
"{} {} {}".format(iz, iy, ix)
import mpi4py.MPI as mpi
comm = mpi.COMM_WORLD
size = comm.Get_size()
rank = comm.Get_rank()
ndim = 2
dims = mpi.Compute_dims(size, [0] * ndim)
topo = comm.Create_cart(dims, periods=False)
print(
f'Process {rank}: coordinate = {topo.coords}, neighbor = {topo.outedges}')
comm.Barrier()
recvbuf = topo.neighbor_alltoall([rank] * 4)
print(rank, "->", recvbuf)
def test_mpifft():
from itertools import product
comm = MPI.COMM_WORLD
dims = (2, 3, 4,)
sizes = (12, 13)
assert MPI.COMM_WORLD.Get_size() < 8, "due to sizes"
types = ''
for t in 'fdg':
if fftw.get_fftw_lib(t):
types += t+t.upper()
grids = {2: (None,),
3: ((-1,), None),
4: ((-1,), None)}
for typecode in types:
for dim in dims:
for shape in product(*([sizes]*dim)):
if dim < 3:
n = min(shape)
if typecode in 'fdg':
n //= 2
n += 1
if n < comm.size:
continue
for grid in grids[dim]:
padding = False
for collapse in (True, False):
for backend in backends:
transforms = None
if dim < 3:
allaxes = [None, (-1,), (-2,),
(-1, -2,), (-2, -1),
(-1, 0), (0, -1),
((0,), (1,))]
elif dim < 4:
allaxes = [None, ((0,), (1, 2)),
((0,), (-2, -1))]
elif dim > 3:
allaxes = [None, ((0,), (1,), (2,), (3,)),
((0,), (1, 2, 3)),
((0,), (1,), (2, 3))]
dctn = functools.partial(fftw.dctn, type=3)
idctn = functools.partial(fftw.idctn, type=3)
if not typecode in 'FDG':
if backend == 'pyfftw':
transforms = {(3,): (pyfftw.builders.rfftn, pyfftw.builders.irfftn),
(2, 3): (pyfftw.builders.rfftn, pyfftw.builders.irfftn),
(1, 2, 3): (pyfftw.builders.rfftn, pyfftw.builders.irfftn),
(0, 1, 2, 3): (pyfftw.builders.rfftn, pyfftw.builders.irfftn)}
else:
transforms = {(3,): (dctn, idctn),
(2, 3): (dctn, idctn),
(1, 2, 3): (dctn, idctn),
(0, 1, 2, 3): (dctn, idctn)}
for axes in allaxes:
# Test also the slab is number interface
_grid = grid
if grid is not None:
ax = -1
if axes is not None:
ax = axes[-1] if isinstance(axes[-1], int) else axes[-1][-1]
_slab = (ax+1) % len(shape)
_grid = [1]*(_slab+1)
_grid[_slab] = 0
_comm = comm
# Test also the comm is Subcomm interfaces
# For PFFT the Subcomm needs to be as long as shape
if len(shape) > 2 and axes is None and grid is None:
_dims = [0] * len(shape)
_dims[-1] = 1 # distribute all but last axis (axes is None)
_comm = comm
if random_true_or_false(comm) == 1:
# then test Subcomm with a MPI.CART argument
_dims = MPI.Compute_dims(comm.Get_size(), _dims)
_comm = comm.Create_cart(_dims)
_dims = None
_comm = Subcomm(_comm, _dims)
#print(typecode, shape, axes, collapse, _grid)
fft = PFFT(_comm, shape, axes=axes, dtype=typecode,
padding=padding, grid=_grid, collapse=collapse,
backend=backend, transforms=transforms)
#if comm.rank == 0:
# grid_ = [c.size for c in fft.subcomm]
# print('grid:{} shape:{} typecode:{} backend:{} axes:{}'
# .format(grid_, shape, typecode, backend, axes))
assert fft.dtype(True) == fft.forward.output_array.dtype
assert fft.dtype(False) == fft.forward.input_array.dtype
assert len(fft.axes) == len(fft.xfftn)
assert len(fft.axes) == len(fft.transfer) + 1
assert (fft.forward.input_pencil.subshape ==
fft.forward.input_array.shape)
assert (fft.forward.output_pencil.subshape ==
fft.forward.output_array.shape)
assert (fft.backward.input_pencil.subshape ==
fft.backward.input_array.shape)
assert (fft.backward.output_pencil.subshape ==
fft.backward.output_array.shape)
assert np.alltrue(np.array(fft.global_shape(True)) == np.array(fft.forward.output_pencil.shape))
assert np.alltrue(np.array(fft.global_shape(False)) == np.array(fft.forward.input_pencil.shape))
ax = -1 if axes is None else axes[-1] if isinstance(axes[-1], int) else axes[-1][-1]
assert fft.forward.input_pencil.substart[ax] == 0
assert fft.backward.output_pencil.substart[ax] == 0
ax = 0 if axes is None else axes[0] if isinstance(axes[0], int) else axes[0][0]
assert fft.forward.output_pencil.substart[ax] == 0
assert fft.backward.input_pencil.substart[ax] == 0
assert fft.dimensions == len(shape)
U = random_like(fft.forward.input_array)
if random_true_or_false(comm) == 1:
F = fft.forward(U)
V = fft.backward(F)
assert allclose(V, U)
else:
fft.forward.input_array[...] = U
fft.forward()
fft.backward()
V = fft.backward.output_array
assert allclose(V, U)
fft.destroy()
padding = [1.5]*len(shape)
for backend in backends:
if dim < 3:
allaxes = [None, (-1,), (-2,),
(-1, -2,), (-2, -1),
(-1, 0), (0, -1),
((0,), (1,))]
elif dim < 4:
allaxes = [None, ((0,), (1,), (2,)),
((0,), (-2,), (-1,))]
elif dim > 3:
allaxes = [None, (0, 1, -2, -1),
((0,), (1,), (2,), (3,))]
for axes in allaxes:
_grid = grid
if grid is not None:
ax = -1
if axes is not None:
ax = axes[-1] if isinstance(axes[-1], int) else axes[-1][-1]
_slab = (ax+1) % len(shape)
_grid = [1]*(_slab+1)
_grid[_slab] = 0
fft = PFFT(comm, shape, axes=axes, dtype=typecode,
padding=padding, grid=_grid, backend=backend)
#if comm.rank == 0:
# grid = [c.size for c in fft.subcomm]
# print('grid:{} shape:{} typecode:{} backend:{} axes:{}'
# .format(grid, shape, typecode, backend, axes))
assert len(fft.axes) == len(fft.xfftn)
assert len(fft.axes) == len(fft.transfer) + 1
assert (fft.forward.input_pencil.subshape ==
fft.forward.input_array.shape)
assert (fft.forward.output_pencil.subshape ==
fft.forward.output_array.shape)
assert (fft.backward.input_pencil.subshape ==
fft.backward.input_array.shape)
assert (fft.backward.output_pencil.subshape ==
fft.backward.output_array.shape)
ax = -1 if axes is None else axes[-1] if isinstance(axes[-1], int) else axes[-1][-1]
assert fft.forward.input_pencil.substart[ax] == 0
assert fft.backward.output_pencil.substart[ax] == 0
ax = 0 if axes is None else axes[0] if isinstance(axes[0], int) else axes[0][0]
assert fft.forward.output_pencil.substart[ax] == 0
assert fft.backward.input_pencil.substart[ax] == 0
U = random_like(fft.forward.input_array)
F = fft.forward(U)
if random_true_or_false(comm) == 1:
Fc = F.copy()
V = fft.backward(F)
F = fft.forward(V)
assert allclose(F, Fc)
else:
fft.backward.input_array[...] = F
fft.backward()
fft.forward()
V = fft.forward.output_array
assert allclose(F, V)
# Test normalization on backward transform instead of default
fft.backward.input_array[...] = F
fft.backward(normalize=True)
fft.forward(normalize=False)
V = fft.forward.output_array
assert allclose(F, V)
fft.destroy()
# Timers
timers = {"projection": 0, "loading": 0, "writing": 0, "total": 0}
timers["total"] = time.time()
# Setup
xmin = 0
xmax = 2 * np.pi
L = xmax - xmin
# MPI setup
minimize_communication = False
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
nprocs = comm.Get_size()
if minimize_communication:
dimensions = MPI.Compute_dims(nprocs, 3)
else: # you won't have to sort data on merge
dimensions = [1, 1, nprocs]
periodicity = (False, False, False)
grid = comm.Create_cart(dimensions, periodicity, reorder=True)
coords = grid.Get_coords(rank)
# Logging information
if rank == 0:
pfx = "fv_{0:d}".format(args.res)
logname = pfx + ".log"
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(name)-12s %(levelname)-8s %(message)s",
datefmt="%m-%d %H:%M",
filename=logname,
from mpi4py import MPI
import numpy as np
# mostly adapted from the unit tests of mpi4py github repo
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
ndims = 2
dims = [0] * ndims
dims = MPI.Compute_dims(size, dims)
periods = [True] * ndims
topo = comm.Create_cart(dims, periods=periods)
coords = topo.Get_coords(rank)
nw, ne = topo.Shift(0, 1)
ns, nn = topo.Shift(1, 1)
localgrid = np.array([[rank] * 8] * 8, dtype='i')
req = []
if rank == 1:
print("BEFORE\n ---------------------")
print(localgrid)
bN = localgrid[:, -1].flatten()
bS = localgrid[:, 0].flatten()
bE = localgrid[-1, :].flatten()
bW = localgrid[0, :].flatten()
bufN = np.zeros_like(bN)
bufS = np.zeros_like(bS)
bufE = np.zeros_like(bE)
bufW = np.zeros_like(bW)
format='%(filename)s:%(lineno)d - %(levelname)s:%(message)s')
log = logging.getLogger("main")
console = logging.StreamHandler()
console.setLevel(level=console_log_level)
log.addHandler(console)
else:
logging.basicConfig(level=console_log_level)
log = logging.getLogger("main")
log.info("Starting configuration.")
###############################################################################
# MPI set up
###############################################################################
log.info("Initialising mpi.cart_comm")
dims_list = [0, 0]
dims = MPI.Compute_dims(MPI.COMM_WORLD.size, dims_list)
periods = [1, 1]
reorder = True
mpi_comm = MPI.COMM_WORLD.Create_cart(dims, periods=periods, reorder=reorder)
log.info("Initialisation of mpi.cart_comm complete")
################################################################################
# System, IBVP and Grid settings
################################################################################
log.info("Starting configuration.")
# How many systems?
num_of_grids = args.nsim
# How many grid points?
Nx = 50
def __init__(self, comm, odir, pid, ndims, decomp, periodic, L, N, method):
# --------------------------------------------------------------
# Unprocessed user-input instance properties
self._pid = pid
self._ndims = ndims
self._config = "Unknown (Base Configuration)"
self._odir = odir
init_comm = comm
# --------------------------------------------------------------
# Make analysis output directory
if init_comm.rank == 0:
try:
os.makedirs(odir)
except OSError as e:
if not os.path.isdir(odir):
raise e
else:
status = e
finally:
if os.path.isdir(odir):
status = 0
else:
status = None
status = init_comm.bcast(status)
if status != 0:
MPI.Finalize()
sys.exit(999)
# --------------------------------------------------------------
# Processed user-input instance properties
if np.iterable(N):
if len(N) == 1:
self._nx = np.array(list(N) * ndims, dtype=np.int)
elif len(N) == ndims:
self._nx = np.array(N, dtype=np.int)
else:
raise IndexError("The length of N must be either 1 or ndims")
else:
self._nx = np.array([N] * ndims, dtype=np.int)
if np.iterable(L):
if len(L) == 1:
self._L = np.array(list(L) * ndims)
elif len(L) == ndims:
self._L = np.array(L)
else:
raise IndexError("The length of L must be either 1 or ndims")
else:
self._L = np.array([L] * ndims, dtype=np.float)
if decomp is None:
self._decomp = 1
elif decomp in [1, 2, 3] and decomp <= ndims:
self._decomp = decomp
else:
raise IndexError("decomp must be 1, 2, 3 or None")
if periodic is None:
self._periodic = tuple([False] * ndims)
elif len(periodic) == ndims:
self._periodic = tuple(periodic)
else:
raise IndexError("Either len(periodic) must be ndims or "
"periodic must be None")
if method == 'central_diff':
self.deriv = self._centdiff_deriv
elif method == 'spline_flux_diff':
self.deriv = self._akima_deriv
elif method == 'ignore':
self.deriv = None
else:
if init_comm.rank == 0:
print("mpiAnalyzer._baseAnalyzer.__init__(): "
"'method' argument not recognized!\n"
"Defaulting to Akima spline flux differencing.")
self.deriv = self._akima_deriv
self._dx = self._L / self._nx
self._Nx = self._nx.prod()
# --------------------------------------------------------------
# MPI domain decomposition
dims = MPI.Compute_dims(init_comm.size, self._decomp)
dims.extend([1] * (ndims - self._decomp))
assert np.all(np.mod(self._nx, dims) == 0)
self._comm = init_comm.Create_cart(dims, self._periodic)
self._nnx = self._nx // dims
self._ixs = self._nnx * self.comm.coords
self._ixe = self._ixs + self._nnx
# --------------------------------------------------------------
# other stuff (DO NOT count on these being permanent!)
self.tol = 1.0e-6
self.prefix = pid + '-'
self.moments_file = '%s%s.moments' % (self.odir, self.prefix)
# Configuration of System
CFLs = [0.5 for i in range(num_of_grids)]
tau = 1
# Select diffop
#raxis_1D_diffop = fd.FD12()
raxis_1D_diffop = fd.FD14()
# Configuration of IBVP
maxIteration = 1000000
###############################################################################
# MPI set up
###############################################################################
log.info("Initialising mpi.cart_comm")
dims = MPI.Compute_dims(MPI.COMM_WORLD.size, [0])
periods = [0]
reorder = True
mpi_comm = MPI.COMM_WORLD.Create_cart(dims, periods=periods, reorder=reorder)
log.info("Initialisation complete")
################################################################################
# Grid construction
################################################################################
# Grid point data
raxis_gdp = [N*2**i for i in range(num_of_grids)]
# Determine the boundary data
ghost_points = (raxis_1D_diffop.ghost_points(),)
#!/usr/bin/env python
from mpi4py import MPI
# Example for creating a cartesian topology
# and identifying your neighbors
comm = MPI.COMM_WORLD
world_rank = comm.Get_rank()
size = comm.Get_size()
ndim = 2
dims = MPI.Compute_dims(size, [0] * ndim)
cart_comm = comm.Create_cart(dims, periods=[True, True], reorder=True)
new_rank = cart_comm.Get_rank()
if new_rank == 0:
print("Cart dim: %s" % (dims))
for i in range(ndim):
for d in (-1, +1):
source, dest = cart_comm.Shift(i, d)
if new_rank == 0:
print("Dir %d, disp %d - Src %d - Dest %d" % (i, d, source, dest))
cart_comm.Free()
def __init__(self, comm, shape, dtype=float, **options):
self.comm = MPI.COMM_NULL
self._comm1 = MPI.COMM_NULL
self._comm2 = MPI.COMM_NULL
self._subarrays1A = []
self._subarrays1B = []
self._subarrays2A = []
self._subarrays2B = []
self._fft1 = self._ifft1 = None
self._fft2 = self._ifft2 = None
self._fft3 = self._ifft3 = None
assert isinstance(comm, MPI.Intracomm)
self.comm = comm
size = comm.Get_size()
rank = comm.Get_rank()
size1, size2 = MPI.Compute_dims(size, 2)
self._comm1 = comm.Split(rank % size2)
self._comm2 = comm.Split(rank // size2)
rank1 = self._comm1.Get_rank()
rank2 = self._comm2.Get_rank()
M, N, P = shape
if np.issubdtype(dtype, np.floating):
assert P % 2 == 0
Q = P // 2 + 1
else:
Q = P
assert M >= size1
assert N >= size1
assert N >= size2
assert Q >= size2
opts = dict(
avoid_copy=True,
overwrite_input=True,
auto_align_input=True,
auto_contiguous=True,
threads=1,
)
opts.update(options)
dtype = np.dtype(dtype)
empty = pyfftw.empty_aligned
fft1 = pyfftw.builders.fft
ifft1 = pyfftw.builders.ifft
fft2 = pyfftw.builders.fft
ifft2 = pyfftw.builders.ifft
if np.issubdtype(dtype, np.floating):
fft3 = pyfftw.builders.rfft
ifft3 = pyfftw.builders.irfft
else:
fft3 = pyfftw.builders.fft
ifft3 = pyfftw.builders.ifft
m = _subsize(M, size1, rank1)
n = _subsize(N, size2, rank2)
dtype = np.dtype(dtype)
U = empty([m, n, P], dtype=dtype)
self._fft3 = fft3(U, axis=2, **opts)
ctype = self._fft3.output_array.dtype
F = self._fft3.output_array
self._ifft3 = ifft3(F, axis=2, **opts)
self._ifft3.update_arrays(F, U)
m = _subsize(M, size1, rank1)
q = _subsize(Q, size2, rank2)
U = empty([m, N, q], dtype=ctype)
self._fft2 = fft2(U, axis=1, **opts)
F = self._fft2.output_array
self._ifft2 = ifft2(F, axis=1, **opts)
self._ifft2.update_arrays(F, U)
n = _subsize(N, size1, rank1)
q = _subsize(Q, size2, rank2)
U = empty([M, n, q], dtype=ctype)
self._fft1 = fft1(U, axis=0, **opts)
F = self._fft1.output_array
self._ifft1 = ifft1(F, axis=0, **opts)
self._ifft1.update_arrays(F, U)
datatype = MPI._typedict[ctype.char]
m = _subsize(M, size1, rank1)
n = _subsize(N, size1, rank1)
q = _subsize(Q, size2, rank2)
self._subarrays1A = [
datatype.Create_subarray([M, n, q], [l, n, q], [s, 0, 0]).Commit()
for l, s in _distribution(M, size1)
]
self._subarrays1B = [
datatype.Create_subarray([m, N, q], [m, l, q], [0, s, 0]).Commit()
for l, s in _distribution(N, size1)
]
self._counts_displs1 = ([1] * size1, [0] * size1)
m = _subsize(M, size1, rank1)
n = _subsize(N, size2, rank2)
q = _subsize(Q, size2, rank2)
self._subarrays2A = [
datatype.Create_subarray([m, N, q], [m, l, q], [0, s, 0]).Commit()
for l, s in _distribution(N, size2)
]
self._subarrays2B = [
datatype.Create_subarray([m, n, Q], [m, n, l], [0, 0, s]).Commit()
for l, s in _distribution(Q, size2)
]
self._counts_displs2 = ([1] * size2, [0] * size2)
sendbuf = np.copy(A[:, 1]).flatten()
recvbuf = np.zeros(A.shape[0])
comm.Send(sendbuf, neighbors_y[0], 2)
comm.Recv(recvbuf, neighbors_y[0], 3)
A[:, 0] = recvbuf
if neighbors_y[1] >= 0: # MPI_PROC_NULL?
sendbuf = np.copy(A[:, -2]).flatten()
recvbuf = np.zeros(A.shape[0])
comm.Send(sendbuf, neighbors_y[1], 3)
comm.Recv(recvbuf, neighbors_y[1], 2)
A[:, -1] = recvbuf
# MPI
nprocs = MPI.COMM_WORLD.Get_size()
dims = MPI.Compute_dims(nprocs, [0, 0])
comm = MPI.COMM_WORLD.Create_cart(dims)
me = comm.Get_rank()
coords = comm.Get_coords(me)
neighbors_x = comm.Shift(direction=0, disp=1)
neighbors_y = comm.Shift(direction=1, disp=1)
# Physics
lam = 1.0 # Thermal conductivity
cp_min = 1.0 # Minimal heat capacity
lx, ly = 10.0, 10.0 # Length of computational domain in dimension x and y
# Numerics
nx, ny = 128, 128 # Number of gridpoints in dimensions x and y
nt = 10000 # Number of time steps
nx_g = dims[0] * (
from mpi4py import MPI import numpy as np # setup communicator and get # procs comm = MPI.COMM_WORLD nprocs = comm.Get_size() # We are solving the Inhomogeneous 1D groundwater flow problem # 0 = d/dx[k(x)dh/dx] # from x = 0 to x = L, i.e. we are solving along a 1 dim grid # To simplify the parallelization, we can use the Cartesian Conv. Function # from MPI. For this example, we could figure this out ourselves pretty easily # but when scaling up, this is really convenient # get dimensions dims = MPI.Compute_dims(nprocs, 1) # setup grid grid = comm.Create_cart(dims) # now we want the gridded rank rank = grid.Get_rank() # the jacobi algorithm is needs the previous and next values # grid.Shift(direction (dim), displacement) # So shift along the zeroth axis, 1 unit in both directions # Shift return -1 for values less than 0 or greater than nprocs left, right = grid.Shift(0,1) # grab the status to be used in sendrecv calls later status = MPI.Status() # setup problem parameters h0 = 1.0 # GW head at x = 0
