def __init__(self):
self.N_p1 = 2
self.N_p2 = 3
self.N_p3 = 4
self.N_q1 = 5
self.N_q2 = 6
self.N_ghost = 1
self.p1_start = self.p2_start = self.p3_start = 0
self.dp1 = 2/self.N_p1
self.dp2 = 2/self.N_p2
self.dp3 = 2/self.N_p3
self.p1, self.p2, self.p3 = self._calculate_p_center()
self.physical_system = type('obj', (object,),
{'moment_exponents': moment_exponents,
'moment_coeffs': moment_coeffs
}
)
self.f = af.randu(self.N_p1 * self.N_p2 * self.N_p3,
self.N_q1 + 2 * self.N_ghost,
self.N_q2 + 2 * self.N_ghost,
dtype = af.Dtype.f64
)
self._comm = PETSc.COMM_WORLD
self._da_dump_f = PETSc.DMDA().create([self.N_q1, self.N_q2],
dof=( self.N_p1
* self.N_p2
* self.N_p3
),
stencil_width = self.N_ghost
)
self._da_dump_moments = PETSc.DMDA().create([self.N_q1, self.N_q2],
dof=len(self.physical_system.\
moment_exponents
),
stencil_width = self.N_ghost
)
self._glob_f = self._da_dump_f.createGlobalVec()
self._glob_f_array = self._glob_f.getArray()
self._glob_moments = self._da_dump_moments.createGlobalVec()
self._glob_moments_array =self._glob_moments.getArray()
PETSc.Object.setName(self._glob_f, 'distribution_function')
PETSc.Object.setName(self._glob_moments, 'moments')
def main():
import petsc4py
from petsc4py import PETSc
n_fine = 5
n_coarse = int((n_fine - 1) / 2) + 1
da_fine = PETSc.DMDA().create([n_fine, n_fine], stencil_width=1)
da_coarse = PETSc.DMDA().create([n_coarse, n_coarse], stencil_width=1)
x_fine = da_fine.createGlobalVec()
xa = da_fine.getVecArray(x_fine)
(xs, xe), (ys, ye) = da_fine.getRanges()
nx, ny = da_fine.getSizes()
for i in range(xs, xe):
for j in range(ys, ye):
# xa[i, j] = 1.0
# xa[i, j] = i / nx
xa[i, j] = np.sin(2 * np.pi * i /
(nx + 1)) * np.sin(2 * np.pi * j / (ny + 1))
da_coarse.setInterpolationType(PETSc.DMDA.InterpolationType.Q1)
B, vec = da_coarse.createInterpolation(da_fine)
# print(B, vec.getArray())
x_coarse = da_coarse.createGlobalVec()
xa = da_coarse.getVecArray(x_coarse)
(xs, xe), (ys, ye) = da_coarse.getRanges()
nx, ny = da_coarse.getSizes()
for i in range(xs, xe):
for j in range(ys, ye):
xa[i, j] = 1.0
# xa[i, j] = i / nx
xa[i, j] = np.sin(2 * np.pi * i /
(nx + 1)) * np.sin(2 * np.pi * j / (ny + 1))
y = da_fine.createGlobalVec()
# x_coarse.pointwiseMult(x_coarse)
# PETSc.Mat.Restrict(B, x_coarse, y)
B.mult(x_coarse, y)
# y.pointwiseMult(vec, y)
# PETSc.VecPointwiseMult()
# print(y.getArray())
# print(x_coarse.getArray())
print((y - x_fine).norm(PETSc.NormType.NORM_INFINITY))
y_coarse = da_coarse.createGlobalVec()
B.multTranspose(x_fine, y_coarse)
y_coarse.pointwiseMult(vec, y_coarse)
print((y_coarse - x_coarse).norm(PETSc.NormType.NORM_INFINITY))
def __init__(self):
self.N_q1 = 32
self.N_q2 = 32
self.N_p1 = 4
self.N_p2 = 5
self.N_p3 = 6
self.p1_start = self.p2_start = self.p3_start = 0
self.dp1 = 2 / self.N_p1
self.dp2 = 2 / self.N_p2
self.dp3 = 2 / self.N_p3
self.p1, self.p2, self.p3 = self._calculate_p_center()
# Creating an object with required attributes for the test:
self.physical_system = type('obj', (object, ), {
'moment_exponents': moment_exponents,
'moment_coeffs': moment_coeffs
})
self.single_mode_evolution = False
self.f = af.randu(self.N_q1,
self.N_q2,
self.N_p1 * self.N_p2 * self.N_p3,
dtype=af.Dtype.f64)
self.Y = 2 * af.fft2(self.f) / (self.N_q1 * self.N_q2)
self._da_dump_f = PETSc.DMDA().create(
[self.N_q1, self.N_q2],
dof=(self.N_p1 * self.N_p2 * self.N_p3),
)
self._da_dump_moments = PETSc.DMDA().create([self.N_q1, self.N_q2],
dof = len(self.physical_system.\
moment_exponents)
)
self._glob_f = self._da_dump_f.createGlobalVec()
self._glob_f_value = self._da_dump_f.getVecArray(self._glob_f)
self._glob_moments = self._da_dump_moments.createGlobalVec()
self._glob_moments_value = self._da_dump_moments.\
getVecArray(self._glob_moments)
PETSc.Object.setName(self._glob_f, 'distribution_function')
PETSc.Object.setName(self._glob_moments, 'moments')
def setup(nonlinear_solver, residual_function, context):
dmda = PETSc.DMDA().create([context.grid.n_x, context.grid.n_y],
dof=1,
stencil_width=1,
stencil_type='star')
nonlinear_solver.setDM(dmda)
n_x = context.grid.n_x
n_y = context.grid.n_y
r = PETSc.Vec().createSeq(n_x * n_y) # residual vector
def wrapped_residual_function(snes, X, f, t, grid, old_grid,
timestep_properties):
residual_function(snes,
X.getArray(readonly=True).reshape((n_y, n_x)), f, t,
grid, old_grid, timestep_properties)
nonlinear_solver.setFunction(wrapped_residual_function, r, context)
def wrapped_nonlinear_solver_callable(initial_guess):
x = PETSc.Vec().createSeq(n_x * n_y) # solution vector
b = PETSc.Vec().createSeq(n_x * n_y) # right-hand side
x.setArray(initial_guess)
b.set(0)
nonlinear_solver.solve(b, x)
solution = x[:].reshape(n_y, n_x)
return solution
return wrapped_nonlinear_solver_callable
def dump_coordinate_info(self, arrays, name, file_name):
self._da_coord_arrays = PETSc.DMDA().create(
[self.N_q1, self.N_q2],
dof=len(arrays),
proc_sizes=(self._nproc_in_q1, self._nproc_in_q2),
ownership_ranges=self._ownership_ranges,
comm=self._comm)
self._glob_coord = self._da_coord_arrays.createGlobalVec()
self._glob_coord_array = self._glob_coord.getArray()
N_g = self.N_ghost
for i in range(len(arrays)):
if (i == 0):
array_to_dump = arrays[0][:, :, N_g:-N_g, N_g:-N_g]
else:
array_to_dump = af.join(0, array_to_dump, arrays[i][:, :, N_g:-N_g,
N_g:-N_g])
af.flat(array_to_dump).to_ndarray(self._glob_coord_array)
PETSc.Object.setName(self._glob_coord, name)
viewer = PETSc.Viewer().createBinary(file_name + '.bin',
'w',
comm=self._comm)
viewer(self._glob_coord)
def __init__(self):
self.q1_start = np.random.randint(0, 5)
self.q2_start = np.random.randint(0, 5)
self.q1_end = np.random.randint(5, 10)
self.q2_end = np.random.randint(5, 10)
self.N_q1 = np.random.randint(16, 32)
self.N_q2 = np.random.randint(16, 32)
self.dq1 = (self.q1_end - self.q1_start) / self.N_q1
self.dq2 = (self.q2_end - self.q2_start) / self.N_q2
N_g = self.N_ghost = np.random.randint(1, 5)
self.q1 = self.q1_start \
* (0.5 + np.arange(-self.N_ghost,
self.N_q1 + self.N_ghost
)
) * self.dq1
self.q2 = self.q2_start \
* (0.5 + np.arange(-self.N_ghost,
self.N_q2 + self.N_ghost
)
) * self.dq2
self.q2, self.q1 = np.meshgrid(self.q2, self.q1)
self.q1, self.q2 = af.to_array(self.q1), af.to_array(self.q2)
self.q1 = af.reorder(self.q1, 2, 0, 1)
self.q2 = af.reorder(self.q2, 2, 0, 1)
self.q1 = af.tile(self.q1, 6)
self.q2 = af.tile(self.q2, 6)
self._da_fields = PETSc.DMDA().create([self.N_q1, self.N_q2],
dof=6,
stencil_width=self.N_ghost,
boundary_type=('periodic',
'periodic'),
stencil_type=1,
)
self._glob_fields = self._da_fields.createGlobalVec()
self._local_fields = self._da_fields.createLocalVec()
self._glob_fields_array = self._glob_fields.getArray()
self._local_fields_array = self._local_fields.getArray()
self.cell_centered_EM_fields = af.constant(0, 6, self.q1.shape[1],
self.q1.shape[2],
dtype=af.Dtype.f64
)
self.cell_centered_EM_fields[:, N_g:-N_g, N_g:-N_g] = \
af.sin(2 * np.pi * self.q1 + 4 * np.pi * self.q2)[:, N_g:-N_g,N_g:-N_g]
self.performance_test_flag = False
def run_mgrit(nt, space_procs, coarsening, freq=1, a=1.0, t_start=0, t_stop=1, cycle='V'):
size = MPI.COMM_WORLD.Get_size()
comm_x, comm_t = split_communicator(MPI.COMM_WORLD, space_procs)
nx = ny = 65
dmda_coarse = PETSc.DMDA().create([nx, ny], stencil_width=1, comm=comm_x)
dmda_fine = dmda_coarse.refine()
t_interval = np.linspace(t_start, t_stop, nt)
time_procs = int(size / space_procs)
problem = [HeatPetsc(dmda=dmda_fine, comm_x=comm_x, freq=freq, a=a, t_interval=t_interval)]
for i in range(0, len(coarsening)):
problem.append(HeatPetsc(dmda=dmda_fine, comm_x=comm_x, freq=freq, a=a,
t_interval=t_interval[::np.prod(coarsening[:i + 1], dtype=int)]))
nested_iteration = True if len(coarsening) > 0 else False
if cycle == 'V':
mgrit = Mgrit(problem=problem, comm_time=comm_t, comm_space=comm_x, nested_iteration=nested_iteration)
else:
mgrit = Mgrit(problem=problem, comm_time=comm_t, comm_space=comm_x, nested_iteration=nested_iteration,
cycle_type='F', cf_iter=0)
info = mgrit.solve()
return info['time_setup'], info['time_solve'], info['time_setup'] + info['time_solve']
def main():
n = 4
da = PETSc.DMDA().create([n, n], stencil_width=1)
rank = PETSc.COMM_WORLD.getRank()
x = da.createGlobalVec()
xa = da.getVecArray(x)
(xs, xe), (ys, ye) = da.getRanges()
for i in range(xs, xe):
for j in range(ys, ye):
xa[i, j] = j * n + i
print('x=', rank, x.getArray(), xs, xe, ys, ye)
A = da.createMatrix()
A.setType('aij') # sparse
A.setFromOptions()
Istart, Iend = A.getOwnershipRange()
for I in range(Istart, Iend):
A[I, I] = 1.0
# communicate off-processor values
# and setup internal data structures
# for performing parallel operations
A.assemblyBegin()
A.assemblyEnd()
res = da.createGlobalVec()
A.mult(x, res)
print(rank, res.getArray())
print((res - x).norm())
def __init__(self, N_q1, N_q2, N_ghost):
self._da_f = \
PETSc.DMDA().create([N_q1, N_q2],
stencil_width=N_ghost,
boundary_type=('periodic', 'periodic')
)
self.N_q1 = N_q1
self.N_q2 = N_q2
self.dq1 = 1 / N_q1
self.dq2 = 1 / N_q2
self.N_ghost = N_ghost
self._A_q1 = 1
self._A_q2 = 1
self.q1_start = self.q2_start = 0
self.q1_center, self.q2_center = calculate_q_center(self)
self.f = af.sin(2 * np.pi * self.q1_center +
4 * np.pi * self.q2_center)
self.performance_test_flag = False
def setUp(self):
self.da = PETSc.DMDA().create(dim=len(self.SIZES),
dof=self.DOF,
sizes=self.SIZES,
boundary_type=self.BOUNDARY,
stencil_type=self.STENCIL,
stencil_width=self.SWIDTH,
comm=self.COMM)
def help(args=None):
import sys, shlex
# program name
try:
prog = sys.argv[0]
except Exception:
prog = getattr(sys, 'executable', 'python')
# arguments
if args is None:
args = sys.argv[1:]
elif isinstance(args, str):
args = shlex.split(args)
else:
args = [str(a) for a in args]
# import and initialize
import petsc4py
petsc4py.init([prog, '-help'] + args)
from petsc4py import PETSc
# help dispatcher
COMM = PETSc.COMM_SELF
if 'vec' in args:
vec = PETSc.Vec().create(comm=COMM)
vec.setSizes(0)
vec.setFromOptions()
vec.destroy()
if 'mat' in args:
mat = PETSc.Mat().create(comm=COMM)
mat.setSizes([0, 0])
mat.setFromOptions()
mat.destroy()
if 'pc' in args:
pc = PETSc.PC().create(comm=COMM)
pc.setFromOptions()
pc.destroy()
if 'ksp' in args:
ksp = PETSc.KSP().create(comm=COMM)
ksp.setFromOptions()
ksp.destroy()
if 'snes' in args:
snes = PETSc.SNES().create(comm=COMM)
snes.setFromOptions()
snes.destroy()
if 'ts' in args:
ts = PETSc.TS().create(comm=COMM)
ts.setFromOptions()
ts.destroy()
if 'tao' in args:
tao = PETSc.TAO().create(comm=COMM)
tao.setFromOptions()
tao.destroy()
if 'dmda' in args:
dmda = PETSc.DMDA().create(comm=COMM)
dmda.setFromOptions()
dmda.destroy()
if 'dmplex' in args:
dmplex = PETSc.DMPlex().create(comm=COMM)
dmplex.setFromOptions()
dmplex.destroy()
def __init__(self, n, L, overlap, comm):
# Build Grid
self.dim = len(n)
self.L = L
self.comm = comm
self.isBnd = lambda x: self.isBoundary(x)
self.da = PETSc.DMDA().create(n, dof=1, stencil_width=overlap)
self.da.setUniformCoordinates(xmax=L[0], ymax=L[1], zmax=L[2])
self.da.setMatType(PETSc.Mat.Type.AIJ)
self.numSub = comm.Get_size()
self.sub2proc = [[] for i in range(self.numSub)]
for i in range(self.numSub):
self.sub2proc[i] = [i, 0]
self.M = 1
# Define Finite Element space
self.fe = DarcyQ1(self.dim)
# Setup global and local matrices + communicators
self.A = self.da.createMatrix()
r, _ = self.A.getLGMap() # Get local to global mapping
self.is_A = PETSc.IS().createGeneral(
r.indices) # Create Index Set for local indices
A_local = self.A.createSubMatrices(
self.is_A)[0] # Construct local submatrix on domain
vglobal = self.da.createGlobalVec()
vlocal = self.da.createLocalVec()
self.scatter_l2g = PETSc.Scatter().create(vlocal, None, vglobal,
self.is_A)
self.A_local = A_local
# Identify boundary nodes
nnodes = int(self.da.getCoordinatesLocal()[:].size / self.dim)
coords = np.transpose(self.da.getCoordinatesLocal()[:].reshape(
(nnodes, self.dim)))
Dirich, Neumann, P2P = checkFaces(self.da, self.isBnd, coords)
# Construct Partition of Unity
self.cS = coarseSpace(self.da, self.A, self.comm, self.scatter_l2g)
self.cS.buildPOU(True)
def main():
import petsc4py
from petsc4py import PETSc
# set MPI communicator
comm = MPI.COMM_WORLD
world_rank = comm.Get_rank()
world_size = comm.Get_size()
if len(sys.argv) == 2:
color = int(world_rank / int(sys.argv[1]))
else:
color = int(world_rank / 1)
space_comm = comm.Split(color=color)
space_rank = space_comm.Get_rank()
space_size = space_comm.Get_size()
if len(sys.argv) == 2:
color = int(world_rank % int(sys.argv[1]))
else:
color = int(world_rank / world_size)
time_comm = comm.Split(color=color)
time_rank = time_comm.Get_rank()
time_size = time_comm.Get_size()
print(
"IDs (world, space, time): %i / %i -- %i / %i -- %i / %i" %
(world_rank, world_size, space_rank, space_size, time_rank, time_size))
n = 7
da = PETSc.DMDA().create([n, n], stencil_width=1, comm=space_comm)
x = da.createGlobalVec()
xa = da.getVecArray(x)
(xs, xe), (ys, ye) = da.getRanges()
for i in range(xs, xe):
for j in range(ys, ye):
xa[i, j] = np.sin(2 * np.pi * (i + 1) /
(n + 1)) * np.sin(2 * np.pi * (j + 1) / (n + 1))
print('x=', x.getArray())
print('x:', x.getSizes(), da.getRanges())
print()
if time_rank == 0:
print('send', time_rank)
time_comm.send(x.getArray(), dest=1, tag=0)
else:
print('recv', time_rank)
y = da.createGlobalVec()
y.setArray(time_comm.recv(source=0, tag=0))
print(type(y.getArray()))
def __init__(self, problem_params, dtype_u, dtype_f):
"""
Initialization routine
Args:
problem_params (dict): custom parameters for the example
dtype_u: mesh data type (will be passed parent class)
dtype_f: mesh data type (will be passed parent class)
"""
# these parameters will be used later, so assert their existence
if 'comm' not in problem_params:
problem_params['comm'] = PETSc.COMM_WORLD
if 'sol_tol' not in problem_params:
problem_params['sol_tol'] = 1E-10
if 'sol_maxiter' not in problem_params:
problem_params['sol_maxiter'] = None
essential_keys = ['nvars', 'nu', 'freq', 'comm']
for key in essential_keys:
if key not in problem_params:
msg = 'need %s to instantiate problem, only got %s' % (key, str(problem_params.keys()))
raise ParameterError(msg)
# make sure parameters have the correct form
if len(problem_params['nvars']) != 2:
raise ProblemError('this is a 2d example, got %s' % problem_params['nvars'])
# create DMDA object which will be used for all grid operations
da = PETSc.DMDA().create([problem_params['nvars'][0], problem_params['nvars'][1]], stencil_width=1,
comm=problem_params['comm'])
# invoke super init, passing number of dofs, dtype_u and dtype_f
super(heat2d_petsc_forced, self).__init__(init=da, dtype_u=dtype_u, dtype_f=dtype_f, params=problem_params)
# compute dx, dy and get local ranges
self.dx = 1.0 / (self.params.nvars[0] - 1)
self.dy = 1.0 / (self.params.nvars[1] - 1)
(self.xs, self.xe), (self.ys, self.ye) = self.init.getRanges()
# compute discretization matrix A and identity
self.A = self.__get_A()
self.Id = self.__get_Id()
# setup solver
self.ksp = PETSc.KSP()
self.ksp.create(comm=self.params.comm)
self.ksp.setType('cg')
pc = self.ksp.getPC()
pc.setType('none')
self.ksp.setInitialGuessNonzero(True)
self.ksp.setFromOptions()
self.ksp.setTolerances(rtol=self.params.sol_tol, atol=self.params.sol_tol, max_it=self.params.sol_maxiter)
def __init__(self, grid, xmin, xmax, ymin, ymax, max_window):
from petsc4py import PETSc
from mpi4py import MPI
comm = MPI.COMM_WORLD
self.comm = comm
self.MPI = MPI
# super(CurieParallel, self).__init__(grid, xmin, xmax, ymin, ymax)
# determine stencil width (should be window size/2)
ny, nx = grid.shape
dx = (xmax - xmin) / nx
dy = (ymax - ymin) / ny
nw = max_window / dx
n2w = int(nw / 2) + 1 # add some buffer
if abs(dx - dy) > 1.0:
warnings.warn(
"node spacing should be identical {}".format((dx, dy)),
RuntimeWarning)
dm = PETSc.DMDA().create(dim=2,
sizes=(nx, ny),
stencil_width=n2w,
comm=comm)
dm.setUniformCoordinates(xmin, xmax, ymin, ymax)
self.dm = dm
self.lgmap = dm.getLGMap()
self.lvec = dm.createLocalVector()
self.gvec = dm.createGlobalVector()
coords = dm.getCoordinatesLocal().array.reshape(-1, 2)
xmin, ymin = coords.min(axis=0)
xmax, ymax = coords.max(axis=0)
self.coords = coords
self.max_window = max_window
(imin, imax), (jmin, jmax) = dm.getGhostRanges()
# now decompose grid for each processor
grid_chunk = grid[jmin:jmax, imin:imax]
super(CurieParallel, self).__init__(grid_chunk, xmin, xmax, ymin, ymax)
reduce_methods = {
'sum': MPI.SUM,
'max': MPI.MAX,
'min': MPI.MIN,
'mean': MPI.SUM
}
self._reduce_methods = reduce_methods
def __init__(self, N):
self.q1_start = 0
self.q2_start = 0
self.q1_end = 1
self.q2_end = 1
self.N_q1 = N
self.N_q2 = N
self.dq1 = (self.q1_end - self.q1_start) / self.N_q1
self.dq2 = (self.q2_end - self.q2_start) / self.N_q2
self.N_ghost = np.random.randint(3, 5)
self.q1 = self.q1_start \
+ (0.5 + np.arange(-self.N_ghost,self.N_q1 + self.N_ghost)) * self.dq1
self.q2 = self.q2_start \
+ (0.5 + np.arange(-self.N_ghost,self.N_q2 + self.N_ghost)) * self.dq2
self.q2, self.q1 = np.meshgrid(self.q2, self.q1)
self.q2, self.q1 = af.to_array(self.q2), af.to_array(self.q1)
self.q1 = af.reorder(self.q1, 2, 0, 1)
self.q2 = af.reorder(self.q2, 2, 0, 1)
self.yee_grid_EM_fields = af.constant(0,
6,
self.q1.shape[1],
self.q1.shape[2],
dtype=af.Dtype.f64)
self._da_fields = PETSc.DMDA().create(
[self.N_q1, self.N_q2],
dof=6,
stencil_width=self.N_ghost,
boundary_type=('periodic', 'periodic'),
stencil_type=1,
)
self._glob_fields = self._da_fields.createGlobalVec()
self._local_fields = self._da_fields.createLocalVec()
self._glob_fields_array = self._glob_fields.getArray()
self._local_fields_array = self._local_fields.getArray()
self.boundary_conditions = type('obj', (object, ), {
'in_q1': 'periodic',
'in_q2': 'periodic'
})
self.performance_test_flag = False
def testDuplicate(self, kargs=kargs):
kargs = dict(kargs)
dim = kargs.pop('dim')
dof = kargs['dof']
boundary = kargs['boundary_type']
stencil = kargs['stencil_type']
width = kargs['stencil_width']
da = PETSc.DMDA().create([8*SCALE]*dim)
newda = da.duplicate(**kargs)
self.assertEqual(newda.dim, da.dim)
self.assertEqual(newda.sizes, da.sizes)
self.assertEqual(newda.proc_sizes,
da.proc_sizes)
self.assertEqual(newda.ranges, da.ranges)
self.assertEqual(newda.corners, da.corners)
if (newda.boundary_type == da.boundary_type
and
newda.stencil_width == da.stencil_width):
self.assertEqual(newda.ghost_ranges,
da.ghost_ranges)
self.assertEqual(newda.ghost_corners,
da.ghost_corners)
if dof is None:
dof = da.dof
if boundary is None:
boundary = da.boundary_type
elif boundary == "none":
boundary = (0,) * dim
elif boundary == "mirror":
boundary = (MIRROR,) * dim
elif boundary == "ghosted":
boundary = (GHOSTED,) * dim
elif boundary == "periodic":
boundary = (PERIODIC,) * dim
elif boundary == "twist":
boundary = (TWIST,) * dim
elif isinstance(boundary, int):
boundary = (boundary,) * dim
if stencil is None:
stencil = da.stencil[0]
if width is None:
width = da.stencil_width
self.assertEqual(newda.dof, dof)
self.assertEqual(newda.boundary_type,
boundary)
if dim == 1:
self.assertEqual(newda.stencil,
(stencil, width))
newda.destroy()
da.destroy()
def main():
dim = 3
proc_sizes = tuple(MPI.Compute_dims(SIZE, dim))
# see lists.mcs.anl.gov/pipermail/petsc-dev/2016-April/018903.html
# manually distribute the unkowns to the processors
# mpiexec -n 3
if SIZE > 1:
if 1:
ownership_ranges = (
np.array([3, 4, 5], dtype='i4'),
np.array([1], dtype='i4'),
np.array([1], dtype='i4')
)
else:
ownership_ranges = (
# proc 0
(3, 4, 5),
# proc 1
(1,),
# proc 2
(1,)
)
else:
ownership_ranges = ((3,), (4,), (5,))
sizes = np.array((
sum(ownership_ranges[0]),
sum(ownership_ranges[1]),
sum(ownership_ranges[2])), dtype='i4'
)
dm = PETSc.DMDA().create(dim=dim, sizes=sizes, proc_sizes=proc_sizes,
ownership_ranges=ownership_ranges,
boundary_type=(PETSc.DMDA.BoundaryType.PERIODIC,
PETSc.DMDA.BoundaryType.PERIODIC,
PETSc.DMDA.BoundaryType.GHOSTED),
stencil_type=PETSc.DMDA.StencilType.BOX,
stencil_width=1, dof=1, comm=PETSc.COMM_WORLD, setup=True)
assert dm.getComm().Get_rank() == RANK
assert dm.getComm().Get_size() == SIZE
print(f'create problem with {sizes.prod()} unknowns split into dim={dim} (sizes={sizes}) split accross {SIZE} procs with distribution {proc_sizes}')
vec_l = dm.createLocalVector()
vec_g = dm.createGlobalVector()
initialise_values(dm, vec_l, vec_g)
result_l, result_g = find_max_grad(dm, vec_l, vec_g)
PETSc.Sys.syncPrint(f'Rank {RANK} had max gradient {result_l} while the global was {result_g}.')
if RANK == 0:
if testme(result_g, dm.getComm()):
print('Result is correct.')
else:
print('Result is incorrect!')
def main():
def output_fcn(self):
# Set path to solution
path = 'results/petsc'
# Create path if not existing
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
# Save solution with corresponding time point to file
np.save(path + '/petsc' + str(self.comm_time_rank) + str(self.comm_space_rank),
[[[self.t[0][i], self.comm_space_rank, self.u[0][i].get_values().getArray()] for i in
self.index_local[0]]])
# Split the communicator into space and time communicator
comm_world = MPI.COMM_WORLD
comm_x, comm_t = split_communicator(comm_world, 4)
# Create PETSc DMDA grids
nx = 129
ny = 129
dmda_coarse = PETSc.DMDA().create([nx, ny], stencil_width=1, comm=comm_x)
dmda_fine = dmda_coarse.refine()
# Set up the problem
heat_petsc_0 = HeatPetsc(dmda=dmda_fine, comm_x=comm_x, freq=1, a=1.0, t_start=0, t_stop=1, nt=33)
heat_petsc_1 = HeatPetsc(dmda=dmda_coarse, comm_x=comm_x, freq=1, a=1.0, t_interval=heat_petsc_0.t[::2])
heat_petsc_2 = HeatPetsc(dmda=dmda_coarse, comm_x=comm_x, freq=1, a=1.0, t_interval=heat_petsc_1.t[::2])
# Setup three-level MGRIT solver with the space and time communicators and
# solve the problem
mgrit = Mgrit(problem=[heat_petsc_0, heat_petsc_1, heat_petsc_2],
transfer=[GridTransferPetsc(fine_prob=dmda_fine, coarse_prob=dmda_coarse), GridTransferCopy()],
comm_time=comm_t, comm_space=comm_x, output_fcn=output_fcn)
info = mgrit.solve()
import time
if comm_t.Get_rank() == 0:
time.sleep(1)
sol = []
path = 'results/petsc/'
for filename in os.listdir(path):
data = np.load(path + filename, allow_pickle=True).tolist()[0]
sol += data
sol = [item for item in sol if item[1] == comm_x.Get_rank()]
sol.sort(key=lambda tup: tup[0])
u_e = heat_petsc_0.u_exact(t=heat_petsc_0.t[-1]).get_values().getArray()
diff = sol[-1][2] - u_e
print('Difference at time point', heat_petsc_0.t[-1], ':',
np.linalg.norm(diff, np.inf), '(space rank',comm_x.Get_rank() , ')')
def run_timestepping(nt, space_procs, freq=1, a=1.0, t_start=0, t_stop=1):
comm_x, comm_t = split_communicator(MPI.COMM_WORLD, space_procs)
nx = ny = 65
dmda_coarse = PETSc.DMDA().create([nx, ny], stencil_width=1, comm=comm_x)
dmda_fine = dmda_coarse.refine()
t_interval = np.linspace(t_start, t_stop, nt)
heat_petsc_0 = HeatPetsc(dmda=dmda_fine, comm_x=comm_x, freq=freq, a=a, t_interval=t_interval)
import time
start = time.time()
sol = heat_petsc_0.vector_t_start
for i in range(1, len(heat_petsc_0.t)):
sol = heat_petsc_0.step(u_start=sol, t_start=heat_petsc_0.t[i - 1], t_stop=heat_petsc_0.t[i])
info = {'time_setup': 0, 'time_solve': time.time() - start}
return info['time_setup'], info['time_solve'], info['time_setup'] + info['time_solve']
def __init__(self):
self.q1_start = np.random.randint(0, 5)
self.q2_start = np.random.randint(0, 5)
self.q1_end = np.random.randint(5, 10)
self.q2_end = np.random.randint(5, 10)
self.N_q1 = np.random.randint(16, 32)
self.N_q2 = np.random.randint(16, 32)
self.dq1 = (self.q1_end - self.q1_start) / self.N_q1
self.dq2 = (self.q2_end - self.q2_start) / self.N_q2
self.N_ghost = np.random.randint(1, 5)
self._da_f = PETSc.DMDA().create([self.N_q1, self.N_q2],
dof=1,
stencil_width=self.N_ghost)
def __init__(self, minCoord, maxCoord, res):
minX, minY, minZ = tuple(minCoord)
maxX, maxY, maxZ = tuple(maxCoord)
resI, resJ, resK = tuple(res)
dm = PETSc.DMDA().create(dim=3, sizes=[resI, resJ, resK], stencil_width=1, comm=comm)
dm.setUniformCoordinates(minX, maxX, minY, maxY, minZ, maxZ)
self.dm = dm
self.lvec = dm.createLocalVector()
self.gvec = dm.createGlobalVector()
self.rhs = dm.createGlobalVector()
self.lgmap = dm.getLGMap()
# Setup matrix sizes
self.sizes = self.gvec.getSizes()
Nx, Ny, Nz = dm.getSizes()
N = Nx*Ny*Nz
dx = (maxX - minX)/(Nx - 1)
dy = (maxY - minY)/(Ny - 1)
dz = (maxZ - minZ)/(Nz - 1)
# include ghost nodes in local domain
(minI, maxI), (minJ, maxJ), (minK, maxK) = dm.getGhostRanges()
nx = maxI - minI
ny = maxJ - minJ
nz = maxK - minK
self.dx, self.dy, self.dz = dx, dy, dz
self.nx, self.ny, self.nz = nx, ny, nz
# local numbering
self.nodes = np.arange(0, nx*ny*nz, dtype=PETSc.IntType)
self._initialise_mesh_variables()
self._initialise_boundary_dictionary()
self._initialise_matrix()
def cavity_flow2D(nx, ny, lidvelocity, grashof, prandtl):
# create application context
# and PETSc nonlinear solver
snes = PETSc.SNES().create()
da = PETSc.DMDA().create([nx,ny],dof=4, stencil_width=1, stencil_type='star')
# set up solution vector
F = da.createGlobalVec()
snes.setFunction(CavityFlow2D.formFunction, F,
args=(da, lidvelocity, prandtl, grashof))
x = da.createGlobalVec()
CavityFlow2D.formInitGuess(x, da, lidvelocity, prandtl, grashof)
snes.setDM(da)
snes.setFromOptions()
# solve the nonlinear problem
snes.solve(None, x)
return x
def __init__(self):
self.q1_start = np.random.randint(0, 5)
self.q2_start = np.random.randint(0, 5)
self.q1_end = np.random.randint(5, 10)
self.q2_end = np.random.randint(5, 10)
self.N_q1 = np.random.randint(16, 32)
self.N_q2 = np.random.randint(16, 32)
self.N_ghost = np.random.randint(1, 5)
self.N_p1 = np.random.randint(16, 32)
self.N_p2 = np.random.randint(16, 32)
self.N_p3 = np.random.randint(16, 32)
self._da_f = PETSc.DMDA().create(
[self.N_q1, self.N_q2],
dof=(self.N_p1 * self.N_p2 * self.N_p3),
stencil_width=self.N_ghost,
)
def __init__(self, N, ncores_x, ncores_y):
self.N = N
self.da = PETSc.DMDA().create(sizes=[N, N, N],
proc_sizes=[ncores_x, ncores_y, 1],
stencil_width=2)
self.comm = self.da.getComm()
self.rank = self.comm.getRank()
if self.rank == 0:
print 'The 3D grid is tiled according to (nproc_x, nproc_y, nproc_z) : '\
+str(self.da.proc_sizes)
# init useful vectors
self.PSI = self.da.createGlobalVec()
self.RHS = self.da.createGlobalVec()
# fill psi
self.fill_PSI()
# create and fill the operator
self.L = self.da.createMat()
self.set_L()
def __init__(self, vs):
if vs.enable_cyclic_x:
boundary_type = ('periodic', 'ghosted')
else:
boundary_type = ('ghosted', 'ghosted')
self._da = PETSc.DMDA().create(
[vs.nx, vs.ny],
stencil_width=1,
stencil_type='star',
comm=rs.mpi_comm,
proc_sizes=rs.num_proc,
boundary_type=boundary_type,
ownership_ranges=[
(vs.nx // rs.num_proc[0], ) * rs.num_proc[0],
(vs.ny // rs.num_proc[1], ) * rs.num_proc[1],
])
self._matrix, self._boundary_fac = self._assemble_poisson_matrix(vs)
petsc_options = PETSc.Options()
# setup krylov method
self._ksp = PETSc.KSP()
self._ksp.create(rs.mpi_comm)
self._ksp.setOperators(self._matrix)
self._ksp.setType('gmres')
self._ksp.setTolerances(atol=0,
rtol=vs.congr_epsilon,
max_it=vs.congr_max_iterations)
# preconditioner
self._ksp.getPC().setType('hypre')
petsc_options['pc_hypre_type'] = 'boomeramg'
petsc_options['pc_hypre_boomeramg_relax_type_all'] = 'SOR/Jacobi'
self._ksp.getPC().setFromOptions()
self._rhs_petsc = self._da.createGlobalVec()
self._sol_petsc = self._da.createGlobalVec()
def main():
dims=tuple(MPI.Compute_dims(MPI.COMM_WORLD.Get_size(),3))
dm = PETSc.DMDA().create(dim=len(dims), ownership_ranges=(numpy.array([3]), numpy.array([4]), numpy.array([5])),
proc_sizes=dims,
boundary_type=(PETSc.DMDA.BoundaryType.PERIODIC,
PETSc.DMDA.BoundaryType.PERIODIC,
PETSc.DMDA.BoundaryType.GHOSTED),
stencil_type=PETSc.DMDA.StencilType.BOX,
stencil_width = 1, dof = 1, comm = PETSc.COMM_WORLD, setup = True)
vec_l=dm.createLocalVector()
vec_g=dm.createGlobalVector()
initialise_values(dm, vec_l, vec_g)
result_l, result_g = find_max_grad(dm, vec_l, vec_g)
PETSc.Sys.syncPrint("Rank {rank} had max gradient {maxgrad_l} while the global was {maxgrad_g}."
.format(
rank=dm.getComm().getRank(),
maxgrad_l=result_l,
maxgrad_g=result_g))
if (dm.getComm().getRank() == 0):
if (testme(result_g, dm.getComm())):
print("Result is correct.")
else:
print("Result is incorrect!")
def LayerCake(nx, ny, Lx, Ly, t, nel_per_layer, plotMesh = True, overlap = 1):
n = [nx, ny, np.sum(nel_per_layer) + 1] # Number of Nodes in each direction.
L = [Lx, Ly, np.sum(nel_per_layer)] # Dimension in x - y plane are set, z dimension will be adjusted according to stacking sequence
da = PETSc.DMDA().create(n, dof=3, stencil_width=overlap)
da.setUniformCoordinates(xmax=L[0], ymax=L[1], zmax=L[2])
da.setMatType(PETSc.Mat.Type.AIJ)
elementCutOffs = [0.0]
for i in range(nel_per_layer.size):
hz = t[i] / nel_per_layer[i]
for j in range(nel_per_layer[i]):
elementCutOffs.append(elementCutOffs[-1] + hz)
nnodes = int(da.getCoordinatesLocal()[:].size/3)
c = da.getCoordinatesLocal()[:]
cnew = da.getCoordinatesLocal().copy()
for i in range(nnodes):
cnew[3 * i + 2] = elementCutOffs[np.int(c[3 * i + 2])]
da.setCoordinates(cnew) # Redefine coordinates in transformed state.
if(plotMesh):
x = da.createGlobalVec()
viewer = PETSc.Viewer().createVTK('initial_Geometry.vts', 'w', comm = PETSc.COMM_WORLD)
x.view(viewer)
viewer.destroy()
return da
def __init__(self, N, ncores_x, ncores_y):
self.N = N
self.da = PETSc.DMDA().create(sizes=[N, N, N],
proc_sizes=[ncores_x, ncores_y, 1],
stencil_width=2)
self.comm = self.da.getComm()
self.rank = self.comm.getRank()
if self.rank == 0:
print 'The 3D grid is tiled according to (nproc_x, nproc_y, nproc_z) : '\
+str(self.da.proc_sizes)
# init useful vectors
self.PSI = self.da.createGlobalVec()
self.RHS = self.da.createGlobalVec()
# fill right-hand-side
self.fill_RHS()
# create and fill the operator
self.L = self.da.createMat()
self.set_L()
# create the solver
self.ksp = PETSc.KSP()
self.ksp.create(PETSc.COMM_WORLD)
self.ksp.setOperators(self.L)
self.ksp.setType('bicg')
self.ksp.setInitialGuessNonzero(True)
self.ksp.setTolerances(max_it=1000)
#
for opt in sys.argv[1:]:
PETSc.Options().setValue(opt, None)
self.ksp.setFromOptions()
pass
def __init__(self, N):
self.N_p1 = N
self.N_p2 = N
self.N_p3 = N
self.N_q1 = 1
self.N_q2 = 1
self.N_ghost = 0
self.dp1 = 20 / self.N_p1
self.dp2 = 20 / self.N_p2
self.dp3 = 20 / self.N_p3
self.p1_start = self.p2_start = self.p3_start = -10
self.p1, self.p2, self.p3 = calculate_p_center(self)
self.q1_center = self.q2_center = np.random.rand(1)
# Creating Dummy Values:
self.cell_centered_EM_fields = np.random.rand(6)
self.B_fields_at_nth_timestep = np.random.rand(3)
self.f = af.exp(-self.p1**2 - 2*self.p2**2 - 3*self.p3**2)
self.physical_system = type('obj', (object, ),
{'params': type('obj', (object, ),{'p_dim':3})}
)
self._da_f = PETSc.DMDA().create([self.N_q1, self.N_q2],
dof=( self.N_p1
* self.N_p2
* self.N_p3
)
)
self.performance_test_flag = False
