def test_nullspace_check(mesh, degree):
V = VectorFunctionSpace(mesh, ('Lagrange', degree))
u, v = TrialFunction(V), TestFunction(V)
E, nu = 2.0e2, 0.3
mu = E / (2.0 * (1.0 + nu))
lmbda = E * nu / ((1.0 + nu) * (1.0 - 2.0 * nu))
def sigma(w, gdim):
return 2.0 * mu * ufl.sym(grad(w)) + lmbda * ufl.tr(
grad(w)) * ufl.Identity(gdim)
a = form(inner(sigma(u, mesh.geometry.dim), grad(v)) * dx)
# Assemble matrix and create compatible vector
A = assemble_matrix(a)
A.assemble()
# Create null space basis and test
nullspace = build_elastic_nullspace(V)
la.orthonormalize(nullspace)
ns = PETSc.NullSpace().create(vectors=nullspace)
assert ns.test(A)
# Create incorrect null space basis and test
nullspace = build_broken_elastic_nullspace(V)
la.orthonormalize(nullspace)
ns = PETSc.NullSpace().create(vectors=nullspace)
assert not ns.test(A)
def build_nullspace(V):
"""Function to build null space for 3D elasticity"""
# Create list of vectors for null space
index_map = V.dofmap.index_map
nullspace_basis = [cpp.la.create_vector(index_map) for i in range(6)]
with ExitStack() as stack:
vec_local = [
stack.enter_context(x.localForm()) for x in nullspace_basis
]
basis = [np.asarray(x) for x in vec_local]
# Build translational null space basis
for i in range(3):
basis[i][V.sub(i).dofmap.list.array()] = 1.0
# Build rotational null space basis
x = V.tabulate_dof_coordinates()
dofs = [V.sub(i).dofmap.list.array() for i in range(3)]
basis[3][dofs[0]] = -x[dofs[0], 1]
basis[3][dofs[1]] = x[dofs[1], 0]
basis[4][dofs[0]] = x[dofs[0], 2]
basis[4][dofs[2]] = -x[dofs[2], 0]
basis[5][dofs[2]] = x[dofs[2], 1]
basis[5][dofs[1]] = -x[dofs[1], 2]
# Create vector space basis and orthogonalize
basis = VectorSpaceBasis(nullspace_basis)
basis.orthonormalize()
_x = [basis[i] for i in range(6)]
nsp = PETSc.NullSpace().create(vectors=_x)
return nsp
def build_nullspace(V):
"""Function to build null space for 3D elasticity"""
# Create list of vectors for null space
index_map = V.dofmap().index_map()
nullspace_basis = [cpp.la.create_vector(index_map) for i in range(6)]
# Build translational null space basis
V.sub(0).dofmap().set(nullspace_basis[0], 1.0)
V.sub(1).dofmap().set(nullspace_basis[1], 1.0)
V.sub(2).dofmap().set(nullspace_basis[2], 1.0)
# Build rotational null space basis
V.sub(0).set_x(nullspace_basis[3], -1.0, 1)
V.sub(1).set_x(nullspace_basis[3], 1.0, 0)
V.sub(0).set_x(nullspace_basis[4], 1.0, 2)
V.sub(2).set_x(nullspace_basis[4], -1.0, 0)
V.sub(2).set_x(nullspace_basis[5], 1.0, 1)
V.sub(1).set_x(nullspace_basis[5], -1.0, 2)
# Create vector space basis and orthogonalize
basis = VectorSpaceBasis(nullspace_basis)
basis.orthonormalize()
_x = [basis[i] for i in range(6)]
nsp = PETSc.NullSpace()
nsp.create(_x)
return nsp
def create(self, pc):
self.diag = None
kspLAMG = PETSc.KSP()
kspLAMG.create(comm=PETSc.COMM_WORLD)
pc = kspLAMG.getPC()
kspLAMG.setType('preonly')
pc.setType('lu')
# pc.setFactorSolverPackage("pastix")
OptDB = PETSc.Options()
OptDB['pc_factor_shift_amount'] = .1
OptDB['pc_factor_mat_ordering_type'] = 'rcm'
OptDB['pc_factor_mat_solver_package'] = 'mumps'
# kspLAMG.setFromOptions()
# kspLAMG.max_it = 1
kspLAMG.setFromOptions()
self.kspLAMG = kspLAMG
# print kspLAMG.view()
nsp = PETSc.NullSpace().create(constant=True)
kspLAMG.setNullSpace(nsp)
kspNLAMG = PETSc.KSP()
kspNLAMG.create(comm=PETSc.COMM_WORLD)
pc = kspNLAMG.getPC()
kspNLAMG.setType('preonly')
pc.setType('lu')
# pc.setFactorSolverPackage("pastix")
# kspNLAMG.max_it = 1
kspNLAMG.setFromOptions()
kspLAMG.setFromOptions()
self.kspNLAMG = kspNLAMG
def build_nullspace(V):
"""Function to build null space for 3D elasticity"""
# Create list of vectors for null space
index_map = V.dofmap.index_map
nullspace_basis = [cpp.la.create_vector(index_map) for i in range(6)]
with ExitStack() as stack:
vec_local = [
stack.enter_context(x.localForm()) for x in nullspace_basis
]
basis = [np.asarray(x) for x in vec_local]
# Build translational null space basis
V.sub(0).dofmap.set(basis[0], 1.0)
V.sub(1).dofmap.set(basis[1], 1.0)
V.sub(2).dofmap.set(basis[2], 1.0)
# Build rotational null space basis
V.sub(0).set_x(basis[3], -1.0, 1)
V.sub(1).set_x(basis[3], 1.0, 0)
V.sub(0).set_x(basis[4], 1.0, 2)
V.sub(2).set_x(basis[4], -1.0, 0)
V.sub(2).set_x(basis[5], 1.0, 1)
V.sub(1).set_x(basis[5], -1.0, 2)
# Create vector space basis and orthogonalize
basis = VectorSpaceBasis(nullspace_basis)
basis.orthonormalize()
_x = [basis[i] for i in range(6)]
nsp = PETSc.NullSpace()
nsp.create(_x)
return nsp
def build_nullspace(V):
"""Function to build PETSc nullspace for 3D elasticity"""
# Create list of vectors for null space
index_map = V.dofmap.index_map
bs = V.dofmap.index_map_bs
ns = [la.create_petsc_vector(index_map, bs) for i in range(6)]
with ExitStack() as stack:
vec_local = [stack.enter_context(x.localForm()) for x in ns]
basis = [np.asarray(x) for x in vec_local]
# Get dof indices for each subspace (x, y and z dofs)
dofs = [V.sub(i).dofmap.list.array for i in range(3)]
# Build translational nullspace basis
for i in range(3):
basis[i][dofs[i]] = 1.0
# Build rotational nullspace basis
x = V.tabulate_dof_coordinates()
dofs_block = V.dofmap.list.array
x0, x1, x2 = x[dofs_block, 0], x[dofs_block, 1], x[dofs_block, 2]
basis[3][dofs[0]] = -x1
basis[3][dofs[1]] = x0
basis[4][dofs[0]] = x2
basis[4][dofs[2]] = -x0
basis[5][dofs[2]] = x1
basis[5][dofs[1]] = -x2
la.orthonormalize(ns)
assert la.is_orthonormal(ns)
return PETSc.NullSpace().create(vectors=ns)
def set_Mat_correctiveMethod(self, csrMat_c, nullspace):
############## matrix
if self.rank == 0:
forwards = [None] * self.cpu
Iminmax = self.A_PETSc_c.getOwnershipRanges()
nf = 0
for nf in range(self.cpu):
Imin, Imax = Iminmax[nf], Iminmax[nf + 1]
forwards[nf] = (csrMat_c[0][Imin:Imax+1]-csrMat_c[0][Imin],\
csrMat_c[1][csrMat_c[0][Imin] : csrMat_c[0][Imax]],\
csrMat_c[2][csrMat_c[0][Imin] : csrMat_c[0][Imax]])
else:
forwards = None
#forward matrix csr data
rows, cols, values = self.comm.scatter(forwards, root=0)
self.A_PETSc_c.createAIJ(size=(self.dim, self.dim),
csr=(rows, cols, values),
comm=self.comm)
self.A_PETSc = (self.A_PETSc_p + self.A_PETSc_c)
if nullspace:
nsp = _PETSc.NullSpace()
nsp.create(constant=True, vectors=(), comm=self.comm)
self.A_PETSc.setNullSpace(nsp)
self.method_PETSc.setOperators(self.A_PETSc)
def set_near_nullspace(self, nullsp, constant=False):
assert self.eqn_subindices is None
assert self.inact_subindices is None
vecs = map(vec, nullsp)
self.nullsp = PETSc.NullSpace().create(vectors=vecs,
constant=constant,
comm=self.comm)
def setUp(self):
u1 = PETSc.Vec().createSeq(3)
u2 = PETSc.Vec().createSeq(3)
u1[0], u1[1], u1[2] = [1, 2, 0]; u1.normalize()
u2[0], u2[1], u2[2] = [2, -1, 0]; u2.normalize()
basis = [u1, u2]
nullsp = PETSc.NullSpace().create(False, basis, comm=PETSc.COMM_SELF)
self.basis = basis
self.nullsp = nullsp
def rigid_motions_nullspace(V: _fem.FunctionSpace):
"""
Function to build nullspace for 2D/3D elasticity.
Parameters:
===========
V
The function space
"""
_x = _fem.Function(V)
# Get geometric dim
gdim = V.mesh.geometry.dim
assert gdim == 2 or gdim == 3
# Set dimension of nullspace
dim = 3 if gdim == 2 else 6
# Create list of vectors for null space
nullspace_basis = [_x.vector.copy() for _ in range(dim)]
with ExitStack() as stack:
vec_local = [stack.enter_context(x.localForm()) for x in nullspace_basis]
basis = [np.asarray(x) for x in vec_local]
dofs = [V.sub(i).dofmap.list.array for i in range(gdim)]
# Build translational null space basis
for i in range(gdim):
basis[i][dofs[i]] = 1.0
# Build rotational null space basis
x = V.tabulate_dof_coordinates()
dofs_block = V.dofmap.list.array
x0, x1, x2 = x[dofs_block, 0], x[dofs_block, 1], x[dofs_block, 2]
if gdim == 2:
basis[2][dofs[0]] = -x1
basis[2][dofs[1]] = x0
elif gdim == 3:
basis[3][dofs[0]] = -x1
basis[3][dofs[1]] = x0
basis[4][dofs[0]] = x2
basis[4][dofs[2]] = -x0
basis[5][dofs[2]] = x1
basis[5][dofs[1]] = -x2
_la.orthonormalize(nullspace_basis)
assert _la.is_orthonormal(nullspace_basis)
return PETSc.NullSpace().create(vectors=nullspace_basis)
def get_nullspace(da, A):
RBM = PETSc.NullSpace().createRigidBody(da.getCoordinates())
rbm_vecs = RBM.getVecs()
(xs, xe), (ys, ye) = da.getRanges()
(gxs, gxe), (gys, gye) = da.getGhostRanges()
# Restriction operator
R = da.createGlobalVec()
Rlocal = da.createLocalVec()
Rlocal_a = da.getVecArray(Rlocal)
Rlocal_a[gxs:xe, gys:ye] = 1
# multiplicity
D = da.createGlobalVec()
Dlocal = da.createLocalVec()
da.localToGlobal(Rlocal, D, addv=PETSc.InsertMode.ADD_VALUES)
da.globalToLocal(D, Dlocal)
work1 = da.createLocalVec()
work2 = da.createLocalVec()
vecs = []
for i in range(mpi.COMM_WORLD.size):
for ivec, rbm_vec in enumerate(rbm_vecs):
vecs.append(da.createGlobalVec())
work1.set(0)
da.globalToLocal(rbm_vec, work2)
if i == mpi.COMM_WORLD.rank:
work1 = work2 * Rlocal / Dlocal
da.localToGlobal(work1, vecs[-1], addv=PETSc.InsertMode.ADD_VALUES)
# orthonormalize
Avecs = []
for vec in vecs:
bcApplyWest_vec(da, vec)
Avecs.append(A * vec)
for i, vec in enumerate(vecs):
alphas = []
for vec_ in Avecs[:i]:
alphas.append(vec.dot(vec_))
for alpha, vec_ in zip(alphas, vecs[:i]):
vec.axpy(-alpha, vec_)
vec.scale(1. / np.sqrt(vec.dot(A * vec)))
Avecs[i] = A * vec
return D, vecs, Avecs
def __init__(self, A, prectype, parameters=None, pdes=1, nullspace=None):
from dolfin import info
from time import time
T = time()
Ad = A.down_cast().mat()
if nullspace:
from block.block_util import isscalar
ns = PETSc.NullSpace()
if isscalar(nullspace):
ns.create(constant=True)
else:
ns.create(constant=False,
vectors=[v.down_cast().vec() for v in nullspace])
try:
Ad.setNearNullSpace(ns)
except:
info(
'failed to set near null space (not supported in petsc4py version)'
)
self.A = A
self.petsc_prec = PETSc.PC()
self.petsc_prec.create()
self.petsc_prec.setType(prectype)
# self.petsc_prec.setOperators(Ad, Ad, PETSc.Mat.Structure.SAME_PRECONDITIONER)
self.petsc_prec.setOperators(Ad, Ad)
# Merge parameters into the options database
if parameters:
origOptions = PETSc.Options().getAll()
for key, val in iter(parameters.items()):
PETSc.Options().setValue(key, val)
# Create preconditioner based on the options database
self.petsc_prec.setFromOptions()
self.petsc_prec.setUp()
# Reset the options database
if parameters:
for key in iter(parameters.keys()):
PETSc.Options().delValue(key)
for key, val in iter(origOptions.items()):
PETSc.Options().setValue(key, val)
info('constructed %s preconditioner in %.2f s' %
(self.__class__.__name__, time() - T))
def __init__(self, cfgfile, runid=None, cfg=None):
'''
Constructor
'''
super().__init__(cfgfile, runid, cfg)
# create VIDA for 2d grid (f, phi and moments)
self.da2 = VIDA().create(dim=1,
dof=self.nv + 4,
sizes=[self.nx],
proc_sizes=[PETSc.COMM_WORLD.getSize()],
boundary_type=('periodic'),
stencil_width=2,
stencil_type='box')
# initialise grid
self.da2.setUniformCoordinates(xmin=0.0, xmax=self.Lx)
# create solution and RHS vector
self.x = self.da2.createGlobalVec()
self.xh = self.da2.createGlobalVec()
self.xn = self.da2.createGlobalVec()
self.b = self.da2.createGlobalVec()
# initialise nullspace basis vector for full solution
# the Poisson equation has a null space of all constant vectors
# that needs to be removed to avoid jumpy potentials
self.xn.set(0.)
x_nvec_arr = self.da2.getGlobalArray(self.xn)
p_nvec_arr = getGlobalArray(self.dax, self.pn)
x_nvec_arr[:, self.nv] = p_nvec_arr
# x_nvec_arr[:, self.nv] = 1.
# self.x_nvec.normalize()
self.nullspace = PETSc.NullSpace().create(constant=False,
vectors=(self.xn, ))
# create placeholder for solver object
self.vlasov_poisson_solver = None
# copy f, p, and moments to solution vector
self.copy_data_to_x()
def __init__(self, cfgfile):
'''
Constructor
'''
# load run config file
cfg = Config(cfgfile)
# timestep setup
self.ht = cfg['grid']['ht'] # timestep size
self.nt = cfg['grid']['nt'] # number of timesteps
self.nsave = cfg['io']['nsave'] # save only every nsave'th timestep
# grid setup
self.nx = cfg['grid']['nx'] # number of points in x
self.ny = cfg['grid']['ny'] # number of points in y
Lx = cfg['grid']['Lx'] # spatial domain in x
x1 = cfg['grid']['x1'] #
x2 = cfg['grid']['x2'] #
Ly = cfg['grid']['Ly'] # spatial domain in y
y1 = cfg['grid']['y1'] #
y2 = cfg['grid']['y2'] #
if x1 != x2:
Lx = x2 - x1
else:
x1 = 0.0
x2 = Lx
if y1 != y2:
Ly = y2 - y1
else:
y1 = 0.0
y2 = Ly
self.hx = Lx / self.nx # gridstep size in x
self.hy = Ly / self.ny # gridstep size in y
self.time = PETSc.Vec().createMPI(1,
PETSc.DECIDE,
comm=PETSc.COMM_WORLD)
self.time.setName('t')
if PETSc.COMM_WORLD.getRank() == 0:
self.time.setValue(0, 0.0)
OptDB = PETSc.Options()
OptDB.setValue('snes_rtol', cfg['solver']['petsc_snes_rtol'])
OptDB.setValue('snes_atol', cfg['solver']['petsc_snes_atol'])
OptDB.setValue('snes_stol', cfg['solver']['petsc_snes_stol'])
OptDB.setValue('snes_max_it', cfg['solver']['petsc_snes_max_iter'])
OptDB.setValue('ksp_rtol', cfg['solver']['petsc_ksp_rtol'])
OptDB.setValue('ksp_atol', cfg['solver']['petsc_ksp_atol'])
OptDB.setValue('ksp_max_it', cfg['solver']['petsc_ksp_max_iter'])
# OptDB.setValue('ksp_monitor', '')
# OptDB.setValue('snes_monitor', '')
#
# OptDB.setValue('log_info', '')
# OptDB.setValue('log_summary', '')
self.snes_rtol = cfg['solver']['petsc_snes_rtol']
self.snes_atol = cfg['solver']['petsc_snes_atol']
self.snes_max_iter = cfg['solver']['petsc_snes_max_iter']
# create DA with single dof
self.da1 = PETSc.DA().create(dim=2,
dof=1,
sizes=[self.nx, self.ny],
proc_sizes=[PETSc.DECIDE, PETSc.DECIDE],
boundary_type=('periodic', 'periodic'),
stencil_width=2,
stencil_type='box')
# create DA (dof = 4 for A, J, P, O)
self.da4 = PETSc.DA().create(dim=2,
dof=4,
sizes=[self.nx, self.ny],
proc_sizes=[PETSc.DECIDE, PETSc.DECIDE],
boundary_type=('periodic', 'periodic'),
stencil_width=2,
stencil_type='box')
# create DA for x grid
self.dax = PETSc.DA().create(dim=1,
dof=1,
sizes=[self.nx],
proc_sizes=[PETSc.DECIDE],
boundary_type=('periodic'))
# create DA for y grid
self.day = PETSc.DA().create(dim=1,
dof=1,
sizes=[self.ny],
proc_sizes=[PETSc.DECIDE],
boundary_type=('periodic'))
# initialise grid
self.da1.setUniformCoordinates(xmin=x1, xmax=x2, ymin=y1, ymax=y2)
self.da4.setUniformCoordinates(xmin=x1, xmax=x2, ymin=y1, ymax=y2)
self.dax.setUniformCoordinates(xmin=x1, xmax=x2)
self.day.setUniformCoordinates(xmin=y1, xmax=y2)
# create RHS vector
self.Ab = self.da1.createGlobalVec()
self.Ob = self.da1.createGlobalVec()
self.Pb = self.da1.createGlobalVec()
# create vectors for magnetic and velocity field
self.A = self.da1.createGlobalVec() # magnetic vector potential A
self.J = self.da1.createGlobalVec() # current density J
self.P = self.da1.createGlobalVec() # streaming function psi
self.O = self.da1.createGlobalVec() # vorticity omega
self.Bx = self.da1.createGlobalVec()
self.By = self.da1.createGlobalVec()
self.Vx = self.da1.createGlobalVec()
self.Vy = self.da1.createGlobalVec()
# set variable names
self.A.setName('A')
self.J.setName('J')
self.P.setName('P')
self.O.setName('O')
self.Bx.setName('Bx')
self.By.setName('By')
self.Vx.setName('Vx')
self.Vy.setName('Vy')
# create nullspace
self.poisson_nullspace = PETSc.NullSpace().create(constant=True)
# create jacobian and matrix objects
self.petsc_vorticity = PETScVorticity(self.da1, self.nx, self.ny,
self.ht, self.hx, self.hy)
self.petsc_ohmslaw = PETScOhmsLaw(self.da1, self.nx, self.ny, self.ht,
self.hx, self.hy)
self.petsc_poisson = PETScPoisson(self.da1, self.nx, self.ny, self.hx,
self.hy)
# initialise vorticity matrix
self.vorticity_matrix = self.da1.createMat()
self.vorticity_matrix.setOption(
self.vorticity_matrix.Option.NEW_NONZERO_ALLOCATION_ERR, False)
self.vorticity_matrix.setUp()
# initialise Ohms's law matrix
self.ohmslaw_matrix = self.da1.createMat()
self.ohmslaw_matrix.setOption(
self.ohmslaw_matrix.Option.NEW_NONZERO_ALLOCATION_ERR, False)
self.ohmslaw_matrix.setUp()
# initialise Poisson matrix
self.poisson_matrix = self.da1.createMat()
self.poisson_matrix.setOption(
self.poisson_matrix.Option.NEW_NONZERO_ALLOCATION_ERR, False)
self.poisson_matrix.setUp()
self.poisson_matrix.setNullSpace(self.poisson_nullspace)
# create nonlinear vorticity solver
self.vorticity_snes = PETSc.SNES().create()
self.vorticity_snes.setType('ksponly')
self.vorticity_snes.setFunction(self.petsc_vorticity.snes_mult,
self.Ob)
self.vorticity_snes.setJacobian(self.update_vorticity_jacobian,
self.vorticity_matrix)
self.vorticity_snes.setFromOptions()
# self.vorticity_snes.getKSP().setType('gmres')
self.vorticity_snes.getKSP().setType('preonly')
self.vorticity_snes.getKSP().getPC().setType('lu')
self.vorticity_snes.getKSP().getPC().setFactorSolverPackage(
solver_package)
# create nonlinear Ohms's law solver
self.ohmslaw_snes = PETSc.SNES().create()
self.ohmslaw_snes.setType('ksponly')
self.ohmslaw_snes.setFunction(self.petsc_ohmslaw.snes_mult, self.Ab)
self.ohmslaw_snes.setJacobian(self.update_ohmslaw_jacobian,
self.ohmslaw_matrix)
self.ohmslaw_snes.setFromOptions()
# self.ohmslaw_snes.getKSP().setType('gmres')
self.ohmslaw_snes.getKSP().setType('preonly')
self.ohmslaw_snes.getKSP().getPC().setType('lu')
self.ohmslaw_snes.getKSP().getPC().setFactorSolverPackage(
solver_package)
# create linear Poisson solver
self.poisson_ksp = PETSc.KSP().create()
self.poisson_ksp.setFromOptions()
self.poisson_ksp.setOperators(self.poisson_matrix)
self.poisson_ksp.setType('preonly')
self.poisson_ksp.getPC().setType('lu')
self.poisson_ksp.getPC().setFactorSolverPackage(solver_package)
# self.poisson_ksp.setNullSpace(self.poisson_nullspace)
self.petsc_poisson.formMat(self.poisson_matrix)
# create derivatives object
self.derivatives = PETScDerivatives(self.da1, self.nx, self.ny,
self.ht, self.hx, self.hy)
# get coordinate vectors
coords_x = self.dax.getCoordinates()
coords_y = self.day.getCoordinates()
# save x coordinate arrays
scatter, xVec = PETSc.Scatter.toAll(coords_x)
scatter.begin(coords_x, xVec, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
scatter.end(coords_x, xVec, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
xGrid = xVec.getValues(range(0, self.nx)).copy()
scatter.destroy()
xVec.destroy()
# save y coordinate arrays
scatter, yVec = PETSc.Scatter.toAll(coords_y)
scatter.begin(coords_y, yVec, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
scatter.end(coords_y, yVec, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
yGrid = yVec.getValues(range(0, self.ny)).copy()
scatter.destroy()
yVec.destroy()
# set initial data
(xs, xe), (ys, ye) = self.da1.getRanges()
A_arr = self.da1.getVecArray(self.A)
P_arr = self.da1.getVecArray(self.P)
init_data = __import__("runs." + cfg['initial_data']['python'],
globals(), locals(),
['magnetic_A', 'velocity_P'], 0)
for i in range(xs, xe):
for j in range(ys, ye):
A_arr[i, j] = init_data.magnetic_A(xGrid[i], yGrid[j], Lx, Ly)
P_arr[i, j] = init_data.velocity_P(xGrid[i], yGrid[j], Lx, Ly)
# Fourier Filtering
self.nfourier = cfg['initial_data']['nfourier']
if self.nfourier >= 0:
# obtain whole A vector everywhere
scatter, Aglobal = PETSc.Scatter.toAll(self.A)
scatter.begin(self.A, Aglobal, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
scatter.end(self.A, Aglobal, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
petsc_indices = self.da1.getAO().app2petsc(
np.arange(self.nx * self.ny, dtype=np.int32))
Ainit = Aglobal.getValues(petsc_indices).copy().reshape(
(self.ny, self.nx))
scatter.destroy()
Aglobal.destroy()
# compute FFT, cut, compute inverse FFT
from scipy.fftpack import rfft, irfft
Afft = rfft(Ainit, axis=1)
Afft[:, 0] = 0.
Afft[:, self.nfourier + 1:] = 0.
A_arr = self.da1.getVecArray(self.A)
A_arr[:, :] = irfft(Afft).T[xs:xe, ys:ye]
# compute current and vorticity
self.derivatives.laplace_vec(self.A, self.J, -1.)
self.derivatives.laplace_vec(self.P, self.O, -1.)
J_arr = self.da1.getVecArray(self.J)
O_arr = self.da1.getVecArray(self.O)
# add perturbations
for i in range(xs, xe):
for j in range(ys, ye):
J_arr[i, j] += init_data.current_perturbation(
xGrid[i], yGrid[j], Lx, Ly)
O_arr[i, j] += init_data.vorticity_perturbation(
xGrid[i], yGrid[j], Lx, Ly)
# create HDF5 output file
self.hdf5_viewer = PETSc.ViewerHDF5().create(
cfg['io']['hdf5_output'],
mode=PETSc.Viewer.Mode.WRITE,
comm=PETSc.COMM_WORLD)
self.hdf5_viewer.pushGroup("/")
# write grid data to hdf5 file
coords_x = self.dax.getCoordinates()
coords_y = self.day.getCoordinates()
coords_x.setName('x')
coords_y.setName('y')
self.hdf5_viewer(coords_x)
self.hdf5_viewer(coords_y)
# write initial data to hdf5 file
self.save_to_hdf5(0)
def ii_PETScOperator(bmat, nullspace):
'''Return an object with mult method which acts like bmat*'''
colspace_vec, rowspace_vec = bmat.create_vec(0), bmat.create_vec(1)
if isinstance(colspace_vec, block_vec):
is_block = True
assert isinstance(rowspace_vec, block_vec)
row_sizes = tuple(bi.size() for bi in colspace_vec)
col_sizes = tuple(xi.size() for xi in rowspace_vec)
else:
is_block = False
assert not isinstance(rowspace_vec, block_vec)
row_sizes = (colspace_vec.size(), )
col_sizes = (rowspace_vec.size(), )
print((is_block, row_sizes, col_sizes))
# if isinstance(bmat, block_base):
# row_sizes, col_sizes = bmat_sizes(bmat)
# is_block = True
# else:
# row_sizes, col_sizes = (bmat.size(0), ), (bmat.size(1), )
# is_block = False
class Foo(object):
def __init__(self, A):
self.A = A
if is_block:
def mult(self, mat, x, y):
'''y = A*x'''
y *= 0
# Now x shall be comming as a nested vector
# Convert
x_bvec = block_vec(list(map(PETScVector, x.getNestSubVecs())))
# Apply
y_bvec = self.A*x_bvec
# Convert back
y.axpy(1., as_petsc_nest(y_bvec))
def multTranspose(self, mat, x, y):
'''y = A.T*x'''
AT = block_transpose(self.A)
y *= 0
# Now x shall be comming as a nested vector
# Convert
x_bvec = block_vec(list(map(PETScVector, x.getNestSubVecs())))
# Apply
y_bvec = AT*x_bvec
# Convert back
y.axpy(1., as_petsc_nest(y_bvec))
# No block
else:
def mult(self, mat, x, y):
'''y = A*x'''
y *= 0
x_bvec = PETScVector(x)
y_bvec = self.A*x_bvec
y.axpy(1., as_petsc(y_bvec))
def multTranspose(self, mat, x, y):
'''y = A.T*x'''
AT = block_transpose(self.A)
y *= 0
x_bvec = PETScVector(x)
y_bvec = AT*x_bvec
y.axpy(1., as_petsc(y_bvec))
mat = PETSc.Mat().createPython([[sum(row_sizes), ]*2, [sum(col_sizes), ]*2])
mat.setPythonContext(Foo(bmat))
if nullspace is not None:
Z = PETSc.NullSpace(constant=True,
vectors=[as_backend_type(v).vec() for v in map(convert, nullspace)],
comm=PETSc.COMM_WORLD)
mat.setNullSpace(Z)
mat.setUp()
return mat
r0, Hiptmairtol, params)
G = HiptmairMatrices[0]
Gt = PETSc.Mat()
G.transpose(out=Gt)
# X = PETScToScipy(X)
# invX = sp.linalg.inv(X)
# invX = PETSc.Mat().createAIJ(size=invX.shape, csr=(invX.indptr, invX.indices, invX.data))
# TT = A + G*Gt
nullVecs = []
for i in range(0, LagrangeF.dim()):
nullVecs.append(Gt.getColumnVector(i))
null_space_final = PETSc.NullSpace()
null_space_final.create(vectors=nullVecs)
T1.setNearNullSpace(null_space_final)
#print (TT-T).view()
# savePETScMat(A, "Matrix/S.mat", "S")
# savePETScMat(Z, "Matrix/Z.mat", "Z")
# savePETScMat(M, "Matrix/M.mat", "M")
# savePETScMat(T, "Matrix/T.mat", "T")
# savePETScMat(T1, "Matrix/T1.mat", "T1")
# ss
# Z = M + G*Gt
# T = PETScToScipy(T)
# Z = PETScToScipy(Z)
# A = PETScToScipy(A)
# M = PETScToScipy(M)
def __init__(self, cfgfile, mode="none"):
'''
Constructor
'''
petsc4py.init(sys.argv)
if PETSc.COMM_WORLD.getRank() == 0:
print("")
print("Reduced MHD 2D")
print("==============")
print("")
# solver mode
self.mode = mode
# set run id to timestamp
self.run_id = datetime.datetime.fromtimestamp(
time.time()).strftime("%y%m%d%H%M%S")
if PETSc.COMM_WORLD.getRank() == 0:
print(" Config: %s" % cfgfile)
# load run config file
self.cfg = Config(cfgfile)
# timestep setup
self.ht = self.cfg['grid']['ht'] # timestep size
self.nt = self.cfg['grid']['nt'] # number of timesteps
self.nsave = self.cfg['io'][
'nsave'] # save only every nsave'th timestep
# grid setup
self.nx = self.cfg['grid']['nx'] # number of points in x
self.ny = self.cfg['grid']['ny'] # number of points in y
self.Lx = self.cfg['grid']['Lx'] # spatial domain in x
x1 = self.cfg['grid']['x1'] #
x2 = self.cfg['grid']['x2'] #
self.Ly = self.cfg['grid']['Ly'] # spatial domain in y
y1 = self.cfg['grid']['y1'] #
y2 = self.cfg['grid']['y2'] #
self.hx = self.cfg['grid']['hx'] # gridstep size in x
self.hy = self.cfg['grid']['hy'] # gridstep size in y
# create time vector
self.time = PETSc.Vec().createMPI(1, comm=PETSc.COMM_WORLD)
self.time.setName('t')
# electron skin depth
self.de = self.cfg['initial_data']['skin_depth']
# double bracket dissipation
self.nu = self.cfg['initial_data']['dissipation']
# set global tolerance
self.tolerance = self.cfg['solver'][
'petsc_snes_atol'] * self.nx * self.ny
# direct solver package
self.solver_package = self.cfg['solver']['lu_solver_package']
# set some PETSc solver options
OptDB = PETSc.Options()
OptDB.setValue('ksp_rtol', self.cfg['solver']['petsc_ksp_rtol'])
OptDB.setValue('ksp_atol', self.cfg['solver']['petsc_ksp_atol'])
OptDB.setValue('ksp_max_it', self.cfg['solver']['petsc_ksp_max_iter'])
OptDB.setValue('pc_type', 'hypre')
OptDB.setValue('pc_hypre_type', 'boomeramg')
# create DA with single dof
self.da1 = PETSc.DA().create(dim=2,
dof=1,
sizes=[self.nx, self.ny],
proc_sizes=[PETSc.DECIDE, PETSc.DECIDE],
boundary_type=('periodic', 'periodic'),
stencil_width=1,
stencil_type='box')
# create DA (dof = 4 for A, J, P, O)
self.da4 = PETSc.DA().create(dim=2,
dof=4,
sizes=[self.nx, self.ny],
proc_sizes=[PETSc.DECIDE, PETSc.DECIDE],
boundary_type=('periodic', 'periodic'),
stencil_width=1,
stencil_type='box')
# create DA for x grid
self.dax = PETSc.DA().create(dim=1,
dof=1,
sizes=[self.nx],
proc_sizes=[PETSc.DECIDE],
boundary_type=('periodic'))
# create DA for y grid
self.day = PETSc.DA().create(dim=1,
dof=1,
sizes=[self.ny],
proc_sizes=[PETSc.DECIDE],
boundary_type=('periodic'))
# initialise grid
self.da1.setUniformCoordinates(xmin=x1, xmax=x2, ymin=y1, ymax=y2)
self.da4.setUniformCoordinates(xmin=x1, xmax=x2, ymin=y1, ymax=y2)
self.dax.setUniformCoordinates(xmin=x1, xmax=x2)
self.day.setUniformCoordinates(xmin=y1, xmax=y2)
# create solution and RHS vector
self.dx = self.da4.createGlobalVec()
self.dy = self.da4.createGlobalVec()
self.x = self.da4.createGlobalVec()
self.b = self.da4.createGlobalVec()
self.f = self.da4.createGlobalVec()
self.Pb = self.da1.createGlobalVec()
self.FA = self.da1.createGlobalVec()
self.FJ = self.da1.createGlobalVec()
self.FP = self.da1.createGlobalVec()
self.FO = self.da1.createGlobalVec()
# create initial guess vectors
self.igFA1 = self.da1.createGlobalVec()
self.igFA2 = self.da1.createGlobalVec()
self.igFO1 = self.da1.createGlobalVec()
self.igFO2 = self.da1.createGlobalVec()
# nullspace vectors
self.x0 = self.da4.createGlobalVec()
self.P0 = self.da1.createGlobalVec()
# create vectors for magnetic and velocity field
self.A = self.da1.createGlobalVec() # magnetic vector potential A
self.J = self.da1.createGlobalVec() # current density J
self.P = self.da1.createGlobalVec() # streaming function psi
self.O = self.da1.createGlobalVec() # vorticity omega
self.Bx = self.da1.createGlobalVec()
self.By = self.da1.createGlobalVec()
self.Vx = self.da1.createGlobalVec()
self.Vy = self.da1.createGlobalVec()
# set variable names
self.A.setName('A')
self.J.setName('J')
self.P.setName('P')
self.O.setName('O')
self.Bx.setName('Bx')
self.By.setName('By')
self.Vx.setName('Vx')
self.Vy.setName('Vy')
# initialise nullspace
self.x0.set(0.)
x0_arr = self.da4.getVecArray(self.x0)[...]
x0_arr[:, :, 2] = 1.
self.x0.assemble()
self.x0.normalize()
self.solver_nullspace = PETSc.NullSpace().create(constant=False,
vectors=(self.x0, ))
self.poisson_nullspace = PETSc.NullSpace().create(constant=True)
# initialise Poisson matrix
self.Pm = self.da1.createMat()
self.Pm.setOption(PETSc.Mat.Option.NEW_NONZERO_ALLOCATION_ERR, False)
self.Pm.setUp()
self.Pm.setNullSpace(self.poisson_nullspace)
# create Poisson solver object
self.petsc_poisson = PETScPoisson(self.da1, self.nx, self.ny, self.hx,
self.hy)
# setup linear Poisson solver
self.poisson_ksp = PETSc.KSP().create()
self.poisson_ksp.setFromOptions()
self.poisson_ksp.setOperators(self.Pm)
self.poisson_ksp.setTolerances(
rtol=self.cfg['solver']['poisson_ksp_rtol'],
atol=self.cfg['solver']['poisson_ksp_atol'],
max_it=self.cfg['solver']['poisson_ksp_max_iter'])
self.poisson_ksp.setType('cg')
self.poisson_ksp.getPC().setType('hypre')
self.poisson_ksp.setUp()
self.petsc_poisson.formMat(self.Pm)
# create derivatives object
self.derivatives = PETScDerivatives(self.da1, self.nx, self.ny,
self.ht, self.hx, self.hy)
# read initial data
if self.cfg["io"]["hdf5_input"] != None and self.cfg["io"][
"hdf5_input"] != "":
if self.cfg["initial_data"]["python"] != None and self.cfg[
"initial_data"]["python"] != "":
if PETSc.COMM_WORLD.getRank() == 0:
print(
"WARNING: Both io.hdf5_input and initial_data.python are set!"
)
print(" Reading initial data from HDF5 file.")
self.read_initial_data_from_hdf5()
else:
self.read_initial_data_from_python()
# copy initial data vectors to x
self.copy_x_from_da1_to_da4()
# create HDF5 output file and write parameters
hdf5_filename = self.cfg['io']['hdf5_output']
last_dot = hdf5_filename.rfind('.')
hdf5_filename = hdf5_filename[:last_dot] + "." + str(
self.run_id) + hdf5_filename[last_dot:]
if PETSc.COMM_WORLD.getRank() == 0:
print(" Output: %s" % hdf5_filename)
hdf5out = h5py.File(hdf5_filename,
"w",
driver="mpio",
comm=PETSc.COMM_WORLD.tompi4py())
hdf5out.attrs["run_id"] = self.run_id
for cfg_group in self.cfg:
for cfg_item in self.cfg[cfg_group]:
if self.cfg[cfg_group][cfg_item] != None:
value = self.cfg[cfg_group][cfg_item]
else:
value = ""
hdf5out.attrs[cfg_group + "." + cfg_item] = value
hdf5out.attrs["solver.solver_mode"] = self.mode
if self.cfg["initial_data"]["python"] != None and self.cfg[
"initial_data"]["python"] != "":
python_input = open(
"examples/" + self.cfg['initial_data']['python'] + ".py", 'r')
python_file = python_input.read()
python_input.close()
else:
python_file = ""
hdf5out.attrs["initial_data.python_file"] = python_file
hdf5out.close()
# create HDF5 viewer
self.hdf5_viewer = PETSc.ViewerHDF5().create(
hdf5_filename,
mode=PETSc.Viewer.Mode.APPEND,
comm=PETSc.COMM_WORLD)
self.hdf5_viewer.pushGroup("/")
# write grid data to hdf5 file
coords_x = self.dax.getCoordinates()
coords_y = self.day.getCoordinates()
coords_x.setName('x')
coords_y.setName('y')
self.hdf5_viewer(coords_x)
self.hdf5_viewer(coords_y)
# write initial data to hdf5 file
self.save_to_hdf5(0)
# output some more information
if PETSc.COMM_WORLD.getRank() == 0:
print("")
print(" nt = %i" % (self.nt))
print(" nx = %i" % (self.nx))
print(" ny = %i" % (self.ny))
print("")
print(" ht = %f" % (self.ht))
print(" hx = %f" % (self.hx))
print(" hy = %f" % (self.hy))
print("")
print(" PETSc SNES rtol = %e" %
self.cfg['solver']['petsc_snes_rtol'])
print(" atol = %e" %
self.cfg['solver']['petsc_snes_atol'])
print(" stol = %e" %
self.cfg['solver']['petsc_snes_stol'])
print(" max iter = %i" %
self.cfg['solver']['petsc_snes_max_iter'])
print("")
print(" PETSc KSP rtol = %e" %
self.cfg['solver']['petsc_ksp_rtol'])
print(" atol = %e" %
self.cfg['solver']['petsc_ksp_atol'])
print(" max iter = %i" %
self.cfg['solver']['petsc_ksp_max_iter'])
print("")
def _create_constant_nullspace(self):
"""Initialize a constant null space. """
self.const_null_space = p4pyPETSc.NullSpace().create(comm=p4pyPETSc.COMM_WORLD,
vectors = (),
constant = True)
# Ths pressure field for this problem is determined only up to a
# constant. We can supply the vector that spans the nullspace and any
# component of the solution in this direction will be eliminated during
# the iterative linear solution process.
# Create nullspace vector
null_vec = dolfinx.fem.create_vector_nest(L)
# Set velocity part to zero and the pressure part to a non-zero constant
null_vecs = null_vec.getNestSubVecs()
null_vecs[0].set(0.0), null_vecs[1].set(1.0)
# Normalize the vector, create a nullspace object, and attach it to the
# matrix
null_vec.normalize()
nsp = PETSc.NullSpace().create(vectors=[null_vec])
assert nsp.test(A)
A.setNullSpace(nsp)
# Now we create a Krylov Subspace Solver ``ksp``. We configure it to use
# the MINRES method, and a block-diagonal preconditioner using PETSc's
# additive fieldsplit type preconditioner::
ksp = PETSc.KSP().create(mesh.mpi_comm())
ksp.setOperators(A, P)
ksp.setType("minres")
ksp.setTolerances(rtol=1e-8)
ksp.getPC().setType("fieldsplit")
ksp.getPC().setFieldSplitType(PETSc.PC.CompositeType.ADDITIVE)
# Define the matrix blocks in the preconditioner with the velocity and
M, C, K = load_qep(folder + '/M.dat', folder + '/C.dat', folder + '/K.dat',
adjoint)
else:
Print("Circulant operators")
M, C, K = load_qep_n(folder + '/M1.dat', folder + '/M2.dat',
folder + '/M3.dat', folder + '/C1.dat',
folder + '/C2.dat', folder + '/C3.dat',
folder + '/K1.dat', folder + '/K2.dat',
folder + '/K3.dat', j, n, adjoint)
# set NullSpace if required
if j == 0:
basis = [
PETSc.Vec().load(PETSc.Viewer().createBinary(folder + '/Mk.dat', 'r'))
]
nullsp = PETSc.NullSpace().create(False, basis)
M.setNullSpace(nullsp)
# Solve QEP
Q = SLEPc.PEP().create()
Q.setOperators([K, C, M])
Q.setProblemType(SLEPc.PEP.ProblemType.GENERAL)
Q.setFromOptions()
Q.solve()
Print()
Print("******************************")
Print("*** SLEPc Solution Results ***")
Print("******************************")
def reference_periodic(tetra: bool,
r_lvl: int = 0,
out_hdf5: h5py.File = None,
xdmf: bool = False,
boomeramg: bool = False,
kspview: bool = False,
degree: int = 1):
# Create mesh and finite element
if tetra:
# Tet setup
N = 3
mesh = create_unit_cube(MPI.COMM_WORLD, N, N, N)
for i in range(r_lvl):
mesh.topology.create_entities(mesh.topology.dim - 2)
mesh = refine(mesh, redistribute=True)
N *= 2
else:
# Hex setup
N = 3
for i in range(r_lvl):
N *= 2
mesh = create_unit_cube(MPI.COMM_WORLD, N, N, N, CellType.hexahedron)
V = FunctionSpace(mesh, ("CG", degree))
# Create Dirichlet boundary condition
def dirichletboundary(x):
return np.logical_or(
np.logical_or(np.isclose(x[1], 0), np.isclose(x[1], 1)),
np.logical_or(np.isclose(x[2], 0), np.isclose(x[2], 1)))
mesh.topology.create_connectivity(2, 1)
geometrical_dofs = locate_dofs_geometrical(V, dirichletboundary)
bc = dirichletbc(PETSc.ScalarType(0), geometrical_dofs, V)
bcs = [bc]
# Define variational problem
u = TrialFunction(V)
v = TestFunction(V)
a = inner(grad(u), grad(v)) * dx
x = SpatialCoordinate(mesh)
dx_ = x[0] - 0.9
dy_ = x[1] - 0.5
dz_ = x[2] - 0.1
f = x[0] * sin(5.0 * pi * x[1]) + 1.0 * exp(
-(dx_ * dx_ + dy_ * dy_ + dz_ * dz_) / 0.02)
rhs = inner(f, v) * dx
# Assemble rhs, RHS and apply lifting
bilinear_form = form(a)
linear_form = form(rhs)
A_org = assemble_matrix(bilinear_form, bcs)
A_org.assemble()
L_org = assemble_vector(linear_form)
apply_lifting(L_org, [bilinear_form], [bcs])
L_org.ghostUpdate(addv=PETSc.InsertMode.ADD_VALUES,
mode=PETSc.ScatterMode.REVERSE)
set_bc(L_org, bcs)
# Create PETSc nullspace
nullspace = PETSc.NullSpace().create(constant=True)
PETSc.Mat.setNearNullSpace(A_org, nullspace)
# Set PETSc options
opts = PETSc.Options()
if boomeramg:
opts["ksp_type"] = "cg"
opts["ksp_rtol"] = 1.0e-5
opts["pc_type"] = "hypre"
opts['pc_hypre_type'] = 'boomeramg'
opts["pc_hypre_boomeramg_max_iter"] = 1
opts["pc_hypre_boomeramg_cycle_type"] = "v"
# opts["pc_hypre_boomeramg_print_statistics"] = 1
else:
opts["ksp_type"] = "cg"
opts["ksp_rtol"] = 1.0e-12
opts["pc_type"] = "gamg"
opts["pc_gamg_type"] = "agg"
opts["pc_gamg_sym_graph"] = True
# Use Chebyshev smoothing for multigrid
opts["mg_levels_ksp_type"] = "richardson"
opts["mg_levels_pc_type"] = "sor"
# opts["help"] = None # List all available options
# opts["ksp_view"] = None # List progress of solver
# Initialize PETSc solver, set options and operator
solver = PETSc.KSP().create(MPI.COMM_WORLD)
solver.setFromOptions()
solver.setOperators(A_org)
# Solve linear problem
u_ = Function(V)
start = perf_counter()
with Timer("Solve"):
solver.solve(L_org, u_.vector)
end = perf_counter()
u_.vector.ghostUpdate(addv=PETSc.InsertMode.INSERT,
mode=PETSc.ScatterMode.FORWARD)
if kspview:
solver.view()
it = solver.getIterationNumber()
num_dofs = V.dofmap.index_map.size_global * V.dofmap.index_map_bs
if out_hdf5 is not None:
d_set = out_hdf5.get("its")
d_set[r_lvl] = it
d_set = out_hdf5.get("num_dofs")
d_set[r_lvl] = num_dofs
d_set = out_hdf5.get("solve_time")
d_set[r_lvl, MPI.COMM_WORLD.rank] = end - start
if MPI.COMM_WORLD.rank == 0:
print("Rlvl {0:d}, Iterations {1:d}".format(r_lvl, it))
# Output solution to XDMF
if xdmf:
ext = "tet" if tetra else "hex"
fname = "results/reference_periodic_{0:d}_{1:s}.xdmf".format(
r_lvl, ext)
u_.name = "u_" + ext + "_unconstrained"
with XDMFFile(MPI.COMM_WORLD, fname, "w") as out_periodic:
out_periodic.write_mesh(mesh)
out_periodic.write_function(
u_, 0.0,
"Xdmf/Domain/" + "Grid[@Name='{0:s}'][1]".format(mesh.name))
def __init__(self, cfgfile, runid=None, cfg=None):
'''
Constructor
'''
assert cfgfile is not None
assert cfgfile is not ""
# if runid is empty use timestamp
if runid == None or runid == "":
runid = datetime.datetime.fromtimestamp(
time.time()).strftime("%y%m%d%H%M%S")
# stencil width
stencil = 2
# load run config file
if cfg != None:
self.cfg = cfg
elif cfgfile != None and len(cfgfile) > 0:
self.cfg = Config(cfgfile)
else:
if PETSc.COMM_WORLD.getRank() == 0:
print("ERROR: No valid config file or object passed.")
sys.exit()
# determine solver modules
if cfg['solver']['method'] == 'explicit':
self.vlasov_module = None
else:
self.vlasov_module = "vlasov.solvers."
if cfg['solver']['mode'] == 'split':
self.vlasov_module += 'vlasov'
else:
self.vlasov_module += self.cfg['solver']['mode']
self.vlasov_module += '.' + "PETSc"
if cfg['solver']['type'] == 'newton' or cfg['solver'][
'type'] == 'nonlinear':
self.vlasov_module += "NL"
if cfg['solver']['mode'] == 'split':
self.vlasov_module += "Vlasov"
# self.vlasov_module += self.cfg['solver']['poisson_bracket']
# if cfg['solver']['preconditioner_type'] != None and cfg['solver']['preconditioner_scheme'] != None:
# self.vlasov_module += self.cfg['solver']['preconditioner_scheme']
self.vlasov_module += self.cfg['solver']['timestepping'].upper()
if not cfg.is_dissipation_none:
if cfg['solver']['dissipation'] == 'double_bracket':
self.vlasov_module += "DB"
self.poisson_module = "vlasov.solvers.poisson.PETScPoisson"
self.poisson_module += self.cfg['solver']['laplace_operator']
# importing solver modules
if PETSc.COMM_WORLD.getRank() == 0:
print("Loading Vlasov solver %s" % (self.vlasov_module))
print("Loading Poisson solver %s" % (self.poisson_module))
print("")
if not self.cfg.is_preconditioner_none():
print("Using Preconditioner %s" %
(self.cfg.get_preconditioner()))
if not self.cfg.is_dissipation_none():
if self.cfg.is_dissipation_collisions():
print("Using Collision Operator %s" %
(self.cfg.get_collision_operator()))
if self.cfg.is_dissipation_double_bracket():
print("Using Double Bracket %s" %
(self.cfg.get_double_bracket()))
print("")
self.vlasov_object = __import__(self.vlasov_module, globals(),
locals(), ['PETScVlasovSolver'], 0)
self.poisson_object = __import__(self.poisson_module, globals(),
locals(), ['PETScPoissonSolver'], 0)
# timestep setup
ht = self.cfg['grid']['ht'] # timestep size
nt = self.cfg['grid']['nt'] # number of timesteps
self.nsave = self.cfg['io'][
'nsave'] # save only every nsave'th timestep
# grid setup
nx = self.cfg['grid']['nx'] # number of points in x
nv = self.cfg['grid']['nv'] # number of points in v
L = self.cfg['grid']['L']
vMax = self.cfg['grid']['vmax']
vMin = self.cfg['grid']['vmin']
if vMin == None:
vMin = -vMax
Lx = L
Lv = vMax - vMin
hx = Lx / nx # gridstep size in x
hv = Lv / (nv - 1) # gridstep size in v
self.time = PETSc.Vec().createMPI(1, 1, comm=PETSc.COMM_WORLD)
self.time.setName('t')
if PETSc.COMM_WORLD.getRank() == 0:
self.time.setValue(0, 0.0)
self.solver_package = self.cfg['solver']['lu_package']
self.nInitial = self.cfg['solver'][
'initial_iter'] # number of iterations for initial guess
self.coll_freq = self.cfg['solver']['coll_freq'] # collision frequency
self.coll_drag = self.cfg['solver']['coll_drag'] # drag factor
self.coll_diff = self.cfg['solver']['coll_diff'] # diff factor
self.charge = self.cfg['initial_data']['charge'] # particle charge
self.mass = self.cfg['initial_data']['mass'] # particle mass
output_directory = self.cfg['io']['output_dir']
if output_directory == None or output_directory == "":
output_directory = "."
tindex = cfgfile.rfind('/')
run_filename = cfgfile[tindex:].replace('.cfg', '.') + runid
hdf_out_filename = output_directory + '/' + run_filename + ".hdf5"
cfg_out_filename = output_directory + '/' + run_filename + ".cfg"
# hdf_in_filename = self.cfg['io']['hdf5_input']
# hdf_out_filename = self.cfg['io']['hdf5_output']
self.cfg.write_current_config(cfg_out_filename)
# set initial guess method
initial_guess_options = {
None: self.initial_guess_none,
"None": self.initial_guess_none,
"": self.initial_guess_none,
"rk4": self.initial_guess_rk4,
"gear": self.initial_guess_gear,
"symplectic2": self.initial_guess_symplectic2,
"symplectic4": self.initial_guess_symplectic4,
}
self.initial_guess_method = initial_guess_options[self.cfg['solver']
['initial_guess']]
# set some PETSc options
OptDB = PETSc.Options()
OptDB.setValue('ksp_rtol', self.cfg['solver']['petsc_ksp_rtol'])
OptDB.setValue('ksp_atol', self.cfg['solver']['petsc_ksp_atol'])
OptDB.setValue('ksp_max_it', self.cfg['solver']['petsc_ksp_max_iter'])
OptDB.setValue('snes_rtol', self.cfg['solver']['petsc_snes_rtol'])
OptDB.setValue('snes_atol', self.cfg['solver']['petsc_snes_atol'])
OptDB.setValue('snes_stol', self.cfg['solver']['petsc_snes_stol'])
OptDB.setValue('snes_max_it',
self.cfg['solver']['petsc_snes_max_iter'])
if PETSc.COMM_WORLD.getRank() == 0:
print("Initialising Distributed Arrays.")
# create DA for 2d grid (f only)
# self.da1 = PETSc.DMDA().create(dim=2, dof=1,
# sizes=[nx, nv],
# proc_sizes=[1, PETSc.COMM_WORLD.getSize()],
# boundary_type=['periodic', 'periodic'],
# stencil_width=stencil,
# stencil_type='box')
# self.da1 = PETSc.DMDA().create(dim=2, dof=1,
# sizes=[nx, nv],
# proc_sizes=[PETSc.COMM_WORLD.getSize(), 1],
# boundary_type=['periodic', 'ghosted'],
# stencil_width=stencil,
# stencil_type='box')
self.da1 = PETSc.DMDA().create(
dim=2,
dof=1,
sizes=[nx, nv],
proc_sizes=[1, PETSc.COMM_WORLD.getSize()],
boundary_type=['periodic', 'ghosted'],
stencil_width=stencil,
stencil_type='box')
# self.da1 = PETSc.DMDA().create(dim=2, dof=1,
# sizes=[nx, nv],
# proc_sizes=[PETSc.DECIDE, 2],
# boundary_type=['periodic', 'ghosted'],
# stencil_width=stencil,
# stencil_type='box')
# self.da1 = PETSc.DMDA().create(dim=2, dof=1,
# sizes=[nx, nv],
# proc_sizes=[PETSc.DECIDE, PETSc.DECIDE],
# boundary_type=['periodic', 'ghosted'],
# stencil_width=stencil,
# stencil_type='box')
# create VIDA for x grid
self.dax = PETSc.DMDA().create(dim=1,
dof=1,
sizes=[nx],
proc_sizes=[PETSc.COMM_WORLD.getSize()],
boundary_type=('periodic'),
stencil_width=stencil,
stencil_type='box')
# create VIDA for y grid
self.day = PETSc.DMDA().create(dim=1,
dof=1,
sizes=[nv],
proc_sizes=[PETSc.COMM_WORLD.getSize()],
boundary_type=('ghosted'),
stencil_width=stencil,
stencil_type='box')
# initialise grid
self.da1.setUniformCoordinates(xmin=0.0, xmax=Lx, ymin=vMin, ymax=vMax)
self.dax.setUniformCoordinates(xmin=0.0, xmax=Lx)
self.day.setUniformCoordinates(xmin=vMin, xmax=vMax)
# get local index ranges
(xs, xe), (ys, ye) = self.da1.getRanges()
(xsx, xex), = self.dax.getRanges()
# get coordinate vectors
coords_x = self.dax.getCoordinates()
coords_v = self.day.getCoordinates()
# save x coordinate arrays
scatter, xVec = PETSc.Scatter.toAll(coords_x)
scatter.begin(coords_x, xVec, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
scatter.end(coords_x, xVec, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
xGrid = xVec.getValues(range(nx)).copy()
scatter.destroy()
xVec.destroy()
# save v coordinate arrays
scatter, vVec = PETSc.Scatter.toAll(coords_v)
scatter.begin(coords_v, vVec, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
scatter.end(coords_v, vVec, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
vGrid = vVec.getValues(range(nv)).copy()
scatter.destroy()
vVec.destroy()
# create grid object
self.grid = Grid().create(xGrid, vGrid, nt, nx, nv, ht, hx, hv,
stencil)
# create vectors for Hamiltonians
self.h0 = self.da1.createGlobalVec() # kinetic Hamiltonian
self.h1c = self.da1.createGlobalVec() # current potential Hamiltonian
self.h2c = self.da1.createGlobalVec() # current external Hamiltonian
self.h1h = self.da1.createGlobalVec() # previous potential Hamiltonian
self.h2h = self.da1.createGlobalVec() # previous external Hamiltonian
# distribution functions
self.fl = self.da1.createGlobalVec() # last (k+1, n )
self.fc = self.da1.createGlobalVec() # current (k+1, n+1)
self.fh = self.da1.createGlobalVec() # history (k)
# distribution function solver vectors
self.fb = self.da1.createGlobalVec() # right hand side
# moments
self.N = self.dax.createGlobalVec() # density
self.U = self.dax.createGlobalVec() # velocity density
self.E = self.dax.createGlobalVec() # energy density
self.A = self.dax.createGlobalVec() # collision factor
# local moments
self.nc = PETSc.Vec().createSeq(nx) # current density
self.uc = PETSc.Vec().createSeq(nx) # current velocity density
self.ec = PETSc.Vec().createSeq(nx) # current energy density
self.ac = PETSc.Vec().createSeq(nx) # current collision factor
self.nh = PETSc.Vec().createSeq(nx) # history density
self.uh = PETSc.Vec().createSeq(nx) # history velocity density
self.eh = PETSc.Vec().createSeq(nx) # history energy density
self.ah = PETSc.Vec().createSeq(nx) # history collision factor
# internal potential
self.pc_int = self.dax.createGlobalVec() # current
self.ph_int = self.dax.createGlobalVec() # history
# external potential
self.pc_ext = self.dax.createGlobalVec() # current
self.ph_ext = self.dax.createGlobalVec() # history
# potential solver vectors
self.pb = self.dax.createGlobalVec() # right hand side
self.pn = self.dax.createGlobalVec() # null vector
# set variable names
self.h0.setName('h0')
self.h1c.setName('h1')
self.h2c.setName('h2')
self.fc.setName('f')
self.pc_int.setName('phi_int')
self.pc_ext.setName('phi_ext')
self.N.setName('n')
self.U.setName('u')
self.E.setName('e')
# initialise nullspace basis vectors
# the Poisson equation has a null space of all constant vectors
# that needs to be removed to avoid jumpy potentials
self.pn.set(1.)
self.pn.normalize()
self.p_nullspace = PETSc.NullSpace().create(constant=False,
vectors=(self.pn, ))
# create Toolbox
self.toolbox = Toolbox(self.da1, self.dax, self.grid)
# initialise kinetic hamiltonian
if PETSc.COMM_WORLD.getRank() == 0:
print("Initialising kinetic Hamiltonian.")
self.toolbox.initialise_kinetic_hamiltonian(self.h0, self.mass)
# create Arakawa initial guess solver object
if PETSc.COMM_WORLD.getRank() == 0:
print("Instantiating Initial Guess Objects.")
self.arakawa_rk4 = PETScArakawaRungeKutta(self.cfg, self.da1,
self.grid, self.h0, self.h1h,
self.h2h, self.nInitial)
self.arakawa_gear = PETScArakawaGear(self.cfg, self.da1, self.grid,
self.h0, self.h1h, self.h2h,
self.nInitial)
self.arakawa_symplectic = PETScArakawaSymplectic(
self.cfg, self.da1, self.grid, self.h0, self.h1h, self.h2h,
self.nInitial)
# create solver dummies
self.vlasov_solver = None
self.poisson_solver = None
self.poisson_ksp = None
if PETSc.COMM_WORLD.getRank() == 0:
print()
print("Run ID: %s" % runid)
print()
print("Config File: %s" % cfgfile)
print("Output File: %s" % hdf_out_filename)
print()
print("nt = %i" % (self.grid.nt))
print("nx = %i" % (self.grid.nx))
print("nv = %i" % (self.grid.nv))
print()
print("ht = %e" % (self.grid.ht))
print("hx = %e" % (self.grid.hx))
print("hv = %e" % (self.grid.hv))
print()
print("xMin = %+12.6e" % (self.grid.xMin()))
print("xMax = %+12.6e" % (self.grid.vMax()))
print("vMin = %+12.6e" % (self.grid.vMin()))
print("vMax = %+12.6e" % (self.grid.vMax()))
print()
print("nu = %7.1e" % (self.coll_freq))
print()
print("CFL = %e" % (self.grid.hx / vMax))
print()
print()
# set initial data
N0 = self.dax.createGlobalVec()
T0 = self.dax.createGlobalVec()
N0.setName('n0')
T0.setName('T0')
n0 = PETSc.Vec().createSeq(nx)
t0 = PETSc.Vec().createSeq(nx)
if self.cfg['initial_data']['distribution_python'] != None:
init_data = __import__(
"runs." + self.cfg['initial_data']['distribution_python'],
globals(), locals(), ['distribution'], 0)
if PETSc.COMM_WORLD.getRank() == 0:
print(
"Initialising distribution function with Python function.")
self.toolbox.initialise_distribution_function(
self.fc, init_data.distribution)
N0.set(0.)
T0.set(0.)
else:
N0_arr = self.dax.getVecArray(N0)
T0_arr = self.dax.getVecArray(T0)
if self.cfg['initial_data']['density_python'] != None:
init_data = __import__(
"runs." + self.cfg['initial_data']['density_python'],
globals(), locals(), ['density'], 0)
if PETSc.COMM_WORLD.getRank() == 0:
print("Initialising density with Python function.")
for i in range(xsx, xex):
N0_arr[i] = init_data.density(self.grid.x[i],
self.grid.xLength())
else:
N0_arr[xsx:xex] = self.cfg['initial_data']['density']
if self.cfg['initial_data']['temperature_python'] != None:
init_data = __import__(
"runs." + self.cfg['initial_data']['temperature_python'],
globals(), locals(), ['temperature'], 0)
if PETSc.COMM_WORLD.getRank() == 0:
print("Initialising temperature with Python function.")
for i in range(xsx, xex):
T0_arr[i] = init_data.temperature(self.grid.x[i])
else:
T0_arr[xsx:xex] = self.cfg['initial_data']['temperature']
if PETSc.COMM_WORLD.getRank() == 0:
print("Initialising distribution function with Maxwellian.")
self.copy_xvec_to_seq(N0, n0)
self.copy_xvec_to_seq(T0, t0)
self.toolbox.initialise_distribution_nT(self.fc, n0, t0)
# Fourier Filtering
nfourier = cfg['initial_data']['nfourier']
if nfourier > 0:
from scipy.fftpack import rfft, irfft
if PETSc.COMM_WORLD.getRank() == 0:
print("Fourier Filtering")
# obtain whole f vector everywhere
scatter, Xglobal = PETSc.Scatter.toAll(self.fc)
scatter.begin(self.fc, Xglobal, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
scatter.end(self.fc, Xglobal, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
petsc_indices = self.da1.getAO().app2petsc(
np.arange(nx * nv, dtype=np.int32))
fTmp = Xglobal.getValues(petsc_indices).copy().reshape((nv, nx)).T
scatter.destroy()
Xglobal.destroy()
# filter distribution function
fFft = rfft(fTmp, axis=0)
fFft[nfourier:, :] = 0.
fTmp = irfft(fFft, axis=0)
# store filtered distribution function
f_arr = self.da1.getVecArray(self.fc)
f_arr[:, :] = fTmp[xs:xe, ys:ye]
# normalise f
self.normalise_distribution_function()
# calculate potential and moments
if PETSc.COMM_WORLD.getRank() == 0:
print("Calculate initial potential and moments.")
self.calculate_moments(output=False)
# initialise Gear History
if self.cfg['solver']['initial_guess'] == "gear":
self.arakawa_gear.initialise_history(self.fc)
# check for external potential
if self.cfg['initial_data']['external_python'] != None:
if PETSc.COMM_WORLD.getRank() == 0:
print("Calculate external potential.")
external_data = __import__(
"runs." + self.cfg['initial_data']['external_python'],
globals(), locals(), ['external'], 0)
self.external = external_data.external
else:
self.external = None
# calculate external potential
self.calculate_external(0.)
# create HDF5 output file
if PETSc.COMM_WORLD.getRank() == 0:
print("Create HDF5 output file.")
# use h5py to store attributes
hdf5out = h5py.File(hdf_out_filename,
'w',
driver='mpio',
comm=PETSc.COMM_WORLD.tompi4py())
hdf5out.attrs['charge'] = self.charge
hdf5out.close()
# create PETSc HDF5 viewer
self.hdf5_viewer = PETSc.ViewerHDF5().create(
hdf_out_filename,
mode=PETSc.Viewer.Mode.APPEND,
comm=PETSc.COMM_WORLD)
self.hdf5_viewer.pushGroup("/")
if PETSc.COMM_WORLD.getRank() == 0:
print("Saving initial data to HDF5.")
# write grid data to hdf5 file
coords_x.setName('x')
coords_v.setName('v')
self.hdf5_viewer(coords_x)
self.hdf5_viewer(coords_v)
# write initial data to hdf5 file
# self.hdf5_viewer(N0)
# self.hdf5_viewer(T0)
# save to hdf5
self.hdf5_viewer.setTimestep(0)
self.save_hdf5_vectors()
if PETSc.COMM_WORLD.getRank() == 0:
print("run_base.py: initialisation done.")
print("")
P = S.ExactPrecond(PP, QQ, L, F, [V, Q])
# # bcp.apply(QQ)
Mass = CP.Assemble(QQ)
# # P = IO.matToSparse(P)
# # plt.spy(P)
# # plt.savefig("plt2")
# # ss
# # # P[W.dim()-1,W.dim()-1] += 1
# # P.assemblyBegin() # Make matrices useable.
# # P.assemblyEnd()
ksp = PETSc.KSP().create()
ksp.setTolerances(1e-5)
ksp.setOperators(
A, P) #.getSubMatrix(u_is,u_is),P.getSubMatrix(u_is,u_is))
nsp = PETSc.NullSpace().create(constant=True)
ksp.setNullSpace(nsp)
# A.destroy()
# P.destroy()
reshist = {}
def monitor(ksp, its, fgnorm):
reshist[its] = fgnorm
ksp.setMonitor(monitor)
OptDB = PETSc.Options()
OptDB['pc_factor_shift_amount'] = "0.1"
OptDB['ksp_monitor_residual'] = ' '
OptDB['pc_factor_mat_ordering_type'] = 'rcm'
OptDB['pc_factor_mat_solver_package'] = 'umfpack'
# kspLAMG.max_it = 1
