def run(self):
OptDB = PETSc.Options()
OptDB.setValue('ksp_monitor', '')
OptDB.setValue('snes_monitor', '')
# OptDB.setValue('snes_lag_preconditioner', 3)
# OptDB.setValue('snes_ls', 'basic')
# initialise matrix
self.A = self.da2.createMat()
self.A.setOption(self.A.Option.NEW_NONZERO_ALLOCATION_ERR, False)
self.A.setUp()
self.A.setNullSpace(self.nullspace)
# initialise Jacobian
self.J = self.da2.createMat()
self.J.setOption(self.J.Option.NEW_NONZERO_ALLOCATION_ERR, False)
self.J.setUp()
self.J.setNullSpace(self.nullspace)
# create Jacobian, Function, and linear Matrix objects
# self.petsc_jacobian_mf = PETScJacobianMatrixFree(self.da1, self.da2, self.dax,
# self.h0, self.vGrid,
# self.nx, self.nv, self.ht, self.hx, self.hv,
# self.charge, coll_freq=self.coll_freq)
self.petsc_solver = PETScSolver(self.da1,
self.da2,
self.dax,
self.h0,
self.vGrid,
self.nx,
self.nv,
self.ht,
self.hx,
self.hv,
self.charge,
coll_freq=self.coll_freq)
# self.petsc_function = PETScFunction(self.da1, self.da2, self.dax,
# self.h0, self.vGrid,
# self.nx, self.nv, self.ht, self.hx, self.hv,
# self.charge, coll_freq=self.coll_freq)
#
# self.petsc_matrix = PETScMatrix(self.da1, self.da2, self.dax,
# self.h0, self.vGrid,
# self.nx, self.nv, self.ht, self.hx, self.hv,
# self.charge)#, coll_freq=self.coll_freq)
# self.poisson_ksp = PETSc.KSP().create()
# self.poisson_ksp.setFromOptions()
# self.poisson_ksp.setOperators(self.poisson_A)
# self.poisson_ksp.setType('cg')
# self.poisson_ksp.getPC().setType('none')
# # self.poisson_ksp.setType('preonly')
# # self.poisson_ksp.getPC().setType('lu')
# # self.poisson_ksp.getPC().setFactorSolverPackage(self.solver_package)
# copy external potential
self.petsc_solver.update_external(self.p_ext)
# self.petsc_matrix.update_external(self.p_ext)
# update solution history
self.petsc_solver.update_history(self.x)
# self.petsc_matrix.update_history(self.f, self.h1, self.p, self.n, self.nu, self.ne, self.u, self.e, self.a)
# initialise matrixfree Jacobian
# self.Jmf = PETSc.Mat().createPython([self.x.getSizes(), self.b.getSizes()],
# context=self.petsc_jacobian_mf,
# comm=PETSc.COMM_WORLD)
# self.Jmf.setUp()
# # create linear solver
# self.snes_linear = PETSc.SNES().create()
# self.snes_linear.setType('ksponly')
# self.snes_linear.setFunction(self.petsc_matrix.snes_mult, self.b)
# self.snes_linear.setJacobian(self.updateMatrix, self.A)
# self.snes_linear.setFromOptions()
# # self.snes_linear.getKSP().setType('gmres')
# # self.snes_linear.getKSP().getPC().setType('bjacobi')
# # self.snes_linear.getKSP().getPC().setFactorSolverPackage(self.solver_package)
# self.snes_linear.getKSP().setType('preonly')
# self.snes_linear.getKSP().getPC().setType('lu')
# self.snes_linear.getKSP().getPC().setFactorSolverPackage(self.solver_package)
# create nonlinear solver
self.snes = PETSc.SNES().create()
self.snes.setFunction(self.petsc_solver.function_snes_mult, self.b)
# self.snes.setJacobian(self.updateJacobian, self.Jmf, self.J)
self.snes.setJacobian(self.updateJacobian, self.J)
self.snes.setFromOptions()
# self.snes.getKSP().setType('gmres')
# self.snes.getKSP().getPC().setType('bjacobi')
self.snes.getKSP().setType('preonly')
self.snes.getKSP().getPC().setType('lu')
self.snes.getKSP().getPC().setFactorSolverPackage(self.solver_package)
# self.snes_nsp = PETSc.NullSpace().create(vectors=(self.x_nvec,))
# self.snes.getKSP().setNullSpace(self.snes_nsp)
for itime in range(1, self.nt + 1):
current_time = self.ht * itime
if PETSc.COMM_WORLD.getRank() == 0:
localtime = time.asctime(time.localtime(time.time()))
print("\nit = %4d, t = %10.4f, %s" %
(itime, current_time, localtime))
print
self.time.setValue(0, current_time)
# calculate external field and copy to matrix, jacobian and function
self.calculate_external(current_time)
# self.petsc_jacobian_mf.update_external(self.p_ext)
self.petsc_solver.update_external(self.p_ext)
# self.petsc_jacobian.update_external(self.p_ext)
# self.petsc_function.update_external(self.p_ext)
# self.petsc_matrix.update_external(self.p_ext)
self.x.copy(self.xh)
self.petsc_solver.function_mult(self.x, self.b)
prev_norm = self.b.norm()
if PETSc.COMM_WORLD.getRank() == 0:
print(
" Previous Step: funcnorm = %24.16E"
% (prev_norm))
# calculate initial guess via RK4
# self.initial_guess_rk4()
# calculate initial guess via Gear
self.initial_guess_gear(itime)
# check if residual went down
# self.petsc_function.mult(self.x, self.b)
# ig_norm = self.b.norm()
#
# if ig_norm > prev_norm:
# self.xh.copy(self.x)
# calculate initial guess via linear solver
# self.initial_guess()
# nonlinear solve
self.snes.solve(None, self.x)
# output some solver info
if PETSc.COMM_WORLD.getRank() == 0:
print(
" Nonlin Solver: %5i iterations, funcnorm = %24.16E" %
(self.snes.getIterationNumber(),
self.snes.getFunctionNorm()))
print()
if self.snes.getConvergedReason() < 0:
if PETSc.COMM_WORLD.getRank() == 0:
print()
print("Solver not converging... %i" %
(self.snes.getConvergedReason()))
print()
# if PETSc.COMM_WORLD.getRank() == 0:
# mat_viewer = PETSc.Viewer().createDraw(size=(800,800), comm=PETSc.COMM_WORLD)
# mat_viewer(self.J)
#
# print
# input('Hit any key to continue.')
# print
# update data vectors
self.copy_x_to_data()
# update history
self.petsc_solver.update_history(self.x)
# self.petsc_matrix.update_history(self.f, self.h1, self.p, self.n, self.nu, self.ne, self.u, self.e, self.a)
self.arakawa_gear.update_history(self.f, self.h1)
# save to hdf5
self.save_to_hdf5(itime)
# # some solver output
phisum = self.p.sum()
#
#
if PETSc.COMM_WORLD.getRank() == 0:
# print(" Solver")
print(" sum(phi) = %24.16E" % (phisum))
import ufl
from fenics import *
from string import Template
import hashlib
from subprocess import Popen, PIPE, check_output
import os
from pathlib import Path
import json
import pdb
import petsc4py
import sys
petsc4py.init(sys.argv)
from petsc4py import PETSc
print(PETSc.Options().getAll())
import argparse
from urllib.parse import unquote
from time import sleep
# class PlateOverNematicFoundationPzero
# class PlateOverNematicFoundationPneg
# class RelaxationOverNematicFoundation
# class ActuationOverNematicFoundation
class PlateProblem(NonlinearProblem):
def __init__(self, z, residual, jacobian, bcs):
NonlinearProblem.__init__(self)
self.z = z
else:
AA, bb = assemble_system(maxwell+ns+CoupleTerm, (Lmaxwell + Lns) - RHSform, bcs)
A = as_backend_type(AA).mat()
zeros = 0*bb.array()
b= as_backend_type(bb).vec()
u = IO.arrayToVec(zeros)
ksp = PETSc.KSP().create()
pc = PETSc.PC().create()
ksp.setOperators(A,P)
if IterType == "Full":
pass
OptDB = PETSc.Options()
OptDB["ksp_type"] = "preonly"
OptDB["pc_type"] = "lu"
OptDB["pc_factor_shift_amount"] = .1
# OptDB["pc_type"] = "ilu"
ksp.setFromOptions()
# ksp.view()
tic()
ksp.solve(b, u)
time = toc()
print time
print ksp.its
SolutionTime = SolutionTime +time
outer = outer + ksp.its
u, p, b, r, eps= Iter.PicardToleranceDecouple(u,x,FSpaces,dim,"inf",iter)
def test_krylov_samg_solver_elasticity():
"Test PETScKrylovSolver with smoothed aggregation AMG"
def build_nullspace(V, x):
"""Function to build null space for 2D elasticity"""
# Create list of vectors for null space
ns = [x.copy() for i in range(3)]
with ExitStack() as stack:
vec_local = [stack.enter_context(x.localForm()) for x in ns]
basis = [np.asarray(x) for x in vec_local]
# Build null space basis
dofs = [V.sub(i).dofmap.list.array for i in range(2)]
for i in range(2):
basis[i][dofs[i]] = 1.0
x = V.tabulate_dof_coordinates()
basis[2][dofs[0]] = -x[dofs[0], 1]
basis[2][dofs[1]] = x[dofs[1], 0]
la.orthonormalize(ns)
return ns
def amg_solve(N, method):
# Elasticity parameters
E = 1.0e9
nu = 0.3
mu = E / (2.0 * (1.0 + nu))
lmbda = E * nu / ((1.0 + nu) * (1.0 - 2.0 * nu))
# Stress computation
def sigma(v):
return 2.0 * mu * sym(grad(v)) + lmbda * tr(sym(
grad(v))) * Identity(2)
# Define problem
mesh = create_unit_square(MPI.COMM_WORLD, N, N)
V = VectorFunctionSpace(mesh, 'Lagrange', 1)
u = TrialFunction(V)
v = TestFunction(V)
facetdim = mesh.topology.dim - 1
bndry_facets = locate_entities_boundary(mesh, facetdim, lambda x: np.full(x.shape[1], True))
bdofs = locate_dofs_topological(V.sub(0), V, facetdim, bndry_facets)
bc = dirichletbc(PETSc.ScalarType(0), bdofs, V.sub(0))
# Forms
a, L = inner(sigma(u), grad(v)) * dx, dot(ufl.as_vector((1.0, 1.0)), v) * dx
# Assemble linear algebra objects
A = assemble_matrix(a, [bc])
A.assemble()
b = assemble_vector(L)
apply_lifting(b, [a], [[bc]])
b.ghostUpdate(addv=PETSc.InsertMode.ADD, mode=PETSc.ScatterMode.REVERSE)
set_bc(b, [bc])
# Create solution function
u = Function(V)
# Create near null space basis and orthonormalize
null_space = build_nullspace(V, u.vector)
# Attached near-null space to matrix
A.set_near_nullspace(null_space)
# Test that basis is orthonormal
assert null_space.is_orthonormal()
# Create PETSC smoothed aggregation AMG preconditioner, and
# create CG solver
solver = PETSc.KSP().create(mesh.comm)
solver.setType("cg")
# Set matrix operator
solver.setOperators(A)
# Compute solution and return number of iterations
return solver.solve(b, u.vector)
# Set some multigrid smoother paramete rs
opts = PETSc.Options()
opts["mg_levels_ksp_type"] = "chebyshev"
opts["mg_levels_pc_type"] = "jacobi"
# Improve estimate of eigenvalues for Chebyshev smoothing
opts["mg_levels_esteig_ksp_type"] = "cg"
opts["mg_levels_ksp_chebyshev_esteig_steps"] = 50
# Build list of smoothed aggregation preconditioners
methods = ["petsc_amg"]
# if "ml_amg" in PETScPreconditioner.preconditioners():
# methods.append("ml_amg")
# Test iteration count with increasing mesh size for each
# preconditioner
for method in methods:
for N in [8, 16, 32, 64]:
print("Testing method '{}' with {} x {} mesh".format(method, N, N))
niter = amg_solve(N, method)
assert niter < 18
def setUp(self):
self.opts = PETSc.Options(self.PREFIX)
def __init__(self,
da,
grid,
bdy_type,
sparam,
verbose=0,
solver='gmres',
pc=None):
''' Setup the PV inversion solver
Parameters
----------
da : petsc DMDA
holds the petsc grid
grid : qgsolver grid object
grid data holder
bdy_type : dict
prescribe vertical and lateral boundary conditions. Examples
bdy_type = {'bottom': 'D', 'top': 'D'} for Dirichlet bdy conditions
bdy_type = {'bottom': 'N_RHO', 'top': 'N_RHO'} for Neumann bdy conditions with RHO
bdy_type = {'bottom': 'N_PSI', 'top': 'N_PSI'} for Neumann bdy conditions using PSI instead of RHO
bdy_type = {'periodic': None} for horizontal periodicity
sparam : ndarray
numpy array containing f^2/N^2
verbose : int, optional
degree of verbosity, 0 means no outputs
solver : str, optional
petsc solver: 'gmres' (default), 'bicg', 'cg'
pc : str, optional
what is default?
preconditionner: 'icc', 'bjacobi', 'asm', 'mg', 'none'
'''
self._verbose = verbose
#
self.bdy_type = bdy_type
if ('periodic' in self.bdy_type) and (self.bdy_type['periodic']):
self.petscBoundaryType = 'periodic'
else:
self.petscBoundaryType = None
# create the operator
self.L = da.createMat()
#
if self._verbose > 0:
print('A PV inversion object is being created')
print(' Operator L declared')
# Fill in operator values
if grid._flag_hgrid_uniform and grid._flag_vgrid_uniform:
self._set_L(self.L, da, grid, sparam)
else:
self._set_L_curv(self.L, da, grid, sparam)
#
if self._verbose > 0:
print(' Operator L filled')
# global vector for PV inversion
self._RHS = da.createGlobalVec()
# create solver
self.ksp = PETSc.KSP()
self.ksp.create(PETSc.COMM_WORLD)
self.ksp.setOperators(self.L)
self.ksp.setType(solver)
self.ksp.setInitialGuessNonzero(True)
if pc is not None:
self.ksp.getPC().setType(pc)
# set tolerances
self.ksp.setTolerances(rtol=1e-4)
self.ksp.setTolerances(max_it=100)
# tests:
#self.ksp.setPCSide(PETSc.PC.Side.R)
#self.ksp.setInitialGuessNonzero(False)
#self.ksp.setTolerances(atol=1e-1)
#self.ksp.setTolerances(max_it=100)
#
for opt in sys.argv[1:]:
PETSc.Options().setValue(opt, None)
self.ksp.setFromOptions()
if self._verbose > 0:
print(' PV inversion is set up')
def xtest_mg_solver_stokes():
mesh0 = UnitCubeMesh(2, 2, 2)
mesh1 = UnitCubeMesh(4, 4, 4)
mesh2 = UnitCubeMesh(8, 8, 8)
Ve = VectorElement("CG", mesh0.ufl_cell(), 2)
Qe = FiniteElement("CG", mesh0.ufl_cell(), 1)
Ze = MixedElement([Ve, Qe])
Z0 = FunctionSpace(mesh0, Ze)
Z1 = FunctionSpace(mesh1, Ze)
Z2 = FunctionSpace(mesh2, Ze)
W = Z2
# Boundaries
def right(x, on_boundary):
return x[0] > (1.0 - DOLFIN_EPS)
def left(x, on_boundary):
return x[0] < DOLFIN_EPS
def top_bottom(x, on_boundary):
return x[1] > 1.0 - DOLFIN_EPS or x[1] < DOLFIN_EPS
# No-slip boundary condition for velocity
noslip = Constant((0.0, 0.0, 0.0))
bc0 = DirichletBC(W.sub(0), noslip, top_bottom)
# Inflow boundary condition for velocity
inflow = Expression(("-sin(x[1]*pi)", "0.0", "0.0"), degree=2)
bc1 = DirichletBC(W.sub(0), inflow, right)
# Collect boundary conditions
bcs = [bc0, bc1]
# Define variational problem
(u, p) = TrialFunctions(W)
(v, q) = TestFunctions(W)
f = Constant((0.0, 0.0, 0.0))
a = inner(grad(u), grad(v)) * dx + div(v) * p * dx + q * div(u) * dx
L = inner(f, v) * dx
# Form for use in constructing preconditioner matrix
b = inner(grad(u), grad(v)) * dx + p * q * dx
# Assemble system
A, bb = assemble_system(a, L, bcs)
# Assemble preconditioner system
P, btmp = assemble_system(b, L, bcs)
spaces = [Z0, Z1, Z2]
dm_collection = PETScDMCollection(spaces)
solver = PETScKrylovSolver()
solver.set_operators(A, P)
PETScOptions.set("ksp_type", "gcr")
PETScOptions.set("pc_type", "mg")
PETScOptions.set("pc_mg_levels", 3)
PETScOptions.set("pc_mg_galerkin")
PETScOptions.set("ksp_monitor_true_residual")
PETScOptions.set("ksp_atol", 1.0e-10)
PETScOptions.set("ksp_rtol", 1.0e-10)
solver.set_from_options()
from petsc4py import PETSc
ksp = solver.ksp()
ksp.setDM(dm_collection.dm())
ksp.setDMActive(False)
x = PETScVector()
solver.solve(x, bb)
# Check multigrid solution against LU solver
solver = LUSolver(A) # noqa
x_lu = PETScVector()
solver.solve(x_lu, bb)
assert round((x - x_lu).norm("l2"), 10) == 0
# Clear all PETSc options
opts = PETSc.Options()
for key in opts.getAll():
opts.delValue(key)
def transient_pipe_flow_1D(
npipes, nx, dof, nphases,
pipe_length,
initial_time,
final_time,
initial_time_step,
dt_min,
dt_max,
initial_solution,
impl_python=False
):
# Time Stepper (TS) for ODE and DAE
# DAE - https://en.wikipedia.org/wiki/Differential_algebraic_equation
# https://www.mcs.anl.gov/petsc/petsc-current/docs/manualpages/TS/
ts = PETSc.TS().create()
#ts.createPython(MyTS(), comm=PETSc.COMM_SELF)
# http://www.mcs.anl.gov/petsc/petsc-current/docs/manualpages/DM/index.html
pipes = []
for i in range(npipes):
boundary_type = PETSc.DMDA.BoundaryType.GHOSTED
da = PETSc.DMDA().create([nx], dof=dof, stencil_width=1, stencil_type='star', boundary_type=boundary_type)
da.setFromOptions()
pipes.append(da)
# Create a redundant DM, there is no petsc4py interface (yet)
# so we created our own wrapper
# http://www.mcs.anl.gov/petsc/petsc-current/docs/manualpages/DM/DMREDUNDANT.html
# dmredundant = PETSc.DM().create()
# dmredundant.setType(dmredundant.Type.REDUNDANT)
# CompositeSimple1D.redundantSetSize(dmredundant, 0, dof)
# dmredundant.setDimension(1)
# dmredundant.setUp()
# http://www.mcs.anl.gov/petsc/petsc-current/docs/manualpages/DM/DMCOMPOSITE.html
dm = PETSc.DMComposite().create()
for pipe in pipes:
dm.addDM(pipe)
# dm.addDM(dmredundant)
# CompositeSimple1D.compositeSetCoupling(dm)
CompositeSimple1D.registerNewSNES()
ts.setDM(dm)
F = dm.createGlobalVec()
if impl_python:
α0 = initial_solution.reshape((nx,dof))[:, nphases:-1]
pde = Flow(dm, nx, dof, pipe_length, nphases, α0)
ts.setIFunction(pde.evalFunction, F)
else:
# http://www.mcs.anl.gov/petsc/petsc-current/docs/manualpages/TS/TSSetIFunction.html
assert False, 'C function not implemented yet!'
# ts.setIFunction(CompositeSimple1D.formFunction, F,
# args=(conductivity, source_term, wall_length, temperature_presc))
snes = ts.getSNES()
snes.setUpdate(pde.updateFunction)
x = dm.createGlobalVec()
x[...] = initial_solution
# http://www.mcs.anl.gov/petsc/petsc-current/docs/manualpages/TS/TSSetDuration.html
ts.setDuration(max_time=final_time, max_steps=None)
# http://www.mcs.anl.gov/petsc/petsc-current/docs/manualpages/TS/TSSetInitialTimeStep.html
ts.setInitialTimeStep(initial_time=initial_time, initial_time_step=initial_time_step)
# http://www.mcs.anl.gov/petsc/petsc-current/docs/manualpages/TS/TSSetProblemType.html
ts.setProblemType(ts.ProblemType.NONLINEAR)
ts.setEquationType(ts.EquationType.IMPLICIT)
prev_sol = initial_solution.copy()
# restart = PreStep(prev_sol)
# ts.setPreStep(restart.prestep)
# ts.setPostStep(restart.poststep)
options = PETSc.Options()
options.setValue('-ts_adapt_dt_min', dt_min)
options.setValue('-ts_adapt_dt_max', dt_max)
ts.setFromOptions()
snes = ts.getSNES()
if options.getString('snes_type') in [snes.Type.VINEWTONRSLS, snes.Type.VINEWTONSSLS]:
# if True:
# snesvi = snes.getCompositeSNES(1)
snesvi = snes
xl = np.zeros((nx, dof))
xl[:,:nphases] = -100
xl[:,-1] = 0
xl[:, nphases:-1] = 0
xu = np.zeros((nx, dof))
xu[:,:nphases] = 100
xu[:,-1] = 1000
xu[:, nphases:-1] = 1
xlVec = dm.createGlobalVec()
xuVec = dm.createGlobalVec()
xlVec.setArray(xl.flatten())
xuVec.setArray(xu.flatten())
snesvi.setVariableBounds(xlVec, xuVec)
ts.solve(x)
# while ts.diverged:
# x[...] = prev_sol.copy()
# ts.setInitialTimeStep(initial_time=initial_time, initial_time_step=0.5*initial_time_step)
# ts.solve(x)
final_dt = ts.getTimeStep()
return x, final_dt
+ δr * M_y * ds(end)
# Optional: linearise weak form
# F = dolfiny.expression.linearise(F, m) # linearise around zero state
# Overall form (as list of forms)
F = dolfiny.function.extract_blocks(F, δm)
# Create output xdmf file -- open in Paraview with Xdmf3ReaderT
ofile = dolfiny.io.XDMFFile(comm, f"{name}.xdmf", "w")
# Write mesh, meshtags
if q <= 2:
ofile.write_mesh_meshtags(mesh, mts)
# Options for PETSc backend
opts = PETSc.Options("beam")
opts["snes_type"] = "newtonls"
opts["snes_linesearch_type"] = "basic"
opts["snes_atol"] = 1.0e-07
opts["snes_rtol"] = 1.0e-07
opts["snes_stol"] = 1.0e-06
opts["snes_max_it"] = 60
opts["ksp_type"] = "preonly"
opts["pc_type"] = "lu"
opts["pc_factor_mat_solver_type"] = "mumps"
opts_global = PETSc.Options()
opts_global["mat_mumps_icntl_14"] = 500
opts_global["mat_mumps_icntl_24"] = 1
# Overall form (as one-form)
F = odeint.discretise_in_time(f)
# Overall form (as list of forms)
F = dolfiny.function.extract_blocks(F, δm)
# Create output xdmf file -- open in Paraview with Xdmf3ReaderT
ofile = dolfiny.io.XDMFFile(comm, f"{name}.xdmf", "w")
# Write mesh, meshtags
ofile.write_mesh_meshtags(mesh, mts)
# Write initial state
ofile.write_function(v, time.value)
ofile.write_function(S, time.value)
ofile.write_function(u, time.value)
# Options for PETSc backend
opts = PETSc.Options(name)
opts["snes_type"] = "newtonls"
opts["snes_linesearch_type"] = "basic"
opts["snes_atol"] = 1.0e-12
opts["snes_rtol"] = 1.0e-09
opts["snes_max_it"] = 12
opts["ksp_type"] = "preonly"
# opts["ksp_view"] = "::ascii_info_detail"
opts["pc_type"] = "lu"
opts["pc_factor_mat_solver_type"] = "mumps"
# opts["pc_factor_mat_solver_type"] = "superlu_dist"
opts_global = PETSc.Options()
opts_global["mat_mumps_icntl_14"] = 150
opts_global["mat_superlu_dist_rowperm"] = "norowperm"
def solve_displ_system(J,
F,
intern_var0,
intern_var1,
expr_compiled,
w0,
w1,
bcs,
df,
dt,
t,
k,
io_callback=None,
refinement_callback=None,
rtol=1.e-6,
atol=1.e-16,
max_its=20):
"""Solve system for displacement"""
rank = MPI.COMM_WORLD.rank
t0 = time()
A = create_matrix(J)
if rank == 0:
logger.info(f"[Timer] Preallocation matrix time {time() - t0:.3f}")
t0 = time()
t0 = time()
b = create_vector(F)
if rank == 0:
logger.info(f"[Timer] Preallocation vector time {time() - t0:.3f}")
with b.localForm() as local:
local.set(0.0)
assemble_vector(b, F)
apply_lifting(b, [J], [bcs])
b.ghostUpdate(addv=PETSc.InsertMode.ADD, mode=PETSc.ScatterMode.REVERSE)
set_bc(b, bcs)
resid_norm0 = b.norm()
if rank == 0:
print(f"Initial DISPL residual: {resid_norm0:.2e}")
iter = 0
# Total number of NR iterations including refinement attempts
iter_total = 0
converged = False
refine = 0
scale = 1.0
rel_resid_norm = 1.0
bcs_init = []
for bc in bcs:
bcs_init += [bc.value.vector.duplicate()]
with bcs_init[-1].localForm() as loc0, bc.value.vector.localForm(
) as loc1:
loc1.copy(loc0)
df0 = np.copy(df.value)
while converged is False:
if iter > max_its or rel_resid_norm > 1.e+2:
refine += 1
iter = 0
if rank == 0:
logger.info(79 * "!")
logger.info(
f"Restarting NR with rel. stepsize: {1.0 / (2 ** refine)}")
with w0["displ"].vector.localForm(
) as w_local, w1["displ"].vector.localForm() as w1_local:
w_local.copy(w1_local)
df.value = df0
scale = 1.0 / 2**refine
if refine > 10:
raise ConvergenceError(
"Inner adaptivity reqiures > 10 refinements.")
# Reset and scale boundary condition
for i, bc in enumerate(bcs_init):
with bcs[i].value.vector.localForm(
) as bcsi_local, bc.localForm() as bc_local:
bc_local.copy(bcsi_local)
bcsi_local.scale(scale)
df.value *= scale
dt.value *= scale
if refinement_callback is not None:
refinement_callback(scale)
if rank == 0:
logger.info("Newton iteration for displ {}".format(iter))
if iter > 0:
for bc in bcs:
with bc.value.vector.localForm() as locvec:
locvec.set(0.0)
A.zeroEntries()
with b.localForm() as local:
local.set(0.0)
size = A.getSize()[0]
local_size = A.getLocalSize()[0]
t0 = time()
assemble_matrix(A, J, bcs)
A.assemble()
_time = time() - t0
Anorm = A.norm()
if rank == 0:
logger.info(f"[Timer] A0 size: {size}, local size: {local_size}")
logger.info(f"[Timer] A0 assembly: {_time:.4f}")
logger.info(f"[Timer] A0 assembly dofs/s: {size / _time:.1f}")
logger.info(f"A0 norm: {Anorm:.4f}")
t0 = time()
assemble_vector(b, F)
apply_lifting(b, [J], [bcs])
b.ghostUpdate(addv=PETSc.InsertMode.ADD,
mode=PETSc.ScatterMode.REVERSE)
set_bc(b, bcs)
bnorm = b.norm()
if rank == 0:
logger.info(f"[Timer] b0 assembly {time() - t0:.4f}")
logger.info(f"b norm: {bnorm:.4f}")
nsp = build_nullspace(w1["displ"].function_space)
ksp = PETSc.KSP()
ksp.create(MPI.COMM_WORLD)
ksp.setOptionsPrefix("disp")
opts = PETSc.Options()
A.setNearNullSpace(nsp)
A.setBlockSize(3)
ksp.setOperators(A)
x = A.createVecRight()
ksp.setFromOptions()
t0 = time()
ksp.solve(b, x)
t1 = time() - t0
opts.view()
xnorm = x.norm()
if rank == 0:
its = ksp.its
t1 = time() - t0
dofsps = int(size / t1)
logger.info(f"[Timer] A0 converged in: {its}")
logger.info(f"[Timer] A0 solve: {t1:.4f}")
logger.info(f"[Timer] A0 solve dofs/s: {dofsps:.1f}")
logger.info(f"Increment norm: {xnorm}")
# TODO: Local axpy segfaults, could ghostUpdate be avoided?
w1["displ"].vector.axpy(1.0, x)
w1["displ"].vector.ghostUpdate(addv=PETSc.InsertMode.INSERT,
mode=PETSc.ScatterMode.FORWARD)
#
# Evaluate true residual and check
#
with b.localForm() as local:
local.set(0.0)
assemble_vector(b, F)
apply_lifting(b, [J], [bcs])
b.ghostUpdate(addv=PETSc.InsertMode.ADD,
mode=PETSc.ScatterMode.REVERSE)
set_bc(b, bcs)
norm = b.norm()
rel_resid_norm = norm / resid_norm0
rel_dx_norm = x.norm() / w1["displ"].vector.norm()
if rank == 0:
logger.info("---")
logger.info(f"Abs. resid norm: {norm:.2e}")
logger.info(f"Rel. dx norm: {rel_dx_norm:.2e}")
logger.info(f"Rel. resid norm: {rel_resid_norm:.2e}")
logger.info("---")
iter += 1
iter_total += 1
if rel_resid_norm < rtol or norm < atol:
if rank == 0:
logger.info(
f"Newton converged in: {iter}, total: {iter_total}")
if io_callback is not None:
io_callback(intern_var1, w1, t.value + dt.value)
return scale
def test_biharmonic():
"""Manufactured biharmonic problem.
Solved using rotated Regge mixed finite element method. This is equivalent
to the Hellan-Herrmann-Johnson (HHJ) finite element method in
two-dimensions."""
mesh = RectangleMesh(MPI.COMM_WORLD, [np.array([0.0, 0.0, 0.0]),
np.array([1.0, 1.0, 0.0])], [32, 32], CellType.triangle)
element = ufl.MixedElement([ufl.FiniteElement("Regge", ufl.triangle, 1),
ufl.FiniteElement("Lagrange", ufl.triangle, 2)])
V = FunctionSpace(mesh, element)
sigma, u = ufl.TrialFunctions(V)
tau, v = ufl.TestFunctions(V)
x = ufl.SpatialCoordinate(mesh)
u_exact = ufl.sin(ufl.pi * x[0]) * ufl.sin(ufl.pi * x[0]) * ufl.sin(ufl.pi * x[1]) * ufl.sin(ufl.pi * x[1])
f_exact = div(grad(div(grad(u_exact))))
sigma_exact = grad(grad(u_exact))
# sigma and tau are tangential-tangential continuous according to the
# H(curl curl) continuity of the Regge space. However, for the biharmonic
# problem we require normal-normal continuity H (div div). Theorem 4.2 of
# Lizao Li's PhD thesis shows that the latter space can be constructed by
# the former through the action of the operator S:
def S(tau):
return tau - ufl.Identity(2) * ufl.tr(tau)
sigma_S = S(sigma)
tau_S = S(tau)
# Discrete duality inner product eq. 4.5 Lizao Li's PhD thesis
def b(tau_S, v):
n = FacetNormal(mesh)
return inner(tau_S, grad(grad(v))) * dx \
- ufl.dot(ufl.dot(tau_S('+'), n('+')), n('+')) * jump(grad(v), n) * dS \
- ufl.dot(ufl.dot(tau_S, n), n) * ufl.dot(grad(v), n) * ds
# Non-symmetric formulation
a = inner(sigma_S, tau_S) * dx - b(tau_S, u) + b(sigma_S, v)
L = inner(f_exact, v) * dx
V_1 = V.sub(1).collapse()
zero_u = Function(V_1)
with zero_u.vector.localForm() as zero_u_local:
zero_u_local.set(0.0)
# Strong (Dirichlet) boundary condition
boundary_facets = locate_entities_boundary(
mesh, mesh.topology.dim - 1, lambda x: np.full(x.shape[1], True, dtype=bool))
boundary_dofs = locate_dofs_topological((V.sub(1), V_1), mesh.topology.dim - 1, boundary_facets)
bcs = [DirichletBC(zero_u, boundary_dofs, V.sub(1))]
A = assemble_matrix(a, bcs=bcs)
A.assemble()
b = assemble_vector(L)
apply_lifting(b, [a], [bcs])
b.ghostUpdate(addv=PETSc.InsertMode.ADD, mode=PETSc.ScatterMode.REVERSE)
# Solve
solver = PETSc.KSP().create(MPI.COMM_WORLD)
PETSc.Options()["ksp_type"] = "preonly"
PETSc.Options()["pc_type"] = "lu"
# PETSc.Options()["pc_factor_mat_solver_type"] = "mumps"
solver.setFromOptions()
solver.setOperators(A)
x_h = Function(V)
solver.solve(b, x_h.vector)
x_h.vector.ghostUpdate(addv=PETSc.InsertMode.INSERT,
mode=PETSc.ScatterMode.FORWARD)
# Recall that x_h has flattened indices.
u_error_numerator = np.sqrt(mesh.mpi_comm().allreduce(assemble_scalar(
inner(u_exact - x_h[4], u_exact - x_h[4]) * dx(mesh, metadata={"quadrature_degree": 5})), op=MPI.SUM))
u_error_denominator = np.sqrt(mesh.mpi_comm().allreduce(assemble_scalar(
inner(u_exact, u_exact) * dx(mesh, metadata={"quadrature_degree": 5})), op=MPI.SUM))
assert(np.absolute(u_error_numerator / u_error_denominator) < 0.05)
# Reconstruct tensor from flattened indices.
# Apply inverse transform. In 2D we have S^{-1} = S.
sigma_h = S(ufl.as_tensor([[x_h[0], x_h[1]], [x_h[2], x_h[3]]]))
sigma_error_numerator = np.sqrt(mesh.mpi_comm().allreduce(assemble_scalar(
inner(sigma_exact - sigma_h, sigma_exact - sigma_h) * dx(mesh, metadata={"quadrature_degree": 5})), op=MPI.SUM))
sigma_error_denominator = np.sqrt(mesh.mpi_comm().allreduce(assemble_scalar(
inner(sigma_exact, sigma_exact) * dx(mesh, metadata={"quadrature_degree": 5})), op=MPI.SUM))
assert(np.absolute(sigma_error_numerator / sigma_error_denominator) < 0.005)
class OptionsManager(object):
# What appeared on the commandline, we should never clear these.
# They will override options passed in as a dict if an
# options_prefix was supplied.
commandline_options = frozenset(PETSc.Options().getAll())
options_object = PETSc.Options()
count = itertools.count()
"""Mixin class that helps with managing setting petsc options.
:arg parameters: The dictionary of parameters to use.
:arg options_prefix: The prefix to look up items in the global
options database (may be ``None``, in which case only entries
from ``parameters`` will be considered. If no trailing
underscore is provided, one is appended. Hence ``foo_`` and
``foo`` are treated equivalently. As an exception, if the
prefix is the empty string, no underscore is appended.
To use this, you must call its constructor to with the parameters
you want in the options database.
You then call :meth:`set_from_options`, passing the PETSc object
you'd like to call ``setFromOptions`` on. Note that this will
actually only call ``setFromOptions`` the first time (so really
this parameters object is a once-per-PETSc-object thing).
So that the runtime monitors which look in the options database
actually see options, you need to ensure that the options database
is populated at the time of a ``SNESSolve`` or ``KSPSolve`` call.
Do that using the :meth:`inserted_options` context manager.
.. code-block:: python3
with self.inserted_options():
self.snes.solve(...)
This ensures that the options database has the relevant entries
for the duration of the ``with`` block, before removing them
afterwards. This is a much more robust way of dealing with the
fixed-size options database than trying to clear it out using
destructors.
This object can also be used only to manage insertion and deletion
into the PETSc options database, by using the context manager.
"""
def __init__(self, parameters, options_prefix):
super().__init__()
if parameters is None:
parameters = {}
else:
# Convert nested dicts
parameters = flatten_parameters(parameters)
if options_prefix is None:
self.options_prefix = "firedrake_%d_" % next(self.count)
self.parameters = parameters
self.to_delete = set(parameters)
else:
if len(options_prefix) and not options_prefix.endswith("_"):
options_prefix += "_"
self.options_prefix = options_prefix
# Remove those options from the dict that were passed on
# the commandline.
self.parameters = {
k: v
for k, v in parameters.items()
if options_prefix + k not in self.commandline_options
}
self.to_delete = set(self.parameters)
# Now update parameters from options, so that they're
# available to solver setup (for, e.g., matrix-free).
# Can't ask for the prefixed guy in the options object,
# since that does not DTRT for flag options.
for k, v in self.options_object.getAll().items():
if k.startswith(self.options_prefix):
self.parameters[k[len(self.options_prefix):]] = v
self._setfromoptions = False
def set_default_parameter(self, key, val):
"""Set a default parameter value.
:arg key: The parameter name
:arg val: The parameter value.
Ensures that the right thing happens cleaning up the options
database.
"""
k = self.options_prefix + key
if k not in self.options_object and key not in self.parameters:
self.parameters[key] = val
self.to_delete.add(key)
def set_from_options(self, petsc_obj):
"""Set up petsc_obj from the options database.
:arg petsc_obj: The PETSc object to call setFromOptions on.
Matt says: "Only ever call setFromOptions once". This
function ensures we do so.
"""
if not self._setfromoptions:
with self.inserted_options():
petsc_obj.setOptionsPrefix(self.options_prefix)
# Call setfromoptions inserting appropriate options into
# the options database.
petsc_obj.setFromOptions()
self._setfromoptions = True
@contextmanager
def inserted_options(self):
"""Context manager inside which the petsc options database
contains the parameters from this object."""
try:
for k, v in self.parameters.items():
self.options_object[self.options_prefix + k] = v
yield
finally:
for k in self.to_delete:
del self.options_object[self.options_prefix + k]
def __init__(self, cfgfile):
'''
Constructor
'''
# load run config file
cfg = Config(cfgfile)
# timestep setup
self.ht = cfg['grid']['ht'] # timestep size
self.nt = cfg['grid']['nt'] # number of timesteps
self.nsave = cfg['io']['nsave'] # save only every nsave'th timestep
# grid setup
nx = cfg['grid']['nx'] # number of points in x
ny = cfg['grid']['ny'] # number of points in y
Lx = cfg['grid']['Lx'] # spatial domain in x
x1 = cfg['grid']['x1'] #
x2 = cfg['grid']['x2'] #
Ly = cfg['grid']['Ly'] # spatial domain in y
y1 = cfg['grid']['y1'] #
y2 = cfg['grid']['y2'] #
if x1 != x2:
Lx = x2-x1
else:
x1 = 0.0
x2 = Lx
if y1 != y2:
Ly = y2-y1
else:
y1 = 0.0
y2 = Ly
self.hx = Lx / nx # gridstep size in x
self.hy = Ly / ny # gridstep size in y
self.time = PETSc.Vec().createMPI(1, PETSc.DECIDE, comm=PETSc.COMM_WORLD)
self.time.setName('t')
if PETSc.COMM_WORLD.getRank() == 0:
self.time.setValue(0, 0.0)
# set some PETSc options
OptDB = PETSc.Options()
OptDB.setValue('ksp_rtol', cfg['solver']['petsc_residual'])
# OptDB.setValue('snes_rtol', 1E-2)
# OptDB.setValue('ksp_max_it', 100)
OptDB.setValue('ksp_max_it', 200)
# OptDB.setValue('ksp_max_it', 1000)
# OptDB.setValue('ksp_max_it', 2000)
# OptDB.setValue('ksp_monitor', '')
# OptDB.setValue('log_info', '')
# OptDB.setValue('log_summary', '')
# create DA with single dof
self.da1 = PETSc.DA().create(dim=2, dof=1,
sizes=[nx, ny],
proc_sizes=[PETSc.DECIDE, PETSc.DECIDE],
boundary_type=('periodic', 'periodic'),
stencil_width=1,
stencil_type='box')
# create DA (dof = 5 for Bx, By, Vx, Vy, P)
self.da4 = PETSc.DA().create(dim=2, dof=5,
sizes=[nx, ny],
proc_sizes=[PETSc.DECIDE, PETSc.DECIDE],
boundary_type=('periodic', 'periodic'),
stencil_width=1,
stencil_type='box')
# create DA for x grid
self.dax = PETSc.DA().create(dim=1, dof=1,
sizes=[nx],
proc_sizes=[PETSc.DECIDE],
boundary_type=('periodic'))
# create DA for y grid
self.day = PETSc.DA().create(dim=1, dof=1,
sizes=[ny],
proc_sizes=[PETSc.DECIDE],
boundary_type=('periodic'))
# initialise grid
self.da1.setUniformCoordinates(xmin=x1, xmax=x2,
ymin=y1, ymax=y2)
self.da4.setUniformCoordinates(xmin=x1, xmax=x2,
ymin=y1, ymax=y2)
self.dax.setUniformCoordinates(xmin=x1, xmax=x2)
self.day.setUniformCoordinates(xmin=y1, xmax=y2)
# create solution and RHS vector
self.dx = self.da4.createGlobalVec()
self.x = self.da4.createGlobalVec()
self.b = self.da4.createGlobalVec()
self.f = self.da4.createGlobalVec()
self.Pb = self.da1.createGlobalVec()
self.localX = self.da4.createLocalVec()
# create global RK4 vectors
self.X1 = self.da4.createGlobalVec()
self.X2 = self.da4.createGlobalVec()
self.X3 = self.da4.createGlobalVec()
self.X4 = self.da4.createGlobalVec()
# create local RK4 vectors
self.localX1 = self.da4.createLocalVec()
self.localX2 = self.da4.createLocalVec()
self.localX3 = self.da4.createLocalVec()
self.localX4 = self.da4.createLocalVec()
# create vectors for magnetic and velocity field
self.Bx = self.da1.createGlobalVec()
self.By = self.da1.createGlobalVec()
self.Vx = self.da1.createGlobalVec()
self.Vy = self.da1.createGlobalVec()
self.P = self.da1.createGlobalVec()
# set variable names
self.x.setName('solver_x')
self.b.setName('solver_b')
self.Bx.setName('Bx')
self.By.setName('By')
self.Vx.setName('Vx')
self.Vy.setName('Vy')
self.P.setName('P')
# create Matrix object
self.petsc_matrix = PETScSolver (self.da1, self.da4, nx, ny, self.ht, self.hx, self.hy)
self.petsc_function = PETScFunction(self.da1, self.da4, nx, ny, self.ht, self.hx, self.hy)
self.petsc_jacobian = PETScJacobian(self.da1, self.da4, nx, ny, self.ht, self.hx, self.hy)
# create sparse matrix
self.J = PETSc.Mat().createPython([self.dx.getSizes(), self.b.getSizes()], comm=PETSc.COMM_WORLD)
self.J.setPythonContext(self.petsc_jacobian)
self.J.setUp()
# create nonlinear solver
self.snes = PETSc.SNES().create()
self.snes.setFunction(self.petsc_function.snes_mult, self.f)
self.snes.setJacobian(self.updateJacobian, self.J)
# self.snes.setUseMF()
self.snes.setFromOptions()
self.snes.getKSP().setType('gmres')
self.snes.getKSP().getPC().setType('none')
# create Preconditioner matrix and solver
self.pc_mat = PETScPreconditioner(self.da1, self.da4, self.P, nx, ny, self.ht, self.hx, self.hy)
# create sparse matrix
self.pc_A = PETSc.Mat().createPython([self.x.getSizes(), self.b.getSizes()], comm=PETSc.COMM_WORLD)
self.pc_A.setPythonContext(self.pc_mat)
self.pc_A.setUp()
# create linear solver and preconditioner
self.pc_ksp = PETSc.KSP().create()
self.pc_ksp.setFromOptions()
self.pc_ksp.setOperators(self.pc_A)
self.pc_ksp.setType(cfg['solver']['petsc_ksp_type'])
self.pc_ksp.setInitialGuessNonzero(True)
self.pc_pc = self.pc_ksp.getPC()
self.pc_pc.setType('none')
# create Poisson matrix and solver
self.poisson_mat = PETScPoissonSolver(self.da1, self.da4, self.x,
nx, ny, self.ht, self.hx, self.hy)
self.poisson_A = PETSc.Mat().createPython([self.P.getSizes(), self.Pb.getSizes()], comm=PETSc.COMM_WORLD)
self.poisson_A.setPythonContext(self.poisson_mat)
self.poisson_A.setUp()
self.poisson_ksp = PETSc.KSP().create()
self.poisson_ksp.setFromOptions()
self.poisson_ksp.setOperators(self.poisson_A)
self.poisson_ksp.setType(cfg['solver']['petsc_ksp_type'])
# self.poisson_ksp.setInitialGuessNonzero(True)
self.poisson_pc = self.poisson_ksp.getPC()
self.poisson_pc.setType('none')
# create Arakawa solver object
# self.mhd_rk4 = PETScRK4(self.da4, nx, ny, self.ht, self.hx, self.hy)
# set initial data
(xs, xe), (ys, ye) = self.da1.getRanges()
coords = self.da1.getCoordinateDA().getVecArray(self.da1.getCoordinates())
# print
# print(self.hx)
# print(coords[1,0][0] - coords[0,0][0])
# print
# print(self.hy)
# print(coords[0,1][1] - coords[0,0][1])
# print
# print(Lx)
# print(coords[-1,0][0]+self.hx)
# print
# print(Ly)
# print(coords[0,-1][1]+self.hy)
# print
x_arr = self.da4.getVecArray(self.x)
Bx_arr = self.da1.getVecArray(self.Bx)
By_arr = self.da1.getVecArray(self.By)
Vx_arr = self.da1.getVecArray(self.Vx)
Vy_arr = self.da1.getVecArray(self.Vy)
P_arr = self.da1.getVecArray(self.P)
if cfg['initial_data']['magnetic_python'] != None:
init_data = __import__("runs." + cfg['initial_data']['magnetic_python'], globals(), locals(), ['magnetic_x', 'magnetic_y'], 0)
for i in range(xs, xe):
for j in range(ys, ye):
Bx_arr[i,j] = init_data.magnetic_x(coords[i,j][0], coords[i,j][1], Lx, Ly)
By_arr[i,j] = init_data.magnetic_y(coords[i,j][0], coords[i,j][1], Lx, Ly)
else:
Bx_arr[xs:xe, ys:ye] = cfg['initial_data']['magnetic']
By_arr[xs:xe, ys:ye] = cfg['initial_data']['magnetic']
if cfg['initial_data']['velocity_python'] != None:
init_data = __import__("runs." + cfg['initial_data']['velocity_python'], globals(), locals(), ['velocity_x', 'velocity_y'], 0)
for i in range(xs, xe):
for j in range(ys, ye):
Vx_arr[i,j] = init_data.velocity_x(coords[i,j][0], coords[i,j][1], Lx, Ly)
Vy_arr[i,j] = init_data.velocity_y(coords[i,j][0], coords[i,j][1], Lx, Ly)
else:
Vx_arr[xs:xe, ys:ye] = cfg['initial_data']['velocity']
Vy_arr[xs:xe, ys:ye] = cfg['initial_data']['velocity']
if cfg['initial_data']['pressure_python'] != None:
init_data = __import__("runs." + cfg['initial_data']['pressure_python'], globals(), locals(), ['pressure', ''], 0)
for i in range(xs, xe):
for j in range(ys, ye):
P_arr[i,j] = init_data.pressure(coords[i,j][0], coords[i,j][1], Lx, Ly) #+ 0.5 * (Bx_arr[i,j]**2 + By_arr[i,j]**2)
# copy distribution function to solution vector
x_arr[xs:xe, ys:ye, 0] = Bx_arr[xs:xe, ys:ye]
x_arr[xs:xe, ys:ye, 1] = By_arr[xs:xe, ys:ye]
x_arr[xs:xe, ys:ye, 2] = Vx_arr[xs:xe, ys:ye]
x_arr[xs:xe, ys:ye, 3] = Vy_arr[xs:xe, ys:ye]
x_arr[xs:xe, ys:ye, 4] = P_arr [xs:xe, ys:ye]
# update solution history
self.petsc_matrix.update_history(self.x)
self.petsc_function.update_history(self.x)
self.petsc_jacobian.update_history(self.x)
# create HDF5 output file
self.hdf5_viewer = PETSc.Viewer().createHDF5(cfg['io']['hdf5_output'],
mode=PETSc.Viewer.Mode.WRITE,
comm=PETSc.COMM_WORLD)
self.hdf5_viewer.HDF5PushGroup("/")
# write grid data to hdf5 file
coords_x = self.dax.getCoordinates()
coords_y = self.day.getCoordinates()
coords_x.setName('x')
coords_y.setName('y')
self.hdf5_viewer(coords_x)
self.hdf5_viewer(coords_y)
# write initial data to hdf5 file
self.hdf5_viewer.HDF5SetTimestep(0)
self.hdf5_viewer(self.time)
# self.hdf5_viewer(self.x)
# self.hdf5_viewer(self.b)
self.hdf5_viewer(self.Bx)
self.hdf5_viewer(self.By)
self.hdf5_viewer(self.Vx)
self.hdf5_viewer(self.Vy)
self.hdf5_viewer(self.P)
def main(*args):
#
# I don't understand why this has to be done as follows, but PETSc
# doesn't get commandline arguments if things are done in different
# order.
#
if args:
args = list(args)
else:
args = sys.argv
commandlineArguments = parse_commandline(args[1:])
#
# this needs to be done before importing PETSc
#
petsc4py.init(args=(args[0:1] + commandlineArguments.petsc))
from KSDG import (box_mesh, makeKSDGSolver, dofremap, KSDGException,
KSDGTimeSeries, random_function, implicitTS,
LigandGroups, ParameterList, default_parameters)
from KSDG.ksdgdebug import log
def logMAIN(*args, **kwargs):
log(*args, system='MAIN', **kwargs)
logMAIN('commandlineArguments.petsc', commandlineArguments.petsc)
# import fenics as fe
import ufl
from petsc4py import PETSc
fe.parameters.parse(sys.argv[0:1] + commandlineArguments.fenics)
fe.parameters['ghost_mode'] = 'shared_facet'
fe.parameters["form_compiler"]["optimize"] = True
fe.parameters["form_compiler"]["cpp_optimize"] = True
catch_signals()
comm = MPI.COMM_WORLD
groups = LigandGroups(commandlineArguments)
params0 = ParameterList(default_parameters) # parameter initial values
params0.add(groups.params())
params0.decode(commandlineArguments.params)
groups.fourier_series()
params0.add(groups.params()) # in case Fourier added any new ones
Vgroups = copy.deepcopy(groups)
Vparams = ParameterList(default_parameters) # for evaluating V
Vparams.add(Vgroups.params())
width = params0['width']
nelements = params0['nelements']
dim = params0['dim']
degree = params0['degree']
periodic = commandlineArguments.periodic
nligands = groups.nligands()
rhomax = params0['rhomax']
cushion = params0['cushion']
t0 = params0['t0']
param_funcs = pfuncs(commandlineArguments, t0=t0, params0=params0)
if commandlineArguments.penalties:
rhopen = grhopen = Upen = gUpen = commandlineArguments.penalties
else:
rhopen = Upen = gUpen = 1.0
grhopen = 10.0
rhopen = params0['rhopen'] or rhopen
params0['rhopen'] = rhopen
grhopen = params0['grhopen'] or grhopen
params0['grhopen'] = grhopen
Upen = params0['Upen'] or Upen
params0['Upen'] = Upen
gUpen = params0['gUpen'] or gUpen
params0['gUpen'] = gUpen
logMAIN('list(groups.ligands())', list(groups.ligands()))
maxscale = params0['maxscale']
if (commandlineArguments.showparams):
for n, p, d, h in params0.params():
print('{n}={val} -- {h}'.format(n=n, val=p(), h=h))
return
logMAIN('params0', params0)
def Vfunc(Us, params={}):
Vparams.update(params) # copy params into ligands
return Vgroups.V(Us) # compute V
def Vtophat(rho, params={}):
tanh = ufl.tanh((rho - params['rhomax']) / params['cushion'])
return params['maxscale'] * params['sigma']**2 / 2 * (tanh + 1)
def Vwitch(rho, params={}):
tanh = ufl.tanh((rho - params['rhomax']) / params['cushion'])
return (params['maxscale'] * params['sigma']**2 / 2 * (tanh + 1) *
(rho / params['rhomax']))
Vcap = Vwitch if commandlineArguments.cappotential == 'witch' else Vtophat
def V2(Us, rho, params={}):
return Vfunc(Us, params=params) + Vcap(rho, params=params)
mesh = box_mesh(width=width, dim=dim, nelements=nelements)
if commandlineArguments.randgrid:
rmesh = box_mesh(width=width,
dim=dim,
nelements=commandlineArguments.randgrid)
else:
rmesh = box_mesh(width=width, dim=dim, nelements=nelements // 2)
fe.parameters["form_compiler"]["representation"] = "uflacs"
fe.parameters['linear_algebra_backend'] = 'PETSc'
np.random.seed(commandlineArguments.seed)
murho0 = params0['Nworms'] / (width**dim)
resuming = commandlineArguments.resume or commandlineArguments.restart
if resuming:
if comm.rank == 0:
cpf = KSDGTimeSeries(resuming, 'r')
tlast = cpf.sorted_times()[-1]
if commandlineArguments.resume:
t0 = tlast
else:
t0 = params0['t0']
lastvec = cpf.retrieve_by_time(tlast)
nlastvec = lastvec.size
vectype = lastvec.dtype
else:
t0 = params0['t0']
nlastvec = None
vectype = None
if commandlineArguments.resume:
#
# if resuming, get t0 from the datafile
#
t0 = comm.bcast(t0, root=0)
nlastvec = comm.bcast(nlastvec, root=0)
vectype = comm.bcast(vectype, root=0)
if comm.rank != 0:
lastvec = np.empty(nlastvec, vectype)
logMAIN('t0', t0)
logMAIN('nlastvec', nlastvec)
logMAIN('vectype', vectype)
U0s = [fe.Constant(0.0) for i in range(nligands)]
ksdg = makeKSDGSolver(degree=degree,
mesh=mesh,
width=width,
nelements=nelements,
t0=t0,
parameters=params0,
param_funcs=param_funcs,
periodic=periodic,
ligands=groups,
solver_type=commandlineArguments.solver,
V=V2,
U0=U0s,
rho0=fe.Constant(0.0))
remap = dofremap(ksdg)
lastvec = np.array(lastvec[np.argsort(remap)])
lastvec = comm.bcast(lastvec, root=0)
nlastvec = lastvec.size
vectype = lastvec.dtype
logMAIN('lastvec', lastvec)
logMAIN('type(lastvec)', type(lastvec))
logMAIN('nlastvec', nlastvec)
logMAIN('vectype', vectype)
dofmap = ksdg.sol.function_space().dofmap()
logMAIN('np.array(dofmap.dofs())', np.array(dofmap.dofs()))
logMAIN('lastvec[np.array(dofmap.dofs())]',
lastvec[np.array(dofmap.dofs())])
ksdg.sol.vector()[:] = lastvec[np.array(dofmap.dofs())]
ksdg.sol.vector().apply('insert')
else:
t0 = params0['t0']
Cd1 = fe.FunctionSpace(mesh, 'CG', degree)
logMAIN('Cd1', Cd1)
rho0 = fe.Function(Cd1)
logMAIN('rho0', rho0)
random_function(rho0,
mesh=rmesh,
periodic=periodic,
mu=murho0,
sigma=params0['srho0'])
if params0['rho0']:
rho0Exp = fe.Expression(params0['rho0'], degree=degree)
rho0Expf = fe.interpolate(rho0Exp, Cd1)
rho0.vector()[:] += rho0Expf.vector()
U0s = [fe.Function(Cd1) for i in range(nligands)]
for i, lig in enumerate(groups.ligands()):
U0s[i].vector()[:] = (lig.s / lig.gamma) * rho0.vector()
ksdg = makeKSDGSolver(degree=degree,
mesh=mesh,
width=width,
nelements=nelements,
t0=t0,
parameters=params0,
param_funcs=param_funcs,
periodic=periodic,
ligands=groups,
solver_type=commandlineArguments.solver,
V=V2,
U0=U0s,
rho0=rho0)
logMAIN('ksdg', str(ksdg))
options = PETSc.Options()
options.setValue('ts_max_snes_failures', 100)
ts = implicitTS(ksdg,
t0=t0,
restart=not bool(resuming),
rtol=params0['rtol'],
atol=params0['atol'],
dt=params0['dt'],
tmax=params0['tmax'],
maxsteps=params0['maxsteps'])
logMAIN('ts', str(ts))
ts.setMonitor(ts.printMonitor)
logMAIN('printMonitor set')
if commandlineArguments.save:
with open(commandlineArguments.save + '_options.txt', 'w') as opts:
optd = vars(commandlineArguments)
for k, v in optd.items():
if k != 'petsc' and k != 'fenics' and k != 'params':
print('--%s=%s' % (k, v), file=opts)
print(params0.str(), file=opts)
try:
popts = optd['petsc']
if popts:
print('--petsc', file=opts)
for popt in popts:
print(popt, file=opts)
print('--', file=opts)
except KeyError:
pass
try:
fopts = optd['fenics']
if fopts:
print('--fenics', file=opts)
for fopt in fopts:
print(fopt, file=opts)
print('--', file=opts)
except KeyError:
pass
saveMonitor, closeMonitor = ts.makeSaveMonitor(
prefix=commandlineArguments.save)
ts.setMonitor(saveMonitor)
logMAIN('saveMonitor set')
if commandlineArguments.check:
ts.setMonitor(ts.checkpointMonitor, (),
{'prefix': commandlineArguments.check})
logMAIN('checkpointMonitor set')
try:
logMAIN('calling ts.solve', ts.solve)
ts.solve()
except KeyboardInterrupt as e:
print('KeyboardInterrupt:', str(e))
except Exception as e:
print('Exception:', str(e))
einfo = sys.exc_info()
sys.excepthook(*einfo)
if commandlineArguments.save:
closeMonitor()
logMAIN('saveMonitor closed')
ts.cleanup()
if MPI.COMM_WORLD.rank == 0:
print("SNES failures = ", ts.getSNESFailures())
def setUp(self, pc):
A, P = pc.getOperators()
print A.size
self.Ct = A.getSubMatrix(self.u_is, self.b_is)
self.C = A.getSubMatrix(self.b_is, self.u_is)
self.D = A.getSubMatrix(self.r_is, self.b_is)
self.Bt = A.getSubMatrix(self.u_is, self.p_is)
self.B = A.getSubMatrix(self.p_is, self.u_is)
self.Dt = A.getSubMatrix(self.b_is, self.r_is)
# print self.Ct.view()
#CFC = sp.csr_matrix( (data,(row,column)), shape=(self.W[1].dim(),self.W[1].dim()) )
#print CFC.shape
#CFC = PETSc.Mat().createAIJ(size=CFC.shape,csr=(CFC.indptr, CFC.indices, CFC.data))
#print CFC.size, self.AA.size
#MX = self.AA+self.F
MX = self.F # MO.StoreMatrix(B,"A")
# print FC.todense()
self.kspF.setType('preonly')
self.kspF.getPC().setType('lu')
self.kspF.setFromOptions()
self.kspF.setPCSide(0)
self.kspA.setType('preonly')
self.kspA.getPC().setType('lu')
self.kspA.setFromOptions()
self.kspA.setPCSide(0)
self.kspQ.setType('preonly')
self.kspQ.getPC().setType('lu')
self.kspQ.setFromOptions()
self.kspQ.setPCSide(0)
self.kspScalar.setType('preonly')
self.kspScalar.getPC().setType('lu')
self.kspScalar.setFromOptions()
self.kspScalar.setPCSide(0)
kspMX = PETSc.KSP()
kspMX.create(comm=PETSc.COMM_WORLD)
pcMX = kspMX.getPC()
kspMX.setType('preonly')
pcMX.setType('lu')
kspMX.setOperators(MX, MX)
OptDB = PETSc.Options()
#OptDB["pc_factor_mat_ordering_type"] = "rcm"
#OptDB["pc_factor_mat_solver_package"] = "mumps"
kspMX.setFromOptions()
self.kspMX = kspMX
# self.kspCGScalar.setType('preonly')
# self.kspCGScalar.getPC().setType('lu')
# self.kspCGScalar.setFromOptions()
# self.kspCGScalar.setPCSide(0)
self.kspVector.setType('preonly')
self.kspVector.getPC().setType('lu')
self.kspVector.setFromOptions()
self.kspVector.setPCSide(0)
print "setup"
from dolfin import *
import numpy as np
from petsc4py import PETSc
from petsc4py.PETSc import Mat
from petsc4py.PETSc import Vec
import pybind11
# set petsc options at beginning
petsc_options = PETSc.Options()
# Use petsc4py to define the smoothers
def direct(Ah, bh):
'''LU factorisation. Ah is the matrix, bh is the rhs'''
ksp = PETSc.KSP().create()
ksp.setOptionsPrefix('coarse_')
yh = bh.duplicate()
ksp.setNormType(PETSc.KSP.NormType.NONE)
pc = ksp.getPC()
pc.setOptionsPrefix('coarse_pc_')
petsc_options['coarse_pc_type'] = 'lu'
ksp.setOperators(Ah)
ksp.setFromOptions()
ksp.solve(bh, yh)
return yh
def smoother(Ag, bg, Ng, igg, ksptype, pctype):
'''Smoother for multigrid. Ag, and bg are the LHS and RHS respectively.
Ng is the number of iterations (usually 1), igg is the initial guess
for the solution.
def __init__(self, cfgfile):
'''
Constructor
'''
super().__init__(cfgfile, mode="lu")
OptDB = PETSc.Options()
OptDB.setValue('ksp_monitor', '')
OptDB.setValue('snes_monitor', '')
# OptDB.setValue('log_info', '')
# OptDB.setValue('log_summary', '')
OptDB.setValue('snes_ls', 'basic')
# OptDB.setValue('snes_ls', 'quadratic')
OptDB.setValue('snes_rtol', self.cfg['solver']['petsc_snes_rtol'])
OptDB.setValue('snes_atol', self.cfg['solver']['petsc_snes_atol'])
OptDB.setValue('snes_stol', self.cfg['solver']['petsc_snes_stol'])
OptDB.setValue('snes_max_it',
self.cfg['solver']['petsc_snes_max_iter'])
OptDB.setValue('ksp_rtol', self.cfg['solver']['petsc_ksp_rtol'])
OptDB.setValue('ksp_atol', self.cfg['solver']['petsc_ksp_atol'])
OptDB.setValue('ksp_max_it', self.cfg['solver']['petsc_ksp_max_iter'])
# OptDB.setValue('mat_superlu_dist_matinput', 'DISTRIBUTED')
# OptDB.setValue('mat_superlu_dist_rowperm', 'NATURAL')
OptDB.setValue('mat_superlu_dist_colperm', 'PARMETIS')
OptDB.setValue('mat_superlu_dist_parsymbfact', 1)
# create Jacobian, Function, and linear Matrix objects
self.petsc_solver = PETScSolver(self.da1, self.da4, self.nx, self.ny,
self.ht, self.hx, self.hy)
# initialise linear matrix
self.M = self.da4.createMat()
self.M.setOption(PETSc.Mat.Option.NEW_NONZERO_ALLOCATION_ERR, False)
self.M.setUp()
# initialise Jacobian
self.Jac = self.da4.createMat()
self.Jac.setOption(PETSc.Mat.Option.NEW_NONZERO_ALLOCATION_ERR, False)
self.Jac.setUp()
# initialise matrixfree Jacobian
self.Jmf = PETSc.Mat().createPython(
[self.x.getSizes(), self.b.getSizes()],
context=self.petsc_solver,
comm=PETSc.COMM_WORLD)
self.Jmf.setUp()
# create nonlinear solver
self.snes = PETSc.SNES().create()
self.snes.setFunction(self.petsc_solver.snes_function, self.f)
self.snes.setJacobian(self.updateJacobian, self.Jmf, self.Jac)
self.snes.setFromOptions()
self.snes.getKSP().setType('preonly')
self.snes.getKSP().getPC().setType('lu')
self.snes.getKSP().getPC().setFactorSolverType(self.solver_package)
# update solution history
self.petsc_solver.update_previous(self.x)
def __init__(self, cfgfile):
'''
Constructor
'''
super().__init__(cfgfile, mode="split")
OptDB = PETSc.Options()
# OptDB.setValue('ksp_monitor', '')
# OptDB.setValue('snes_monitor', '')
#
# OptDB.setValue('log_info', '')
# OptDB.setValue('log_summary', '')
OptDB.setValue('ksp_rtol', self.cfg['solver']['petsc_ksp_rtol'])
OptDB.setValue('ksp_atol', self.cfg['solver']['petsc_ksp_atol'])
OptDB.setValue('ksp_max_it', self.cfg['solver']['petsc_ksp_max_iter'])
# OptDB.setValue('ksp_initial_guess_nonzero', 1)
OptDB.setValue('pc_type', 'hypre')
OptDB.setValue('pc_hypre_type', 'boomeramg')
OptDB.setValue('pc_hypre_boomeramg_max_iter', 2)
# OptDB.setValue('pc_hypre_boomeramg_max_levels', 6)
# OptDB.setValue('pc_hypre_boomeramg_tol', 1e-7)
# create DA (dof = 2 for A, P)
self.da2 = PETSc.DA().create(dim=2,
dof=2,
sizes=[self.nx, self.ny],
proc_sizes=[PETSc.DECIDE, PETSc.DECIDE],
boundary_type=('periodic', 'periodic'),
stencil_width=1,
stencil_type='box')
# create solution and RHS vector
self.dx2 = self.da2.createGlobalVec()
self.dy2 = self.da2.createGlobalVec()
self.b = self.da2.createGlobalVec()
self.Ad = self.da1.createGlobalVec()
self.Jd = self.da1.createGlobalVec()
self.Pd = self.da1.createGlobalVec()
self.Od = self.da1.createGlobalVec()
# create Jacobian, Function, and linear Matrix objects
self.petsc_precon = PETScPreconditioner(self.da1, self.da2, self.nx,
self.ny, self.ht, self.hx,
self.hy)
# self.petsc_solver2 = PETScSolverDOF2(self.da1, self.da2, self.nx, self.ny, self.ht, self.hx, self.hy)
self.petsc_solver2 = PETScSolverDOF2(self.da1, self.da2, self.nx,
self.ny, self.ht, self.hx,
self.hy, self.petsc_precon)
self.petsc_solver = PETScSolver(self.da1, self.da4, self.nx, self.ny,
self.ht, self.hx, self.hy)
self.petsc_precon.set_tolerances(
poisson_rtol=self.cfg['solver']['pc_poisson_rtol'],
poisson_atol=self.cfg['solver']['pc_poisson_atol'],
poisson_max_it=self.cfg['solver']['pc_poisson_max_iter'],
parabol_rtol=self.cfg['solver']['pc_parabol_rtol'],
parabol_atol=self.cfg['solver']['pc_parabol_atol'],
parabol_max_it=self.cfg['solver']['pc_parabol_max_iter'],
jacobi_max_it=self.cfg['solver']['pc_jacobi_max_iter'])
# initialise matrixfree Jacobian
self.Jmf = PETSc.Mat().createPython(
[self.b.getSizes(), self.b.getSizes()],
context=self.petsc_solver2,
comm=PETSc.COMM_WORLD)
self.Jmf.setUp()
# create linear solver
self.ksp = PETSc.KSP().create()
self.ksp.setFromOptions()
self.ksp.setOperators(self.Jmf)
self.ksp.setInitialGuessNonzero(True)
self.ksp.setType('fgmres')
self.ksp.getPC().setType('none')
# update solution history
self.petsc_solver.update_previous(self.x)
self.petsc_solver2.update_previous(self.A, self.J, self.P, self.O)
def solve(A, b, u, params, Fspace, SolveType, IterType, OuterTol, InnerTol,
HiptmairMatrices, Hiptmairtol, KSPlinearfluids, Fp, kspF):
if SolveType == "Direct":
ksp = PETSc.KSP()
ksp.create(comm=PETSc.COMM_WORLD)
pc = ksp.getPC()
ksp.setType('preonly')
pc.setType('lu')
OptDB = PETSc.Options()
OptDB['pc_factor_mat_solver_package'] = "pastix"
OptDB['pc_factor_mat_ordering_type'] = "rcm"
ksp.setFromOptions()
scale = b.norm()
b = b / scale
ksp.setOperators(A, A)
del A
ksp.solve(b, u)
# Mits +=dodim
u = u * scale
MO.PrintStr("Number iterations = " + str(ksp.its), 60, "+", "\n\n",
"\n\n")
return u, ksp.its, 0
elif SolveType == "Direct-class":
ksp = PETSc.KSP()
ksp.create(comm=PETSc.COMM_WORLD)
pc = ksp.getPC()
ksp.setType('gmres')
pc.setType('none')
ksp.setFromOptions()
scale = b.norm()
b = b / scale
ksp.setOperators(A, A)
del A
ksp.solve(b, u)
# Mits +=dodim
u = u * scale
MO.PrintStr("Number iterations = " + str(ksp.its), 60, "+", "\n\n",
"\n\n")
return u, ksp.its, 0
else:
# u = b.duplicate()
if IterType == "Full":
ksp = PETSc.KSP()
ksp.create(comm=PETSc.COMM_WORLD)
pc = ksp.getPC()
ksp.setType('fgmres')
pc.setType('python')
pc.setType(PETSc.PC.Type.PYTHON)
# FSpace = [Velocity,Magnetic,Pressure,Lagrange]
reshist = {}
def monitor(ksp, its, fgnorm):
reshist[its] = fgnorm
print its, " OUTER:", fgnorm
# ksp.setMonitor(monitor)
ksp.max_it = 1000
W = Fspace
FFSS = [W.sub(0), W.sub(1), W.sub(2), W.sub(3)]
pc.setPythonContext(
MHDprec.InnerOuterMAGNETICinverse(
FFSS, kspF, KSPlinearfluids[0], KSPlinearfluids[1], Fp,
HiptmairMatrices[3], HiptmairMatrices[4],
HiptmairMatrices[2], HiptmairMatrices[0],
HiptmairMatrices[1], HiptmairMatrices[6], Hiptmairtol))
#OptDB = PETSc.Options()
# OptDB['pc_factor_mat_solver_package'] = "mumps"
# OptDB['pc_factor_mat_ordering_type'] = "rcm"
# ksp.setFromOptions()
scale = b.norm()
b = b / scale
ksp.setOperators(A, A)
del A
ksp.solve(b, u)
# Mits +=dodim
u = u * scale
MO.PrintStr("Number iterations = " + str(ksp.its), 60, "+", "\n\n",
"\n\n")
return u, ksp.its, 0
IS = MO.IndexSet(Fspace, '2by2')
M_is = IS[1]
NS_is = IS[0]
kspNS = PETSc.KSP().create()
kspM = PETSc.KSP().create()
kspNS.setTolerances(OuterTol)
kspNS.setOperators(A.getSubMatrix(NS_is, NS_is))
kspM.setOperators(A.getSubMatrix(M_is, M_is))
# print P.symmetric
if IterType == "MD":
kspNS.setType('gmres')
kspNS.max_it = 500
pcNS = kspNS.getPC()
pcNS.setType(PETSc.PC.Type.PYTHON)
pcNS.setPythonContext(
NSpreconditioner.NSPCD(
MixedFunctionSpace([Fspace.sub(0),
Fspace.sub(1)]), kspF,
KSPlinearfluids[0], KSPlinearfluids[1], Fp))
elif IterType == "CD":
kspNS.setType('minres')
pcNS = kspNS.getPC()
pcNS.setType(PETSc.PC.Type.PYTHON)
Q = KSPlinearfluids[1].getOperators()[0]
Q = 1. / params[2] * Q
KSPlinearfluids[1].setOperators(Q, Q)
pcNS.setPythonContext(
StokesPrecond.MHDApprox(
MixedFunctionSpace([Fspace.sub(0),
Fspace.sub(1)]), kspF,
KSPlinearfluids[1]))
reshist = {}
def monitor(ksp, its, fgnorm):
reshist[its] = fgnorm
print fgnorm
# kspNS.setMonitor(monitor)
uNS = u.getSubVector(NS_is)
bNS = b.getSubVector(NS_is)
# print kspNS.view()
scale = bNS.norm()
bNS = bNS / scale
print bNS.norm()
kspNS.solve(bNS, uNS)
uNS = uNS * scale
NSits = kspNS.its
kspNS.destroy()
# for line in reshist.values():
# print line
kspM.setFromOptions()
kspM.setType(kspM.Type.MINRES)
kspM.setTolerances(InnerTol)
pcM = kspM.getPC()
pcM.setType(PETSc.PC.Type.PYTHON)
pcM.setPythonContext(
MP.Hiptmair(MixedFunctionSpace([Fspace.sub(2),
Fspace.sub(3)]),
HiptmairMatrices[3], HiptmairMatrices[4],
HiptmairMatrices[2], HiptmairMatrices[0],
HiptmairMatrices[1], HiptmairMatrices[6], Hiptmairtol))
uM = u.getSubVector(M_is)
bM = b.getSubVector(M_is)
scale = bM.norm()
bM = bM / scale
print bM.norm()
kspM.solve(bM, uM)
uM = uM * scale
Mits = kspM.its
kspM.destroy()
u = IO.arrayToVec(np.concatenate([uNS.array, uM.array]))
MO.PrintStr("Number of M iterations = " + str(Mits), 60, "+", "\n\n",
"\n\n")
MO.PrintStr("Number of NS/S iterations = " + str(NSits), 60, "+",
"\n\n", "\n\n")
return u, NSits, Mits
def setup_method(self, method):
OptDB = PETSc.Options()
OptDB.setValue("ksp_type", "cg")
OptDB.setValue("pc_type", "gamg")
reload(poisson_3d_p)
def condition_number(A, method='simplified'):
"""
Estimate the condition number of the matrix A
"""
if method == 'simplified':
# Calculate max(abs(A))/min(abs(A))
amin, amax = 1e10, -1e10
for irow in range(A.size(0)):
_indices, values = A.getrow(irow)
aa = abs(values)
amax = max(amax, aa.max())
aa[aa == 0] = amax
amin = min(amin, aa.min())
amin = dolfin.MPI.min(dolfin.MPI.comm_world, float(amin))
amax = dolfin.MPI.max(dolfin.MPI.comm_world, float(amax))
return amax / amin
elif method == 'numpy':
from numpy.linalg import cond
A = mat_to_scipy_csr(A).todense()
return cond(A)
elif method == 'SLEPc':
from petsc4py import PETSc
from slepc4py import SLEPc
# Get the petc4py matrix
PA = dolfin.as_backend_type(A).mat()
# Calculate the largest and smallest singular value
opts = {
'svd_type': 'cross',
'svd_eps_type': 'gd',
# 'help': 'svd_type'
}
with petsc_options(opts):
S = SLEPc.SVD()
S.create()
S.setOperator(PA)
S.setFromOptions()
S.setDimensions(1, PETSc.DEFAULT, PETSc.DEFAULT)
S.setWhichSingularTriplets(SLEPc.SVD.Which.LARGEST)
S.solve()
if S.getConverged() == 1:
sigma_1 = S.getSingularTriplet(0)
else:
raise ValueError(
'Could not find the highest singular value (%d)' %
S.getConvergedReason())
print('Highest singular value:', sigma_1)
S.setWhichSingularTriplets(SLEPc.SVD.Which.SMALLEST)
S.solve()
if S.getConverged() == 1:
sigma_n = S.getSingularTriplet(0)
else:
raise ValueError(
'Could not find the lowest singular value (%d)' %
S.getConvergedReason())
print('Lowest singular value:', sigma_n)
print(PETSc.Options().getAll())
print(PETSc.Options().getAll())
return sigma_1 / sigma_n
def __init__(self, cfgfile):
'''
Constructor
'''
# load run config file
cfg = Config(cfgfile)
# timestep setup
self.ht = cfg['grid']['ht'] # timestep size
self.nt = cfg['grid']['nt'] # number of timesteps
self.nsave = cfg['io']['nsave'] # save only every nsave'th timestep
# grid setup
self.nx = cfg['grid']['nx'] # number of points in x
self.ny = cfg['grid']['ny'] # number of points in y
Lx = cfg['grid']['Lx'] # spatial domain in x
x1 = cfg['grid']['x1'] #
x2 = cfg['grid']['x2'] #
Ly = cfg['grid']['Ly'] # spatial domain in y
y1 = cfg['grid']['y1'] #
y2 = cfg['grid']['y2'] #
if x1 != x2:
Lx = x2 - x1
else:
x1 = 0.0
x2 = Lx
if y1 != y2:
Ly = y2 - y1
else:
y1 = 0.0
y2 = Ly
self.hx = Lx / self.nx # gridstep size in x
self.hy = Ly / self.ny # gridstep size in y
self.time = PETSc.Vec().createMPI(1,
PETSc.DECIDE,
comm=PETSc.COMM_WORLD)
self.time.setName('t')
if PETSc.COMM_WORLD.getRank() == 0:
self.time.setValue(0, 0.0)
OptDB = PETSc.Options()
OptDB.setValue('snes_rtol', cfg['solver']['petsc_snes_rtol'])
OptDB.setValue('snes_atol', cfg['solver']['petsc_snes_atol'])
OptDB.setValue('snes_stol', cfg['solver']['petsc_snes_stol'])
OptDB.setValue('snes_max_it', cfg['solver']['petsc_snes_max_iter'])
OptDB.setValue('ksp_rtol', cfg['solver']['petsc_ksp_rtol'])
OptDB.setValue('ksp_atol', cfg['solver']['petsc_ksp_atol'])
OptDB.setValue('ksp_max_it', cfg['solver']['petsc_ksp_max_iter'])
# OptDB.setValue('ksp_monitor', '')
# OptDB.setValue('snes_monitor', '')
#
# OptDB.setValue('log_info', '')
# OptDB.setValue('log_summary', '')
self.snes_rtol = cfg['solver']['petsc_snes_rtol']
self.snes_atol = cfg['solver']['petsc_snes_atol']
self.snes_max_iter = cfg['solver']['petsc_snes_max_iter']
# create DA with single dof
self.da1 = PETSc.DA().create(dim=2,
dof=1,
sizes=[self.nx, self.ny],
proc_sizes=[PETSc.DECIDE, PETSc.DECIDE],
boundary_type=('periodic', 'periodic'),
stencil_width=2,
stencil_type='box')
# create DA (dof = 4 for A, J, P, O)
self.da4 = PETSc.DA().create(dim=2,
dof=4,
sizes=[self.nx, self.ny],
proc_sizes=[PETSc.DECIDE, PETSc.DECIDE],
boundary_type=('periodic', 'periodic'),
stencil_width=2,
stencil_type='box')
# create DA for x grid
self.dax = PETSc.DA().create(dim=1,
dof=1,
sizes=[self.nx],
proc_sizes=[PETSc.DECIDE],
boundary_type=('periodic'))
# create DA for y grid
self.day = PETSc.DA().create(dim=1,
dof=1,
sizes=[self.ny],
proc_sizes=[PETSc.DECIDE],
boundary_type=('periodic'))
# initialise grid
self.da1.setUniformCoordinates(xmin=x1, xmax=x2, ymin=y1, ymax=y2)
self.da4.setUniformCoordinates(xmin=x1, xmax=x2, ymin=y1, ymax=y2)
self.dax.setUniformCoordinates(xmin=x1, xmax=x2)
self.day.setUniformCoordinates(xmin=y1, xmax=y2)
# create RHS vector
self.Ab = self.da1.createGlobalVec()
self.Ob = self.da1.createGlobalVec()
self.Pb = self.da1.createGlobalVec()
# create vectors for magnetic and velocity field
self.A = self.da1.createGlobalVec() # magnetic vector potential A
self.J = self.da1.createGlobalVec() # current density J
self.P = self.da1.createGlobalVec() # streaming function psi
self.O = self.da1.createGlobalVec() # vorticity omega
self.Bx = self.da1.createGlobalVec()
self.By = self.da1.createGlobalVec()
self.Vx = self.da1.createGlobalVec()
self.Vy = self.da1.createGlobalVec()
# set variable names
self.A.setName('A')
self.J.setName('J')
self.P.setName('P')
self.O.setName('O')
self.Bx.setName('Bx')
self.By.setName('By')
self.Vx.setName('Vx')
self.Vy.setName('Vy')
# create nullspace
self.poisson_nullspace = PETSc.NullSpace().create(constant=True)
# create jacobian and matrix objects
self.petsc_vorticity = PETScVorticity(self.da1, self.nx, self.ny,
self.ht, self.hx, self.hy)
self.petsc_ohmslaw = PETScOhmsLaw(self.da1, self.nx, self.ny, self.ht,
self.hx, self.hy)
self.petsc_poisson = PETScPoisson(self.da1, self.nx, self.ny, self.hx,
self.hy)
# initialise vorticity matrix
self.vorticity_matrix = self.da1.createMat()
self.vorticity_matrix.setOption(
self.vorticity_matrix.Option.NEW_NONZERO_ALLOCATION_ERR, False)
self.vorticity_matrix.setUp()
# initialise Ohms's law matrix
self.ohmslaw_matrix = self.da1.createMat()
self.ohmslaw_matrix.setOption(
self.ohmslaw_matrix.Option.NEW_NONZERO_ALLOCATION_ERR, False)
self.ohmslaw_matrix.setUp()
# initialise Poisson matrix
self.poisson_matrix = self.da1.createMat()
self.poisson_matrix.setOption(
self.poisson_matrix.Option.NEW_NONZERO_ALLOCATION_ERR, False)
self.poisson_matrix.setUp()
self.poisson_matrix.setNullSpace(self.poisson_nullspace)
# create nonlinear vorticity solver
self.vorticity_snes = PETSc.SNES().create()
self.vorticity_snes.setType('ksponly')
self.vorticity_snes.setFunction(self.petsc_vorticity.snes_mult,
self.Ob)
self.vorticity_snes.setJacobian(self.update_vorticity_jacobian,
self.vorticity_matrix)
self.vorticity_snes.setFromOptions()
# self.vorticity_snes.getKSP().setType('gmres')
self.vorticity_snes.getKSP().setType('preonly')
self.vorticity_snes.getKSP().getPC().setType('lu')
self.vorticity_snes.getKSP().getPC().setFactorSolverPackage(
solver_package)
# create nonlinear Ohms's law solver
self.ohmslaw_snes = PETSc.SNES().create()
self.ohmslaw_snes.setType('ksponly')
self.ohmslaw_snes.setFunction(self.petsc_ohmslaw.snes_mult, self.Ab)
self.ohmslaw_snes.setJacobian(self.update_ohmslaw_jacobian,
self.ohmslaw_matrix)
self.ohmslaw_snes.setFromOptions()
# self.ohmslaw_snes.getKSP().setType('gmres')
self.ohmslaw_snes.getKSP().setType('preonly')
self.ohmslaw_snes.getKSP().getPC().setType('lu')
self.ohmslaw_snes.getKSP().getPC().setFactorSolverPackage(
solver_package)
# create linear Poisson solver
self.poisson_ksp = PETSc.KSP().create()
self.poisson_ksp.setFromOptions()
self.poisson_ksp.setOperators(self.poisson_matrix)
self.poisson_ksp.setType('preonly')
self.poisson_ksp.getPC().setType('lu')
self.poisson_ksp.getPC().setFactorSolverPackage(solver_package)
# self.poisson_ksp.setNullSpace(self.poisson_nullspace)
self.petsc_poisson.formMat(self.poisson_matrix)
# create derivatives object
self.derivatives = PETScDerivatives(self.da1, self.nx, self.ny,
self.ht, self.hx, self.hy)
# get coordinate vectors
coords_x = self.dax.getCoordinates()
coords_y = self.day.getCoordinates()
# save x coordinate arrays
scatter, xVec = PETSc.Scatter.toAll(coords_x)
scatter.begin(coords_x, xVec, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
scatter.end(coords_x, xVec, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
xGrid = xVec.getValues(range(0, self.nx)).copy()
scatter.destroy()
xVec.destroy()
# save y coordinate arrays
scatter, yVec = PETSc.Scatter.toAll(coords_y)
scatter.begin(coords_y, yVec, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
scatter.end(coords_y, yVec, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
yGrid = yVec.getValues(range(0, self.ny)).copy()
scatter.destroy()
yVec.destroy()
# set initial data
(xs, xe), (ys, ye) = self.da1.getRanges()
A_arr = self.da1.getVecArray(self.A)
P_arr = self.da1.getVecArray(self.P)
init_data = __import__("runs." + cfg['initial_data']['python'],
globals(), locals(),
['magnetic_A', 'velocity_P'], 0)
for i in range(xs, xe):
for j in range(ys, ye):
A_arr[i, j] = init_data.magnetic_A(xGrid[i], yGrid[j], Lx, Ly)
P_arr[i, j] = init_data.velocity_P(xGrid[i], yGrid[j], Lx, Ly)
# Fourier Filtering
self.nfourier = cfg['initial_data']['nfourier']
if self.nfourier >= 0:
# obtain whole A vector everywhere
scatter, Aglobal = PETSc.Scatter.toAll(self.A)
scatter.begin(self.A, Aglobal, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
scatter.end(self.A, Aglobal, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
petsc_indices = self.da1.getAO().app2petsc(
np.arange(self.nx * self.ny, dtype=np.int32))
Ainit = Aglobal.getValues(petsc_indices).copy().reshape(
(self.ny, self.nx))
scatter.destroy()
Aglobal.destroy()
# compute FFT, cut, compute inverse FFT
from scipy.fftpack import rfft, irfft
Afft = rfft(Ainit, axis=1)
Afft[:, 0] = 0.
Afft[:, self.nfourier + 1:] = 0.
A_arr = self.da1.getVecArray(self.A)
A_arr[:, :] = irfft(Afft).T[xs:xe, ys:ye]
# compute current and vorticity
self.derivatives.laplace_vec(self.A, self.J, -1.)
self.derivatives.laplace_vec(self.P, self.O, -1.)
J_arr = self.da1.getVecArray(self.J)
O_arr = self.da1.getVecArray(self.O)
# add perturbations
for i in range(xs, xe):
for j in range(ys, ye):
J_arr[i, j] += init_data.current_perturbation(
xGrid[i], yGrid[j], Lx, Ly)
O_arr[i, j] += init_data.vorticity_perturbation(
xGrid[i], yGrid[j], Lx, Ly)
# create HDF5 output file
self.hdf5_viewer = PETSc.ViewerHDF5().create(
cfg['io']['hdf5_output'],
mode=PETSc.Viewer.Mode.WRITE,
comm=PETSc.COMM_WORLD)
self.hdf5_viewer.pushGroup("/")
# write grid data to hdf5 file
coords_x = self.dax.getCoordinates()
coords_y = self.day.getCoordinates()
coords_x.setName('x')
coords_y.setName('y')
self.hdf5_viewer(coords_x)
self.hdf5_viewer(coords_y)
# write initial data to hdf5 file
self.save_to_hdf5(0)
def __init__(self, cfgfile):
'''
Constructor
'''
# stencil = 1
stencil = 2
# load run config file
cfg = Config(cfgfile)
# set some PETSc options
OptDB = PETSc.Options()
# OptDB.setValue('snes_lag_preconditioner', 5)
self.residual = cfg['solver']['petsc_residual']
# OptDB.setValue('ksp_max_it', 2)
# OptDB.setValue('ksp_convergence_test', 'skip')
# OptDB.setValue('ksp_monitor', '')
OptDB.setValue('snes_monitor', '')
# timestep setup
self.ht = cfg['grid']['ht'] # timestep size
self.nt = cfg['grid']['nt'] # number of timesteps
self.nsave = cfg['io']['nsave'] # save only every nsave'th timestep
# grid setup
nx = cfg['grid']['nx'] # number of points in x
ny = cfg['grid']['ny'] # number of points in y
Lx = cfg['grid']['Lx'] # spatial domain in x
x1 = cfg['grid']['x1'] #
x2 = cfg['grid']['x2'] #
Ly = cfg['grid']['Ly'] # spatial domain in y
y1 = cfg['grid']['y1'] #
y2 = cfg['grid']['y2'] #
self.nx = nx
self.ny = ny
if x1 != x2:
Lx = x2-x1
else:
x1 = 0.0
x2 = Lx
if y1 != y2:
Ly = y2-y1
else:
y1 = 0.0
y2 = Ly
self.hx = Lx / nx # gridstep size in x
self.hy = Ly / ny # gridstep size in y
if PETSc.COMM_WORLD.getRank() == 0:
print()
print("nt = %i" % (self.nt))
print("nx = %i" % (self.nx))
print("ny = %i" % (self.ny))
print()
print("ht = %e" % (self.ht))
print("hx = %e" % (self.hx))
print("hy = %e" % (self.hy))
print()
print("Lx = %e" % (Lx))
print("Ly = %e" % (Ly))
print()
self.time = PETSc.Vec().createMPI(1, PETSc.DECIDE, comm=PETSc.COMM_WORLD)
self.time.setName('t')
if PETSc.COMM_WORLD.getRank() == 0:
self.time.setValue(0, 0.0)
# create DA with single dof
self.da1 = PETSc.DA().create(dim=2, dof=1,
sizes=[nx, ny],
proc_sizes=[PETSc.DECIDE, PETSc.DECIDE],
boundary_type=('periodic', 'periodic'),
stencil_width=stencil,
stencil_type='box')
# create DA (dof = 4 for Bx, By, Vx, Vy)
self.da4 = PETSc.DA().create(dim=2, dof=4,
sizes=[nx, ny],
proc_sizes=[PETSc.DECIDE, PETSc.DECIDE],
boundary_type=('periodic', 'periodic'),
stencil_width=stencil,
stencil_type='box')
# create DA (dof = 5 for Bx, By, Vx, Vy, P)
self.da5 = PETSc.DA().create(dim=2, dof=5,
sizes=[nx, ny],
proc_sizes=[PETSc.DECIDE, PETSc.DECIDE],
boundary_type=('periodic', 'periodic'),
stencil_width=stencil,
stencil_type='box')
# create DA for x grid
self.dax = PETSc.DA().create(dim=1, dof=1,
sizes=[nx],
proc_sizes=[PETSc.DECIDE],
boundary_type=('periodic'))
# create DA for y grid
self.day = PETSc.DA().create(dim=1, dof=1,
sizes=[ny],
proc_sizes=[PETSc.DECIDE],
boundary_type=('periodic'))
# initialise grid
self.da1.setUniformCoordinates(xmin=x1, xmax=x2,
ymin=y1, ymax=y2)
self.da4.setUniformCoordinates(xmin=x1, xmax=x2,
ymin=y1, ymax=y2)
self.da5.setUniformCoordinates(xmin=x1, xmax=x2,
ymin=y1, ymax=y2)
self.dax.setUniformCoordinates(xmin=x1, xmax=x2)
self.day.setUniformCoordinates(xmin=y1, xmax=y2)
# create solution and RHS vector
self.f = self.da5.createGlobalVec()
self.x = self.da5.createGlobalVec()
self.b = self.da5.createGlobalVec()
# create global RK4 vectors
self.Y = self.da5.createGlobalVec()
self.X0 = self.da5.createGlobalVec()
self.X1 = self.da5.createGlobalVec()
self.X2 = self.da5.createGlobalVec()
self.X3 = self.da5.createGlobalVec()
self.X4 = self.da5.createGlobalVec()
# create local RK4 vectors
self.localX0 = self.da5.createLocalVec()
self.localX1 = self.da5.createLocalVec()
self.localX2 = self.da5.createLocalVec()
self.localX3 = self.da5.createLocalVec()
self.localX4 = self.da5.createLocalVec()
# self.localP = self.da1.createLocalVec()
# create vectors for magnetic and velocity field
self.Bx = self.da1.createGlobalVec()
self.By = self.da1.createGlobalVec()
self.Vx = self.da1.createGlobalVec()
self.Vy = self.da1.createGlobalVec()
self.P = self.da1.createGlobalVec()
# create local vectors for initialisation of pressure
self.localBx = self.da1.createLocalVec()
self.localBy = self.da1.createLocalVec()
self.localVx = self.da1.createLocalVec()
self.localVy = self.da1.createLocalVec()
# set variable names
self.Bx.setName('Bx')
self.By.setName('By')
self.Vx.setName('Vx')
self.Vy.setName('Vy')
self.P.setName('P')
# create Matrix object
self.petsc_matrix = PETScMatrix (self.da1, self.da5, nx, ny, self.ht, self.hx, self.hy)
self.petsc_jacobian = PETScJacobian(self.da1, self.da5, nx, ny, self.ht, self.hx, self.hy)
self.petsc_function = PETScFunction(self.da1, self.da5, nx, ny, self.ht, self.hx, self.hy)
# self.petsc_jacobian_4d = PETSc_MHD_NL_Jacobian_Matrix.PETScJacobian(self.da1, self.da5, nx, ny, self.ht, self.hx, self.hy)
# initialise matrix
self.A = self.da5.createMat()
self.A.setOption(self.A.Option.NEW_NONZERO_ALLOCATION_ERR, False)
self.A.setUp()
# create jacobian
self.J = self.da5.createMat()
self.J.setOption(self.J.Option.NEW_NONZERO_ALLOCATION_ERR, False)
self.J.setUp()
# create nonlinear solver
self.snes = PETSc.SNES().create()
self.snes.setType('ksponly')
self.snes.setFunction(self.petsc_function.snes_mult, self.f)
self.snes.setJacobian(self.updateJacobian, self.J)
self.snes.setFromOptions()
self.snes.getKSP().setType('preonly')
# self.snes.getKSP().setType('gmres')
self.snes.getKSP().getPC().setType('lu')
# self.snes.getKSP().getPC().setFactorSolverPackage('superlu_dist')
self.snes.getKSP().getPC().setFactorSolverPackage('mumps')
self.ksp = None
# set initial data
(xs, xe), (ys, ye) = self.da1.getRanges()
coords = self.da1.getCoordinatesLocal()
Bx_arr = self.da1.getVecArray(self.Bx)
By_arr = self.da1.getVecArray(self.By)
Vx_arr = self.da1.getVecArray(self.Vx)
Vy_arr = self.da1.getVecArray(self.Vy)
if cfg['initial_data']['magnetic_python'] != None:
init_data = __import__("runs." + cfg['initial_data']['magnetic_python'], globals(), locals(), ['magnetic_x', 'magnetic_y'], 0)
for i in range(xs, xe):
for j in range(ys, ye):
Bx_arr[i,j] = init_data.magnetic_x(coords[i,j][0], coords[i,j][1] + 0.5 * self.hy, Lx, Ly)
By_arr[i,j] = init_data.magnetic_y(coords[i,j][0] + 0.5 * self.hx, coords[i,j][1], Lx, Ly)
else:
Bx_arr[xs:xe, ys:ye] = cfg['initial_data']['magnetic']
By_arr[xs:xe, ys:ye] = cfg['initial_data']['magnetic']
if cfg['initial_data']['velocity_python'] != None:
init_data = __import__("runs." + cfg['initial_data']['velocity_python'], globals(), locals(), ['velocity_x', 'velocity_y'], 0)
for i in range(xs, xe):
for j in range(ys, ye):
Vx_arr[i,j] = init_data.velocity_x(coords[i,j][0], coords[i,j][1] + 0.5 * self.hy, Lx, Ly)
Vy_arr[i,j] = init_data.velocity_y(coords[i,j][0] + 0.5 * self.hx, coords[i,j][1], Lx, Ly)
else:
Vx_arr[xs:xe, ys:ye] = cfg['initial_data']['velocity']
Vy_arr[xs:xe, ys:ye] = cfg['initial_data']['velocity']
if cfg['initial_data']['pressure_python'] != None:
init_data = __import__("runs." + cfg['initial_data']['pressure_python'], globals(), locals(), ['pressure', ''], 0)
x_arr = self.da5.getVecArray(self.x)
x_arr[xs:xe, ys:ye, 0] = Vx_arr[xs:xe, ys:ye]
x_arr[xs:xe, ys:ye, 1] = Vy_arr[xs:xe, ys:ye]
x_arr[xs:xe, ys:ye, 2] = Bx_arr[xs:xe, ys:ye]
x_arr[xs:xe, ys:ye, 3] = By_arr[xs:xe, ys:ye]
self.da1.globalToLocal(self.Bx, self.localBx)
self.da1.globalToLocal(self.By, self.localBy)
self.da1.globalToLocal(self.Vx, self.localVx)
self.da1.globalToLocal(self.Vy, self.localVy)
Bx_arr = self.da1.getVecArray(self.localBx)
By_arr = self.da1.getVecArray(self.localBy)
Vx_arr = self.da1.getVecArray(self.localVx)
Vy_arr = self.da1.getVecArray(self.localVy)
P_arr = self.da1.getVecArray(self.P)
for i in range(xs, xe):
for j in range(ys, ye):
P_arr[i,j] = init_data.pressure(coords[i,j][0] + 0.5 * self.hx, coords[i,j][1] + 0.5 * self.hy, Lx, Ly)
# P_arr[i,j] = init_data.pressure(coords[i,j][0] + 0.5 * self.hx, coords[i,j][1] + 0.5 * self.hy, Lx, Ly) \
# + 0.5 * (0.25 * (Bx_arr[i,j] + Bx_arr[i+1,j])**2 + 0.25 * (By_arr[i,j] + By_arr[i,j+1])**2)
# P_arr[i,j] = init_data.pressure(coords[i,j][0] + 0.5 * self.hx, coords[i,j][1] + 0.5 * self.hy, Lx, Ly) \
# + 0.5 * (0.25 * (Vx_arr[i,j] + Vx_arr[i+1,j])**2 + 0.25 * (Vy_arr[i,j] + Vy_arr[i,j+1])**2)
# P_arr[i,j] = init_data.pressure(coords[i,j][0] + 0.5 * self.hx, coords[i,j][1] + 0.5 * self.hy, Lx, Ly) \
# + 0.5 * (0.25 * (Vx_arr[i,j] + Vx_arr[i+1,j])**2 + 0.25 * (Vy_arr[i,j] + Vy_arr[i,j+1])**2) \
# - 1.0 * (0.25 * (Bx_arr[i,j] + Bx_arr[i+1,j])**2 + 0.25 * (By_arr[i,j] + By_arr[i,j+1])**2)
# copy distribution function to solution vector
x_arr = self.da5.getVecArray(self.x)
x_arr[xs:xe, ys:ye, 4] = P_arr [xs:xe, ys:ye]
# update solution history
# self.petsc_matrix.update_history(self.x)
self.petsc_jacobian.update_history(self.x)
self.petsc_function.update_history(self.x)
# create HDF5 output file
self.hdf5_viewer = PETSc.ViewerHDF5().create(cfg['io']['hdf5_output'],
mode=PETSc.Viewer.Mode.WRITE,
comm=PETSc.COMM_WORLD)
self.hdf5_viewer.pushGroup("/")
# write grid data to hdf5 file
coords_x = self.dax.getCoordinates()
coords_y = self.day.getCoordinates()
coords_x.setName('x')
coords_y.setName('y')
self.hdf5_viewer(coords_x)
self.hdf5_viewer(coords_y)
# write initial data to hdf5 file
self.hdf5_viewer.setTimestep(0)
self.save_hdf5_vectors()
def __init__(self, cfgfile, runid=None, cfg=None):
'''
Constructor
'''
assert cfgfile is not None
assert cfgfile is not ""
# if runid is empty use timestamp
if runid == None or runid == "":
runid = datetime.datetime.fromtimestamp(
time.time()).strftime("%y%m%d%H%M%S")
# stencil width
stencil = 2
# load run config file
if cfg != None:
self.cfg = cfg
elif cfgfile != None and len(cfgfile) > 0:
self.cfg = Config(cfgfile)
else:
if PETSc.COMM_WORLD.getRank() == 0:
print("ERROR: No valid config file or object passed.")
sys.exit()
# determine solver modules
if cfg['solver']['method'] == 'explicit':
self.vlasov_module = None
else:
self.vlasov_module = "vlasov.solvers."
if cfg['solver']['mode'] == 'split':
self.vlasov_module += 'vlasov'
else:
self.vlasov_module += self.cfg['solver']['mode']
self.vlasov_module += '.' + "PETSc"
if cfg['solver']['type'] == 'newton' or cfg['solver'][
'type'] == 'nonlinear':
self.vlasov_module += "NL"
if cfg['solver']['mode'] == 'split':
self.vlasov_module += "Vlasov"
# self.vlasov_module += self.cfg['solver']['poisson_bracket']
# if cfg['solver']['preconditioner_type'] != None and cfg['solver']['preconditioner_scheme'] != None:
# self.vlasov_module += self.cfg['solver']['preconditioner_scheme']
self.vlasov_module += self.cfg['solver']['timestepping'].upper()
if not cfg.is_dissipation_none:
if cfg['solver']['dissipation'] == 'double_bracket':
self.vlasov_module += "DB"
self.poisson_module = "vlasov.solvers.poisson.PETScPoisson"
self.poisson_module += self.cfg['solver']['laplace_operator']
# importing solver modules
if PETSc.COMM_WORLD.getRank() == 0:
print("Loading Vlasov solver %s" % (self.vlasov_module))
print("Loading Poisson solver %s" % (self.poisson_module))
print("")
if not self.cfg.is_preconditioner_none():
print("Using Preconditioner %s" %
(self.cfg.get_preconditioner()))
if not self.cfg.is_dissipation_none():
if self.cfg.is_dissipation_collisions():
print("Using Collision Operator %s" %
(self.cfg.get_collision_operator()))
if self.cfg.is_dissipation_double_bracket():
print("Using Double Bracket %s" %
(self.cfg.get_double_bracket()))
print("")
self.vlasov_object = __import__(self.vlasov_module, globals(),
locals(), ['PETScVlasovSolver'], 0)
self.poisson_object = __import__(self.poisson_module, globals(),
locals(), ['PETScPoissonSolver'], 0)
# timestep setup
ht = self.cfg['grid']['ht'] # timestep size
nt = self.cfg['grid']['nt'] # number of timesteps
self.nsave = self.cfg['io'][
'nsave'] # save only every nsave'th timestep
# grid setup
nx = self.cfg['grid']['nx'] # number of points in x
nv = self.cfg['grid']['nv'] # number of points in v
L = self.cfg['grid']['L']
vMax = self.cfg['grid']['vmax']
vMin = self.cfg['grid']['vmin']
if vMin == None:
vMin = -vMax
Lx = L
Lv = vMax - vMin
hx = Lx / nx # gridstep size in x
hv = Lv / (nv - 1) # gridstep size in v
self.time = PETSc.Vec().createMPI(1, 1, comm=PETSc.COMM_WORLD)
self.time.setName('t')
if PETSc.COMM_WORLD.getRank() == 0:
self.time.setValue(0, 0.0)
self.solver_package = self.cfg['solver']['lu_package']
self.nInitial = self.cfg['solver'][
'initial_iter'] # number of iterations for initial guess
self.coll_freq = self.cfg['solver']['coll_freq'] # collision frequency
self.coll_drag = self.cfg['solver']['coll_drag'] # drag factor
self.coll_diff = self.cfg['solver']['coll_diff'] # diff factor
self.charge = self.cfg['initial_data']['charge'] # particle charge
self.mass = self.cfg['initial_data']['mass'] # particle mass
output_directory = self.cfg['io']['output_dir']
if output_directory == None or output_directory == "":
output_directory = "."
tindex = cfgfile.rfind('/')
run_filename = cfgfile[tindex:].replace('.cfg', '.') + runid
hdf_out_filename = output_directory + '/' + run_filename + ".hdf5"
cfg_out_filename = output_directory + '/' + run_filename + ".cfg"
# hdf_in_filename = self.cfg['io']['hdf5_input']
# hdf_out_filename = self.cfg['io']['hdf5_output']
self.cfg.write_current_config(cfg_out_filename)
# set initial guess method
initial_guess_options = {
None: self.initial_guess_none,
"None": self.initial_guess_none,
"": self.initial_guess_none,
"rk4": self.initial_guess_rk4,
"gear": self.initial_guess_gear,
"symplectic2": self.initial_guess_symplectic2,
"symplectic4": self.initial_guess_symplectic4,
}
self.initial_guess_method = initial_guess_options[self.cfg['solver']
['initial_guess']]
# set some PETSc options
OptDB = PETSc.Options()
OptDB.setValue('ksp_rtol', self.cfg['solver']['petsc_ksp_rtol'])
OptDB.setValue('ksp_atol', self.cfg['solver']['petsc_ksp_atol'])
OptDB.setValue('ksp_max_it', self.cfg['solver']['petsc_ksp_max_iter'])
OptDB.setValue('snes_rtol', self.cfg['solver']['petsc_snes_rtol'])
OptDB.setValue('snes_atol', self.cfg['solver']['petsc_snes_atol'])
OptDB.setValue('snes_stol', self.cfg['solver']['petsc_snes_stol'])
OptDB.setValue('snes_max_it',
self.cfg['solver']['petsc_snes_max_iter'])
if PETSc.COMM_WORLD.getRank() == 0:
print("Initialising Distributed Arrays.")
# create DA for 2d grid (f only)
# self.da1 = PETSc.DMDA().create(dim=2, dof=1,
# sizes=[nx, nv],
# proc_sizes=[1, PETSc.COMM_WORLD.getSize()],
# boundary_type=['periodic', 'periodic'],
# stencil_width=stencil,
# stencil_type='box')
# self.da1 = PETSc.DMDA().create(dim=2, dof=1,
# sizes=[nx, nv],
# proc_sizes=[PETSc.COMM_WORLD.getSize(), 1],
# boundary_type=['periodic', 'ghosted'],
# stencil_width=stencil,
# stencil_type='box')
self.da1 = PETSc.DMDA().create(
dim=2,
dof=1,
sizes=[nx, nv],
proc_sizes=[1, PETSc.COMM_WORLD.getSize()],
boundary_type=['periodic', 'ghosted'],
stencil_width=stencil,
stencil_type='box')
# self.da1 = PETSc.DMDA().create(dim=2, dof=1,
# sizes=[nx, nv],
# proc_sizes=[PETSc.DECIDE, 2],
# boundary_type=['periodic', 'ghosted'],
# stencil_width=stencil,
# stencil_type='box')
# self.da1 = PETSc.DMDA().create(dim=2, dof=1,
# sizes=[nx, nv],
# proc_sizes=[PETSc.DECIDE, PETSc.DECIDE],
# boundary_type=['periodic', 'ghosted'],
# stencil_width=stencil,
# stencil_type='box')
# create VIDA for x grid
self.dax = PETSc.DMDA().create(dim=1,
dof=1,
sizes=[nx],
proc_sizes=[PETSc.COMM_WORLD.getSize()],
boundary_type=('periodic'),
stencil_width=stencil,
stencil_type='box')
# create VIDA for y grid
self.day = PETSc.DMDA().create(dim=1,
dof=1,
sizes=[nv],
proc_sizes=[PETSc.COMM_WORLD.getSize()],
boundary_type=('ghosted'),
stencil_width=stencil,
stencil_type='box')
# initialise grid
self.da1.setUniformCoordinates(xmin=0.0, xmax=Lx, ymin=vMin, ymax=vMax)
self.dax.setUniformCoordinates(xmin=0.0, xmax=Lx)
self.day.setUniformCoordinates(xmin=vMin, xmax=vMax)
# get local index ranges
(xs, xe), (ys, ye) = self.da1.getRanges()
(xsx, xex), = self.dax.getRanges()
# get coordinate vectors
coords_x = self.dax.getCoordinates()
coords_v = self.day.getCoordinates()
# save x coordinate arrays
scatter, xVec = PETSc.Scatter.toAll(coords_x)
scatter.begin(coords_x, xVec, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
scatter.end(coords_x, xVec, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
xGrid = xVec.getValues(range(nx)).copy()
scatter.destroy()
xVec.destroy()
# save v coordinate arrays
scatter, vVec = PETSc.Scatter.toAll(coords_v)
scatter.begin(coords_v, vVec, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
scatter.end(coords_v, vVec, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
vGrid = vVec.getValues(range(nv)).copy()
scatter.destroy()
vVec.destroy()
# create grid object
self.grid = Grid().create(xGrid, vGrid, nt, nx, nv, ht, hx, hv,
stencil)
# create vectors for Hamiltonians
self.h0 = self.da1.createGlobalVec() # kinetic Hamiltonian
self.h1c = self.da1.createGlobalVec() # current potential Hamiltonian
self.h2c = self.da1.createGlobalVec() # current external Hamiltonian
self.h1h = self.da1.createGlobalVec() # previous potential Hamiltonian
self.h2h = self.da1.createGlobalVec() # previous external Hamiltonian
# distribution functions
self.fl = self.da1.createGlobalVec() # last (k+1, n )
self.fc = self.da1.createGlobalVec() # current (k+1, n+1)
self.fh = self.da1.createGlobalVec() # history (k)
# distribution function solver vectors
self.fb = self.da1.createGlobalVec() # right hand side
# moments
self.N = self.dax.createGlobalVec() # density
self.U = self.dax.createGlobalVec() # velocity density
self.E = self.dax.createGlobalVec() # energy density
self.A = self.dax.createGlobalVec() # collision factor
# local moments
self.nc = PETSc.Vec().createSeq(nx) # current density
self.uc = PETSc.Vec().createSeq(nx) # current velocity density
self.ec = PETSc.Vec().createSeq(nx) # current energy density
self.ac = PETSc.Vec().createSeq(nx) # current collision factor
self.nh = PETSc.Vec().createSeq(nx) # history density
self.uh = PETSc.Vec().createSeq(nx) # history velocity density
self.eh = PETSc.Vec().createSeq(nx) # history energy density
self.ah = PETSc.Vec().createSeq(nx) # history collision factor
# internal potential
self.pc_int = self.dax.createGlobalVec() # current
self.ph_int = self.dax.createGlobalVec() # history
# external potential
self.pc_ext = self.dax.createGlobalVec() # current
self.ph_ext = self.dax.createGlobalVec() # history
# potential solver vectors
self.pb = self.dax.createGlobalVec() # right hand side
self.pn = self.dax.createGlobalVec() # null vector
# set variable names
self.h0.setName('h0')
self.h1c.setName('h1')
self.h2c.setName('h2')
self.fc.setName('f')
self.pc_int.setName('phi_int')
self.pc_ext.setName('phi_ext')
self.N.setName('n')
self.U.setName('u')
self.E.setName('e')
# initialise nullspace basis vectors
# the Poisson equation has a null space of all constant vectors
# that needs to be removed to avoid jumpy potentials
self.pn.set(1.)
self.pn.normalize()
self.p_nullspace = PETSc.NullSpace().create(constant=False,
vectors=(self.pn, ))
# create Toolbox
self.toolbox = Toolbox(self.da1, self.dax, self.grid)
# initialise kinetic hamiltonian
if PETSc.COMM_WORLD.getRank() == 0:
print("Initialising kinetic Hamiltonian.")
self.toolbox.initialise_kinetic_hamiltonian(self.h0, self.mass)
# create Arakawa initial guess solver object
if PETSc.COMM_WORLD.getRank() == 0:
print("Instantiating Initial Guess Objects.")
self.arakawa_rk4 = PETScArakawaRungeKutta(self.cfg, self.da1,
self.grid, self.h0, self.h1h,
self.h2h, self.nInitial)
self.arakawa_gear = PETScArakawaGear(self.cfg, self.da1, self.grid,
self.h0, self.h1h, self.h2h,
self.nInitial)
self.arakawa_symplectic = PETScArakawaSymplectic(
self.cfg, self.da1, self.grid, self.h0, self.h1h, self.h2h,
self.nInitial)
# create solver dummies
self.vlasov_solver = None
self.poisson_solver = None
self.poisson_ksp = None
if PETSc.COMM_WORLD.getRank() == 0:
print()
print("Run ID: %s" % runid)
print()
print("Config File: %s" % cfgfile)
print("Output File: %s" % hdf_out_filename)
print()
print("nt = %i" % (self.grid.nt))
print("nx = %i" % (self.grid.nx))
print("nv = %i" % (self.grid.nv))
print()
print("ht = %e" % (self.grid.ht))
print("hx = %e" % (self.grid.hx))
print("hv = %e" % (self.grid.hv))
print()
print("xMin = %+12.6e" % (self.grid.xMin()))
print("xMax = %+12.6e" % (self.grid.vMax()))
print("vMin = %+12.6e" % (self.grid.vMin()))
print("vMax = %+12.6e" % (self.grid.vMax()))
print()
print("nu = %7.1e" % (self.coll_freq))
print()
print("CFL = %e" % (self.grid.hx / vMax))
print()
print()
# set initial data
N0 = self.dax.createGlobalVec()
T0 = self.dax.createGlobalVec()
N0.setName('n0')
T0.setName('T0')
n0 = PETSc.Vec().createSeq(nx)
t0 = PETSc.Vec().createSeq(nx)
if self.cfg['initial_data']['distribution_python'] != None:
init_data = __import__(
"runs." + self.cfg['initial_data']['distribution_python'],
globals(), locals(), ['distribution'], 0)
if PETSc.COMM_WORLD.getRank() == 0:
print(
"Initialising distribution function with Python function.")
self.toolbox.initialise_distribution_function(
self.fc, init_data.distribution)
N0.set(0.)
T0.set(0.)
else:
N0_arr = self.dax.getVecArray(N0)
T0_arr = self.dax.getVecArray(T0)
if self.cfg['initial_data']['density_python'] != None:
init_data = __import__(
"runs." + self.cfg['initial_data']['density_python'],
globals(), locals(), ['density'], 0)
if PETSc.COMM_WORLD.getRank() == 0:
print("Initialising density with Python function.")
for i in range(xsx, xex):
N0_arr[i] = init_data.density(self.grid.x[i],
self.grid.xLength())
else:
N0_arr[xsx:xex] = self.cfg['initial_data']['density']
if self.cfg['initial_data']['temperature_python'] != None:
init_data = __import__(
"runs." + self.cfg['initial_data']['temperature_python'],
globals(), locals(), ['temperature'], 0)
if PETSc.COMM_WORLD.getRank() == 0:
print("Initialising temperature with Python function.")
for i in range(xsx, xex):
T0_arr[i] = init_data.temperature(self.grid.x[i])
else:
T0_arr[xsx:xex] = self.cfg['initial_data']['temperature']
if PETSc.COMM_WORLD.getRank() == 0:
print("Initialising distribution function with Maxwellian.")
self.copy_xvec_to_seq(N0, n0)
self.copy_xvec_to_seq(T0, t0)
self.toolbox.initialise_distribution_nT(self.fc, n0, t0)
# Fourier Filtering
nfourier = cfg['initial_data']['nfourier']
if nfourier > 0:
from scipy.fftpack import rfft, irfft
if PETSc.COMM_WORLD.getRank() == 0:
print("Fourier Filtering")
# obtain whole f vector everywhere
scatter, Xglobal = PETSc.Scatter.toAll(self.fc)
scatter.begin(self.fc, Xglobal, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
scatter.end(self.fc, Xglobal, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
petsc_indices = self.da1.getAO().app2petsc(
np.arange(nx * nv, dtype=np.int32))
fTmp = Xglobal.getValues(petsc_indices).copy().reshape((nv, nx)).T
scatter.destroy()
Xglobal.destroy()
# filter distribution function
fFft = rfft(fTmp, axis=0)
fFft[nfourier:, :] = 0.
fTmp = irfft(fFft, axis=0)
# store filtered distribution function
f_arr = self.da1.getVecArray(self.fc)
f_arr[:, :] = fTmp[xs:xe, ys:ye]
# normalise f
self.normalise_distribution_function()
# calculate potential and moments
if PETSc.COMM_WORLD.getRank() == 0:
print("Calculate initial potential and moments.")
self.calculate_moments(output=False)
# initialise Gear History
if self.cfg['solver']['initial_guess'] == "gear":
self.arakawa_gear.initialise_history(self.fc)
# check for external potential
if self.cfg['initial_data']['external_python'] != None:
if PETSc.COMM_WORLD.getRank() == 0:
print("Calculate external potential.")
external_data = __import__(
"runs." + self.cfg['initial_data']['external_python'],
globals(), locals(), ['external'], 0)
self.external = external_data.external
else:
self.external = None
# calculate external potential
self.calculate_external(0.)
# create HDF5 output file
if PETSc.COMM_WORLD.getRank() == 0:
print("Create HDF5 output file.")
# use h5py to store attributes
hdf5out = h5py.File(hdf_out_filename,
'w',
driver='mpio',
comm=PETSc.COMM_WORLD.tompi4py())
hdf5out.attrs['charge'] = self.charge
hdf5out.close()
# create PETSc HDF5 viewer
self.hdf5_viewer = PETSc.ViewerHDF5().create(
hdf_out_filename,
mode=PETSc.Viewer.Mode.APPEND,
comm=PETSc.COMM_WORLD)
self.hdf5_viewer.pushGroup("/")
if PETSc.COMM_WORLD.getRank() == 0:
print("Saving initial data to HDF5.")
# write grid data to hdf5 file
coords_x.setName('x')
coords_v.setName('v')
self.hdf5_viewer(coords_x)
self.hdf5_viewer(coords_v)
# write initial data to hdf5 file
# self.hdf5_viewer(N0)
# self.hdf5_viewer(T0)
# save to hdf5
self.hdf5_viewer.setTimestep(0)
self.save_hdf5_vectors()
if PETSc.COMM_WORLD.getRank() == 0:
print("run_base.py: initialisation done.")
print("")
def __init__(self, A_IS):
"""
Initialize the domain decomposition preconditioner, multipreconditioner and coarse space with its operators
Parameters
==========
A_IS : petsc.Mat
The matrix of the problem in IS format. A must be a symmetric positive definite matrix
with symmetric positive semi-definite submatrices
PETSc.Options
=============
PCBNN_switchtoASM :Bool
Default is False
If True then the domain decomposition preconditioner is the BNN preconditioner. If false then the domain
decomposition precondition is the Additive Schwarz preconditioner with minimal overlap.
PCBNN_kscaling : Bool
Default is True.
If true then kscaling (partition of unity that is proportional to the diagonal of the submatrices of A)
is used when a partition of unity is required. Otherwise multiplicity scaling is used when a partition
of unity is required. This may occur in two occasions:
- to scale the local BNN matrices if PCBNN_switchtoASM=True,
- in the GenEO eigenvalue problem for eigmin if PCBNN_switchtoASM=False and PCBNN_GenEO=True with
PCBNN_GenEO_eigmin > 0 (see projection.__init__ for the meaning of these options).
PCBNN_verbose : Bool
Default is False.
If True, some information about the preconditioners is printed when the code is executed.
PCBNN_GenEO : Bool
Default is False.
If True then the coarse space is enriched by solving local generalized eigenvalue problems.
PCBNN_CoarseProjection :True
Default is True
If False then there is no coarse projection: Two level Additive Schwarz or One-level preconditioner depending on PCBNN_addCoarseSolve
If True, the coarse projection is applied: Projected preconditioner of hybrid preconditioner depending on PCBNN_addCoarseSolve
PCBNN_addCoarseSolve : False
Default is True
If True then (R0t A0\R0 r) is added to the preconditioned residual
False corresponds to the projected preconditioner (need to choose initial guess accordingly) (or the one level preconditioner if PCBNN_CoarseProjection = False)
True corresponds to the hybrid preconditioner (or the fully additive preconditioner if PCBNN_CoarseProjection = False)
"""
OptDB = PETSc.Options()
self.switchtoASM = OptDB.getBool('PCBNN_switchtoASM', False) #use Additive Schwarz as a preconditioner instead of BNN
self.kscaling = OptDB.getBool('PCBNN_kscaling', True) #kscaling if true, multiplicity scaling if false
self.verbose = OptDB.getBool('PCBNN_verbose', False)
self.GenEO = OptDB.getBool('PCBNN_GenEO', True)
self.addCS = OptDB.getBool('PCBNN_addCoarseSolve', False)
self.projCS = OptDB.getBool('PCBNN_CoarseProjection', True)
#extract Neumann matrix from A in IS format
Ms = A_IS.copy().getISLocalMat()
# convert A_IS from matis to mpiaij
A_mpiaij = A_IS.convertISToAIJ()
r, _ = A_mpiaij.getLGMap() #r, _ = A_IS.getLGMap()
is_A = PETSc.IS().createGeneral(r.indices)
# extract exact local solver
As = A_mpiaij.createSubMatrices(is_A)[0]
vglobal, _ = A_mpiaij.getVecs()
vlocal, _ = Ms.getVecs()
scatter_l2g = PETSc.Scatter().create(vlocal, None, vglobal, is_A)
#compute the multiplicity of each degree
vlocal.set(1.)
vglobal.set(0.)
scatter_l2g(vlocal, vglobal, PETSc.InsertMode.ADD_VALUES)
scatter_l2g(vglobal, vlocal, PETSc.InsertMode.INSERT_VALUES, PETSc.ScatterMode.SCATTER_REVERSE)
NULL,mult_max = vglobal.max()
# k-scaling or multiplicity scaling of the local (non-assembled) matrix
if self.kscaling == False:
Ms.diagonalScale(vlocal,vlocal)
else:
v1 = As.getDiagonal()
v2 = Ms.getDiagonal()
Ms.diagonalScale(v1/v2, v1/v2)
# the default local solver is the scaled non assembled local matrix (as in BNN)
if self.switchtoASM:
Atildes = As
if mpi.COMM_WORLD.rank == 0:
print('The user has chosen to switch to Additive Schwarz instead of BNN.')
else: #(default)
Atildes = Ms
ksp_Atildes = PETSc.KSP().create(comm=PETSc.COMM_SELF)
ksp_Atildes.setOptionsPrefix("ksp_Atildes_")
ksp_Atildes.setOperators(Atildes)
ksp_Atildes.setType('preonly')
pc_Atildes = ksp_Atildes.getPC()
pc_Atildes.setType('cholesky')
pc_Atildes.setFactorSolverType('mumps')
ksp_Atildes.setFromOptions()
self.A = A_mpiaij
self.Ms = Ms
self.As = As
self.ksp_Atildes = ksp_Atildes
self.works_1 = vlocal.copy()
self.works_2 = self.works_1.copy()
self.scatter_l2g = scatter_l2g
self.mult_max = mult_max
self.minV0 = minimal_V0(self.ksp_Atildes)
if self.GenEO == True:
GenEOV0 = GenEO_V0(self.ksp_Atildes,self.Ms,self.As,self.mult_max,self.minV0.V0s)
self.V0s = GenEOV0.V0s
else:
self.V0s = self.minV0.V0s
self.proj = coarse_operators(self.V0s,self.A,self.scatter_l2g,vlocal)
ksp.setOperators(A, P)
ksp.setTolerances(rtol=1e-8)
ksp.setType("minres")
ksp.getPC().setType("fieldsplit")
ksp.getPC().setFieldSplitType(PETSc.PC.CompositeType.ADDITIVE)
ksp.getPC().setFieldSplitIS(("u", is_u), ("p", is_p))
# Configure velocity and pressure sub KSPs
ksp_u, ksp_p = ksp.getPC().getFieldSplitSubKSP()
ksp_u.setType("preonly")
ksp_u.getPC().setType("gamg")
ksp_p.setType("preonly")
ksp_p.getPC().setType("jacobi")
# Monitor the convergence of the KSP
opts = PETSc.Options()
# opts["ksp_monitor"] = None
# opts["ksp_view"] = None
ksp.setFromOptions()
# We also need to create a block vector,``x``, to store the (full)
# solution, which we initialize using the block RHS form ``L``.
# Compute solution
x = A.createVecRight()
ksp.solve(b, x)
# Create Functions and scatter x solution
u, p = Function(V), Function(Q)
offset = V_map.size_local * V_map.block_size
# work some magic to put the distributed matrix together
P.assemble()
if (J != P):
J.assemble() # matrix-free operator
# our non-zero pattern stays the same for all iterations, so we tell PETSc that
# (it allows some optimisations inside PETSc)
return PETSc.Mat.Structure.SAME_NONZERO_PATTERN
stype = PETSc.DMDA.StencilType.BOX
bx = PETSc.DMDA.BoundaryType.GHOSTED
by = PETSc.DMDA.BoundaryType.GHOSTED
bz = PETSc.DMDA.BoundaryType.GHOSTED
comm = PETSc.COMM_WORLD
rank = comm.rank
OptDB = PETSc.Options() # get PETSc option DB
# M = OptDB.getInt('M', 11)
# N = OptDB.getInt('N', 7)
# P = OptDB.getInt('P', 5)
M, N, P = -11, -7, -5
m = OptDB.getInt('m', PETSc.DECIDE)
n = OptDB.getInt('n', PETSc.DECIDE)
p = OptDB.getInt('p', PETSc.DECIDE)
dm = PETSc.DMDA().create(dim=3, sizes=(M, N, P), proc_sizes=(m, n, p),
boundary_type=(bx, by, bz), stencil_type=stype,
stencil_width=1, dof=1, comm=comm, setup=False)
dm.setFromOptions()
dm.setUp()
convdiff_problem = convection_diffusion(dm, {"dx": 0.1, "dy": 0.1, "dz": 0.1})
# prepare the initial conditions
def setup_method(self, method):
self.petsc_options = PETSc.Options()
self.petsc_options.clear()
self._scriptdir = os.path.dirname(__file__)
# Overall form (as one-form)
F = odeint.discretise_in_time(f)
# Overall form (as list of forms)
F = dolfiny.function.extract_blocks(F, δm)
# Create output xdmf file -- open in Paraview with Xdmf3ReaderT
ofile = dolfiny.io.XDMFFile(comm, f"{name}.xdmf", "w")
# Write mesh, meshtags
ofile.write_mesh_meshtags(mesh, mts)
# Write initial state
ofile.write_function(v, time.value)
ofile.write_function(p, time.value)
# Options for PETSc backend
opts = PETSc.Options("bingham")
opts["snes_type"] = "newtonls"
opts["snes_linesearch_type"] = "basic"
opts["snes_rtol"] = 1.0e-08
opts["snes_max_it"] = 12
opts["ksp_type"] = "preonly"
opts["pc_type"] = "lu"
opts["pc_factor_mat_solver_type"] = "mumps"
opts_global = PETSc.Options()
opts_global["mat_mumps_icntl_14"] = 500
opts_global["mat_mumps_icntl_24"] = 1
# Create nonlinear problem: SNES
problem = dolfiny.snesblockproblem.SNESBlockProblem(F, m, prefix="bingham")
