def __init__(self, V, two_way=True):
t0 = time()
# Keep sub dimensions
self.ns = V.sub(0).dim()
self.nf = V.sub(1).dim()
self.np = V.sub(2).dim()
# Also dofmaps
self.dofmap_s = V.sub(0).dofmap().dofs()
self.dofmap_f = V.sub(1).dofmap().dofs()
self.dofmap_p = V.sub(2).dofmap().dofs()
self.dofmap_fp = sorted(self.dofmap_f + self.dofmap_p)
# Note that f and p dofmaps are used for the fieldsplit Preconditioner
# in the 2-way splittings, so they are still useful but bear a different meaning.
# All gather the global fp dofmap
if two_way:
comm = MPI.COMM_WORLD
dofs_fp_global = self.dofmap_fp.copy()
dofs_fp_global = comm.allgather(dofs_fp_global)
dofs_fp_global = np.array(tuple(
chain(*dofs_fp_global))) # Concat and sort
# Replace global dofmaps with f-p dofmaps.
self.dofmap_f, self.dofmap_p = get_local_fp_dofs(
dofs_fp_global, self.dofmap_f, self.dofmap_p)
# and Index Sets
self.is_s = PETSc.IS().createGeneral(self.dofmap_s)
self.is_f = PETSc.IS().createGeneral(self.dofmap_f)
self.is_p = PETSc.IS().createGeneral(self.dofmap_p)
self.is_fp = PETSc.IS().createGeneral(self.dofmap_fp)
parprint(
"---- [Indexes] computed local indices in {:.3f}s".format(time() -
t0))
def solve(self):
"""
Solve poromechanics problem. Problem loads are assumed to give a function when evaluated at a specific time. For example:
f_vol = lambda t: Constant((1,1))
"""
# All functions start as 0 (for now), so no modifications are required.
t0_simulation = time()
if self.output_solutions:
self.export(self.t0)
current_time = time()
iterations = []
while self.t < self.tf:
self.t += self.dt
its = self.solve_time_step(self.t)
parprint("-------- Solved time t={:.2f}. {} iterations in {:.2f}s".
format(self.t, its,
time() - current_time))
if self.output_solutions:
self.export(self.t)
current_time = time()
parprint("Total simulation time = {}s\n".format(time() -
t0_simulation))
def __init__(self, parameters, mesh, parser):
super().__init__(parameters, mesh, parser)
V = df.FunctionSpace(
self.mesh,
df.MixedElement(
df.VectorElement('CG', self.mesh.ufl_cell(),
parameters["fe degree solid"]),
df.VectorElement('CG', self.mesh.ufl_cell(),
parameters["fe degree fluid"]),
df.FiniteElement('CG', self.mesh.ufl_cell(),
parameters["fe degree pressure"])))
self.V = V
self.two_way = True
self.three_way = False
if "3-way" in self.parameters["pc type"]:
self.two_way = False
self.three_way = True
parprint("---- Problem dofs={}, h={}, solving with {} procs".format(
V.dim(), mesh.hmin(), MPI.COMM_WORLD.size))
self.assembler = PoromechanicsAssembler(parameters, V, self.three_way)
self.index_map = IndexSet(V, self.two_way)
# Start by assembling system matrices
self.assembler.assemble()
self.sol = df.Function(V)
self.us_nm1, self.uf_nm1, self.p_nm1 = self.sol.split(True)
self.us_nm2 = self.us_nm1.copy(True)
self.first_timestep = True
def create_solver(self, A, b, PC):
t0_create = time()
b = self.b.vec()
self.dummy = b.copy()
self.dummy_s = PETSc.Vec().create()
self.dummy_f = PETSc.Vec().create()
self.dummy_p = PETSc.Vec().create()
b.getSubVector(self.index_map.is_s, self.dummy_s)
b.getSubVector(self.index_map.is_f, self.dummy_f)
b.getSubVector(self.index_map.is_p, self.dummy_p)
b.restoreSubVector(self.index_map.is_s, self.dummy_s)
b.restoreSubVector(self.index_map.is_f, self.dummy_f)
b.restoreSubVector(self.index_map.is_p, self.dummy_p)
solver_type = self.parameters["solver type"]
atol = self.parameters["solver atol"]
rtol = self.parameters["solver rtol"]
maxiter = self.parameters["solver maxiter"]
monitor_convergence = self.parameters["solver monitor"]
if self.parameters["solver type"] == "aar":
order = self.parameters["AAR order"]
p = self.parameters["AAR p"]
omega = self.parameters["AAR omega"]
beta = self.parameters["AAR beta"]
self.solver = AAR(order,
p,
omega,
beta,
self.A.mat(),
x0=None,
pc=self.PC,
atol=atol,
rtol=rtol,
maxiter=maxiter,
monitor_convergence=monitor_convergence)
else:
solver = PETSc.KSP().create()
solver.setOptionsPrefix("global_")
# solver.setInitialGuessNonzero(True)
solver.setOperators(self.A.mat())
solver.setType(self.parameters["solver type"])
solver.setTolerances(rtol, atol, 1e20, maxiter)
solver.setPC(self.PC)
if solver_type == "gmres":
solver.setGMRESRestart(maxiter)
solver.setFromOptions()
self.solver = solver
parprint("---- [Solver] Solver created in {}s".format(time() -
t0_create))
def print_timings(self):
parprint("\n===== Timing preconditioner: {:.3f}s".format(self.t_total))
if self.flag_3_way:
parprint("\tSolid solver: {:.3f}s\n\tFluid solver: {:.3f}s\n\tPressure solver: {:.3f}s".format(
self.t_solid, self.t_fluid, self.t_press))
else:
parprint(
"\tSolid solver: {:.3f}s\n\tFluid-pressure solver: {:.3f}s".format(self.t_solid, self.t_fluid))
parprint("\n\tAllocation time: {:.3f}".format(self.t_alloc))
def set_up(self):
t0_setup = time()
# Create linear solver
solver_type = self.parameters["solver type"]
atol = self.parameters["solver atol"]
rtol = self.parameters["solver rtol"]
maxiter = self.parameters["solver maxiter"]
monitor_convergence = self.parameters["solver monitor"]
# Prepare elements for convergence test:
args = None
# index_map, b, dummy, dummy_s, dummy_f, dummy_p, b0_s, b0_f, b0_p
b = self.b.vec()
dummy = b.copy()
b_s = PETSc.Vec().create()
b_f = PETSc.Vec().create()
b_p = PETSc.Vec().create()
b.getSubVector(self.index_map.is_s, b_s)
dummy_s = b_s.copy()
b.getSubVector(self.index_map.is_f, b_f)
dummy_f = b_f.copy()
b.getSubVector(self.index_map.is_p, b_p)
dummy_p = b_p.copy()
b0_s = b_s.norm()
b0_f = b_f.norm()
b0_p = b_p.norm()
b.restoreSubVector(self.index_map.is_s, b_s)
b.restoreSubVector(self.index_map.is_f, b_f)
b.restoreSubVector(self.index_map.is_p, b_p)
# if b0_s < 1e-13:
# b0_s = 1
# if b0_f < 1e-13:
# b0_f = 1
# if b0_p < 1e-13:
# b0_p = 1
kwargs = {
'index_map': self.index_map,
'b': b,
'dummy': dummy,
'dummy_subs': (dummy_s, dummy_f, dummy_p),
'b0_norms': (b0_s, b0_f, b0_p),
'monitor': monitor_convergence
}
parprint("---- [Solver] Solver set up in {}s".format(time() -
t0_setup))
def setUp(self, pc):
t0_setup = time()
# create local ksp and pc contexts
self.create_solvers()
# Create temp block vectors used in apply()
self.allocate_temp_vectors()
# Extract sub-matrices
self.allocate_submatrices()
for solver, mat in zip(self.ksps_elliptic, self.matrices_elliptic):
self.setup_elliptic_solver(solver, mat)
if not self.flag_3_way:
if self.inner_pc_type == "lu":
self.setup_elliptic_solver(self.ksp_fp, self.Mfp_fp)
else:
self.setup_fieldsplit(self.ksp_fp, self.Mfp_fp)
parprint("---- [Preconditioner] Set up in {}s".format(time() - t0_setup))
def set_bcs(self, bcs, bcs_diff):
"""
Set boundary conditions to both physics. Assumed to be constant.
"""
t0 = time()
self.bcs = bcs
self.bcs_diff = bcs_diff
# Create map for pressure dofs, used in 3way CC preconditioner
dofs_p = self.V.sub(2).dofmap().dofs()
bcs_sub_pressure = []
for b in bcs_diff:
bc_vals = b.get_boundary_values().keys()
for i, dof in enumerate(dofs_p):
if dof in bc_vals:
bcs_sub_pressure.append(i)
self.bcs_sub_pressure = bcs_sub_pressure
parprint(
"---- [BC] Created inverse pressure BC in {:.3f}s".format(time() -
t0))
def getRHS(self, t, us_nm1, us_nm2, uf_nm1, p_nm1):
"""
Get Dolfin::PETScVector RHS at time t
"""
t0_rhs = time()
v, w, q = TestFunctions(self.V)
# Compute solid residual
rhs_s_n = dot(self.fs_sur(t), v) * self.dsNs + self.phis * \
self.rhos * dot(self.fs_vol(t), v) * dx
lhs_s_n = dot(self.rhos * self.idt**2 * self.phis * (-2. * us_nm1 + us_nm2), v) * \
dx - self.phi0**2 * dot(self.ikf * (- self.idt * (- us_nm1)), v) * dx
r_s = rhs_s_n - lhs_s_n
# Compute fluid residual
rhs_f_n = dot(self.ff_sur(t), w) * self.dsNf + self.phi0 * \
self.rhof * dot(self.ff_vol(t), w) * dx
lhs_f = dot(self.rhof * self.idt * self.phi0 * (- uf_nm1), w) + self.phi0**2 * \
dot(self.ikf * (-self.idt * (-us_nm1)), w)
lhs_f_n = lhs_f * dx
r_f = rhs_f_n - lhs_f_n
# Compute pressure residual
rhs_p_n = 1 / self.rhof * self.p_source(t) * q * dx
D_sf = div(self.idt * self.phis * (-us_nm1)) * q
M_p = self.phis**2 / Constant(self.ks * self.dt) * (-p_nm1) * q
lhs_p_n = (M_p + D_sf) * dx
r_p = rhs_p_n - lhs_p_n
# assemble(rhs_s_n + rhs_f_n + rhs_p_n, tensor=self.b)
assemble(self.adim_s * rhs_s_n + self.adim_f * rhs_f_n +
self.adim_p * rhs_p_n,
tensor=self.b)
parprint("---- [Assembler] Assembly RHS = {}s".format(time() - t0_rhs))
return self.b
def converged(_ksp, _it, _rnorm, *args, **kwargs):
"""
args must have: index_map, dummy, dummy_s, dummy_f, dummy_p, b0_s, b0_f, b0_p.
dummy is used to avoid allocation of new vector for residual. [is is somewhere in PETSc...?]
"""
dummy = kwargs['dummy']
index_map = kwargs['index_map']
dummy_s, dummy_f, dummy_p = kwargs['dummy_subs']
b0_s, b0_f, b0_p = kwargs['b0_norms']
normalize = max(b0_s, b0_f, b0_p)
_ksp.buildResidual(dummy)
# Get residual subcomponents
dummy.getSubVector(index_map.is_s, dummy_s)
dummy.getSubVector(index_map.is_f, dummy_f)
dummy.getSubVector(index_map.is_p, dummy_p)
res_s_a = dummy_s.norm(PETSc.NormType.NORM_INFINITY)
res_s_r = res_s_a / normalize # /b0_s
res_f_a = dummy_f.norm(PETSc.NormType.NORM_INFINITY)
res_f_r = res_f_a / normalize # /b0_f
res_p_a = dummy_p.norm(PETSc.NormType.NORM_INFINITY)
res_p_r = res_p_a / normalize # /b0_p
error_abs = max(res_s_a, res_f_a, res_p_a)
error_rel = max(res_s_r, res_f_r, res_p_r)
if kwargs['monitor']:
width = 11
if _it == 0:
parprint("KSP errors: {}, {}, {}, {}, {}, {}".format(
'abs_s'.rjust(width), 'abs_f'.rjust(width),
'abs_p'.rjust(width), 'rel_s'.rjust(width),
'rel_f'.rjust(width), 'rel_p'.rjust(width)))
parprint("KSP it {}: {:.5e}, {:.5e}, {:.5e}, {:.5e}, {:.5e}, {:.5e}".
format(_it, res_s_a, res_f_a, res_p_a, res_s_r, res_f_r,
res_p_r))
if error_abs < _ksp.atol or error_rel < _ksp.rtol:
# Convergence
parprint("---- [Solver] Converged")
return 1
elif _it > _ksp.max_it or error_abs > _ksp.divtol:
# Divergence
return -1
else:
# Continue
return 0
def print_timings(self):
parprint("\n===== Timing Solver: {:.3f}s".format(self.t_total))
phi0 = 0.1
mu_s = 4000
lmbda = 700
ks = 1e6
kf = 1e-7
dt = 0.1
t0 = 0.0
tf = 0.1
maxiter = 1000
betas = -0.5
betaf = 0.
betap = 1.
# FE space
V = FunctionSpace(mesh, VectorElement('CG', tetrahedron, degree_s))
parprint("Dofs = {}".format(V.dim()))
sol = Function(V)
us_nm1 = Function(V)
us_nm2 = Function(V)
# BCs
bcs = [
DirichletBC(V.sub(0), Constant(0), markers, XM),
DirichletBC(V.sub(1), Constant(0), markers, YM),
DirichletBC(V.sub(2), Constant(0), markers, ZM)
]
def fs_sur(t):
return Constant(-1e3 * 0.9 * (1 - exp(-(t**2) / 0.25))) * FacetNormal(mesh)
def assemble(self):
t0_assemble = time()
def hooke(ten):
return 2 * self.mu_s * ten + self.lmbda * tr(ten) * Identity(
self.dim)
def eps(vec):
return sym(grad(vec))
us, vf, p = TrialFunctions(self.V)
v, w, q = TestFunctions(self.V)
dx = Measure('dx', domain=self.mesh)
# First base matrix
a_s = (self.rhos * self.idt**2 * self.phis * dot(us, v) +
inner(hooke(eps(us)), eps(v)) - p * div(self.phis * v) -
self.phi0**2 * dot(self.ikf * (vf - self.idt * us), v)) * dx
a_f = (self.rhof * self.idt * self.phi0 * dot(vf, w) + 2. * self.mu_f *
inner(self.phi0 * eps(vf), eps(w)) - p * div(self.phi0 * w) +
self.phi0**2 * dot(self.ikf * (vf - self.idt * us), w)) * dx
a_p = (self.phis**2 * self.idt / self.ks * p * q +
div(self.phi0 * vf) * q +
div(self.phis * self.idt * us) * q) * dx
a_p_diff = Constant(0.0) * p * q * dx
assemble(self.adim_s * a_s + self.adim_f * a_f + self.adim_p * a_p,
tensor=self.A)
# Then, preconditioner matrices (FS and DIFF)
if self.prec_type == "undrained":
N = self.ks / self.phis**2
a_s = (self.rhos * self.idt**2 * self.phis * dot(us, v) +
inner(hooke(eps(us)), eps(v)) +
N * div(self.phis * us) * div(self.phis * v) -
self.phi0**2 * dot(self.ikf * (-self.idt * us), v)) * dx
a_f = (self.rhof * self.idt * self.phi0 * dot(vf, w) +
2. * self.mu_f * inner(self.phi0 * eps(vf), eps(w)) -
p * div(self.phi0 * w) +
self.phi0**2 * dot(self.ikf * (vf - self.idt * us), w)) * dx
a_p = (self.phis**2 * self.idt / self.ks * p * q +
div(self.phi0 * vf) * q +
div(self.phis * self.idt * us) * q) * dx
a_p_diff = 0 * q * dx
elif self.prec_type == "undrained 3-way":
N = self.ks / self.phis**2
a_s = (self.rhos * self.idt**2 * self.phis * dot(us, v) +
inner(hooke(eps(us)), eps(v)) +
N * div(self.phis * us) * div(self.phis * v) -
self.phi0**2 * dot(self.ikf * (-self.idt * us), v)) * dx
a_f = (self.rhof * self.idt * self.phi0 * dot(vf, w) +
2. * self.mu_f * inner(self.phi0 * eps(vf), eps(w)) -
p * div(self.phi0 * w) +
self.phi0**2 * dot(self.ikf * (vf - self.idt * us), w)) * dx
beta_CC1 = self.phi0 / Constant(2. * self.mu_f / self.dim)
beta_CC2 = inv(self.rhof * self.idt / self.phi0 + self.ikf)
a_p = (self.phis**2 * self.idt / self.ks * p * q +
beta_CC1 * p * q) * dx
a_p_diff = (self.phis**2 * self.idt / self.ks * p * q +
dot(beta_CC2 * grad(p), grad(q))) * dx
elif self.prec_type == "diagonal":
beta_s_hat = self.betas
a_s = (self.rhos * self.idt**2 * self.phis * dot(us, v) +
inner(hooke(eps(us)), eps(v)) - p * div(self.phis * v) -
self.phi0**2 *
dot(self.ikf * (vf -
(1. + beta_s_hat) * self.idt * us), v)) * dx
beta_f_hat = self.betaf
a_f = (
self.rhof * self.idt * self.phi0 * dot(vf, w) +
2. * self.mu_f * inner(self.phi0 * eps(vf), eps(w)) -
p * div(self.phi0 * w) +
(1. + beta_f_hat) * self.phi0**2 * dot(self.ikf * vf, w)) * dx
beta_p_hat = self.betap
beta_p = beta_p_hat * self.phis**2 / \
(self.dt * (2. * self.mu_s / self.dim + self.lmbda))
a_p = (self.phis**2 * self.idt / self.ks * p * q + beta_p * p * q +
div(self.phi0 * vf) * q) * dx
a_p_diff = 0.0 * q * dx
elif self.prec_type == "diagonal 3-way":
beta_s_hat = self.betas
a_s = (self.rhos * self.idt**2 * self.phis * dot(us, v) +
inner(hooke(eps(us)), eps(v)) - p * div(self.phis * v) -
self.phi0**2 *
dot(self.ikf * (vf -
(1. + beta_s_hat) * self.idt * us), v)) * dx
beta_f_hat = self.betaf
a_f = (self.rhof * self.idt * self.phi0 * dot(vf, w) +
2. * self.mu_f * self.phi0 * inner(eps(vf), eps(w)) -
p * div(self.phi0 * w) +
(1. + beta_f_hat) * self.phi0**2 * dot(self.ikf *
(vf), w)) * dx
beta_p_hat = self.betap
beta_p = beta_p_hat * self.phis**2 / \
Constant(self.dt * (2. * self.mu_s / self.dim + self.lmbda))
beta_CC1 = self.phi0 / Constant(2. * self.mu_f / self.dim)
beta_CC2 = inv(self.rhof * self.idt / self.phi0 + self.ikf)
a_p = (self.phis**2 * self.idt / self.ks * p * q +
(beta_p + beta_CC1) * p * q) * dx
a_p_diff = (self.phis**2 * self.idt / self.ks * p * q +
beta_p * p * q + dot(beta_CC2 * grad(p), grad(q))) * dx
elif self.prec_type == "diagonal 3-way-II":
beta_s_hat = self.betas
a_s = (self.rhos * self.idt**2 * self.phis * dot(us, v) +
inner(hooke(eps(us)), eps(v)) - p * div(self.phis * v) -
self.phi0**2 *
dot(self.ikf * (vf -
(1. + beta_s_hat) * self.idt * us), v)) * dx
beta_f_hat = self.betaf
beta_p_hat = self.betap
beta_p = beta_p_hat * self.phis**2 / \
(self.dt * (2. * self.mu_s / self.dim + self.lmbda))
a_f = (
self.rhof * self.idt * self.phi0 * dot(vf, w) +
2. * self.mu_f * inner(self.phi0 * eps(vf), eps(w)) + 1. /
(self.phis**2 * self.idt / self.ks + beta_p) *
div(self.phi0 * vf) * div(self.phi0 * w) +
(1. + beta_f_hat) * self.phi0**2 * dot(self.ikf * vf, w)) * dx
a_p = (self.phis**2 * self.idt / self.ks * p * q + beta_p * p * q +
div(self.phi0 * vf) * q) * dx
a_p_diff = 0.0 * q * dx
else:
a_s = a_s
a_f = a_f
a_p = a_p
assemble(self.adim_s * a_s + self.adim_f * a_f + self.adim_p * a_p,
tensor=self.P)
if self.three_way:
assemble(self.adim_s * a_s + self.adim_f * a_f +
self.adim_p * a_p_diff,
tensor=self.P_diff)
parprint(
"---- [Assembler] Assembly A, P time = {}s".format(time() -
t0_assemble))
def solve(self, b, sol):
if not self.x0:
self.x0 = b.copy()
self.x0.zeroEntries()
# Initialize current vectors
self.temp_vec = None
self.xk = self.x0.copy()
self.fk = b.copy()
self.fk.axpy(-1, self.matA * self.x0)
self.delta_xk = b.copy()
self.delta_fk = b.copy()
self.temp_vec = b.copy()
# Init global vectors and scatterer
self.scatter, aux = PETSc.Scatter.toZero(b)
self.d_fk0 = aux.copy()
self.d_fk0.zeroEntries()
self.fk0 = aux.copy()
self.fk0.zeroEntries()
error0 = self.fk.norm()
err_abs = error0
err_rel = 1
it = 0
alpha = None
current_type = ""
while err_abs > self.atol and err_rel > self.rtol and it < self.maxiter:
self.fk.copy(self.delta_fk)
self.xk.copy(self.delta_xk)
self.update_residual(b) # Update fk value
self.delta_fk.aypx(-1, self.fk)
self.F.append(self.delta_fk.copy())
if len(self.F) > self.order:
self.F.pop(0)
# Update global vectors on first cpu
self.scatter.scatter(self.delta_fk, self.d_fk0)
self.scatter.scatter(self.fk, self.fk0)
self.F0.append(self.d_fk0.copy())
if len(self.F0) > self.order:
self.F0.pop(0)
if self.fk.norm() < 1e-14:
pass
# If not a natural number or first iteration
elif it == 0 or self.order == 0 or (it + 1) / self.p % 1 > 0:
current_type = "R"
self.xk.axpy(self.omega, self.fk)
else:
current_type = "A"
mk = min(self.order, it)
# Process only on first core, then scatter alpha
if self.rank == 0:
F = np.vstack(self.F0).T
Q, R = np.linalg.qr(F)
rhs = self.fk0
alpha = np.linalg.solve(R, -Q.T @ rhs)
else:
alpha = None
alpha = self.comm.bcast(alpha, root=0)
self.xk.axpy(self.beta, self.fk)
for i in range(mk):
self.xk.axpy(alpha[i], self.X[i] + self.beta * self.F[i])
self.delta_xk.aypx(-1, self.xk)
self.X.append(self.delta_xk.copy())
if len(self.X) > self.order:
self.X.pop(0)
err_abs = self.fk.norm()
err_rel = err_abs / error0
it += 1
if self.monitor_convergence:
parprint("---- Iteration [{}] {:3}\tabs={:1.2e}\trel={:1.2e}".
format(current_type, it, err_abs, err_rel))
# Update solution and return number of iterations
self.xk.copy(sol)
self.it = it
return it
