def run(self): OptDB = PETSc.Options() OptDB.setValue('ksp_monitor', '') OptDB.setValue('snes_monitor', '') # OptDB.setValue('snes_lag_preconditioner', 3) # OptDB.setValue('snes_ls', 'basic') # initialise matrix self.A = self.da2.createMat() self.A.setOption(self.A.Option.NEW_NONZERO_ALLOCATION_ERR, False) self.A.setUp() self.A.setNullSpace(self.nullspace) # initialise Jacobian self.J = self.da2.createMat() self.J.setOption(self.J.Option.NEW_NONZERO_ALLOCATION_ERR, False) self.J.setUp() self.J.setNullSpace(self.nullspace) # create Jacobian, Function, and linear Matrix objects # self.petsc_jacobian_mf = PETScJacobianMatrixFree(self.da1, self.da2, self.dax, # self.h0, self.vGrid, # self.nx, self.nv, self.ht, self.hx, self.hv, # self.charge, coll_freq=self.coll_freq) self.petsc_solver = PETScSolver(self.da1, self.da2, self.dax, self.h0, self.vGrid, self.nx, self.nv, self.ht, self.hx, self.hv, self.charge, coll_freq=self.coll_freq) # self.petsc_function = PETScFunction(self.da1, self.da2, self.dax, # self.h0, self.vGrid, # self.nx, self.nv, self.ht, self.hx, self.hv, # self.charge, coll_freq=self.coll_freq) # # self.petsc_matrix = PETScMatrix(self.da1, self.da2, self.dax, # self.h0, self.vGrid, # self.nx, self.nv, self.ht, self.hx, self.hv, # self.charge)#, coll_freq=self.coll_freq) # self.poisson_ksp = PETSc.KSP().create() # self.poisson_ksp.setFromOptions() # self.poisson_ksp.setOperators(self.poisson_A) # self.poisson_ksp.setType('cg') # self.poisson_ksp.getPC().setType('none') # # self.poisson_ksp.setType('preonly') # # self.poisson_ksp.getPC().setType('lu') # # self.poisson_ksp.getPC().setFactorSolverPackage(self.solver_package) # copy external potential self.petsc_solver.update_external(self.p_ext) # self.petsc_matrix.update_external(self.p_ext) # update solution history self.petsc_solver.update_history(self.x) # self.petsc_matrix.update_history(self.f, self.h1, self.p, self.n, self.nu, self.ne, self.u, self.e, self.a) # initialise matrixfree Jacobian # self.Jmf = PETSc.Mat().createPython([self.x.getSizes(), self.b.getSizes()], # context=self.petsc_jacobian_mf, # comm=PETSc.COMM_WORLD) # self.Jmf.setUp() # # create linear solver # self.snes_linear = PETSc.SNES().create() # self.snes_linear.setType('ksponly') # self.snes_linear.setFunction(self.petsc_matrix.snes_mult, self.b) # self.snes_linear.setJacobian(self.updateMatrix, self.A) # self.snes_linear.setFromOptions() # # self.snes_linear.getKSP().setType('gmres') # # self.snes_linear.getKSP().getPC().setType('bjacobi') # # self.snes_linear.getKSP().getPC().setFactorSolverPackage(self.solver_package) # self.snes_linear.getKSP().setType('preonly') # self.snes_linear.getKSP().getPC().setType('lu') # self.snes_linear.getKSP().getPC().setFactorSolverPackage(self.solver_package) # create nonlinear solver self.snes = PETSc.SNES().create() self.snes.setFunction(self.petsc_solver.function_snes_mult, self.b) # self.snes.setJacobian(self.updateJacobian, self.Jmf, self.J) self.snes.setJacobian(self.updateJacobian, self.J) self.snes.setFromOptions() # self.snes.getKSP().setType('gmres') # self.snes.getKSP().getPC().setType('bjacobi') self.snes.getKSP().setType('preonly') self.snes.getKSP().getPC().setType('lu') self.snes.getKSP().getPC().setFactorSolverPackage(self.solver_package) # self.snes_nsp = PETSc.NullSpace().create(vectors=(self.x_nvec,)) # self.snes.getKSP().setNullSpace(self.snes_nsp) for itime in range(1, self.nt + 1): current_time = self.ht * itime if PETSc.COMM_WORLD.getRank() == 0: localtime = time.asctime(time.localtime(time.time())) print("\nit = %4d, t = %10.4f, %s" % (itime, current_time, localtime)) print self.time.setValue(0, current_time) # calculate external field and copy to matrix, jacobian and function self.calculate_external(current_time) # self.petsc_jacobian_mf.update_external(self.p_ext) self.petsc_solver.update_external(self.p_ext) # self.petsc_jacobian.update_external(self.p_ext) # self.petsc_function.update_external(self.p_ext) # self.petsc_matrix.update_external(self.p_ext) self.x.copy(self.xh) self.petsc_solver.function_mult(self.x, self.b) prev_norm = self.b.norm() if PETSc.COMM_WORLD.getRank() == 0: print( " Previous Step: funcnorm = %24.16E" % (prev_norm)) # calculate initial guess via RK4 # self.initial_guess_rk4() # calculate initial guess via Gear self.initial_guess_gear(itime) # check if residual went down # self.petsc_function.mult(self.x, self.b) # ig_norm = self.b.norm() # # if ig_norm > prev_norm: # self.xh.copy(self.x) # calculate initial guess via linear solver # self.initial_guess() # nonlinear solve self.snes.solve(None, self.x) # output some solver info if PETSc.COMM_WORLD.getRank() == 0: print( " Nonlin Solver: %5i iterations, funcnorm = %24.16E" % (self.snes.getIterationNumber(), self.snes.getFunctionNorm())) print() if self.snes.getConvergedReason() < 0: if PETSc.COMM_WORLD.getRank() == 0: print() print("Solver not converging... %i" % (self.snes.getConvergedReason())) print() # if PETSc.COMM_WORLD.getRank() == 0: # mat_viewer = PETSc.Viewer().createDraw(size=(800,800), comm=PETSc.COMM_WORLD) # mat_viewer(self.J) # # print # input('Hit any key to continue.') # print # update data vectors self.copy_x_to_data() # update history self.petsc_solver.update_history(self.x) # self.petsc_matrix.update_history(self.f, self.h1, self.p, self.n, self.nu, self.ne, self.u, self.e, self.a) self.arakawa_gear.update_history(self.f, self.h1) # save to hdf5 self.save_to_hdf5(itime) # # some solver output phisum = self.p.sum() # # if PETSc.COMM_WORLD.getRank() == 0: # print(" Solver") print(" sum(phi) = %24.16E" % (phisum))
import ufl from fenics import * from string import Template import hashlib from subprocess import Popen, PIPE, check_output import os from pathlib import Path import json import pdb import petsc4py import sys petsc4py.init(sys.argv) from petsc4py import PETSc print(PETSc.Options().getAll()) import argparse from urllib.parse import unquote from time import sleep # class PlateOverNematicFoundationPzero # class PlateOverNematicFoundationPneg # class RelaxationOverNematicFoundation # class ActuationOverNematicFoundation class PlateProblem(NonlinearProblem): def __init__(self, z, residual, jacobian, bcs): NonlinearProblem.__init__(self) self.z = z
else: AA, bb = assemble_system(maxwell+ns+CoupleTerm, (Lmaxwell + Lns) - RHSform, bcs) A = as_backend_type(AA).mat() zeros = 0*bb.array() b= as_backend_type(bb).vec() u = IO.arrayToVec(zeros) ksp = PETSc.KSP().create() pc = PETSc.PC().create() ksp.setOperators(A,P) if IterType == "Full": pass OptDB = PETSc.Options() OptDB["ksp_type"] = "preonly" OptDB["pc_type"] = "lu" OptDB["pc_factor_shift_amount"] = .1 # OptDB["pc_type"] = "ilu" ksp.setFromOptions() # ksp.view() tic() ksp.solve(b, u) time = toc() print time print ksp.its SolutionTime = SolutionTime +time outer = outer + ksp.its u, p, b, r, eps= Iter.PicardToleranceDecouple(u,x,FSpaces,dim,"inf",iter)
def test_krylov_samg_solver_elasticity(): "Test PETScKrylovSolver with smoothed aggregation AMG" def build_nullspace(V, x): """Function to build null space for 2D elasticity""" # Create list of vectors for null space ns = [x.copy() for i in range(3)] with ExitStack() as stack: vec_local = [stack.enter_context(x.localForm()) for x in ns] basis = [np.asarray(x) for x in vec_local] # Build null space basis dofs = [V.sub(i).dofmap.list.array for i in range(2)] for i in range(2): basis[i][dofs[i]] = 1.0 x = V.tabulate_dof_coordinates() basis[2][dofs[0]] = -x[dofs[0], 1] basis[2][dofs[1]] = x[dofs[1], 0] la.orthonormalize(ns) return ns def amg_solve(N, method): # Elasticity parameters E = 1.0e9 nu = 0.3 mu = E / (2.0 * (1.0 + nu)) lmbda = E * nu / ((1.0 + nu) * (1.0 - 2.0 * nu)) # Stress computation def sigma(v): return 2.0 * mu * sym(grad(v)) + lmbda * tr(sym( grad(v))) * Identity(2) # Define problem mesh = create_unit_square(MPI.COMM_WORLD, N, N) V = VectorFunctionSpace(mesh, 'Lagrange', 1) u = TrialFunction(V) v = TestFunction(V) facetdim = mesh.topology.dim - 1 bndry_facets = locate_entities_boundary(mesh, facetdim, lambda x: np.full(x.shape[1], True)) bdofs = locate_dofs_topological(V.sub(0), V, facetdim, bndry_facets) bc = dirichletbc(PETSc.ScalarType(0), bdofs, V.sub(0)) # Forms a, L = inner(sigma(u), grad(v)) * dx, dot(ufl.as_vector((1.0, 1.0)), v) * dx # Assemble linear algebra objects A = assemble_matrix(a, [bc]) A.assemble() b = assemble_vector(L) apply_lifting(b, [a], [[bc]]) b.ghostUpdate(addv=PETSc.InsertMode.ADD, mode=PETSc.ScatterMode.REVERSE) set_bc(b, [bc]) # Create solution function u = Function(V) # Create near null space basis and orthonormalize null_space = build_nullspace(V, u.vector) # Attached near-null space to matrix A.set_near_nullspace(null_space) # Test that basis is orthonormal assert null_space.is_orthonormal() # Create PETSC smoothed aggregation AMG preconditioner, and # create CG solver solver = PETSc.KSP().create(mesh.comm) solver.setType("cg") # Set matrix operator solver.setOperators(A) # Compute solution and return number of iterations return solver.solve(b, u.vector) # Set some multigrid smoother paramete rs opts = PETSc.Options() opts["mg_levels_ksp_type"] = "chebyshev" opts["mg_levels_pc_type"] = "jacobi" # Improve estimate of eigenvalues for Chebyshev smoothing opts["mg_levels_esteig_ksp_type"] = "cg" opts["mg_levels_ksp_chebyshev_esteig_steps"] = 50 # Build list of smoothed aggregation preconditioners methods = ["petsc_amg"] # if "ml_amg" in PETScPreconditioner.preconditioners(): # methods.append("ml_amg") # Test iteration count with increasing mesh size for each # preconditioner for method in methods: for N in [8, 16, 32, 64]: print("Testing method '{}' with {} x {} mesh".format(method, N, N)) niter = amg_solve(N, method) assert niter < 18
def setUp(self): self.opts = PETSc.Options(self.PREFIX)
def __init__(self, da, grid, bdy_type, sparam, verbose=0, solver='gmres', pc=None): ''' Setup the PV inversion solver Parameters ---------- da : petsc DMDA holds the petsc grid grid : qgsolver grid object grid data holder bdy_type : dict prescribe vertical and lateral boundary conditions. Examples bdy_type = {'bottom': 'D', 'top': 'D'} for Dirichlet bdy conditions bdy_type = {'bottom': 'N_RHO', 'top': 'N_RHO'} for Neumann bdy conditions with RHO bdy_type = {'bottom': 'N_PSI', 'top': 'N_PSI'} for Neumann bdy conditions using PSI instead of RHO bdy_type = {'periodic': None} for horizontal periodicity sparam : ndarray numpy array containing f^2/N^2 verbose : int, optional degree of verbosity, 0 means no outputs solver : str, optional petsc solver: 'gmres' (default), 'bicg', 'cg' pc : str, optional what is default? preconditionner: 'icc', 'bjacobi', 'asm', 'mg', 'none' ''' self._verbose = verbose # self.bdy_type = bdy_type if ('periodic' in self.bdy_type) and (self.bdy_type['periodic']): self.petscBoundaryType = 'periodic' else: self.petscBoundaryType = None # create the operator self.L = da.createMat() # if self._verbose > 0: print('A PV inversion object is being created') print(' Operator L declared') # Fill in operator values if grid._flag_hgrid_uniform and grid._flag_vgrid_uniform: self._set_L(self.L, da, grid, sparam) else: self._set_L_curv(self.L, da, grid, sparam) # if self._verbose > 0: print(' Operator L filled') # global vector for PV inversion self._RHS = da.createGlobalVec() # create solver self.ksp = PETSc.KSP() self.ksp.create(PETSc.COMM_WORLD) self.ksp.setOperators(self.L) self.ksp.setType(solver) self.ksp.setInitialGuessNonzero(True) if pc is not None: self.ksp.getPC().setType(pc) # set tolerances self.ksp.setTolerances(rtol=1e-4) self.ksp.setTolerances(max_it=100) # tests: #self.ksp.setPCSide(PETSc.PC.Side.R) #self.ksp.setInitialGuessNonzero(False) #self.ksp.setTolerances(atol=1e-1) #self.ksp.setTolerances(max_it=100) # for opt in sys.argv[1:]: PETSc.Options().setValue(opt, None) self.ksp.setFromOptions() if self._verbose > 0: print(' PV inversion is set up')
def xtest_mg_solver_stokes(): mesh0 = UnitCubeMesh(2, 2, 2) mesh1 = UnitCubeMesh(4, 4, 4) mesh2 = UnitCubeMesh(8, 8, 8) Ve = VectorElement("CG", mesh0.ufl_cell(), 2) Qe = FiniteElement("CG", mesh0.ufl_cell(), 1) Ze = MixedElement([Ve, Qe]) Z0 = FunctionSpace(mesh0, Ze) Z1 = FunctionSpace(mesh1, Ze) Z2 = FunctionSpace(mesh2, Ze) W = Z2 # Boundaries def right(x, on_boundary): return x[0] > (1.0 - DOLFIN_EPS) def left(x, on_boundary): return x[0] < DOLFIN_EPS def top_bottom(x, on_boundary): return x[1] > 1.0 - DOLFIN_EPS or x[1] < DOLFIN_EPS # No-slip boundary condition for velocity noslip = Constant((0.0, 0.0, 0.0)) bc0 = DirichletBC(W.sub(0), noslip, top_bottom) # Inflow boundary condition for velocity inflow = Expression(("-sin(x[1]*pi)", "0.0", "0.0"), degree=2) bc1 = DirichletBC(W.sub(0), inflow, right) # Collect boundary conditions bcs = [bc0, bc1] # Define variational problem (u, p) = TrialFunctions(W) (v, q) = TestFunctions(W) f = Constant((0.0, 0.0, 0.0)) a = inner(grad(u), grad(v)) * dx + div(v) * p * dx + q * div(u) * dx L = inner(f, v) * dx # Form for use in constructing preconditioner matrix b = inner(grad(u), grad(v)) * dx + p * q * dx # Assemble system A, bb = assemble_system(a, L, bcs) # Assemble preconditioner system P, btmp = assemble_system(b, L, bcs) spaces = [Z0, Z1, Z2] dm_collection = PETScDMCollection(spaces) solver = PETScKrylovSolver() solver.set_operators(A, P) PETScOptions.set("ksp_type", "gcr") PETScOptions.set("pc_type", "mg") PETScOptions.set("pc_mg_levels", 3) PETScOptions.set("pc_mg_galerkin") PETScOptions.set("ksp_monitor_true_residual") PETScOptions.set("ksp_atol", 1.0e-10) PETScOptions.set("ksp_rtol", 1.0e-10) solver.set_from_options() from petsc4py import PETSc ksp = solver.ksp() ksp.setDM(dm_collection.dm()) ksp.setDMActive(False) x = PETScVector() solver.solve(x, bb) # Check multigrid solution against LU solver solver = LUSolver(A) # noqa x_lu = PETScVector() solver.solve(x_lu, bb) assert round((x - x_lu).norm("l2"), 10) == 0 # Clear all PETSc options opts = PETSc.Options() for key in opts.getAll(): opts.delValue(key)
def transient_pipe_flow_1D( npipes, nx, dof, nphases, pipe_length, initial_time, final_time, initial_time_step, dt_min, dt_max, initial_solution, impl_python=False ): # Time Stepper (TS) for ODE and DAE # DAE - https://en.wikipedia.org/wiki/Differential_algebraic_equation # https://www.mcs.anl.gov/petsc/petsc-current/docs/manualpages/TS/ ts = PETSc.TS().create() #ts.createPython(MyTS(), comm=PETSc.COMM_SELF) # http://www.mcs.anl.gov/petsc/petsc-current/docs/manualpages/DM/index.html pipes = [] for i in range(npipes): boundary_type = PETSc.DMDA.BoundaryType.GHOSTED da = PETSc.DMDA().create([nx], dof=dof, stencil_width=1, stencil_type='star', boundary_type=boundary_type) da.setFromOptions() pipes.append(da) # Create a redundant DM, there is no petsc4py interface (yet) # so we created our own wrapper # http://www.mcs.anl.gov/petsc/petsc-current/docs/manualpages/DM/DMREDUNDANT.html # dmredundant = PETSc.DM().create() # dmredundant.setType(dmredundant.Type.REDUNDANT) # CompositeSimple1D.redundantSetSize(dmredundant, 0, dof) # dmredundant.setDimension(1) # dmredundant.setUp() # http://www.mcs.anl.gov/petsc/petsc-current/docs/manualpages/DM/DMCOMPOSITE.html dm = PETSc.DMComposite().create() for pipe in pipes: dm.addDM(pipe) # dm.addDM(dmredundant) # CompositeSimple1D.compositeSetCoupling(dm) CompositeSimple1D.registerNewSNES() ts.setDM(dm) F = dm.createGlobalVec() if impl_python: α0 = initial_solution.reshape((nx,dof))[:, nphases:-1] pde = Flow(dm, nx, dof, pipe_length, nphases, α0) ts.setIFunction(pde.evalFunction, F) else: # http://www.mcs.anl.gov/petsc/petsc-current/docs/manualpages/TS/TSSetIFunction.html assert False, 'C function not implemented yet!' # ts.setIFunction(CompositeSimple1D.formFunction, F, # args=(conductivity, source_term, wall_length, temperature_presc)) snes = ts.getSNES() snes.setUpdate(pde.updateFunction) x = dm.createGlobalVec() x[...] = initial_solution # http://www.mcs.anl.gov/petsc/petsc-current/docs/manualpages/TS/TSSetDuration.html ts.setDuration(max_time=final_time, max_steps=None) # http://www.mcs.anl.gov/petsc/petsc-current/docs/manualpages/TS/TSSetInitialTimeStep.html ts.setInitialTimeStep(initial_time=initial_time, initial_time_step=initial_time_step) # http://www.mcs.anl.gov/petsc/petsc-current/docs/manualpages/TS/TSSetProblemType.html ts.setProblemType(ts.ProblemType.NONLINEAR) ts.setEquationType(ts.EquationType.IMPLICIT) prev_sol = initial_solution.copy() # restart = PreStep(prev_sol) # ts.setPreStep(restart.prestep) # ts.setPostStep(restart.poststep) options = PETSc.Options() options.setValue('-ts_adapt_dt_min', dt_min) options.setValue('-ts_adapt_dt_max', dt_max) ts.setFromOptions() snes = ts.getSNES() if options.getString('snes_type') in [snes.Type.VINEWTONRSLS, snes.Type.VINEWTONSSLS]: # if True: # snesvi = snes.getCompositeSNES(1) snesvi = snes xl = np.zeros((nx, dof)) xl[:,:nphases] = -100 xl[:,-1] = 0 xl[:, nphases:-1] = 0 xu = np.zeros((nx, dof)) xu[:,:nphases] = 100 xu[:,-1] = 1000 xu[:, nphases:-1] = 1 xlVec = dm.createGlobalVec() xuVec = dm.createGlobalVec() xlVec.setArray(xl.flatten()) xuVec.setArray(xu.flatten()) snesvi.setVariableBounds(xlVec, xuVec) ts.solve(x) # while ts.diverged: # x[...] = prev_sol.copy() # ts.setInitialTimeStep(initial_time=initial_time, initial_time_step=0.5*initial_time_step) # ts.solve(x) final_dt = ts.getTimeStep() return x, final_dt
+ δr * M_y * ds(end) # Optional: linearise weak form # F = dolfiny.expression.linearise(F, m) # linearise around zero state # Overall form (as list of forms) F = dolfiny.function.extract_blocks(F, δm) # Create output xdmf file -- open in Paraview with Xdmf3ReaderT ofile = dolfiny.io.XDMFFile(comm, f"{name}.xdmf", "w") # Write mesh, meshtags if q <= 2: ofile.write_mesh_meshtags(mesh, mts) # Options for PETSc backend opts = PETSc.Options("beam") opts["snes_type"] = "newtonls" opts["snes_linesearch_type"] = "basic" opts["snes_atol"] = 1.0e-07 opts["snes_rtol"] = 1.0e-07 opts["snes_stol"] = 1.0e-06 opts["snes_max_it"] = 60 opts["ksp_type"] = "preonly" opts["pc_type"] = "lu" opts["pc_factor_mat_solver_type"] = "mumps" opts_global = PETSc.Options() opts_global["mat_mumps_icntl_14"] = 500 opts_global["mat_mumps_icntl_24"] = 1
# Overall form (as one-form) F = odeint.discretise_in_time(f) # Overall form (as list of forms) F = dolfiny.function.extract_blocks(F, δm) # Create output xdmf file -- open in Paraview with Xdmf3ReaderT ofile = dolfiny.io.XDMFFile(comm, f"{name}.xdmf", "w") # Write mesh, meshtags ofile.write_mesh_meshtags(mesh, mts) # Write initial state ofile.write_function(v, time.value) ofile.write_function(S, time.value) ofile.write_function(u, time.value) # Options for PETSc backend opts = PETSc.Options(name) opts["snes_type"] = "newtonls" opts["snes_linesearch_type"] = "basic" opts["snes_atol"] = 1.0e-12 opts["snes_rtol"] = 1.0e-09 opts["snes_max_it"] = 12 opts["ksp_type"] = "preonly" # opts["ksp_view"] = "::ascii_info_detail" opts["pc_type"] = "lu" opts["pc_factor_mat_solver_type"] = "mumps" # opts["pc_factor_mat_solver_type"] = "superlu_dist" opts_global = PETSc.Options() opts_global["mat_mumps_icntl_14"] = 150 opts_global["mat_superlu_dist_rowperm"] = "norowperm"
def solve_displ_system(J, F, intern_var0, intern_var1, expr_compiled, w0, w1, bcs, df, dt, t, k, io_callback=None, refinement_callback=None, rtol=1.e-6, atol=1.e-16, max_its=20): """Solve system for displacement""" rank = MPI.COMM_WORLD.rank t0 = time() A = create_matrix(J) if rank == 0: logger.info(f"[Timer] Preallocation matrix time {time() - t0:.3f}") t0 = time() t0 = time() b = create_vector(F) if rank == 0: logger.info(f"[Timer] Preallocation vector time {time() - t0:.3f}") with b.localForm() as local: local.set(0.0) assemble_vector(b, F) apply_lifting(b, [J], [bcs]) b.ghostUpdate(addv=PETSc.InsertMode.ADD, mode=PETSc.ScatterMode.REVERSE) set_bc(b, bcs) resid_norm0 = b.norm() if rank == 0: print(f"Initial DISPL residual: {resid_norm0:.2e}") iter = 0 # Total number of NR iterations including refinement attempts iter_total = 0 converged = False refine = 0 scale = 1.0 rel_resid_norm = 1.0 bcs_init = [] for bc in bcs: bcs_init += [bc.value.vector.duplicate()] with bcs_init[-1].localForm() as loc0, bc.value.vector.localForm( ) as loc1: loc1.copy(loc0) df0 = np.copy(df.value) while converged is False: if iter > max_its or rel_resid_norm > 1.e+2: refine += 1 iter = 0 if rank == 0: logger.info(79 * "!") logger.info( f"Restarting NR with rel. stepsize: {1.0 / (2 ** refine)}") with w0["displ"].vector.localForm( ) as w_local, w1["displ"].vector.localForm() as w1_local: w_local.copy(w1_local) df.value = df0 scale = 1.0 / 2**refine if refine > 10: raise ConvergenceError( "Inner adaptivity reqiures > 10 refinements.") # Reset and scale boundary condition for i, bc in enumerate(bcs_init): with bcs[i].value.vector.localForm( ) as bcsi_local, bc.localForm() as bc_local: bc_local.copy(bcsi_local) bcsi_local.scale(scale) df.value *= scale dt.value *= scale if refinement_callback is not None: refinement_callback(scale) if rank == 0: logger.info("Newton iteration for displ {}".format(iter)) if iter > 0: for bc in bcs: with bc.value.vector.localForm() as locvec: locvec.set(0.0) A.zeroEntries() with b.localForm() as local: local.set(0.0) size = A.getSize()[0] local_size = A.getLocalSize()[0] t0 = time() assemble_matrix(A, J, bcs) A.assemble() _time = time() - t0 Anorm = A.norm() if rank == 0: logger.info(f"[Timer] A0 size: {size}, local size: {local_size}") logger.info(f"[Timer] A0 assembly: {_time:.4f}") logger.info(f"[Timer] A0 assembly dofs/s: {size / _time:.1f}") logger.info(f"A0 norm: {Anorm:.4f}") t0 = time() assemble_vector(b, F) apply_lifting(b, [J], [bcs]) b.ghostUpdate(addv=PETSc.InsertMode.ADD, mode=PETSc.ScatterMode.REVERSE) set_bc(b, bcs) bnorm = b.norm() if rank == 0: logger.info(f"[Timer] b0 assembly {time() - t0:.4f}") logger.info(f"b norm: {bnorm:.4f}") nsp = build_nullspace(w1["displ"].function_space) ksp = PETSc.KSP() ksp.create(MPI.COMM_WORLD) ksp.setOptionsPrefix("disp") opts = PETSc.Options() A.setNearNullSpace(nsp) A.setBlockSize(3) ksp.setOperators(A) x = A.createVecRight() ksp.setFromOptions() t0 = time() ksp.solve(b, x) t1 = time() - t0 opts.view() xnorm = x.norm() if rank == 0: its = ksp.its t1 = time() - t0 dofsps = int(size / t1) logger.info(f"[Timer] A0 converged in: {its}") logger.info(f"[Timer] A0 solve: {t1:.4f}") logger.info(f"[Timer] A0 solve dofs/s: {dofsps:.1f}") logger.info(f"Increment norm: {xnorm}") # TODO: Local axpy segfaults, could ghostUpdate be avoided? w1["displ"].vector.axpy(1.0, x) w1["displ"].vector.ghostUpdate(addv=PETSc.InsertMode.INSERT, mode=PETSc.ScatterMode.FORWARD) # # Evaluate true residual and check # with b.localForm() as local: local.set(0.0) assemble_vector(b, F) apply_lifting(b, [J], [bcs]) b.ghostUpdate(addv=PETSc.InsertMode.ADD, mode=PETSc.ScatterMode.REVERSE) set_bc(b, bcs) norm = b.norm() rel_resid_norm = norm / resid_norm0 rel_dx_norm = x.norm() / w1["displ"].vector.norm() if rank == 0: logger.info("---") logger.info(f"Abs. resid norm: {norm:.2e}") logger.info(f"Rel. dx norm: {rel_dx_norm:.2e}") logger.info(f"Rel. resid norm: {rel_resid_norm:.2e}") logger.info("---") iter += 1 iter_total += 1 if rel_resid_norm < rtol or norm < atol: if rank == 0: logger.info( f"Newton converged in: {iter}, total: {iter_total}") if io_callback is not None: io_callback(intern_var1, w1, t.value + dt.value) return scale
def test_biharmonic(): """Manufactured biharmonic problem. Solved using rotated Regge mixed finite element method. This is equivalent to the Hellan-Herrmann-Johnson (HHJ) finite element method in two-dimensions.""" mesh = RectangleMesh(MPI.COMM_WORLD, [np.array([0.0, 0.0, 0.0]), np.array([1.0, 1.0, 0.0])], [32, 32], CellType.triangle) element = ufl.MixedElement([ufl.FiniteElement("Regge", ufl.triangle, 1), ufl.FiniteElement("Lagrange", ufl.triangle, 2)]) V = FunctionSpace(mesh, element) sigma, u = ufl.TrialFunctions(V) tau, v = ufl.TestFunctions(V) x = ufl.SpatialCoordinate(mesh) u_exact = ufl.sin(ufl.pi * x[0]) * ufl.sin(ufl.pi * x[0]) * ufl.sin(ufl.pi * x[1]) * ufl.sin(ufl.pi * x[1]) f_exact = div(grad(div(grad(u_exact)))) sigma_exact = grad(grad(u_exact)) # sigma and tau are tangential-tangential continuous according to the # H(curl curl) continuity of the Regge space. However, for the biharmonic # problem we require normal-normal continuity H (div div). Theorem 4.2 of # Lizao Li's PhD thesis shows that the latter space can be constructed by # the former through the action of the operator S: def S(tau): return tau - ufl.Identity(2) * ufl.tr(tau) sigma_S = S(sigma) tau_S = S(tau) # Discrete duality inner product eq. 4.5 Lizao Li's PhD thesis def b(tau_S, v): n = FacetNormal(mesh) return inner(tau_S, grad(grad(v))) * dx \ - ufl.dot(ufl.dot(tau_S('+'), n('+')), n('+')) * jump(grad(v), n) * dS \ - ufl.dot(ufl.dot(tau_S, n), n) * ufl.dot(grad(v), n) * ds # Non-symmetric formulation a = inner(sigma_S, tau_S) * dx - b(tau_S, u) + b(sigma_S, v) L = inner(f_exact, v) * dx V_1 = V.sub(1).collapse() zero_u = Function(V_1) with zero_u.vector.localForm() as zero_u_local: zero_u_local.set(0.0) # Strong (Dirichlet) boundary condition boundary_facets = locate_entities_boundary( mesh, mesh.topology.dim - 1, lambda x: np.full(x.shape[1], True, dtype=bool)) boundary_dofs = locate_dofs_topological((V.sub(1), V_1), mesh.topology.dim - 1, boundary_facets) bcs = [DirichletBC(zero_u, boundary_dofs, V.sub(1))] A = assemble_matrix(a, bcs=bcs) A.assemble() b = assemble_vector(L) apply_lifting(b, [a], [bcs]) b.ghostUpdate(addv=PETSc.InsertMode.ADD, mode=PETSc.ScatterMode.REVERSE) # Solve solver = PETSc.KSP().create(MPI.COMM_WORLD) PETSc.Options()["ksp_type"] = "preonly" PETSc.Options()["pc_type"] = "lu" # PETSc.Options()["pc_factor_mat_solver_type"] = "mumps" solver.setFromOptions() solver.setOperators(A) x_h = Function(V) solver.solve(b, x_h.vector) x_h.vector.ghostUpdate(addv=PETSc.InsertMode.INSERT, mode=PETSc.ScatterMode.FORWARD) # Recall that x_h has flattened indices. u_error_numerator = np.sqrt(mesh.mpi_comm().allreduce(assemble_scalar( inner(u_exact - x_h[4], u_exact - x_h[4]) * dx(mesh, metadata={"quadrature_degree": 5})), op=MPI.SUM)) u_error_denominator = np.sqrt(mesh.mpi_comm().allreduce(assemble_scalar( inner(u_exact, u_exact) * dx(mesh, metadata={"quadrature_degree": 5})), op=MPI.SUM)) assert(np.absolute(u_error_numerator / u_error_denominator) < 0.05) # Reconstruct tensor from flattened indices. # Apply inverse transform. In 2D we have S^{-1} = S. sigma_h = S(ufl.as_tensor([[x_h[0], x_h[1]], [x_h[2], x_h[3]]])) sigma_error_numerator = np.sqrt(mesh.mpi_comm().allreduce(assemble_scalar( inner(sigma_exact - sigma_h, sigma_exact - sigma_h) * dx(mesh, metadata={"quadrature_degree": 5})), op=MPI.SUM)) sigma_error_denominator = np.sqrt(mesh.mpi_comm().allreduce(assemble_scalar( inner(sigma_exact, sigma_exact) * dx(mesh, metadata={"quadrature_degree": 5})), op=MPI.SUM)) assert(np.absolute(sigma_error_numerator / sigma_error_denominator) < 0.005)
class OptionsManager(object): # What appeared on the commandline, we should never clear these. # They will override options passed in as a dict if an # options_prefix was supplied. commandline_options = frozenset(PETSc.Options().getAll()) options_object = PETSc.Options() count = itertools.count() """Mixin class that helps with managing setting petsc options. :arg parameters: The dictionary of parameters to use. :arg options_prefix: The prefix to look up items in the global options database (may be ``None``, in which case only entries from ``parameters`` will be considered. If no trailing underscore is provided, one is appended. Hence ``foo_`` and ``foo`` are treated equivalently. As an exception, if the prefix is the empty string, no underscore is appended. To use this, you must call its constructor to with the parameters you want in the options database. You then call :meth:`set_from_options`, passing the PETSc object you'd like to call ``setFromOptions`` on. Note that this will actually only call ``setFromOptions`` the first time (so really this parameters object is a once-per-PETSc-object thing). So that the runtime monitors which look in the options database actually see options, you need to ensure that the options database is populated at the time of a ``SNESSolve`` or ``KSPSolve`` call. Do that using the :meth:`inserted_options` context manager. .. code-block:: python3 with self.inserted_options(): self.snes.solve(...) This ensures that the options database has the relevant entries for the duration of the ``with`` block, before removing them afterwards. This is a much more robust way of dealing with the fixed-size options database than trying to clear it out using destructors. This object can also be used only to manage insertion and deletion into the PETSc options database, by using the context manager. """ def __init__(self, parameters, options_prefix): super().__init__() if parameters is None: parameters = {} else: # Convert nested dicts parameters = flatten_parameters(parameters) if options_prefix is None: self.options_prefix = "firedrake_%d_" % next(self.count) self.parameters = parameters self.to_delete = set(parameters) else: if len(options_prefix) and not options_prefix.endswith("_"): options_prefix += "_" self.options_prefix = options_prefix # Remove those options from the dict that were passed on # the commandline. self.parameters = { k: v for k, v in parameters.items() if options_prefix + k not in self.commandline_options } self.to_delete = set(self.parameters) # Now update parameters from options, so that they're # available to solver setup (for, e.g., matrix-free). # Can't ask for the prefixed guy in the options object, # since that does not DTRT for flag options. for k, v in self.options_object.getAll().items(): if k.startswith(self.options_prefix): self.parameters[k[len(self.options_prefix):]] = v self._setfromoptions = False def set_default_parameter(self, key, val): """Set a default parameter value. :arg key: The parameter name :arg val: The parameter value. Ensures that the right thing happens cleaning up the options database. """ k = self.options_prefix + key if k not in self.options_object and key not in self.parameters: self.parameters[key] = val self.to_delete.add(key) def set_from_options(self, petsc_obj): """Set up petsc_obj from the options database. :arg petsc_obj: The PETSc object to call setFromOptions on. Matt says: "Only ever call setFromOptions once". This function ensures we do so. """ if not self._setfromoptions: with self.inserted_options(): petsc_obj.setOptionsPrefix(self.options_prefix) # Call setfromoptions inserting appropriate options into # the options database. petsc_obj.setFromOptions() self._setfromoptions = True @contextmanager def inserted_options(self): """Context manager inside which the petsc options database contains the parameters from this object.""" try: for k, v in self.parameters.items(): self.options_object[self.options_prefix + k] = v yield finally: for k in self.to_delete: del self.options_object[self.options_prefix + k]
def __init__(self, cfgfile): ''' Constructor ''' # load run config file cfg = Config(cfgfile) # timestep setup self.ht = cfg['grid']['ht'] # timestep size self.nt = cfg['grid']['nt'] # number of timesteps self.nsave = cfg['io']['nsave'] # save only every nsave'th timestep # grid setup nx = cfg['grid']['nx'] # number of points in x ny = cfg['grid']['ny'] # number of points in y Lx = cfg['grid']['Lx'] # spatial domain in x x1 = cfg['grid']['x1'] # x2 = cfg['grid']['x2'] # Ly = cfg['grid']['Ly'] # spatial domain in y y1 = cfg['grid']['y1'] # y2 = cfg['grid']['y2'] # if x1 != x2: Lx = x2-x1 else: x1 = 0.0 x2 = Lx if y1 != y2: Ly = y2-y1 else: y1 = 0.0 y2 = Ly self.hx = Lx / nx # gridstep size in x self.hy = Ly / ny # gridstep size in y self.time = PETSc.Vec().createMPI(1, PETSc.DECIDE, comm=PETSc.COMM_WORLD) self.time.setName('t') if PETSc.COMM_WORLD.getRank() == 0: self.time.setValue(0, 0.0) # set some PETSc options OptDB = PETSc.Options() OptDB.setValue('ksp_rtol', cfg['solver']['petsc_residual']) # OptDB.setValue('snes_rtol', 1E-2) # OptDB.setValue('ksp_max_it', 100) OptDB.setValue('ksp_max_it', 200) # OptDB.setValue('ksp_max_it', 1000) # OptDB.setValue('ksp_max_it', 2000) # OptDB.setValue('ksp_monitor', '') # OptDB.setValue('log_info', '') # OptDB.setValue('log_summary', '') # create DA with single dof self.da1 = PETSc.DA().create(dim=2, dof=1, sizes=[nx, ny], proc_sizes=[PETSc.DECIDE, PETSc.DECIDE], boundary_type=('periodic', 'periodic'), stencil_width=1, stencil_type='box') # create DA (dof = 5 for Bx, By, Vx, Vy, P) self.da4 = PETSc.DA().create(dim=2, dof=5, sizes=[nx, ny], proc_sizes=[PETSc.DECIDE, PETSc.DECIDE], boundary_type=('periodic', 'periodic'), stencil_width=1, stencil_type='box') # create DA for x grid self.dax = PETSc.DA().create(dim=1, dof=1, sizes=[nx], proc_sizes=[PETSc.DECIDE], boundary_type=('periodic')) # create DA for y grid self.day = PETSc.DA().create(dim=1, dof=1, sizes=[ny], proc_sizes=[PETSc.DECIDE], boundary_type=('periodic')) # initialise grid self.da1.setUniformCoordinates(xmin=x1, xmax=x2, ymin=y1, ymax=y2) self.da4.setUniformCoordinates(xmin=x1, xmax=x2, ymin=y1, ymax=y2) self.dax.setUniformCoordinates(xmin=x1, xmax=x2) self.day.setUniformCoordinates(xmin=y1, xmax=y2) # create solution and RHS vector self.dx = self.da4.createGlobalVec() self.x = self.da4.createGlobalVec() self.b = self.da4.createGlobalVec() self.f = self.da4.createGlobalVec() self.Pb = self.da1.createGlobalVec() self.localX = self.da4.createLocalVec() # create global RK4 vectors self.X1 = self.da4.createGlobalVec() self.X2 = self.da4.createGlobalVec() self.X3 = self.da4.createGlobalVec() self.X4 = self.da4.createGlobalVec() # create local RK4 vectors self.localX1 = self.da4.createLocalVec() self.localX2 = self.da4.createLocalVec() self.localX3 = self.da4.createLocalVec() self.localX4 = self.da4.createLocalVec() # create vectors for magnetic and velocity field self.Bx = self.da1.createGlobalVec() self.By = self.da1.createGlobalVec() self.Vx = self.da1.createGlobalVec() self.Vy = self.da1.createGlobalVec() self.P = self.da1.createGlobalVec() # set variable names self.x.setName('solver_x') self.b.setName('solver_b') self.Bx.setName('Bx') self.By.setName('By') self.Vx.setName('Vx') self.Vy.setName('Vy') self.P.setName('P') # create Matrix object self.petsc_matrix = PETScSolver (self.da1, self.da4, nx, ny, self.ht, self.hx, self.hy) self.petsc_function = PETScFunction(self.da1, self.da4, nx, ny, self.ht, self.hx, self.hy) self.petsc_jacobian = PETScJacobian(self.da1, self.da4, nx, ny, self.ht, self.hx, self.hy) # create sparse matrix self.J = PETSc.Mat().createPython([self.dx.getSizes(), self.b.getSizes()], comm=PETSc.COMM_WORLD) self.J.setPythonContext(self.petsc_jacobian) self.J.setUp() # create nonlinear solver self.snes = PETSc.SNES().create() self.snes.setFunction(self.petsc_function.snes_mult, self.f) self.snes.setJacobian(self.updateJacobian, self.J) # self.snes.setUseMF() self.snes.setFromOptions() self.snes.getKSP().setType('gmres') self.snes.getKSP().getPC().setType('none') # create Preconditioner matrix and solver self.pc_mat = PETScPreconditioner(self.da1, self.da4, self.P, nx, ny, self.ht, self.hx, self.hy) # create sparse matrix self.pc_A = PETSc.Mat().createPython([self.x.getSizes(), self.b.getSizes()], comm=PETSc.COMM_WORLD) self.pc_A.setPythonContext(self.pc_mat) self.pc_A.setUp() # create linear solver and preconditioner self.pc_ksp = PETSc.KSP().create() self.pc_ksp.setFromOptions() self.pc_ksp.setOperators(self.pc_A) self.pc_ksp.setType(cfg['solver']['petsc_ksp_type']) self.pc_ksp.setInitialGuessNonzero(True) self.pc_pc = self.pc_ksp.getPC() self.pc_pc.setType('none') # create Poisson matrix and solver self.poisson_mat = PETScPoissonSolver(self.da1, self.da4, self.x, nx, ny, self.ht, self.hx, self.hy) self.poisson_A = PETSc.Mat().createPython([self.P.getSizes(), self.Pb.getSizes()], comm=PETSc.COMM_WORLD) self.poisson_A.setPythonContext(self.poisson_mat) self.poisson_A.setUp() self.poisson_ksp = PETSc.KSP().create() self.poisson_ksp.setFromOptions() self.poisson_ksp.setOperators(self.poisson_A) self.poisson_ksp.setType(cfg['solver']['petsc_ksp_type']) # self.poisson_ksp.setInitialGuessNonzero(True) self.poisson_pc = self.poisson_ksp.getPC() self.poisson_pc.setType('none') # create Arakawa solver object # self.mhd_rk4 = PETScRK4(self.da4, nx, ny, self.ht, self.hx, self.hy) # set initial data (xs, xe), (ys, ye) = self.da1.getRanges() coords = self.da1.getCoordinateDA().getVecArray(self.da1.getCoordinates()) # print # print(self.hx) # print(coords[1,0][0] - coords[0,0][0]) # print # print(self.hy) # print(coords[0,1][1] - coords[0,0][1]) # print # print(Lx) # print(coords[-1,0][0]+self.hx) # print # print(Ly) # print(coords[0,-1][1]+self.hy) # print x_arr = self.da4.getVecArray(self.x) Bx_arr = self.da1.getVecArray(self.Bx) By_arr = self.da1.getVecArray(self.By) Vx_arr = self.da1.getVecArray(self.Vx) Vy_arr = self.da1.getVecArray(self.Vy) P_arr = self.da1.getVecArray(self.P) if cfg['initial_data']['magnetic_python'] != None: init_data = __import__("runs." + cfg['initial_data']['magnetic_python'], globals(), locals(), ['magnetic_x', 'magnetic_y'], 0) for i in range(xs, xe): for j in range(ys, ye): Bx_arr[i,j] = init_data.magnetic_x(coords[i,j][0], coords[i,j][1], Lx, Ly) By_arr[i,j] = init_data.magnetic_y(coords[i,j][0], coords[i,j][1], Lx, Ly) else: Bx_arr[xs:xe, ys:ye] = cfg['initial_data']['magnetic'] By_arr[xs:xe, ys:ye] = cfg['initial_data']['magnetic'] if cfg['initial_data']['velocity_python'] != None: init_data = __import__("runs." + cfg['initial_data']['velocity_python'], globals(), locals(), ['velocity_x', 'velocity_y'], 0) for i in range(xs, xe): for j in range(ys, ye): Vx_arr[i,j] = init_data.velocity_x(coords[i,j][0], coords[i,j][1], Lx, Ly) Vy_arr[i,j] = init_data.velocity_y(coords[i,j][0], coords[i,j][1], Lx, Ly) else: Vx_arr[xs:xe, ys:ye] = cfg['initial_data']['velocity'] Vy_arr[xs:xe, ys:ye] = cfg['initial_data']['velocity'] if cfg['initial_data']['pressure_python'] != None: init_data = __import__("runs." + cfg['initial_data']['pressure_python'], globals(), locals(), ['pressure', ''], 0) for i in range(xs, xe): for j in range(ys, ye): P_arr[i,j] = init_data.pressure(coords[i,j][0], coords[i,j][1], Lx, Ly) #+ 0.5 * (Bx_arr[i,j]**2 + By_arr[i,j]**2) # copy distribution function to solution vector x_arr[xs:xe, ys:ye, 0] = Bx_arr[xs:xe, ys:ye] x_arr[xs:xe, ys:ye, 1] = By_arr[xs:xe, ys:ye] x_arr[xs:xe, ys:ye, 2] = Vx_arr[xs:xe, ys:ye] x_arr[xs:xe, ys:ye, 3] = Vy_arr[xs:xe, ys:ye] x_arr[xs:xe, ys:ye, 4] = P_arr [xs:xe, ys:ye] # update solution history self.petsc_matrix.update_history(self.x) self.petsc_function.update_history(self.x) self.petsc_jacobian.update_history(self.x) # create HDF5 output file self.hdf5_viewer = PETSc.Viewer().createHDF5(cfg['io']['hdf5_output'], mode=PETSc.Viewer.Mode.WRITE, comm=PETSc.COMM_WORLD) self.hdf5_viewer.HDF5PushGroup("/") # write grid data to hdf5 file coords_x = self.dax.getCoordinates() coords_y = self.day.getCoordinates() coords_x.setName('x') coords_y.setName('y') self.hdf5_viewer(coords_x) self.hdf5_viewer(coords_y) # write initial data to hdf5 file self.hdf5_viewer.HDF5SetTimestep(0) self.hdf5_viewer(self.time) # self.hdf5_viewer(self.x) # self.hdf5_viewer(self.b) self.hdf5_viewer(self.Bx) self.hdf5_viewer(self.By) self.hdf5_viewer(self.Vx) self.hdf5_viewer(self.Vy) self.hdf5_viewer(self.P)
def main(*args): # # I don't understand why this has to be done as follows, but PETSc # doesn't get commandline arguments if things are done in different # order. # if args: args = list(args) else: args = sys.argv commandlineArguments = parse_commandline(args[1:]) # # this needs to be done before importing PETSc # petsc4py.init(args=(args[0:1] + commandlineArguments.petsc)) from KSDG import (box_mesh, makeKSDGSolver, dofremap, KSDGException, KSDGTimeSeries, random_function, implicitTS, LigandGroups, ParameterList, default_parameters) from KSDG.ksdgdebug import log def logMAIN(*args, **kwargs): log(*args, system='MAIN', **kwargs) logMAIN('commandlineArguments.petsc', commandlineArguments.petsc) # import fenics as fe import ufl from petsc4py import PETSc fe.parameters.parse(sys.argv[0:1] + commandlineArguments.fenics) fe.parameters['ghost_mode'] = 'shared_facet' fe.parameters["form_compiler"]["optimize"] = True fe.parameters["form_compiler"]["cpp_optimize"] = True catch_signals() comm = MPI.COMM_WORLD groups = LigandGroups(commandlineArguments) params0 = ParameterList(default_parameters) # parameter initial values params0.add(groups.params()) params0.decode(commandlineArguments.params) groups.fourier_series() params0.add(groups.params()) # in case Fourier added any new ones Vgroups = copy.deepcopy(groups) Vparams = ParameterList(default_parameters) # for evaluating V Vparams.add(Vgroups.params()) width = params0['width'] nelements = params0['nelements'] dim = params0['dim'] degree = params0['degree'] periodic = commandlineArguments.periodic nligands = groups.nligands() rhomax = params0['rhomax'] cushion = params0['cushion'] t0 = params0['t0'] param_funcs = pfuncs(commandlineArguments, t0=t0, params0=params0) if commandlineArguments.penalties: rhopen = grhopen = Upen = gUpen = commandlineArguments.penalties else: rhopen = Upen = gUpen = 1.0 grhopen = 10.0 rhopen = params0['rhopen'] or rhopen params0['rhopen'] = rhopen grhopen = params0['grhopen'] or grhopen params0['grhopen'] = grhopen Upen = params0['Upen'] or Upen params0['Upen'] = Upen gUpen = params0['gUpen'] or gUpen params0['gUpen'] = gUpen logMAIN('list(groups.ligands())', list(groups.ligands())) maxscale = params0['maxscale'] if (commandlineArguments.showparams): for n, p, d, h in params0.params(): print('{n}={val} -- {h}'.format(n=n, val=p(), h=h)) return logMAIN('params0', params0) def Vfunc(Us, params={}): Vparams.update(params) # copy params into ligands return Vgroups.V(Us) # compute V def Vtophat(rho, params={}): tanh = ufl.tanh((rho - params['rhomax']) / params['cushion']) return params['maxscale'] * params['sigma']**2 / 2 * (tanh + 1) def Vwitch(rho, params={}): tanh = ufl.tanh((rho - params['rhomax']) / params['cushion']) return (params['maxscale'] * params['sigma']**2 / 2 * (tanh + 1) * (rho / params['rhomax'])) Vcap = Vwitch if commandlineArguments.cappotential == 'witch' else Vtophat def V2(Us, rho, params={}): return Vfunc(Us, params=params) + Vcap(rho, params=params) mesh = box_mesh(width=width, dim=dim, nelements=nelements) if commandlineArguments.randgrid: rmesh = box_mesh(width=width, dim=dim, nelements=commandlineArguments.randgrid) else: rmesh = box_mesh(width=width, dim=dim, nelements=nelements // 2) fe.parameters["form_compiler"]["representation"] = "uflacs" fe.parameters['linear_algebra_backend'] = 'PETSc' np.random.seed(commandlineArguments.seed) murho0 = params0['Nworms'] / (width**dim) resuming = commandlineArguments.resume or commandlineArguments.restart if resuming: if comm.rank == 0: cpf = KSDGTimeSeries(resuming, 'r') tlast = cpf.sorted_times()[-1] if commandlineArguments.resume: t0 = tlast else: t0 = params0['t0'] lastvec = cpf.retrieve_by_time(tlast) nlastvec = lastvec.size vectype = lastvec.dtype else: t0 = params0['t0'] nlastvec = None vectype = None if commandlineArguments.resume: # # if resuming, get t0 from the datafile # t0 = comm.bcast(t0, root=0) nlastvec = comm.bcast(nlastvec, root=0) vectype = comm.bcast(vectype, root=0) if comm.rank != 0: lastvec = np.empty(nlastvec, vectype) logMAIN('t0', t0) logMAIN('nlastvec', nlastvec) logMAIN('vectype', vectype) U0s = [fe.Constant(0.0) for i in range(nligands)] ksdg = makeKSDGSolver(degree=degree, mesh=mesh, width=width, nelements=nelements, t0=t0, parameters=params0, param_funcs=param_funcs, periodic=periodic, ligands=groups, solver_type=commandlineArguments.solver, V=V2, U0=U0s, rho0=fe.Constant(0.0)) remap = dofremap(ksdg) lastvec = np.array(lastvec[np.argsort(remap)]) lastvec = comm.bcast(lastvec, root=0) nlastvec = lastvec.size vectype = lastvec.dtype logMAIN('lastvec', lastvec) logMAIN('type(lastvec)', type(lastvec)) logMAIN('nlastvec', nlastvec) logMAIN('vectype', vectype) dofmap = ksdg.sol.function_space().dofmap() logMAIN('np.array(dofmap.dofs())', np.array(dofmap.dofs())) logMAIN('lastvec[np.array(dofmap.dofs())]', lastvec[np.array(dofmap.dofs())]) ksdg.sol.vector()[:] = lastvec[np.array(dofmap.dofs())] ksdg.sol.vector().apply('insert') else: t0 = params0['t0'] Cd1 = fe.FunctionSpace(mesh, 'CG', degree) logMAIN('Cd1', Cd1) rho0 = fe.Function(Cd1) logMAIN('rho0', rho0) random_function(rho0, mesh=rmesh, periodic=periodic, mu=murho0, sigma=params0['srho0']) if params0['rho0']: rho0Exp = fe.Expression(params0['rho0'], degree=degree) rho0Expf = fe.interpolate(rho0Exp, Cd1) rho0.vector()[:] += rho0Expf.vector() U0s = [fe.Function(Cd1) for i in range(nligands)] for i, lig in enumerate(groups.ligands()): U0s[i].vector()[:] = (lig.s / lig.gamma) * rho0.vector() ksdg = makeKSDGSolver(degree=degree, mesh=mesh, width=width, nelements=nelements, t0=t0, parameters=params0, param_funcs=param_funcs, periodic=periodic, ligands=groups, solver_type=commandlineArguments.solver, V=V2, U0=U0s, rho0=rho0) logMAIN('ksdg', str(ksdg)) options = PETSc.Options() options.setValue('ts_max_snes_failures', 100) ts = implicitTS(ksdg, t0=t0, restart=not bool(resuming), rtol=params0['rtol'], atol=params0['atol'], dt=params0['dt'], tmax=params0['tmax'], maxsteps=params0['maxsteps']) logMAIN('ts', str(ts)) ts.setMonitor(ts.printMonitor) logMAIN('printMonitor set') if commandlineArguments.save: with open(commandlineArguments.save + '_options.txt', 'w') as opts: optd = vars(commandlineArguments) for k, v in optd.items(): if k != 'petsc' and k != 'fenics' and k != 'params': print('--%s=%s' % (k, v), file=opts) print(params0.str(), file=opts) try: popts = optd['petsc'] if popts: print('--petsc', file=opts) for popt in popts: print(popt, file=opts) print('--', file=opts) except KeyError: pass try: fopts = optd['fenics'] if fopts: print('--fenics', file=opts) for fopt in fopts: print(fopt, file=opts) print('--', file=opts) except KeyError: pass saveMonitor, closeMonitor = ts.makeSaveMonitor( prefix=commandlineArguments.save) ts.setMonitor(saveMonitor) logMAIN('saveMonitor set') if commandlineArguments.check: ts.setMonitor(ts.checkpointMonitor, (), {'prefix': commandlineArguments.check}) logMAIN('checkpointMonitor set') try: logMAIN('calling ts.solve', ts.solve) ts.solve() except KeyboardInterrupt as e: print('KeyboardInterrupt:', str(e)) except Exception as e: print('Exception:', str(e)) einfo = sys.exc_info() sys.excepthook(*einfo) if commandlineArguments.save: closeMonitor() logMAIN('saveMonitor closed') ts.cleanup() if MPI.COMM_WORLD.rank == 0: print("SNES failures = ", ts.getSNESFailures())
def setUp(self, pc): A, P = pc.getOperators() print A.size self.Ct = A.getSubMatrix(self.u_is, self.b_is) self.C = A.getSubMatrix(self.b_is, self.u_is) self.D = A.getSubMatrix(self.r_is, self.b_is) self.Bt = A.getSubMatrix(self.u_is, self.p_is) self.B = A.getSubMatrix(self.p_is, self.u_is) self.Dt = A.getSubMatrix(self.b_is, self.r_is) # print self.Ct.view() #CFC = sp.csr_matrix( (data,(row,column)), shape=(self.W[1].dim(),self.W[1].dim()) ) #print CFC.shape #CFC = PETSc.Mat().createAIJ(size=CFC.shape,csr=(CFC.indptr, CFC.indices, CFC.data)) #print CFC.size, self.AA.size #MX = self.AA+self.F MX = self.F # MO.StoreMatrix(B,"A") # print FC.todense() self.kspF.setType('preonly') self.kspF.getPC().setType('lu') self.kspF.setFromOptions() self.kspF.setPCSide(0) self.kspA.setType('preonly') self.kspA.getPC().setType('lu') self.kspA.setFromOptions() self.kspA.setPCSide(0) self.kspQ.setType('preonly') self.kspQ.getPC().setType('lu') self.kspQ.setFromOptions() self.kspQ.setPCSide(0) self.kspScalar.setType('preonly') self.kspScalar.getPC().setType('lu') self.kspScalar.setFromOptions() self.kspScalar.setPCSide(0) kspMX = PETSc.KSP() kspMX.create(comm=PETSc.COMM_WORLD) pcMX = kspMX.getPC() kspMX.setType('preonly') pcMX.setType('lu') kspMX.setOperators(MX, MX) OptDB = PETSc.Options() #OptDB["pc_factor_mat_ordering_type"] = "rcm" #OptDB["pc_factor_mat_solver_package"] = "mumps" kspMX.setFromOptions() self.kspMX = kspMX # self.kspCGScalar.setType('preonly') # self.kspCGScalar.getPC().setType('lu') # self.kspCGScalar.setFromOptions() # self.kspCGScalar.setPCSide(0) self.kspVector.setType('preonly') self.kspVector.getPC().setType('lu') self.kspVector.setFromOptions() self.kspVector.setPCSide(0) print "setup"
from dolfin import * import numpy as np from petsc4py import PETSc from petsc4py.PETSc import Mat from petsc4py.PETSc import Vec import pybind11 # set petsc options at beginning petsc_options = PETSc.Options() # Use petsc4py to define the smoothers def direct(Ah, bh): '''LU factorisation. Ah is the matrix, bh is the rhs''' ksp = PETSc.KSP().create() ksp.setOptionsPrefix('coarse_') yh = bh.duplicate() ksp.setNormType(PETSc.KSP.NormType.NONE) pc = ksp.getPC() pc.setOptionsPrefix('coarse_pc_') petsc_options['coarse_pc_type'] = 'lu' ksp.setOperators(Ah) ksp.setFromOptions() ksp.solve(bh, yh) return yh def smoother(Ag, bg, Ng, igg, ksptype, pctype): '''Smoother for multigrid. Ag, and bg are the LHS and RHS respectively. Ng is the number of iterations (usually 1), igg is the initial guess for the solution.
def __init__(self, cfgfile): ''' Constructor ''' super().__init__(cfgfile, mode="lu") OptDB = PETSc.Options() OptDB.setValue('ksp_monitor', '') OptDB.setValue('snes_monitor', '') # OptDB.setValue('log_info', '') # OptDB.setValue('log_summary', '') OptDB.setValue('snes_ls', 'basic') # OptDB.setValue('snes_ls', 'quadratic') OptDB.setValue('snes_rtol', self.cfg['solver']['petsc_snes_rtol']) OptDB.setValue('snes_atol', self.cfg['solver']['petsc_snes_atol']) OptDB.setValue('snes_stol', self.cfg['solver']['petsc_snes_stol']) OptDB.setValue('snes_max_it', self.cfg['solver']['petsc_snes_max_iter']) OptDB.setValue('ksp_rtol', self.cfg['solver']['petsc_ksp_rtol']) OptDB.setValue('ksp_atol', self.cfg['solver']['petsc_ksp_atol']) OptDB.setValue('ksp_max_it', self.cfg['solver']['petsc_ksp_max_iter']) # OptDB.setValue('mat_superlu_dist_matinput', 'DISTRIBUTED') # OptDB.setValue('mat_superlu_dist_rowperm', 'NATURAL') OptDB.setValue('mat_superlu_dist_colperm', 'PARMETIS') OptDB.setValue('mat_superlu_dist_parsymbfact', 1) # create Jacobian, Function, and linear Matrix objects self.petsc_solver = PETScSolver(self.da1, self.da4, self.nx, self.ny, self.ht, self.hx, self.hy) # initialise linear matrix self.M = self.da4.createMat() self.M.setOption(PETSc.Mat.Option.NEW_NONZERO_ALLOCATION_ERR, False) self.M.setUp() # initialise Jacobian self.Jac = self.da4.createMat() self.Jac.setOption(PETSc.Mat.Option.NEW_NONZERO_ALLOCATION_ERR, False) self.Jac.setUp() # initialise matrixfree Jacobian self.Jmf = PETSc.Mat().createPython( [self.x.getSizes(), self.b.getSizes()], context=self.petsc_solver, comm=PETSc.COMM_WORLD) self.Jmf.setUp() # create nonlinear solver self.snes = PETSc.SNES().create() self.snes.setFunction(self.petsc_solver.snes_function, self.f) self.snes.setJacobian(self.updateJacobian, self.Jmf, self.Jac) self.snes.setFromOptions() self.snes.getKSP().setType('preonly') self.snes.getKSP().getPC().setType('lu') self.snes.getKSP().getPC().setFactorSolverType(self.solver_package) # update solution history self.petsc_solver.update_previous(self.x)
def __init__(self, cfgfile): ''' Constructor ''' super().__init__(cfgfile, mode="split") OptDB = PETSc.Options() # OptDB.setValue('ksp_monitor', '') # OptDB.setValue('snes_monitor', '') # # OptDB.setValue('log_info', '') # OptDB.setValue('log_summary', '') OptDB.setValue('ksp_rtol', self.cfg['solver']['petsc_ksp_rtol']) OptDB.setValue('ksp_atol', self.cfg['solver']['petsc_ksp_atol']) OptDB.setValue('ksp_max_it', self.cfg['solver']['petsc_ksp_max_iter']) # OptDB.setValue('ksp_initial_guess_nonzero', 1) OptDB.setValue('pc_type', 'hypre') OptDB.setValue('pc_hypre_type', 'boomeramg') OptDB.setValue('pc_hypre_boomeramg_max_iter', 2) # OptDB.setValue('pc_hypre_boomeramg_max_levels', 6) # OptDB.setValue('pc_hypre_boomeramg_tol', 1e-7) # create DA (dof = 2 for A, P) self.da2 = PETSc.DA().create(dim=2, dof=2, sizes=[self.nx, self.ny], proc_sizes=[PETSc.DECIDE, PETSc.DECIDE], boundary_type=('periodic', 'periodic'), stencil_width=1, stencil_type='box') # create solution and RHS vector self.dx2 = self.da2.createGlobalVec() self.dy2 = self.da2.createGlobalVec() self.b = self.da2.createGlobalVec() self.Ad = self.da1.createGlobalVec() self.Jd = self.da1.createGlobalVec() self.Pd = self.da1.createGlobalVec() self.Od = self.da1.createGlobalVec() # create Jacobian, Function, and linear Matrix objects self.petsc_precon = PETScPreconditioner(self.da1, self.da2, self.nx, self.ny, self.ht, self.hx, self.hy) # self.petsc_solver2 = PETScSolverDOF2(self.da1, self.da2, self.nx, self.ny, self.ht, self.hx, self.hy) self.petsc_solver2 = PETScSolverDOF2(self.da1, self.da2, self.nx, self.ny, self.ht, self.hx, self.hy, self.petsc_precon) self.petsc_solver = PETScSolver(self.da1, self.da4, self.nx, self.ny, self.ht, self.hx, self.hy) self.petsc_precon.set_tolerances( poisson_rtol=self.cfg['solver']['pc_poisson_rtol'], poisson_atol=self.cfg['solver']['pc_poisson_atol'], poisson_max_it=self.cfg['solver']['pc_poisson_max_iter'], parabol_rtol=self.cfg['solver']['pc_parabol_rtol'], parabol_atol=self.cfg['solver']['pc_parabol_atol'], parabol_max_it=self.cfg['solver']['pc_parabol_max_iter'], jacobi_max_it=self.cfg['solver']['pc_jacobi_max_iter']) # initialise matrixfree Jacobian self.Jmf = PETSc.Mat().createPython( [self.b.getSizes(), self.b.getSizes()], context=self.petsc_solver2, comm=PETSc.COMM_WORLD) self.Jmf.setUp() # create linear solver self.ksp = PETSc.KSP().create() self.ksp.setFromOptions() self.ksp.setOperators(self.Jmf) self.ksp.setInitialGuessNonzero(True) self.ksp.setType('fgmres') self.ksp.getPC().setType('none') # update solution history self.petsc_solver.update_previous(self.x) self.petsc_solver2.update_previous(self.A, self.J, self.P, self.O)
def solve(A, b, u, params, Fspace, SolveType, IterType, OuterTol, InnerTol, HiptmairMatrices, Hiptmairtol, KSPlinearfluids, Fp, kspF): if SolveType == "Direct": ksp = PETSc.KSP() ksp.create(comm=PETSc.COMM_WORLD) pc = ksp.getPC() ksp.setType('preonly') pc.setType('lu') OptDB = PETSc.Options() OptDB['pc_factor_mat_solver_package'] = "pastix" OptDB['pc_factor_mat_ordering_type'] = "rcm" ksp.setFromOptions() scale = b.norm() b = b / scale ksp.setOperators(A, A) del A ksp.solve(b, u) # Mits +=dodim u = u * scale MO.PrintStr("Number iterations = " + str(ksp.its), 60, "+", "\n\n", "\n\n") return u, ksp.its, 0 elif SolveType == "Direct-class": ksp = PETSc.KSP() ksp.create(comm=PETSc.COMM_WORLD) pc = ksp.getPC() ksp.setType('gmres') pc.setType('none') ksp.setFromOptions() scale = b.norm() b = b / scale ksp.setOperators(A, A) del A ksp.solve(b, u) # Mits +=dodim u = u * scale MO.PrintStr("Number iterations = " + str(ksp.its), 60, "+", "\n\n", "\n\n") return u, ksp.its, 0 else: # u = b.duplicate() if IterType == "Full": ksp = PETSc.KSP() ksp.create(comm=PETSc.COMM_WORLD) pc = ksp.getPC() ksp.setType('fgmres') pc.setType('python') pc.setType(PETSc.PC.Type.PYTHON) # FSpace = [Velocity,Magnetic,Pressure,Lagrange] reshist = {} def monitor(ksp, its, fgnorm): reshist[its] = fgnorm print its, " OUTER:", fgnorm # ksp.setMonitor(monitor) ksp.max_it = 1000 W = Fspace FFSS = [W.sub(0), W.sub(1), W.sub(2), W.sub(3)] pc.setPythonContext( MHDprec.InnerOuterMAGNETICinverse( FFSS, kspF, KSPlinearfluids[0], KSPlinearfluids[1], Fp, HiptmairMatrices[3], HiptmairMatrices[4], HiptmairMatrices[2], HiptmairMatrices[0], HiptmairMatrices[1], HiptmairMatrices[6], Hiptmairtol)) #OptDB = PETSc.Options() # OptDB['pc_factor_mat_solver_package'] = "mumps" # OptDB['pc_factor_mat_ordering_type'] = "rcm" # ksp.setFromOptions() scale = b.norm() b = b / scale ksp.setOperators(A, A) del A ksp.solve(b, u) # Mits +=dodim u = u * scale MO.PrintStr("Number iterations = " + str(ksp.its), 60, "+", "\n\n", "\n\n") return u, ksp.its, 0 IS = MO.IndexSet(Fspace, '2by2') M_is = IS[1] NS_is = IS[0] kspNS = PETSc.KSP().create() kspM = PETSc.KSP().create() kspNS.setTolerances(OuterTol) kspNS.setOperators(A.getSubMatrix(NS_is, NS_is)) kspM.setOperators(A.getSubMatrix(M_is, M_is)) # print P.symmetric if IterType == "MD": kspNS.setType('gmres') kspNS.max_it = 500 pcNS = kspNS.getPC() pcNS.setType(PETSc.PC.Type.PYTHON) pcNS.setPythonContext( NSpreconditioner.NSPCD( MixedFunctionSpace([Fspace.sub(0), Fspace.sub(1)]), kspF, KSPlinearfluids[0], KSPlinearfluids[1], Fp)) elif IterType == "CD": kspNS.setType('minres') pcNS = kspNS.getPC() pcNS.setType(PETSc.PC.Type.PYTHON) Q = KSPlinearfluids[1].getOperators()[0] Q = 1. / params[2] * Q KSPlinearfluids[1].setOperators(Q, Q) pcNS.setPythonContext( StokesPrecond.MHDApprox( MixedFunctionSpace([Fspace.sub(0), Fspace.sub(1)]), kspF, KSPlinearfluids[1])) reshist = {} def monitor(ksp, its, fgnorm): reshist[its] = fgnorm print fgnorm # kspNS.setMonitor(monitor) uNS = u.getSubVector(NS_is) bNS = b.getSubVector(NS_is) # print kspNS.view() scale = bNS.norm() bNS = bNS / scale print bNS.norm() kspNS.solve(bNS, uNS) uNS = uNS * scale NSits = kspNS.its kspNS.destroy() # for line in reshist.values(): # print line kspM.setFromOptions() kspM.setType(kspM.Type.MINRES) kspM.setTolerances(InnerTol) pcM = kspM.getPC() pcM.setType(PETSc.PC.Type.PYTHON) pcM.setPythonContext( MP.Hiptmair(MixedFunctionSpace([Fspace.sub(2), Fspace.sub(3)]), HiptmairMatrices[3], HiptmairMatrices[4], HiptmairMatrices[2], HiptmairMatrices[0], HiptmairMatrices[1], HiptmairMatrices[6], Hiptmairtol)) uM = u.getSubVector(M_is) bM = b.getSubVector(M_is) scale = bM.norm() bM = bM / scale print bM.norm() kspM.solve(bM, uM) uM = uM * scale Mits = kspM.its kspM.destroy() u = IO.arrayToVec(np.concatenate([uNS.array, uM.array])) MO.PrintStr("Number of M iterations = " + str(Mits), 60, "+", "\n\n", "\n\n") MO.PrintStr("Number of NS/S iterations = " + str(NSits), 60, "+", "\n\n", "\n\n") return u, NSits, Mits
def setup_method(self, method): OptDB = PETSc.Options() OptDB.setValue("ksp_type", "cg") OptDB.setValue("pc_type", "gamg") reload(poisson_3d_p)
def condition_number(A, method='simplified'): """ Estimate the condition number of the matrix A """ if method == 'simplified': # Calculate max(abs(A))/min(abs(A)) amin, amax = 1e10, -1e10 for irow in range(A.size(0)): _indices, values = A.getrow(irow) aa = abs(values) amax = max(amax, aa.max()) aa[aa == 0] = amax amin = min(amin, aa.min()) amin = dolfin.MPI.min(dolfin.MPI.comm_world, float(amin)) amax = dolfin.MPI.max(dolfin.MPI.comm_world, float(amax)) return amax / amin elif method == 'numpy': from numpy.linalg import cond A = mat_to_scipy_csr(A).todense() return cond(A) elif method == 'SLEPc': from petsc4py import PETSc from slepc4py import SLEPc # Get the petc4py matrix PA = dolfin.as_backend_type(A).mat() # Calculate the largest and smallest singular value opts = { 'svd_type': 'cross', 'svd_eps_type': 'gd', # 'help': 'svd_type' } with petsc_options(opts): S = SLEPc.SVD() S.create() S.setOperator(PA) S.setFromOptions() S.setDimensions(1, PETSc.DEFAULT, PETSc.DEFAULT) S.setWhichSingularTriplets(SLEPc.SVD.Which.LARGEST) S.solve() if S.getConverged() == 1: sigma_1 = S.getSingularTriplet(0) else: raise ValueError( 'Could not find the highest singular value (%d)' % S.getConvergedReason()) print('Highest singular value:', sigma_1) S.setWhichSingularTriplets(SLEPc.SVD.Which.SMALLEST) S.solve() if S.getConverged() == 1: sigma_n = S.getSingularTriplet(0) else: raise ValueError( 'Could not find the lowest singular value (%d)' % S.getConvergedReason()) print('Lowest singular value:', sigma_n) print(PETSc.Options().getAll()) print(PETSc.Options().getAll()) return sigma_1 / sigma_n
def __init__(self, cfgfile): ''' Constructor ''' # load run config file cfg = Config(cfgfile) # timestep setup self.ht = cfg['grid']['ht'] # timestep size self.nt = cfg['grid']['nt'] # number of timesteps self.nsave = cfg['io']['nsave'] # save only every nsave'th timestep # grid setup self.nx = cfg['grid']['nx'] # number of points in x self.ny = cfg['grid']['ny'] # number of points in y Lx = cfg['grid']['Lx'] # spatial domain in x x1 = cfg['grid']['x1'] # x2 = cfg['grid']['x2'] # Ly = cfg['grid']['Ly'] # spatial domain in y y1 = cfg['grid']['y1'] # y2 = cfg['grid']['y2'] # if x1 != x2: Lx = x2 - x1 else: x1 = 0.0 x2 = Lx if y1 != y2: Ly = y2 - y1 else: y1 = 0.0 y2 = Ly self.hx = Lx / self.nx # gridstep size in x self.hy = Ly / self.ny # gridstep size in y self.time = PETSc.Vec().createMPI(1, PETSc.DECIDE, comm=PETSc.COMM_WORLD) self.time.setName('t') if PETSc.COMM_WORLD.getRank() == 0: self.time.setValue(0, 0.0) OptDB = PETSc.Options() OptDB.setValue('snes_rtol', cfg['solver']['petsc_snes_rtol']) OptDB.setValue('snes_atol', cfg['solver']['petsc_snes_atol']) OptDB.setValue('snes_stol', cfg['solver']['petsc_snes_stol']) OptDB.setValue('snes_max_it', cfg['solver']['petsc_snes_max_iter']) OptDB.setValue('ksp_rtol', cfg['solver']['petsc_ksp_rtol']) OptDB.setValue('ksp_atol', cfg['solver']['petsc_ksp_atol']) OptDB.setValue('ksp_max_it', cfg['solver']['petsc_ksp_max_iter']) # OptDB.setValue('ksp_monitor', '') # OptDB.setValue('snes_monitor', '') # # OptDB.setValue('log_info', '') # OptDB.setValue('log_summary', '') self.snes_rtol = cfg['solver']['petsc_snes_rtol'] self.snes_atol = cfg['solver']['petsc_snes_atol'] self.snes_max_iter = cfg['solver']['petsc_snes_max_iter'] # create DA with single dof self.da1 = PETSc.DA().create(dim=2, dof=1, sizes=[self.nx, self.ny], proc_sizes=[PETSc.DECIDE, PETSc.DECIDE], boundary_type=('periodic', 'periodic'), stencil_width=2, stencil_type='box') # create DA (dof = 4 for A, J, P, O) self.da4 = PETSc.DA().create(dim=2, dof=4, sizes=[self.nx, self.ny], proc_sizes=[PETSc.DECIDE, PETSc.DECIDE], boundary_type=('periodic', 'periodic'), stencil_width=2, stencil_type='box') # create DA for x grid self.dax = PETSc.DA().create(dim=1, dof=1, sizes=[self.nx], proc_sizes=[PETSc.DECIDE], boundary_type=('periodic')) # create DA for y grid self.day = PETSc.DA().create(dim=1, dof=1, sizes=[self.ny], proc_sizes=[PETSc.DECIDE], boundary_type=('periodic')) # initialise grid self.da1.setUniformCoordinates(xmin=x1, xmax=x2, ymin=y1, ymax=y2) self.da4.setUniformCoordinates(xmin=x1, xmax=x2, ymin=y1, ymax=y2) self.dax.setUniformCoordinates(xmin=x1, xmax=x2) self.day.setUniformCoordinates(xmin=y1, xmax=y2) # create RHS vector self.Ab = self.da1.createGlobalVec() self.Ob = self.da1.createGlobalVec() self.Pb = self.da1.createGlobalVec() # create vectors for magnetic and velocity field self.A = self.da1.createGlobalVec() # magnetic vector potential A self.J = self.da1.createGlobalVec() # current density J self.P = self.da1.createGlobalVec() # streaming function psi self.O = self.da1.createGlobalVec() # vorticity omega self.Bx = self.da1.createGlobalVec() self.By = self.da1.createGlobalVec() self.Vx = self.da1.createGlobalVec() self.Vy = self.da1.createGlobalVec() # set variable names self.A.setName('A') self.J.setName('J') self.P.setName('P') self.O.setName('O') self.Bx.setName('Bx') self.By.setName('By') self.Vx.setName('Vx') self.Vy.setName('Vy') # create nullspace self.poisson_nullspace = PETSc.NullSpace().create(constant=True) # create jacobian and matrix objects self.petsc_vorticity = PETScVorticity(self.da1, self.nx, self.ny, self.ht, self.hx, self.hy) self.petsc_ohmslaw = PETScOhmsLaw(self.da1, self.nx, self.ny, self.ht, self.hx, self.hy) self.petsc_poisson = PETScPoisson(self.da1, self.nx, self.ny, self.hx, self.hy) # initialise vorticity matrix self.vorticity_matrix = self.da1.createMat() self.vorticity_matrix.setOption( self.vorticity_matrix.Option.NEW_NONZERO_ALLOCATION_ERR, False) self.vorticity_matrix.setUp() # initialise Ohms's law matrix self.ohmslaw_matrix = self.da1.createMat() self.ohmslaw_matrix.setOption( self.ohmslaw_matrix.Option.NEW_NONZERO_ALLOCATION_ERR, False) self.ohmslaw_matrix.setUp() # initialise Poisson matrix self.poisson_matrix = self.da1.createMat() self.poisson_matrix.setOption( self.poisson_matrix.Option.NEW_NONZERO_ALLOCATION_ERR, False) self.poisson_matrix.setUp() self.poisson_matrix.setNullSpace(self.poisson_nullspace) # create nonlinear vorticity solver self.vorticity_snes = PETSc.SNES().create() self.vorticity_snes.setType('ksponly') self.vorticity_snes.setFunction(self.petsc_vorticity.snes_mult, self.Ob) self.vorticity_snes.setJacobian(self.update_vorticity_jacobian, self.vorticity_matrix) self.vorticity_snes.setFromOptions() # self.vorticity_snes.getKSP().setType('gmres') self.vorticity_snes.getKSP().setType('preonly') self.vorticity_snes.getKSP().getPC().setType('lu') self.vorticity_snes.getKSP().getPC().setFactorSolverPackage( solver_package) # create nonlinear Ohms's law solver self.ohmslaw_snes = PETSc.SNES().create() self.ohmslaw_snes.setType('ksponly') self.ohmslaw_snes.setFunction(self.petsc_ohmslaw.snes_mult, self.Ab) self.ohmslaw_snes.setJacobian(self.update_ohmslaw_jacobian, self.ohmslaw_matrix) self.ohmslaw_snes.setFromOptions() # self.ohmslaw_snes.getKSP().setType('gmres') self.ohmslaw_snes.getKSP().setType('preonly') self.ohmslaw_snes.getKSP().getPC().setType('lu') self.ohmslaw_snes.getKSP().getPC().setFactorSolverPackage( solver_package) # create linear Poisson solver self.poisson_ksp = PETSc.KSP().create() self.poisson_ksp.setFromOptions() self.poisson_ksp.setOperators(self.poisson_matrix) self.poisson_ksp.setType('preonly') self.poisson_ksp.getPC().setType('lu') self.poisson_ksp.getPC().setFactorSolverPackage(solver_package) # self.poisson_ksp.setNullSpace(self.poisson_nullspace) self.petsc_poisson.formMat(self.poisson_matrix) # create derivatives object self.derivatives = PETScDerivatives(self.da1, self.nx, self.ny, self.ht, self.hx, self.hy) # get coordinate vectors coords_x = self.dax.getCoordinates() coords_y = self.day.getCoordinates() # save x coordinate arrays scatter, xVec = PETSc.Scatter.toAll(coords_x) scatter.begin(coords_x, xVec, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) scatter.end(coords_x, xVec, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) xGrid = xVec.getValues(range(0, self.nx)).copy() scatter.destroy() xVec.destroy() # save y coordinate arrays scatter, yVec = PETSc.Scatter.toAll(coords_y) scatter.begin(coords_y, yVec, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) scatter.end(coords_y, yVec, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) yGrid = yVec.getValues(range(0, self.ny)).copy() scatter.destroy() yVec.destroy() # set initial data (xs, xe), (ys, ye) = self.da1.getRanges() A_arr = self.da1.getVecArray(self.A) P_arr = self.da1.getVecArray(self.P) init_data = __import__("runs." + cfg['initial_data']['python'], globals(), locals(), ['magnetic_A', 'velocity_P'], 0) for i in range(xs, xe): for j in range(ys, ye): A_arr[i, j] = init_data.magnetic_A(xGrid[i], yGrid[j], Lx, Ly) P_arr[i, j] = init_data.velocity_P(xGrid[i], yGrid[j], Lx, Ly) # Fourier Filtering self.nfourier = cfg['initial_data']['nfourier'] if self.nfourier >= 0: # obtain whole A vector everywhere scatter, Aglobal = PETSc.Scatter.toAll(self.A) scatter.begin(self.A, Aglobal, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) scatter.end(self.A, Aglobal, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) petsc_indices = self.da1.getAO().app2petsc( np.arange(self.nx * self.ny, dtype=np.int32)) Ainit = Aglobal.getValues(petsc_indices).copy().reshape( (self.ny, self.nx)) scatter.destroy() Aglobal.destroy() # compute FFT, cut, compute inverse FFT from scipy.fftpack import rfft, irfft Afft = rfft(Ainit, axis=1) Afft[:, 0] = 0. Afft[:, self.nfourier + 1:] = 0. A_arr = self.da1.getVecArray(self.A) A_arr[:, :] = irfft(Afft).T[xs:xe, ys:ye] # compute current and vorticity self.derivatives.laplace_vec(self.A, self.J, -1.) self.derivatives.laplace_vec(self.P, self.O, -1.) J_arr = self.da1.getVecArray(self.J) O_arr = self.da1.getVecArray(self.O) # add perturbations for i in range(xs, xe): for j in range(ys, ye): J_arr[i, j] += init_data.current_perturbation( xGrid[i], yGrid[j], Lx, Ly) O_arr[i, j] += init_data.vorticity_perturbation( xGrid[i], yGrid[j], Lx, Ly) # create HDF5 output file self.hdf5_viewer = PETSc.ViewerHDF5().create( cfg['io']['hdf5_output'], mode=PETSc.Viewer.Mode.WRITE, comm=PETSc.COMM_WORLD) self.hdf5_viewer.pushGroup("/") # write grid data to hdf5 file coords_x = self.dax.getCoordinates() coords_y = self.day.getCoordinates() coords_x.setName('x') coords_y.setName('y') self.hdf5_viewer(coords_x) self.hdf5_viewer(coords_y) # write initial data to hdf5 file self.save_to_hdf5(0)
def __init__(self, cfgfile): ''' Constructor ''' # stencil = 1 stencil = 2 # load run config file cfg = Config(cfgfile) # set some PETSc options OptDB = PETSc.Options() # OptDB.setValue('snes_lag_preconditioner', 5) self.residual = cfg['solver']['petsc_residual'] # OptDB.setValue('ksp_max_it', 2) # OptDB.setValue('ksp_convergence_test', 'skip') # OptDB.setValue('ksp_monitor', '') OptDB.setValue('snes_monitor', '') # timestep setup self.ht = cfg['grid']['ht'] # timestep size self.nt = cfg['grid']['nt'] # number of timesteps self.nsave = cfg['io']['nsave'] # save only every nsave'th timestep # grid setup nx = cfg['grid']['nx'] # number of points in x ny = cfg['grid']['ny'] # number of points in y Lx = cfg['grid']['Lx'] # spatial domain in x x1 = cfg['grid']['x1'] # x2 = cfg['grid']['x2'] # Ly = cfg['grid']['Ly'] # spatial domain in y y1 = cfg['grid']['y1'] # y2 = cfg['grid']['y2'] # self.nx = nx self.ny = ny if x1 != x2: Lx = x2-x1 else: x1 = 0.0 x2 = Lx if y1 != y2: Ly = y2-y1 else: y1 = 0.0 y2 = Ly self.hx = Lx / nx # gridstep size in x self.hy = Ly / ny # gridstep size in y if PETSc.COMM_WORLD.getRank() == 0: print() print("nt = %i" % (self.nt)) print("nx = %i" % (self.nx)) print("ny = %i" % (self.ny)) print() print("ht = %e" % (self.ht)) print("hx = %e" % (self.hx)) print("hy = %e" % (self.hy)) print() print("Lx = %e" % (Lx)) print("Ly = %e" % (Ly)) print() self.time = PETSc.Vec().createMPI(1, PETSc.DECIDE, comm=PETSc.COMM_WORLD) self.time.setName('t') if PETSc.COMM_WORLD.getRank() == 0: self.time.setValue(0, 0.0) # create DA with single dof self.da1 = PETSc.DA().create(dim=2, dof=1, sizes=[nx, ny], proc_sizes=[PETSc.DECIDE, PETSc.DECIDE], boundary_type=('periodic', 'periodic'), stencil_width=stencil, stencil_type='box') # create DA (dof = 4 for Bx, By, Vx, Vy) self.da4 = PETSc.DA().create(dim=2, dof=4, sizes=[nx, ny], proc_sizes=[PETSc.DECIDE, PETSc.DECIDE], boundary_type=('periodic', 'periodic'), stencil_width=stencil, stencil_type='box') # create DA (dof = 5 for Bx, By, Vx, Vy, P) self.da5 = PETSc.DA().create(dim=2, dof=5, sizes=[nx, ny], proc_sizes=[PETSc.DECIDE, PETSc.DECIDE], boundary_type=('periodic', 'periodic'), stencil_width=stencil, stencil_type='box') # create DA for x grid self.dax = PETSc.DA().create(dim=1, dof=1, sizes=[nx], proc_sizes=[PETSc.DECIDE], boundary_type=('periodic')) # create DA for y grid self.day = PETSc.DA().create(dim=1, dof=1, sizes=[ny], proc_sizes=[PETSc.DECIDE], boundary_type=('periodic')) # initialise grid self.da1.setUniformCoordinates(xmin=x1, xmax=x2, ymin=y1, ymax=y2) self.da4.setUniformCoordinates(xmin=x1, xmax=x2, ymin=y1, ymax=y2) self.da5.setUniformCoordinates(xmin=x1, xmax=x2, ymin=y1, ymax=y2) self.dax.setUniformCoordinates(xmin=x1, xmax=x2) self.day.setUniformCoordinates(xmin=y1, xmax=y2) # create solution and RHS vector self.f = self.da5.createGlobalVec() self.x = self.da5.createGlobalVec() self.b = self.da5.createGlobalVec() # create global RK4 vectors self.Y = self.da5.createGlobalVec() self.X0 = self.da5.createGlobalVec() self.X1 = self.da5.createGlobalVec() self.X2 = self.da5.createGlobalVec() self.X3 = self.da5.createGlobalVec() self.X4 = self.da5.createGlobalVec() # create local RK4 vectors self.localX0 = self.da5.createLocalVec() self.localX1 = self.da5.createLocalVec() self.localX2 = self.da5.createLocalVec() self.localX3 = self.da5.createLocalVec() self.localX4 = self.da5.createLocalVec() # self.localP = self.da1.createLocalVec() # create vectors for magnetic and velocity field self.Bx = self.da1.createGlobalVec() self.By = self.da1.createGlobalVec() self.Vx = self.da1.createGlobalVec() self.Vy = self.da1.createGlobalVec() self.P = self.da1.createGlobalVec() # create local vectors for initialisation of pressure self.localBx = self.da1.createLocalVec() self.localBy = self.da1.createLocalVec() self.localVx = self.da1.createLocalVec() self.localVy = self.da1.createLocalVec() # set variable names self.Bx.setName('Bx') self.By.setName('By') self.Vx.setName('Vx') self.Vy.setName('Vy') self.P.setName('P') # create Matrix object self.petsc_matrix = PETScMatrix (self.da1, self.da5, nx, ny, self.ht, self.hx, self.hy) self.petsc_jacobian = PETScJacobian(self.da1, self.da5, nx, ny, self.ht, self.hx, self.hy) self.petsc_function = PETScFunction(self.da1, self.da5, nx, ny, self.ht, self.hx, self.hy) # self.petsc_jacobian_4d = PETSc_MHD_NL_Jacobian_Matrix.PETScJacobian(self.da1, self.da5, nx, ny, self.ht, self.hx, self.hy) # initialise matrix self.A = self.da5.createMat() self.A.setOption(self.A.Option.NEW_NONZERO_ALLOCATION_ERR, False) self.A.setUp() # create jacobian self.J = self.da5.createMat() self.J.setOption(self.J.Option.NEW_NONZERO_ALLOCATION_ERR, False) self.J.setUp() # create nonlinear solver self.snes = PETSc.SNES().create() self.snes.setType('ksponly') self.snes.setFunction(self.petsc_function.snes_mult, self.f) self.snes.setJacobian(self.updateJacobian, self.J) self.snes.setFromOptions() self.snes.getKSP().setType('preonly') # self.snes.getKSP().setType('gmres') self.snes.getKSP().getPC().setType('lu') # self.snes.getKSP().getPC().setFactorSolverPackage('superlu_dist') self.snes.getKSP().getPC().setFactorSolverPackage('mumps') self.ksp = None # set initial data (xs, xe), (ys, ye) = self.da1.getRanges() coords = self.da1.getCoordinatesLocal() Bx_arr = self.da1.getVecArray(self.Bx) By_arr = self.da1.getVecArray(self.By) Vx_arr = self.da1.getVecArray(self.Vx) Vy_arr = self.da1.getVecArray(self.Vy) if cfg['initial_data']['magnetic_python'] != None: init_data = __import__("runs." + cfg['initial_data']['magnetic_python'], globals(), locals(), ['magnetic_x', 'magnetic_y'], 0) for i in range(xs, xe): for j in range(ys, ye): Bx_arr[i,j] = init_data.magnetic_x(coords[i,j][0], coords[i,j][1] + 0.5 * self.hy, Lx, Ly) By_arr[i,j] = init_data.magnetic_y(coords[i,j][0] + 0.5 * self.hx, coords[i,j][1], Lx, Ly) else: Bx_arr[xs:xe, ys:ye] = cfg['initial_data']['magnetic'] By_arr[xs:xe, ys:ye] = cfg['initial_data']['magnetic'] if cfg['initial_data']['velocity_python'] != None: init_data = __import__("runs." + cfg['initial_data']['velocity_python'], globals(), locals(), ['velocity_x', 'velocity_y'], 0) for i in range(xs, xe): for j in range(ys, ye): Vx_arr[i,j] = init_data.velocity_x(coords[i,j][0], coords[i,j][1] + 0.5 * self.hy, Lx, Ly) Vy_arr[i,j] = init_data.velocity_y(coords[i,j][0] + 0.5 * self.hx, coords[i,j][1], Lx, Ly) else: Vx_arr[xs:xe, ys:ye] = cfg['initial_data']['velocity'] Vy_arr[xs:xe, ys:ye] = cfg['initial_data']['velocity'] if cfg['initial_data']['pressure_python'] != None: init_data = __import__("runs." + cfg['initial_data']['pressure_python'], globals(), locals(), ['pressure', ''], 0) x_arr = self.da5.getVecArray(self.x) x_arr[xs:xe, ys:ye, 0] = Vx_arr[xs:xe, ys:ye] x_arr[xs:xe, ys:ye, 1] = Vy_arr[xs:xe, ys:ye] x_arr[xs:xe, ys:ye, 2] = Bx_arr[xs:xe, ys:ye] x_arr[xs:xe, ys:ye, 3] = By_arr[xs:xe, ys:ye] self.da1.globalToLocal(self.Bx, self.localBx) self.da1.globalToLocal(self.By, self.localBy) self.da1.globalToLocal(self.Vx, self.localVx) self.da1.globalToLocal(self.Vy, self.localVy) Bx_arr = self.da1.getVecArray(self.localBx) By_arr = self.da1.getVecArray(self.localBy) Vx_arr = self.da1.getVecArray(self.localVx) Vy_arr = self.da1.getVecArray(self.localVy) P_arr = self.da1.getVecArray(self.P) for i in range(xs, xe): for j in range(ys, ye): P_arr[i,j] = init_data.pressure(coords[i,j][0] + 0.5 * self.hx, coords[i,j][1] + 0.5 * self.hy, Lx, Ly) # P_arr[i,j] = init_data.pressure(coords[i,j][0] + 0.5 * self.hx, coords[i,j][1] + 0.5 * self.hy, Lx, Ly) \ # + 0.5 * (0.25 * (Bx_arr[i,j] + Bx_arr[i+1,j])**2 + 0.25 * (By_arr[i,j] + By_arr[i,j+1])**2) # P_arr[i,j] = init_data.pressure(coords[i,j][0] + 0.5 * self.hx, coords[i,j][1] + 0.5 * self.hy, Lx, Ly) \ # + 0.5 * (0.25 * (Vx_arr[i,j] + Vx_arr[i+1,j])**2 + 0.25 * (Vy_arr[i,j] + Vy_arr[i,j+1])**2) # P_arr[i,j] = init_data.pressure(coords[i,j][0] + 0.5 * self.hx, coords[i,j][1] + 0.5 * self.hy, Lx, Ly) \ # + 0.5 * (0.25 * (Vx_arr[i,j] + Vx_arr[i+1,j])**2 + 0.25 * (Vy_arr[i,j] + Vy_arr[i,j+1])**2) \ # - 1.0 * (0.25 * (Bx_arr[i,j] + Bx_arr[i+1,j])**2 + 0.25 * (By_arr[i,j] + By_arr[i,j+1])**2) # copy distribution function to solution vector x_arr = self.da5.getVecArray(self.x) x_arr[xs:xe, ys:ye, 4] = P_arr [xs:xe, ys:ye] # update solution history # self.petsc_matrix.update_history(self.x) self.petsc_jacobian.update_history(self.x) self.petsc_function.update_history(self.x) # create HDF5 output file self.hdf5_viewer = PETSc.ViewerHDF5().create(cfg['io']['hdf5_output'], mode=PETSc.Viewer.Mode.WRITE, comm=PETSc.COMM_WORLD) self.hdf5_viewer.pushGroup("/") # write grid data to hdf5 file coords_x = self.dax.getCoordinates() coords_y = self.day.getCoordinates() coords_x.setName('x') coords_y.setName('y') self.hdf5_viewer(coords_x) self.hdf5_viewer(coords_y) # write initial data to hdf5 file self.hdf5_viewer.setTimestep(0) self.save_hdf5_vectors()
def __init__(self, cfgfile, runid=None, cfg=None): ''' Constructor ''' assert cfgfile is not None assert cfgfile is not "" # if runid is empty use timestamp if runid == None or runid == "": runid = datetime.datetime.fromtimestamp( time.time()).strftime("%y%m%d%H%M%S") # stencil width stencil = 2 # load run config file if cfg != None: self.cfg = cfg elif cfgfile != None and len(cfgfile) > 0: self.cfg = Config(cfgfile) else: if PETSc.COMM_WORLD.getRank() == 0: print("ERROR: No valid config file or object passed.") sys.exit() # determine solver modules if cfg['solver']['method'] == 'explicit': self.vlasov_module = None else: self.vlasov_module = "vlasov.solvers." if cfg['solver']['mode'] == 'split': self.vlasov_module += 'vlasov' else: self.vlasov_module += self.cfg['solver']['mode'] self.vlasov_module += '.' + "PETSc" if cfg['solver']['type'] == 'newton' or cfg['solver'][ 'type'] == 'nonlinear': self.vlasov_module += "NL" if cfg['solver']['mode'] == 'split': self.vlasov_module += "Vlasov" # self.vlasov_module += self.cfg['solver']['poisson_bracket'] # if cfg['solver']['preconditioner_type'] != None and cfg['solver']['preconditioner_scheme'] != None: # self.vlasov_module += self.cfg['solver']['preconditioner_scheme'] self.vlasov_module += self.cfg['solver']['timestepping'].upper() if not cfg.is_dissipation_none: if cfg['solver']['dissipation'] == 'double_bracket': self.vlasov_module += "DB" self.poisson_module = "vlasov.solvers.poisson.PETScPoisson" self.poisson_module += self.cfg['solver']['laplace_operator'] # importing solver modules if PETSc.COMM_WORLD.getRank() == 0: print("Loading Vlasov solver %s" % (self.vlasov_module)) print("Loading Poisson solver %s" % (self.poisson_module)) print("") if not self.cfg.is_preconditioner_none(): print("Using Preconditioner %s" % (self.cfg.get_preconditioner())) if not self.cfg.is_dissipation_none(): if self.cfg.is_dissipation_collisions(): print("Using Collision Operator %s" % (self.cfg.get_collision_operator())) if self.cfg.is_dissipation_double_bracket(): print("Using Double Bracket %s" % (self.cfg.get_double_bracket())) print("") self.vlasov_object = __import__(self.vlasov_module, globals(), locals(), ['PETScVlasovSolver'], 0) self.poisson_object = __import__(self.poisson_module, globals(), locals(), ['PETScPoissonSolver'], 0) # timestep setup ht = self.cfg['grid']['ht'] # timestep size nt = self.cfg['grid']['nt'] # number of timesteps self.nsave = self.cfg['io'][ 'nsave'] # save only every nsave'th timestep # grid setup nx = self.cfg['grid']['nx'] # number of points in x nv = self.cfg['grid']['nv'] # number of points in v L = self.cfg['grid']['L'] vMax = self.cfg['grid']['vmax'] vMin = self.cfg['grid']['vmin'] if vMin == None: vMin = -vMax Lx = L Lv = vMax - vMin hx = Lx / nx # gridstep size in x hv = Lv / (nv - 1) # gridstep size in v self.time = PETSc.Vec().createMPI(1, 1, comm=PETSc.COMM_WORLD) self.time.setName('t') if PETSc.COMM_WORLD.getRank() == 0: self.time.setValue(0, 0.0) self.solver_package = self.cfg['solver']['lu_package'] self.nInitial = self.cfg['solver'][ 'initial_iter'] # number of iterations for initial guess self.coll_freq = self.cfg['solver']['coll_freq'] # collision frequency self.coll_drag = self.cfg['solver']['coll_drag'] # drag factor self.coll_diff = self.cfg['solver']['coll_diff'] # diff factor self.charge = self.cfg['initial_data']['charge'] # particle charge self.mass = self.cfg['initial_data']['mass'] # particle mass output_directory = self.cfg['io']['output_dir'] if output_directory == None or output_directory == "": output_directory = "." tindex = cfgfile.rfind('/') run_filename = cfgfile[tindex:].replace('.cfg', '.') + runid hdf_out_filename = output_directory + '/' + run_filename + ".hdf5" cfg_out_filename = output_directory + '/' + run_filename + ".cfg" # hdf_in_filename = self.cfg['io']['hdf5_input'] # hdf_out_filename = self.cfg['io']['hdf5_output'] self.cfg.write_current_config(cfg_out_filename) # set initial guess method initial_guess_options = { None: self.initial_guess_none, "None": self.initial_guess_none, "": self.initial_guess_none, "rk4": self.initial_guess_rk4, "gear": self.initial_guess_gear, "symplectic2": self.initial_guess_symplectic2, "symplectic4": self.initial_guess_symplectic4, } self.initial_guess_method = initial_guess_options[self.cfg['solver'] ['initial_guess']] # set some PETSc options OptDB = PETSc.Options() OptDB.setValue('ksp_rtol', self.cfg['solver']['petsc_ksp_rtol']) OptDB.setValue('ksp_atol', self.cfg['solver']['petsc_ksp_atol']) OptDB.setValue('ksp_max_it', self.cfg['solver']['petsc_ksp_max_iter']) OptDB.setValue('snes_rtol', self.cfg['solver']['petsc_snes_rtol']) OptDB.setValue('snes_atol', self.cfg['solver']['petsc_snes_atol']) OptDB.setValue('snes_stol', self.cfg['solver']['petsc_snes_stol']) OptDB.setValue('snes_max_it', self.cfg['solver']['petsc_snes_max_iter']) if PETSc.COMM_WORLD.getRank() == 0: print("Initialising Distributed Arrays.") # create DA for 2d grid (f only) # self.da1 = PETSc.DMDA().create(dim=2, dof=1, # sizes=[nx, nv], # proc_sizes=[1, PETSc.COMM_WORLD.getSize()], # boundary_type=['periodic', 'periodic'], # stencil_width=stencil, # stencil_type='box') # self.da1 = PETSc.DMDA().create(dim=2, dof=1, # sizes=[nx, nv], # proc_sizes=[PETSc.COMM_WORLD.getSize(), 1], # boundary_type=['periodic', 'ghosted'], # stencil_width=stencil, # stencil_type='box') self.da1 = PETSc.DMDA().create( dim=2, dof=1, sizes=[nx, nv], proc_sizes=[1, PETSc.COMM_WORLD.getSize()], boundary_type=['periodic', 'ghosted'], stencil_width=stencil, stencil_type='box') # self.da1 = PETSc.DMDA().create(dim=2, dof=1, # sizes=[nx, nv], # proc_sizes=[PETSc.DECIDE, 2], # boundary_type=['periodic', 'ghosted'], # stencil_width=stencil, # stencil_type='box') # self.da1 = PETSc.DMDA().create(dim=2, dof=1, # sizes=[nx, nv], # proc_sizes=[PETSc.DECIDE, PETSc.DECIDE], # boundary_type=['periodic', 'ghosted'], # stencil_width=stencil, # stencil_type='box') # create VIDA for x grid self.dax = PETSc.DMDA().create(dim=1, dof=1, sizes=[nx], proc_sizes=[PETSc.COMM_WORLD.getSize()], boundary_type=('periodic'), stencil_width=stencil, stencil_type='box') # create VIDA for y grid self.day = PETSc.DMDA().create(dim=1, dof=1, sizes=[nv], proc_sizes=[PETSc.COMM_WORLD.getSize()], boundary_type=('ghosted'), stencil_width=stencil, stencil_type='box') # initialise grid self.da1.setUniformCoordinates(xmin=0.0, xmax=Lx, ymin=vMin, ymax=vMax) self.dax.setUniformCoordinates(xmin=0.0, xmax=Lx) self.day.setUniformCoordinates(xmin=vMin, xmax=vMax) # get local index ranges (xs, xe), (ys, ye) = self.da1.getRanges() (xsx, xex), = self.dax.getRanges() # get coordinate vectors coords_x = self.dax.getCoordinates() coords_v = self.day.getCoordinates() # save x coordinate arrays scatter, xVec = PETSc.Scatter.toAll(coords_x) scatter.begin(coords_x, xVec, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) scatter.end(coords_x, xVec, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) xGrid = xVec.getValues(range(nx)).copy() scatter.destroy() xVec.destroy() # save v coordinate arrays scatter, vVec = PETSc.Scatter.toAll(coords_v) scatter.begin(coords_v, vVec, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) scatter.end(coords_v, vVec, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) vGrid = vVec.getValues(range(nv)).copy() scatter.destroy() vVec.destroy() # create grid object self.grid = Grid().create(xGrid, vGrid, nt, nx, nv, ht, hx, hv, stencil) # create vectors for Hamiltonians self.h0 = self.da1.createGlobalVec() # kinetic Hamiltonian self.h1c = self.da1.createGlobalVec() # current potential Hamiltonian self.h2c = self.da1.createGlobalVec() # current external Hamiltonian self.h1h = self.da1.createGlobalVec() # previous potential Hamiltonian self.h2h = self.da1.createGlobalVec() # previous external Hamiltonian # distribution functions self.fl = self.da1.createGlobalVec() # last (k+1, n ) self.fc = self.da1.createGlobalVec() # current (k+1, n+1) self.fh = self.da1.createGlobalVec() # history (k) # distribution function solver vectors self.fb = self.da1.createGlobalVec() # right hand side # moments self.N = self.dax.createGlobalVec() # density self.U = self.dax.createGlobalVec() # velocity density self.E = self.dax.createGlobalVec() # energy density self.A = self.dax.createGlobalVec() # collision factor # local moments self.nc = PETSc.Vec().createSeq(nx) # current density self.uc = PETSc.Vec().createSeq(nx) # current velocity density self.ec = PETSc.Vec().createSeq(nx) # current energy density self.ac = PETSc.Vec().createSeq(nx) # current collision factor self.nh = PETSc.Vec().createSeq(nx) # history density self.uh = PETSc.Vec().createSeq(nx) # history velocity density self.eh = PETSc.Vec().createSeq(nx) # history energy density self.ah = PETSc.Vec().createSeq(nx) # history collision factor # internal potential self.pc_int = self.dax.createGlobalVec() # current self.ph_int = self.dax.createGlobalVec() # history # external potential self.pc_ext = self.dax.createGlobalVec() # current self.ph_ext = self.dax.createGlobalVec() # history # potential solver vectors self.pb = self.dax.createGlobalVec() # right hand side self.pn = self.dax.createGlobalVec() # null vector # set variable names self.h0.setName('h0') self.h1c.setName('h1') self.h2c.setName('h2') self.fc.setName('f') self.pc_int.setName('phi_int') self.pc_ext.setName('phi_ext') self.N.setName('n') self.U.setName('u') self.E.setName('e') # initialise nullspace basis vectors # the Poisson equation has a null space of all constant vectors # that needs to be removed to avoid jumpy potentials self.pn.set(1.) self.pn.normalize() self.p_nullspace = PETSc.NullSpace().create(constant=False, vectors=(self.pn, )) # create Toolbox self.toolbox = Toolbox(self.da1, self.dax, self.grid) # initialise kinetic hamiltonian if PETSc.COMM_WORLD.getRank() == 0: print("Initialising kinetic Hamiltonian.") self.toolbox.initialise_kinetic_hamiltonian(self.h0, self.mass) # create Arakawa initial guess solver object if PETSc.COMM_WORLD.getRank() == 0: print("Instantiating Initial Guess Objects.") self.arakawa_rk4 = PETScArakawaRungeKutta(self.cfg, self.da1, self.grid, self.h0, self.h1h, self.h2h, self.nInitial) self.arakawa_gear = PETScArakawaGear(self.cfg, self.da1, self.grid, self.h0, self.h1h, self.h2h, self.nInitial) self.arakawa_symplectic = PETScArakawaSymplectic( self.cfg, self.da1, self.grid, self.h0, self.h1h, self.h2h, self.nInitial) # create solver dummies self.vlasov_solver = None self.poisson_solver = None self.poisson_ksp = None if PETSc.COMM_WORLD.getRank() == 0: print() print("Run ID: %s" % runid) print() print("Config File: %s" % cfgfile) print("Output File: %s" % hdf_out_filename) print() print("nt = %i" % (self.grid.nt)) print("nx = %i" % (self.grid.nx)) print("nv = %i" % (self.grid.nv)) print() print("ht = %e" % (self.grid.ht)) print("hx = %e" % (self.grid.hx)) print("hv = %e" % (self.grid.hv)) print() print("xMin = %+12.6e" % (self.grid.xMin())) print("xMax = %+12.6e" % (self.grid.vMax())) print("vMin = %+12.6e" % (self.grid.vMin())) print("vMax = %+12.6e" % (self.grid.vMax())) print() print("nu = %7.1e" % (self.coll_freq)) print() print("CFL = %e" % (self.grid.hx / vMax)) print() print() # set initial data N0 = self.dax.createGlobalVec() T0 = self.dax.createGlobalVec() N0.setName('n0') T0.setName('T0') n0 = PETSc.Vec().createSeq(nx) t0 = PETSc.Vec().createSeq(nx) if self.cfg['initial_data']['distribution_python'] != None: init_data = __import__( "runs." + self.cfg['initial_data']['distribution_python'], globals(), locals(), ['distribution'], 0) if PETSc.COMM_WORLD.getRank() == 0: print( "Initialising distribution function with Python function.") self.toolbox.initialise_distribution_function( self.fc, init_data.distribution) N0.set(0.) T0.set(0.) else: N0_arr = self.dax.getVecArray(N0) T0_arr = self.dax.getVecArray(T0) if self.cfg['initial_data']['density_python'] != None: init_data = __import__( "runs." + self.cfg['initial_data']['density_python'], globals(), locals(), ['density'], 0) if PETSc.COMM_WORLD.getRank() == 0: print("Initialising density with Python function.") for i in range(xsx, xex): N0_arr[i] = init_data.density(self.grid.x[i], self.grid.xLength()) else: N0_arr[xsx:xex] = self.cfg['initial_data']['density'] if self.cfg['initial_data']['temperature_python'] != None: init_data = __import__( "runs." + self.cfg['initial_data']['temperature_python'], globals(), locals(), ['temperature'], 0) if PETSc.COMM_WORLD.getRank() == 0: print("Initialising temperature with Python function.") for i in range(xsx, xex): T0_arr[i] = init_data.temperature(self.grid.x[i]) else: T0_arr[xsx:xex] = self.cfg['initial_data']['temperature'] if PETSc.COMM_WORLD.getRank() == 0: print("Initialising distribution function with Maxwellian.") self.copy_xvec_to_seq(N0, n0) self.copy_xvec_to_seq(T0, t0) self.toolbox.initialise_distribution_nT(self.fc, n0, t0) # Fourier Filtering nfourier = cfg['initial_data']['nfourier'] if nfourier > 0: from scipy.fftpack import rfft, irfft if PETSc.COMM_WORLD.getRank() == 0: print("Fourier Filtering") # obtain whole f vector everywhere scatter, Xglobal = PETSc.Scatter.toAll(self.fc) scatter.begin(self.fc, Xglobal, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) scatter.end(self.fc, Xglobal, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) petsc_indices = self.da1.getAO().app2petsc( np.arange(nx * nv, dtype=np.int32)) fTmp = Xglobal.getValues(petsc_indices).copy().reshape((nv, nx)).T scatter.destroy() Xglobal.destroy() # filter distribution function fFft = rfft(fTmp, axis=0) fFft[nfourier:, :] = 0. fTmp = irfft(fFft, axis=0) # store filtered distribution function f_arr = self.da1.getVecArray(self.fc) f_arr[:, :] = fTmp[xs:xe, ys:ye] # normalise f self.normalise_distribution_function() # calculate potential and moments if PETSc.COMM_WORLD.getRank() == 0: print("Calculate initial potential and moments.") self.calculate_moments(output=False) # initialise Gear History if self.cfg['solver']['initial_guess'] == "gear": self.arakawa_gear.initialise_history(self.fc) # check for external potential if self.cfg['initial_data']['external_python'] != None: if PETSc.COMM_WORLD.getRank() == 0: print("Calculate external potential.") external_data = __import__( "runs." + self.cfg['initial_data']['external_python'], globals(), locals(), ['external'], 0) self.external = external_data.external else: self.external = None # calculate external potential self.calculate_external(0.) # create HDF5 output file if PETSc.COMM_WORLD.getRank() == 0: print("Create HDF5 output file.") # use h5py to store attributes hdf5out = h5py.File(hdf_out_filename, 'w', driver='mpio', comm=PETSc.COMM_WORLD.tompi4py()) hdf5out.attrs['charge'] = self.charge hdf5out.close() # create PETSc HDF5 viewer self.hdf5_viewer = PETSc.ViewerHDF5().create( hdf_out_filename, mode=PETSc.Viewer.Mode.APPEND, comm=PETSc.COMM_WORLD) self.hdf5_viewer.pushGroup("/") if PETSc.COMM_WORLD.getRank() == 0: print("Saving initial data to HDF5.") # write grid data to hdf5 file coords_x.setName('x') coords_v.setName('v') self.hdf5_viewer(coords_x) self.hdf5_viewer(coords_v) # write initial data to hdf5 file # self.hdf5_viewer(N0) # self.hdf5_viewer(T0) # save to hdf5 self.hdf5_viewer.setTimestep(0) self.save_hdf5_vectors() if PETSc.COMM_WORLD.getRank() == 0: print("run_base.py: initialisation done.") print("")
def __init__(self, A_IS): """ Initialize the domain decomposition preconditioner, multipreconditioner and coarse space with its operators Parameters ========== A_IS : petsc.Mat The matrix of the problem in IS format. A must be a symmetric positive definite matrix with symmetric positive semi-definite submatrices PETSc.Options ============= PCBNN_switchtoASM :Bool Default is False If True then the domain decomposition preconditioner is the BNN preconditioner. If false then the domain decomposition precondition is the Additive Schwarz preconditioner with minimal overlap. PCBNN_kscaling : Bool Default is True. If true then kscaling (partition of unity that is proportional to the diagonal of the submatrices of A) is used when a partition of unity is required. Otherwise multiplicity scaling is used when a partition of unity is required. This may occur in two occasions: - to scale the local BNN matrices if PCBNN_switchtoASM=True, - in the GenEO eigenvalue problem for eigmin if PCBNN_switchtoASM=False and PCBNN_GenEO=True with PCBNN_GenEO_eigmin > 0 (see projection.__init__ for the meaning of these options). PCBNN_verbose : Bool Default is False. If True, some information about the preconditioners is printed when the code is executed. PCBNN_GenEO : Bool Default is False. If True then the coarse space is enriched by solving local generalized eigenvalue problems. PCBNN_CoarseProjection :True Default is True If False then there is no coarse projection: Two level Additive Schwarz or One-level preconditioner depending on PCBNN_addCoarseSolve If True, the coarse projection is applied: Projected preconditioner of hybrid preconditioner depending on PCBNN_addCoarseSolve PCBNN_addCoarseSolve : False Default is True If True then (R0t A0\R0 r) is added to the preconditioned residual False corresponds to the projected preconditioner (need to choose initial guess accordingly) (or the one level preconditioner if PCBNN_CoarseProjection = False) True corresponds to the hybrid preconditioner (or the fully additive preconditioner if PCBNN_CoarseProjection = False) """ OptDB = PETSc.Options() self.switchtoASM = OptDB.getBool('PCBNN_switchtoASM', False) #use Additive Schwarz as a preconditioner instead of BNN self.kscaling = OptDB.getBool('PCBNN_kscaling', True) #kscaling if true, multiplicity scaling if false self.verbose = OptDB.getBool('PCBNN_verbose', False) self.GenEO = OptDB.getBool('PCBNN_GenEO', True) self.addCS = OptDB.getBool('PCBNN_addCoarseSolve', False) self.projCS = OptDB.getBool('PCBNN_CoarseProjection', True) #extract Neumann matrix from A in IS format Ms = A_IS.copy().getISLocalMat() # convert A_IS from matis to mpiaij A_mpiaij = A_IS.convertISToAIJ() r, _ = A_mpiaij.getLGMap() #r, _ = A_IS.getLGMap() is_A = PETSc.IS().createGeneral(r.indices) # extract exact local solver As = A_mpiaij.createSubMatrices(is_A)[0] vglobal, _ = A_mpiaij.getVecs() vlocal, _ = Ms.getVecs() scatter_l2g = PETSc.Scatter().create(vlocal, None, vglobal, is_A) #compute the multiplicity of each degree vlocal.set(1.) vglobal.set(0.) scatter_l2g(vlocal, vglobal, PETSc.InsertMode.ADD_VALUES) scatter_l2g(vglobal, vlocal, PETSc.InsertMode.INSERT_VALUES, PETSc.ScatterMode.SCATTER_REVERSE) NULL,mult_max = vglobal.max() # k-scaling or multiplicity scaling of the local (non-assembled) matrix if self.kscaling == False: Ms.diagonalScale(vlocal,vlocal) else: v1 = As.getDiagonal() v2 = Ms.getDiagonal() Ms.diagonalScale(v1/v2, v1/v2) # the default local solver is the scaled non assembled local matrix (as in BNN) if self.switchtoASM: Atildes = As if mpi.COMM_WORLD.rank == 0: print('The user has chosen to switch to Additive Schwarz instead of BNN.') else: #(default) Atildes = Ms ksp_Atildes = PETSc.KSP().create(comm=PETSc.COMM_SELF) ksp_Atildes.setOptionsPrefix("ksp_Atildes_") ksp_Atildes.setOperators(Atildes) ksp_Atildes.setType('preonly') pc_Atildes = ksp_Atildes.getPC() pc_Atildes.setType('cholesky') pc_Atildes.setFactorSolverType('mumps') ksp_Atildes.setFromOptions() self.A = A_mpiaij self.Ms = Ms self.As = As self.ksp_Atildes = ksp_Atildes self.works_1 = vlocal.copy() self.works_2 = self.works_1.copy() self.scatter_l2g = scatter_l2g self.mult_max = mult_max self.minV0 = minimal_V0(self.ksp_Atildes) if self.GenEO == True: GenEOV0 = GenEO_V0(self.ksp_Atildes,self.Ms,self.As,self.mult_max,self.minV0.V0s) self.V0s = GenEOV0.V0s else: self.V0s = self.minV0.V0s self.proj = coarse_operators(self.V0s,self.A,self.scatter_l2g,vlocal)
ksp.setOperators(A, P) ksp.setTolerances(rtol=1e-8) ksp.setType("minres") ksp.getPC().setType("fieldsplit") ksp.getPC().setFieldSplitType(PETSc.PC.CompositeType.ADDITIVE) ksp.getPC().setFieldSplitIS(("u", is_u), ("p", is_p)) # Configure velocity and pressure sub KSPs ksp_u, ksp_p = ksp.getPC().getFieldSplitSubKSP() ksp_u.setType("preonly") ksp_u.getPC().setType("gamg") ksp_p.setType("preonly") ksp_p.getPC().setType("jacobi") # Monitor the convergence of the KSP opts = PETSc.Options() # opts["ksp_monitor"] = None # opts["ksp_view"] = None ksp.setFromOptions() # We also need to create a block vector,``x``, to store the (full) # solution, which we initialize using the block RHS form ``L``. # Compute solution x = A.createVecRight() ksp.solve(b, x) # Create Functions and scatter x solution u, p = Function(V), Function(Q) offset = V_map.size_local * V_map.block_size
# work some magic to put the distributed matrix together P.assemble() if (J != P): J.assemble() # matrix-free operator # our non-zero pattern stays the same for all iterations, so we tell PETSc that # (it allows some optimisations inside PETSc) return PETSc.Mat.Structure.SAME_NONZERO_PATTERN stype = PETSc.DMDA.StencilType.BOX bx = PETSc.DMDA.BoundaryType.GHOSTED by = PETSc.DMDA.BoundaryType.GHOSTED bz = PETSc.DMDA.BoundaryType.GHOSTED comm = PETSc.COMM_WORLD rank = comm.rank OptDB = PETSc.Options() # get PETSc option DB # M = OptDB.getInt('M', 11) # N = OptDB.getInt('N', 7) # P = OptDB.getInt('P', 5) M, N, P = -11, -7, -5 m = OptDB.getInt('m', PETSc.DECIDE) n = OptDB.getInt('n', PETSc.DECIDE) p = OptDB.getInt('p', PETSc.DECIDE) dm = PETSc.DMDA().create(dim=3, sizes=(M, N, P), proc_sizes=(m, n, p), boundary_type=(bx, by, bz), stencil_type=stype, stencil_width=1, dof=1, comm=comm, setup=False) dm.setFromOptions() dm.setUp() convdiff_problem = convection_diffusion(dm, {"dx": 0.1, "dy": 0.1, "dz": 0.1}) # prepare the initial conditions
def setup_method(self, method): self.petsc_options = PETSc.Options() self.petsc_options.clear() self._scriptdir = os.path.dirname(__file__)
# Overall form (as one-form) F = odeint.discretise_in_time(f) # Overall form (as list of forms) F = dolfiny.function.extract_blocks(F, δm) # Create output xdmf file -- open in Paraview with Xdmf3ReaderT ofile = dolfiny.io.XDMFFile(comm, f"{name}.xdmf", "w") # Write mesh, meshtags ofile.write_mesh_meshtags(mesh, mts) # Write initial state ofile.write_function(v, time.value) ofile.write_function(p, time.value) # Options for PETSc backend opts = PETSc.Options("bingham") opts["snes_type"] = "newtonls" opts["snes_linesearch_type"] = "basic" opts["snes_rtol"] = 1.0e-08 opts["snes_max_it"] = 12 opts["ksp_type"] = "preonly" opts["pc_type"] = "lu" opts["pc_factor_mat_solver_type"] = "mumps" opts_global = PETSc.Options() opts_global["mat_mumps_icntl_14"] = 500 opts_global["mat_mumps_icntl_24"] = 1 # Create nonlinear problem: SNES problem = dolfiny.snesblockproblem.SNESBlockProblem(F, m, prefix="bingham")