def test_nullspace_check(mesh, degree): V = VectorFunctionSpace(mesh, ('Lagrange', degree)) u, v = TrialFunction(V), TestFunction(V) E, nu = 2.0e2, 0.3 mu = E / (2.0 * (1.0 + nu)) lmbda = E * nu / ((1.0 + nu) * (1.0 - 2.0 * nu)) def sigma(w, gdim): return 2.0 * mu * ufl.sym(grad(w)) + lmbda * ufl.tr( grad(w)) * ufl.Identity(gdim) a = form(inner(sigma(u, mesh.geometry.dim), grad(v)) * dx) # Assemble matrix and create compatible vector A = assemble_matrix(a) A.assemble() # Create null space basis and test nullspace = build_elastic_nullspace(V) la.orthonormalize(nullspace) ns = PETSc.NullSpace().create(vectors=nullspace) assert ns.test(A) # Create incorrect null space basis and test nullspace = build_broken_elastic_nullspace(V) la.orthonormalize(nullspace) ns = PETSc.NullSpace().create(vectors=nullspace) assert not ns.test(A)
def build_nullspace(V): """Function to build null space for 3D elasticity""" # Create list of vectors for null space index_map = V.dofmap.index_map nullspace_basis = [cpp.la.create_vector(index_map) for i in range(6)] with ExitStack() as stack: vec_local = [ stack.enter_context(x.localForm()) for x in nullspace_basis ] basis = [np.asarray(x) for x in vec_local] # Build translational null space basis for i in range(3): basis[i][V.sub(i).dofmap.list.array()] = 1.0 # Build rotational null space basis x = V.tabulate_dof_coordinates() dofs = [V.sub(i).dofmap.list.array() for i in range(3)] basis[3][dofs[0]] = -x[dofs[0], 1] basis[3][dofs[1]] = x[dofs[1], 0] basis[4][dofs[0]] = x[dofs[0], 2] basis[4][dofs[2]] = -x[dofs[2], 0] basis[5][dofs[2]] = x[dofs[2], 1] basis[5][dofs[1]] = -x[dofs[1], 2] # Create vector space basis and orthogonalize basis = VectorSpaceBasis(nullspace_basis) basis.orthonormalize() _x = [basis[i] for i in range(6)] nsp = PETSc.NullSpace().create(vectors=_x) return nsp
def build_nullspace(V): """Function to build null space for 3D elasticity""" # Create list of vectors for null space index_map = V.dofmap().index_map() nullspace_basis = [cpp.la.create_vector(index_map) for i in range(6)] # Build translational null space basis V.sub(0).dofmap().set(nullspace_basis[0], 1.0) V.sub(1).dofmap().set(nullspace_basis[1], 1.0) V.sub(2).dofmap().set(nullspace_basis[2], 1.0) # Build rotational null space basis V.sub(0).set_x(nullspace_basis[3], -1.0, 1) V.sub(1).set_x(nullspace_basis[3], 1.0, 0) V.sub(0).set_x(nullspace_basis[4], 1.0, 2) V.sub(2).set_x(nullspace_basis[4], -1.0, 0) V.sub(2).set_x(nullspace_basis[5], 1.0, 1) V.sub(1).set_x(nullspace_basis[5], -1.0, 2) # Create vector space basis and orthogonalize basis = VectorSpaceBasis(nullspace_basis) basis.orthonormalize() _x = [basis[i] for i in range(6)] nsp = PETSc.NullSpace() nsp.create(_x) return nsp
def create(self, pc): self.diag = None kspLAMG = PETSc.KSP() kspLAMG.create(comm=PETSc.COMM_WORLD) pc = kspLAMG.getPC() kspLAMG.setType('preonly') pc.setType('lu') # pc.setFactorSolverPackage("pastix") OptDB = PETSc.Options() OptDB['pc_factor_shift_amount'] = .1 OptDB['pc_factor_mat_ordering_type'] = 'rcm' OptDB['pc_factor_mat_solver_package'] = 'mumps' # kspLAMG.setFromOptions() # kspLAMG.max_it = 1 kspLAMG.setFromOptions() self.kspLAMG = kspLAMG # print kspLAMG.view() nsp = PETSc.NullSpace().create(constant=True) kspLAMG.setNullSpace(nsp) kspNLAMG = PETSc.KSP() kspNLAMG.create(comm=PETSc.COMM_WORLD) pc = kspNLAMG.getPC() kspNLAMG.setType('preonly') pc.setType('lu') # pc.setFactorSolverPackage("pastix") # kspNLAMG.max_it = 1 kspNLAMG.setFromOptions() kspLAMG.setFromOptions() self.kspNLAMG = kspNLAMG
def build_nullspace(V): """Function to build null space for 3D elasticity""" # Create list of vectors for null space index_map = V.dofmap.index_map nullspace_basis = [cpp.la.create_vector(index_map) for i in range(6)] with ExitStack() as stack: vec_local = [ stack.enter_context(x.localForm()) for x in nullspace_basis ] basis = [np.asarray(x) for x in vec_local] # Build translational null space basis V.sub(0).dofmap.set(basis[0], 1.0) V.sub(1).dofmap.set(basis[1], 1.0) V.sub(2).dofmap.set(basis[2], 1.0) # Build rotational null space basis V.sub(0).set_x(basis[3], -1.0, 1) V.sub(1).set_x(basis[3], 1.0, 0) V.sub(0).set_x(basis[4], 1.0, 2) V.sub(2).set_x(basis[4], -1.0, 0) V.sub(2).set_x(basis[5], 1.0, 1) V.sub(1).set_x(basis[5], -1.0, 2) # Create vector space basis and orthogonalize basis = VectorSpaceBasis(nullspace_basis) basis.orthonormalize() _x = [basis[i] for i in range(6)] nsp = PETSc.NullSpace() nsp.create(_x) return nsp
def build_nullspace(V): """Function to build PETSc nullspace for 3D elasticity""" # Create list of vectors for null space index_map = V.dofmap.index_map bs = V.dofmap.index_map_bs ns = [la.create_petsc_vector(index_map, bs) for i in range(6)] with ExitStack() as stack: vec_local = [stack.enter_context(x.localForm()) for x in ns] basis = [np.asarray(x) for x in vec_local] # Get dof indices for each subspace (x, y and z dofs) dofs = [V.sub(i).dofmap.list.array for i in range(3)] # Build translational nullspace basis for i in range(3): basis[i][dofs[i]] = 1.0 # Build rotational nullspace basis x = V.tabulate_dof_coordinates() dofs_block = V.dofmap.list.array x0, x1, x2 = x[dofs_block, 0], x[dofs_block, 1], x[dofs_block, 2] basis[3][dofs[0]] = -x1 basis[3][dofs[1]] = x0 basis[4][dofs[0]] = x2 basis[4][dofs[2]] = -x0 basis[5][dofs[2]] = x1 basis[5][dofs[1]] = -x2 la.orthonormalize(ns) assert la.is_orthonormal(ns) return PETSc.NullSpace().create(vectors=ns)
def set_Mat_correctiveMethod(self, csrMat_c, nullspace): ############## matrix if self.rank == 0: forwards = [None] * self.cpu Iminmax = self.A_PETSc_c.getOwnershipRanges() nf = 0 for nf in range(self.cpu): Imin, Imax = Iminmax[nf], Iminmax[nf + 1] forwards[nf] = (csrMat_c[0][Imin:Imax+1]-csrMat_c[0][Imin],\ csrMat_c[1][csrMat_c[0][Imin] : csrMat_c[0][Imax]],\ csrMat_c[2][csrMat_c[0][Imin] : csrMat_c[0][Imax]]) else: forwards = None #forward matrix csr data rows, cols, values = self.comm.scatter(forwards, root=0) self.A_PETSc_c.createAIJ(size=(self.dim, self.dim), csr=(rows, cols, values), comm=self.comm) self.A_PETSc = (self.A_PETSc_p + self.A_PETSc_c) if nullspace: nsp = _PETSc.NullSpace() nsp.create(constant=True, vectors=(), comm=self.comm) self.A_PETSc.setNullSpace(nsp) self.method_PETSc.setOperators(self.A_PETSc)
def set_near_nullspace(self, nullsp, constant=False): assert self.eqn_subindices is None assert self.inact_subindices is None vecs = map(vec, nullsp) self.nullsp = PETSc.NullSpace().create(vectors=vecs, constant=constant, comm=self.comm)
def setUp(self): u1 = PETSc.Vec().createSeq(3) u2 = PETSc.Vec().createSeq(3) u1[0], u1[1], u1[2] = [1, 2, 0]; u1.normalize() u2[0], u2[1], u2[2] = [2, -1, 0]; u2.normalize() basis = [u1, u2] nullsp = PETSc.NullSpace().create(False, basis, comm=PETSc.COMM_SELF) self.basis = basis self.nullsp = nullsp
def rigid_motions_nullspace(V: _fem.FunctionSpace): """ Function to build nullspace for 2D/3D elasticity. Parameters: =========== V The function space """ _x = _fem.Function(V) # Get geometric dim gdim = V.mesh.geometry.dim assert gdim == 2 or gdim == 3 # Set dimension of nullspace dim = 3 if gdim == 2 else 6 # Create list of vectors for null space nullspace_basis = [_x.vector.copy() for _ in range(dim)] with ExitStack() as stack: vec_local = [stack.enter_context(x.localForm()) for x in nullspace_basis] basis = [np.asarray(x) for x in vec_local] dofs = [V.sub(i).dofmap.list.array for i in range(gdim)] # Build translational null space basis for i in range(gdim): basis[i][dofs[i]] = 1.0 # Build rotational null space basis x = V.tabulate_dof_coordinates() dofs_block = V.dofmap.list.array x0, x1, x2 = x[dofs_block, 0], x[dofs_block, 1], x[dofs_block, 2] if gdim == 2: basis[2][dofs[0]] = -x1 basis[2][dofs[1]] = x0 elif gdim == 3: basis[3][dofs[0]] = -x1 basis[3][dofs[1]] = x0 basis[4][dofs[0]] = x2 basis[4][dofs[2]] = -x0 basis[5][dofs[2]] = x1 basis[5][dofs[1]] = -x2 _la.orthonormalize(nullspace_basis) assert _la.is_orthonormal(nullspace_basis) return PETSc.NullSpace().create(vectors=nullspace_basis)
def get_nullspace(da, A): RBM = PETSc.NullSpace().createRigidBody(da.getCoordinates()) rbm_vecs = RBM.getVecs() (xs, xe), (ys, ye) = da.getRanges() (gxs, gxe), (gys, gye) = da.getGhostRanges() # Restriction operator R = da.createGlobalVec() Rlocal = da.createLocalVec() Rlocal_a = da.getVecArray(Rlocal) Rlocal_a[gxs:xe, gys:ye] = 1 # multiplicity D = da.createGlobalVec() Dlocal = da.createLocalVec() da.localToGlobal(Rlocal, D, addv=PETSc.InsertMode.ADD_VALUES) da.globalToLocal(D, Dlocal) work1 = da.createLocalVec() work2 = da.createLocalVec() vecs = [] for i in range(mpi.COMM_WORLD.size): for ivec, rbm_vec in enumerate(rbm_vecs): vecs.append(da.createGlobalVec()) work1.set(0) da.globalToLocal(rbm_vec, work2) if i == mpi.COMM_WORLD.rank: work1 = work2 * Rlocal / Dlocal da.localToGlobal(work1, vecs[-1], addv=PETSc.InsertMode.ADD_VALUES) # orthonormalize Avecs = [] for vec in vecs: bcApplyWest_vec(da, vec) Avecs.append(A * vec) for i, vec in enumerate(vecs): alphas = [] for vec_ in Avecs[:i]: alphas.append(vec.dot(vec_)) for alpha, vec_ in zip(alphas, vecs[:i]): vec.axpy(-alpha, vec_) vec.scale(1. / np.sqrt(vec.dot(A * vec))) Avecs[i] = A * vec return D, vecs, Avecs
def __init__(self, A, prectype, parameters=None, pdes=1, nullspace=None): from dolfin import info from time import time T = time() Ad = A.down_cast().mat() if nullspace: from block.block_util import isscalar ns = PETSc.NullSpace() if isscalar(nullspace): ns.create(constant=True) else: ns.create(constant=False, vectors=[v.down_cast().vec() for v in nullspace]) try: Ad.setNearNullSpace(ns) except: info( 'failed to set near null space (not supported in petsc4py version)' ) self.A = A self.petsc_prec = PETSc.PC() self.petsc_prec.create() self.petsc_prec.setType(prectype) # self.petsc_prec.setOperators(Ad, Ad, PETSc.Mat.Structure.SAME_PRECONDITIONER) self.petsc_prec.setOperators(Ad, Ad) # Merge parameters into the options database if parameters: origOptions = PETSc.Options().getAll() for key, val in iter(parameters.items()): PETSc.Options().setValue(key, val) # Create preconditioner based on the options database self.petsc_prec.setFromOptions() self.petsc_prec.setUp() # Reset the options database if parameters: for key in iter(parameters.keys()): PETSc.Options().delValue(key) for key, val in iter(origOptions.items()): PETSc.Options().setValue(key, val) info('constructed %s preconditioner in %.2f s' % (self.__class__.__name__, time() - T))
def __init__(self, cfgfile, runid=None, cfg=None): ''' Constructor ''' super().__init__(cfgfile, runid, cfg) # create VIDA for 2d grid (f, phi and moments) self.da2 = VIDA().create(dim=1, dof=self.nv + 4, sizes=[self.nx], proc_sizes=[PETSc.COMM_WORLD.getSize()], boundary_type=('periodic'), stencil_width=2, stencil_type='box') # initialise grid self.da2.setUniformCoordinates(xmin=0.0, xmax=self.Lx) # create solution and RHS vector self.x = self.da2.createGlobalVec() self.xh = self.da2.createGlobalVec() self.xn = self.da2.createGlobalVec() self.b = self.da2.createGlobalVec() # initialise nullspace basis vector for full solution # the Poisson equation has a null space of all constant vectors # that needs to be removed to avoid jumpy potentials self.xn.set(0.) x_nvec_arr = self.da2.getGlobalArray(self.xn) p_nvec_arr = getGlobalArray(self.dax, self.pn) x_nvec_arr[:, self.nv] = p_nvec_arr # x_nvec_arr[:, self.nv] = 1. # self.x_nvec.normalize() self.nullspace = PETSc.NullSpace().create(constant=False, vectors=(self.xn, )) # create placeholder for solver object self.vlasov_poisson_solver = None # copy f, p, and moments to solution vector self.copy_data_to_x()
def __init__(self, cfgfile): ''' Constructor ''' # load run config file cfg = Config(cfgfile) # timestep setup self.ht = cfg['grid']['ht'] # timestep size self.nt = cfg['grid']['nt'] # number of timesteps self.nsave = cfg['io']['nsave'] # save only every nsave'th timestep # grid setup self.nx = cfg['grid']['nx'] # number of points in x self.ny = cfg['grid']['ny'] # number of points in y Lx = cfg['grid']['Lx'] # spatial domain in x x1 = cfg['grid']['x1'] # x2 = cfg['grid']['x2'] # Ly = cfg['grid']['Ly'] # spatial domain in y y1 = cfg['grid']['y1'] # y2 = cfg['grid']['y2'] # if x1 != x2: Lx = x2 - x1 else: x1 = 0.0 x2 = Lx if y1 != y2: Ly = y2 - y1 else: y1 = 0.0 y2 = Ly self.hx = Lx / self.nx # gridstep size in x self.hy = Ly / self.ny # gridstep size in y self.time = PETSc.Vec().createMPI(1, PETSc.DECIDE, comm=PETSc.COMM_WORLD) self.time.setName('t') if PETSc.COMM_WORLD.getRank() == 0: self.time.setValue(0, 0.0) OptDB = PETSc.Options() OptDB.setValue('snes_rtol', cfg['solver']['petsc_snes_rtol']) OptDB.setValue('snes_atol', cfg['solver']['petsc_snes_atol']) OptDB.setValue('snes_stol', cfg['solver']['petsc_snes_stol']) OptDB.setValue('snes_max_it', cfg['solver']['petsc_snes_max_iter']) OptDB.setValue('ksp_rtol', cfg['solver']['petsc_ksp_rtol']) OptDB.setValue('ksp_atol', cfg['solver']['petsc_ksp_atol']) OptDB.setValue('ksp_max_it', cfg['solver']['petsc_ksp_max_iter']) # OptDB.setValue('ksp_monitor', '') # OptDB.setValue('snes_monitor', '') # # OptDB.setValue('log_info', '') # OptDB.setValue('log_summary', '') self.snes_rtol = cfg['solver']['petsc_snes_rtol'] self.snes_atol = cfg['solver']['petsc_snes_atol'] self.snes_max_iter = cfg['solver']['petsc_snes_max_iter'] # create DA with single dof self.da1 = PETSc.DA().create(dim=2, dof=1, sizes=[self.nx, self.ny], proc_sizes=[PETSc.DECIDE, PETSc.DECIDE], boundary_type=('periodic', 'periodic'), stencil_width=2, stencil_type='box') # create DA (dof = 4 for A, J, P, O) self.da4 = PETSc.DA().create(dim=2, dof=4, sizes=[self.nx, self.ny], proc_sizes=[PETSc.DECIDE, PETSc.DECIDE], boundary_type=('periodic', 'periodic'), stencil_width=2, stencil_type='box') # create DA for x grid self.dax = PETSc.DA().create(dim=1, dof=1, sizes=[self.nx], proc_sizes=[PETSc.DECIDE], boundary_type=('periodic')) # create DA for y grid self.day = PETSc.DA().create(dim=1, dof=1, sizes=[self.ny], proc_sizes=[PETSc.DECIDE], boundary_type=('periodic')) # initialise grid self.da1.setUniformCoordinates(xmin=x1, xmax=x2, ymin=y1, ymax=y2) self.da4.setUniformCoordinates(xmin=x1, xmax=x2, ymin=y1, ymax=y2) self.dax.setUniformCoordinates(xmin=x1, xmax=x2) self.day.setUniformCoordinates(xmin=y1, xmax=y2) # create RHS vector self.Ab = self.da1.createGlobalVec() self.Ob = self.da1.createGlobalVec() self.Pb = self.da1.createGlobalVec() # create vectors for magnetic and velocity field self.A = self.da1.createGlobalVec() # magnetic vector potential A self.J = self.da1.createGlobalVec() # current density J self.P = self.da1.createGlobalVec() # streaming function psi self.O = self.da1.createGlobalVec() # vorticity omega self.Bx = self.da1.createGlobalVec() self.By = self.da1.createGlobalVec() self.Vx = self.da1.createGlobalVec() self.Vy = self.da1.createGlobalVec() # set variable names self.A.setName('A') self.J.setName('J') self.P.setName('P') self.O.setName('O') self.Bx.setName('Bx') self.By.setName('By') self.Vx.setName('Vx') self.Vy.setName('Vy') # create nullspace self.poisson_nullspace = PETSc.NullSpace().create(constant=True) # create jacobian and matrix objects self.petsc_vorticity = PETScVorticity(self.da1, self.nx, self.ny, self.ht, self.hx, self.hy) self.petsc_ohmslaw = PETScOhmsLaw(self.da1, self.nx, self.ny, self.ht, self.hx, self.hy) self.petsc_poisson = PETScPoisson(self.da1, self.nx, self.ny, self.hx, self.hy) # initialise vorticity matrix self.vorticity_matrix = self.da1.createMat() self.vorticity_matrix.setOption( self.vorticity_matrix.Option.NEW_NONZERO_ALLOCATION_ERR, False) self.vorticity_matrix.setUp() # initialise Ohms's law matrix self.ohmslaw_matrix = self.da1.createMat() self.ohmslaw_matrix.setOption( self.ohmslaw_matrix.Option.NEW_NONZERO_ALLOCATION_ERR, False) self.ohmslaw_matrix.setUp() # initialise Poisson matrix self.poisson_matrix = self.da1.createMat() self.poisson_matrix.setOption( self.poisson_matrix.Option.NEW_NONZERO_ALLOCATION_ERR, False) self.poisson_matrix.setUp() self.poisson_matrix.setNullSpace(self.poisson_nullspace) # create nonlinear vorticity solver self.vorticity_snes = PETSc.SNES().create() self.vorticity_snes.setType('ksponly') self.vorticity_snes.setFunction(self.petsc_vorticity.snes_mult, self.Ob) self.vorticity_snes.setJacobian(self.update_vorticity_jacobian, self.vorticity_matrix) self.vorticity_snes.setFromOptions() # self.vorticity_snes.getKSP().setType('gmres') self.vorticity_snes.getKSP().setType('preonly') self.vorticity_snes.getKSP().getPC().setType('lu') self.vorticity_snes.getKSP().getPC().setFactorSolverPackage( solver_package) # create nonlinear Ohms's law solver self.ohmslaw_snes = PETSc.SNES().create() self.ohmslaw_snes.setType('ksponly') self.ohmslaw_snes.setFunction(self.petsc_ohmslaw.snes_mult, self.Ab) self.ohmslaw_snes.setJacobian(self.update_ohmslaw_jacobian, self.ohmslaw_matrix) self.ohmslaw_snes.setFromOptions() # self.ohmslaw_snes.getKSP().setType('gmres') self.ohmslaw_snes.getKSP().setType('preonly') self.ohmslaw_snes.getKSP().getPC().setType('lu') self.ohmslaw_snes.getKSP().getPC().setFactorSolverPackage( solver_package) # create linear Poisson solver self.poisson_ksp = PETSc.KSP().create() self.poisson_ksp.setFromOptions() self.poisson_ksp.setOperators(self.poisson_matrix) self.poisson_ksp.setType('preonly') self.poisson_ksp.getPC().setType('lu') self.poisson_ksp.getPC().setFactorSolverPackage(solver_package) # self.poisson_ksp.setNullSpace(self.poisson_nullspace) self.petsc_poisson.formMat(self.poisson_matrix) # create derivatives object self.derivatives = PETScDerivatives(self.da1, self.nx, self.ny, self.ht, self.hx, self.hy) # get coordinate vectors coords_x = self.dax.getCoordinates() coords_y = self.day.getCoordinates() # save x coordinate arrays scatter, xVec = PETSc.Scatter.toAll(coords_x) scatter.begin(coords_x, xVec, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) scatter.end(coords_x, xVec, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) xGrid = xVec.getValues(range(0, self.nx)).copy() scatter.destroy() xVec.destroy() # save y coordinate arrays scatter, yVec = PETSc.Scatter.toAll(coords_y) scatter.begin(coords_y, yVec, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) scatter.end(coords_y, yVec, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) yGrid = yVec.getValues(range(0, self.ny)).copy() scatter.destroy() yVec.destroy() # set initial data (xs, xe), (ys, ye) = self.da1.getRanges() A_arr = self.da1.getVecArray(self.A) P_arr = self.da1.getVecArray(self.P) init_data = __import__("runs." + cfg['initial_data']['python'], globals(), locals(), ['magnetic_A', 'velocity_P'], 0) for i in range(xs, xe): for j in range(ys, ye): A_arr[i, j] = init_data.magnetic_A(xGrid[i], yGrid[j], Lx, Ly) P_arr[i, j] = init_data.velocity_P(xGrid[i], yGrid[j], Lx, Ly) # Fourier Filtering self.nfourier = cfg['initial_data']['nfourier'] if self.nfourier >= 0: # obtain whole A vector everywhere scatter, Aglobal = PETSc.Scatter.toAll(self.A) scatter.begin(self.A, Aglobal, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) scatter.end(self.A, Aglobal, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) petsc_indices = self.da1.getAO().app2petsc( np.arange(self.nx * self.ny, dtype=np.int32)) Ainit = Aglobal.getValues(petsc_indices).copy().reshape( (self.ny, self.nx)) scatter.destroy() Aglobal.destroy() # compute FFT, cut, compute inverse FFT from scipy.fftpack import rfft, irfft Afft = rfft(Ainit, axis=1) Afft[:, 0] = 0. Afft[:, self.nfourier + 1:] = 0. A_arr = self.da1.getVecArray(self.A) A_arr[:, :] = irfft(Afft).T[xs:xe, ys:ye] # compute current and vorticity self.derivatives.laplace_vec(self.A, self.J, -1.) self.derivatives.laplace_vec(self.P, self.O, -1.) J_arr = self.da1.getVecArray(self.J) O_arr = self.da1.getVecArray(self.O) # add perturbations for i in range(xs, xe): for j in range(ys, ye): J_arr[i, j] += init_data.current_perturbation( xGrid[i], yGrid[j], Lx, Ly) O_arr[i, j] += init_data.vorticity_perturbation( xGrid[i], yGrid[j], Lx, Ly) # create HDF5 output file self.hdf5_viewer = PETSc.ViewerHDF5().create( cfg['io']['hdf5_output'], mode=PETSc.Viewer.Mode.WRITE, comm=PETSc.COMM_WORLD) self.hdf5_viewer.pushGroup("/") # write grid data to hdf5 file coords_x = self.dax.getCoordinates() coords_y = self.day.getCoordinates() coords_x.setName('x') coords_y.setName('y') self.hdf5_viewer(coords_x) self.hdf5_viewer(coords_y) # write initial data to hdf5 file self.save_to_hdf5(0)
def ii_PETScOperator(bmat, nullspace): '''Return an object with mult method which acts like bmat*''' colspace_vec, rowspace_vec = bmat.create_vec(0), bmat.create_vec(1) if isinstance(colspace_vec, block_vec): is_block = True assert isinstance(rowspace_vec, block_vec) row_sizes = tuple(bi.size() for bi in colspace_vec) col_sizes = tuple(xi.size() for xi in rowspace_vec) else: is_block = False assert not isinstance(rowspace_vec, block_vec) row_sizes = (colspace_vec.size(), ) col_sizes = (rowspace_vec.size(), ) print((is_block, row_sizes, col_sizes)) # if isinstance(bmat, block_base): # row_sizes, col_sizes = bmat_sizes(bmat) # is_block = True # else: # row_sizes, col_sizes = (bmat.size(0), ), (bmat.size(1), ) # is_block = False class Foo(object): def __init__(self, A): self.A = A if is_block: def mult(self, mat, x, y): '''y = A*x''' y *= 0 # Now x shall be comming as a nested vector # Convert x_bvec = block_vec(list(map(PETScVector, x.getNestSubVecs()))) # Apply y_bvec = self.A*x_bvec # Convert back y.axpy(1., as_petsc_nest(y_bvec)) def multTranspose(self, mat, x, y): '''y = A.T*x''' AT = block_transpose(self.A) y *= 0 # Now x shall be comming as a nested vector # Convert x_bvec = block_vec(list(map(PETScVector, x.getNestSubVecs()))) # Apply y_bvec = AT*x_bvec # Convert back y.axpy(1., as_petsc_nest(y_bvec)) # No block else: def mult(self, mat, x, y): '''y = A*x''' y *= 0 x_bvec = PETScVector(x) y_bvec = self.A*x_bvec y.axpy(1., as_petsc(y_bvec)) def multTranspose(self, mat, x, y): '''y = A.T*x''' AT = block_transpose(self.A) y *= 0 x_bvec = PETScVector(x) y_bvec = AT*x_bvec y.axpy(1., as_petsc(y_bvec)) mat = PETSc.Mat().createPython([[sum(row_sizes), ]*2, [sum(col_sizes), ]*2]) mat.setPythonContext(Foo(bmat)) if nullspace is not None: Z = PETSc.NullSpace(constant=True, vectors=[as_backend_type(v).vec() for v in map(convert, nullspace)], comm=PETSc.COMM_WORLD) mat.setNullSpace(Z) mat.setUp() return mat
r0, Hiptmairtol, params) G = HiptmairMatrices[0] Gt = PETSc.Mat() G.transpose(out=Gt) # X = PETScToScipy(X) # invX = sp.linalg.inv(X) # invX = PETSc.Mat().createAIJ(size=invX.shape, csr=(invX.indptr, invX.indices, invX.data)) # TT = A + G*Gt nullVecs = [] for i in range(0, LagrangeF.dim()): nullVecs.append(Gt.getColumnVector(i)) null_space_final = PETSc.NullSpace() null_space_final.create(vectors=nullVecs) T1.setNearNullSpace(null_space_final) #print (TT-T).view() # savePETScMat(A, "Matrix/S.mat", "S") # savePETScMat(Z, "Matrix/Z.mat", "Z") # savePETScMat(M, "Matrix/M.mat", "M") # savePETScMat(T, "Matrix/T.mat", "T") # savePETScMat(T1, "Matrix/T1.mat", "T1") # ss # Z = M + G*Gt # T = PETScToScipy(T) # Z = PETScToScipy(Z) # A = PETScToScipy(A) # M = PETScToScipy(M)
def __init__(self, cfgfile, mode="none"): ''' Constructor ''' petsc4py.init(sys.argv) if PETSc.COMM_WORLD.getRank() == 0: print("") print("Reduced MHD 2D") print("==============") print("") # solver mode self.mode = mode # set run id to timestamp self.run_id = datetime.datetime.fromtimestamp( time.time()).strftime("%y%m%d%H%M%S") if PETSc.COMM_WORLD.getRank() == 0: print(" Config: %s" % cfgfile) # load run config file self.cfg = Config(cfgfile) # timestep setup self.ht = self.cfg['grid']['ht'] # timestep size self.nt = self.cfg['grid']['nt'] # number of timesteps self.nsave = self.cfg['io'][ 'nsave'] # save only every nsave'th timestep # grid setup self.nx = self.cfg['grid']['nx'] # number of points in x self.ny = self.cfg['grid']['ny'] # number of points in y self.Lx = self.cfg['grid']['Lx'] # spatial domain in x x1 = self.cfg['grid']['x1'] # x2 = self.cfg['grid']['x2'] # self.Ly = self.cfg['grid']['Ly'] # spatial domain in y y1 = self.cfg['grid']['y1'] # y2 = self.cfg['grid']['y2'] # self.hx = self.cfg['grid']['hx'] # gridstep size in x self.hy = self.cfg['grid']['hy'] # gridstep size in y # create time vector self.time = PETSc.Vec().createMPI(1, comm=PETSc.COMM_WORLD) self.time.setName('t') # electron skin depth self.de = self.cfg['initial_data']['skin_depth'] # double bracket dissipation self.nu = self.cfg['initial_data']['dissipation'] # set global tolerance self.tolerance = self.cfg['solver'][ 'petsc_snes_atol'] * self.nx * self.ny # direct solver package self.solver_package = self.cfg['solver']['lu_solver_package'] # set some PETSc solver options OptDB = PETSc.Options() OptDB.setValue('ksp_rtol', self.cfg['solver']['petsc_ksp_rtol']) OptDB.setValue('ksp_atol', self.cfg['solver']['petsc_ksp_atol']) OptDB.setValue('ksp_max_it', self.cfg['solver']['petsc_ksp_max_iter']) OptDB.setValue('pc_type', 'hypre') OptDB.setValue('pc_hypre_type', 'boomeramg') # create DA with single dof self.da1 = PETSc.DA().create(dim=2, dof=1, sizes=[self.nx, self.ny], proc_sizes=[PETSc.DECIDE, PETSc.DECIDE], boundary_type=('periodic', 'periodic'), stencil_width=1, stencil_type='box') # create DA (dof = 4 for A, J, P, O) self.da4 = PETSc.DA().create(dim=2, dof=4, sizes=[self.nx, self.ny], proc_sizes=[PETSc.DECIDE, PETSc.DECIDE], boundary_type=('periodic', 'periodic'), stencil_width=1, stencil_type='box') # create DA for x grid self.dax = PETSc.DA().create(dim=1, dof=1, sizes=[self.nx], proc_sizes=[PETSc.DECIDE], boundary_type=('periodic')) # create DA for y grid self.day = PETSc.DA().create(dim=1, dof=1, sizes=[self.ny], proc_sizes=[PETSc.DECIDE], boundary_type=('periodic')) # initialise grid self.da1.setUniformCoordinates(xmin=x1, xmax=x2, ymin=y1, ymax=y2) self.da4.setUniformCoordinates(xmin=x1, xmax=x2, ymin=y1, ymax=y2) self.dax.setUniformCoordinates(xmin=x1, xmax=x2) self.day.setUniformCoordinates(xmin=y1, xmax=y2) # create solution and RHS vector self.dx = self.da4.createGlobalVec() self.dy = self.da4.createGlobalVec() self.x = self.da4.createGlobalVec() self.b = self.da4.createGlobalVec() self.f = self.da4.createGlobalVec() self.Pb = self.da1.createGlobalVec() self.FA = self.da1.createGlobalVec() self.FJ = self.da1.createGlobalVec() self.FP = self.da1.createGlobalVec() self.FO = self.da1.createGlobalVec() # create initial guess vectors self.igFA1 = self.da1.createGlobalVec() self.igFA2 = self.da1.createGlobalVec() self.igFO1 = self.da1.createGlobalVec() self.igFO2 = self.da1.createGlobalVec() # nullspace vectors self.x0 = self.da4.createGlobalVec() self.P0 = self.da1.createGlobalVec() # create vectors for magnetic and velocity field self.A = self.da1.createGlobalVec() # magnetic vector potential A self.J = self.da1.createGlobalVec() # current density J self.P = self.da1.createGlobalVec() # streaming function psi self.O = self.da1.createGlobalVec() # vorticity omega self.Bx = self.da1.createGlobalVec() self.By = self.da1.createGlobalVec() self.Vx = self.da1.createGlobalVec() self.Vy = self.da1.createGlobalVec() # set variable names self.A.setName('A') self.J.setName('J') self.P.setName('P') self.O.setName('O') self.Bx.setName('Bx') self.By.setName('By') self.Vx.setName('Vx') self.Vy.setName('Vy') # initialise nullspace self.x0.set(0.) x0_arr = self.da4.getVecArray(self.x0)[...] x0_arr[:, :, 2] = 1. self.x0.assemble() self.x0.normalize() self.solver_nullspace = PETSc.NullSpace().create(constant=False, vectors=(self.x0, )) self.poisson_nullspace = PETSc.NullSpace().create(constant=True) # initialise Poisson matrix self.Pm = self.da1.createMat() self.Pm.setOption(PETSc.Mat.Option.NEW_NONZERO_ALLOCATION_ERR, False) self.Pm.setUp() self.Pm.setNullSpace(self.poisson_nullspace) # create Poisson solver object self.petsc_poisson = PETScPoisson(self.da1, self.nx, self.ny, self.hx, self.hy) # setup linear Poisson solver self.poisson_ksp = PETSc.KSP().create() self.poisson_ksp.setFromOptions() self.poisson_ksp.setOperators(self.Pm) self.poisson_ksp.setTolerances( rtol=self.cfg['solver']['poisson_ksp_rtol'], atol=self.cfg['solver']['poisson_ksp_atol'], max_it=self.cfg['solver']['poisson_ksp_max_iter']) self.poisson_ksp.setType('cg') self.poisson_ksp.getPC().setType('hypre') self.poisson_ksp.setUp() self.petsc_poisson.formMat(self.Pm) # create derivatives object self.derivatives = PETScDerivatives(self.da1, self.nx, self.ny, self.ht, self.hx, self.hy) # read initial data if self.cfg["io"]["hdf5_input"] != None and self.cfg["io"][ "hdf5_input"] != "": if self.cfg["initial_data"]["python"] != None and self.cfg[ "initial_data"]["python"] != "": if PETSc.COMM_WORLD.getRank() == 0: print( "WARNING: Both io.hdf5_input and initial_data.python are set!" ) print(" Reading initial data from HDF5 file.") self.read_initial_data_from_hdf5() else: self.read_initial_data_from_python() # copy initial data vectors to x self.copy_x_from_da1_to_da4() # create HDF5 output file and write parameters hdf5_filename = self.cfg['io']['hdf5_output'] last_dot = hdf5_filename.rfind('.') hdf5_filename = hdf5_filename[:last_dot] + "." + str( self.run_id) + hdf5_filename[last_dot:] if PETSc.COMM_WORLD.getRank() == 0: print(" Output: %s" % hdf5_filename) hdf5out = h5py.File(hdf5_filename, "w", driver="mpio", comm=PETSc.COMM_WORLD.tompi4py()) hdf5out.attrs["run_id"] = self.run_id for cfg_group in self.cfg: for cfg_item in self.cfg[cfg_group]: if self.cfg[cfg_group][cfg_item] != None: value = self.cfg[cfg_group][cfg_item] else: value = "" hdf5out.attrs[cfg_group + "." + cfg_item] = value hdf5out.attrs["solver.solver_mode"] = self.mode if self.cfg["initial_data"]["python"] != None and self.cfg[ "initial_data"]["python"] != "": python_input = open( "examples/" + self.cfg['initial_data']['python'] + ".py", 'r') python_file = python_input.read() python_input.close() else: python_file = "" hdf5out.attrs["initial_data.python_file"] = python_file hdf5out.close() # create HDF5 viewer self.hdf5_viewer = PETSc.ViewerHDF5().create( hdf5_filename, mode=PETSc.Viewer.Mode.APPEND, comm=PETSc.COMM_WORLD) self.hdf5_viewer.pushGroup("/") # write grid data to hdf5 file coords_x = self.dax.getCoordinates() coords_y = self.day.getCoordinates() coords_x.setName('x') coords_y.setName('y') self.hdf5_viewer(coords_x) self.hdf5_viewer(coords_y) # write initial data to hdf5 file self.save_to_hdf5(0) # output some more information if PETSc.COMM_WORLD.getRank() == 0: print("") print(" nt = %i" % (self.nt)) print(" nx = %i" % (self.nx)) print(" ny = %i" % (self.ny)) print("") print(" ht = %f" % (self.ht)) print(" hx = %f" % (self.hx)) print(" hy = %f" % (self.hy)) print("") print(" PETSc SNES rtol = %e" % self.cfg['solver']['petsc_snes_rtol']) print(" atol = %e" % self.cfg['solver']['petsc_snes_atol']) print(" stol = %e" % self.cfg['solver']['petsc_snes_stol']) print(" max iter = %i" % self.cfg['solver']['petsc_snes_max_iter']) print("") print(" PETSc KSP rtol = %e" % self.cfg['solver']['petsc_ksp_rtol']) print(" atol = %e" % self.cfg['solver']['petsc_ksp_atol']) print(" max iter = %i" % self.cfg['solver']['petsc_ksp_max_iter']) print("")
def _create_constant_nullspace(self): """Initialize a constant null space. """ self.const_null_space = p4pyPETSc.NullSpace().create(comm=p4pyPETSc.COMM_WORLD, vectors = (), constant = True)
# Ths pressure field for this problem is determined only up to a # constant. We can supply the vector that spans the nullspace and any # component of the solution in this direction will be eliminated during # the iterative linear solution process. # Create nullspace vector null_vec = dolfinx.fem.create_vector_nest(L) # Set velocity part to zero and the pressure part to a non-zero constant null_vecs = null_vec.getNestSubVecs() null_vecs[0].set(0.0), null_vecs[1].set(1.0) # Normalize the vector, create a nullspace object, and attach it to the # matrix null_vec.normalize() nsp = PETSc.NullSpace().create(vectors=[null_vec]) assert nsp.test(A) A.setNullSpace(nsp) # Now we create a Krylov Subspace Solver ``ksp``. We configure it to use # the MINRES method, and a block-diagonal preconditioner using PETSc's # additive fieldsplit type preconditioner:: ksp = PETSc.KSP().create(mesh.mpi_comm()) ksp.setOperators(A, P) ksp.setType("minres") ksp.setTolerances(rtol=1e-8) ksp.getPC().setType("fieldsplit") ksp.getPC().setFieldSplitType(PETSc.PC.CompositeType.ADDITIVE) # Define the matrix blocks in the preconditioner with the velocity and
M, C, K = load_qep(folder + '/M.dat', folder + '/C.dat', folder + '/K.dat', adjoint) else: Print("Circulant operators") M, C, K = load_qep_n(folder + '/M1.dat', folder + '/M2.dat', folder + '/M3.dat', folder + '/C1.dat', folder + '/C2.dat', folder + '/C3.dat', folder + '/K1.dat', folder + '/K2.dat', folder + '/K3.dat', j, n, adjoint) # set NullSpace if required if j == 0: basis = [ PETSc.Vec().load(PETSc.Viewer().createBinary(folder + '/Mk.dat', 'r')) ] nullsp = PETSc.NullSpace().create(False, basis) M.setNullSpace(nullsp) # Solve QEP Q = SLEPc.PEP().create() Q.setOperators([K, C, M]) Q.setProblemType(SLEPc.PEP.ProblemType.GENERAL) Q.setFromOptions() Q.solve() Print() Print("******************************") Print("*** SLEPc Solution Results ***") Print("******************************")
def reference_periodic(tetra: bool, r_lvl: int = 0, out_hdf5: h5py.File = None, xdmf: bool = False, boomeramg: bool = False, kspview: bool = False, degree: int = 1): # Create mesh and finite element if tetra: # Tet setup N = 3 mesh = create_unit_cube(MPI.COMM_WORLD, N, N, N) for i in range(r_lvl): mesh.topology.create_entities(mesh.topology.dim - 2) mesh = refine(mesh, redistribute=True) N *= 2 else: # Hex setup N = 3 for i in range(r_lvl): N *= 2 mesh = create_unit_cube(MPI.COMM_WORLD, N, N, N, CellType.hexahedron) V = FunctionSpace(mesh, ("CG", degree)) # Create Dirichlet boundary condition def dirichletboundary(x): return np.logical_or( np.logical_or(np.isclose(x[1], 0), np.isclose(x[1], 1)), np.logical_or(np.isclose(x[2], 0), np.isclose(x[2], 1))) mesh.topology.create_connectivity(2, 1) geometrical_dofs = locate_dofs_geometrical(V, dirichletboundary) bc = dirichletbc(PETSc.ScalarType(0), geometrical_dofs, V) bcs = [bc] # Define variational problem u = TrialFunction(V) v = TestFunction(V) a = inner(grad(u), grad(v)) * dx x = SpatialCoordinate(mesh) dx_ = x[0] - 0.9 dy_ = x[1] - 0.5 dz_ = x[2] - 0.1 f = x[0] * sin(5.0 * pi * x[1]) + 1.0 * exp( -(dx_ * dx_ + dy_ * dy_ + dz_ * dz_) / 0.02) rhs = inner(f, v) * dx # Assemble rhs, RHS and apply lifting bilinear_form = form(a) linear_form = form(rhs) A_org = assemble_matrix(bilinear_form, bcs) A_org.assemble() L_org = assemble_vector(linear_form) apply_lifting(L_org, [bilinear_form], [bcs]) L_org.ghostUpdate(addv=PETSc.InsertMode.ADD_VALUES, mode=PETSc.ScatterMode.REVERSE) set_bc(L_org, bcs) # Create PETSc nullspace nullspace = PETSc.NullSpace().create(constant=True) PETSc.Mat.setNearNullSpace(A_org, nullspace) # Set PETSc options opts = PETSc.Options() if boomeramg: opts["ksp_type"] = "cg" opts["ksp_rtol"] = 1.0e-5 opts["pc_type"] = "hypre" opts['pc_hypre_type'] = 'boomeramg' opts["pc_hypre_boomeramg_max_iter"] = 1 opts["pc_hypre_boomeramg_cycle_type"] = "v" # opts["pc_hypre_boomeramg_print_statistics"] = 1 else: opts["ksp_type"] = "cg" opts["ksp_rtol"] = 1.0e-12 opts["pc_type"] = "gamg" opts["pc_gamg_type"] = "agg" opts["pc_gamg_sym_graph"] = True # Use Chebyshev smoothing for multigrid opts["mg_levels_ksp_type"] = "richardson" opts["mg_levels_pc_type"] = "sor" # opts["help"] = None # List all available options # opts["ksp_view"] = None # List progress of solver # Initialize PETSc solver, set options and operator solver = PETSc.KSP().create(MPI.COMM_WORLD) solver.setFromOptions() solver.setOperators(A_org) # Solve linear problem u_ = Function(V) start = perf_counter() with Timer("Solve"): solver.solve(L_org, u_.vector) end = perf_counter() u_.vector.ghostUpdate(addv=PETSc.InsertMode.INSERT, mode=PETSc.ScatterMode.FORWARD) if kspview: solver.view() it = solver.getIterationNumber() num_dofs = V.dofmap.index_map.size_global * V.dofmap.index_map_bs if out_hdf5 is not None: d_set = out_hdf5.get("its") d_set[r_lvl] = it d_set = out_hdf5.get("num_dofs") d_set[r_lvl] = num_dofs d_set = out_hdf5.get("solve_time") d_set[r_lvl, MPI.COMM_WORLD.rank] = end - start if MPI.COMM_WORLD.rank == 0: print("Rlvl {0:d}, Iterations {1:d}".format(r_lvl, it)) # Output solution to XDMF if xdmf: ext = "tet" if tetra else "hex" fname = "results/reference_periodic_{0:d}_{1:s}.xdmf".format( r_lvl, ext) u_.name = "u_" + ext + "_unconstrained" with XDMFFile(MPI.COMM_WORLD, fname, "w") as out_periodic: out_periodic.write_mesh(mesh) out_periodic.write_function( u_, 0.0, "Xdmf/Domain/" + "Grid[@Name='{0:s}'][1]".format(mesh.name))
def __init__(self, cfgfile, runid=None, cfg=None): ''' Constructor ''' assert cfgfile is not None assert cfgfile is not "" # if runid is empty use timestamp if runid == None or runid == "": runid = datetime.datetime.fromtimestamp( time.time()).strftime("%y%m%d%H%M%S") # stencil width stencil = 2 # load run config file if cfg != None: self.cfg = cfg elif cfgfile != None and len(cfgfile) > 0: self.cfg = Config(cfgfile) else: if PETSc.COMM_WORLD.getRank() == 0: print("ERROR: No valid config file or object passed.") sys.exit() # determine solver modules if cfg['solver']['method'] == 'explicit': self.vlasov_module = None else: self.vlasov_module = "vlasov.solvers." if cfg['solver']['mode'] == 'split': self.vlasov_module += 'vlasov' else: self.vlasov_module += self.cfg['solver']['mode'] self.vlasov_module += '.' + "PETSc" if cfg['solver']['type'] == 'newton' or cfg['solver'][ 'type'] == 'nonlinear': self.vlasov_module += "NL" if cfg['solver']['mode'] == 'split': self.vlasov_module += "Vlasov" # self.vlasov_module += self.cfg['solver']['poisson_bracket'] # if cfg['solver']['preconditioner_type'] != None and cfg['solver']['preconditioner_scheme'] != None: # self.vlasov_module += self.cfg['solver']['preconditioner_scheme'] self.vlasov_module += self.cfg['solver']['timestepping'].upper() if not cfg.is_dissipation_none: if cfg['solver']['dissipation'] == 'double_bracket': self.vlasov_module += "DB" self.poisson_module = "vlasov.solvers.poisson.PETScPoisson" self.poisson_module += self.cfg['solver']['laplace_operator'] # importing solver modules if PETSc.COMM_WORLD.getRank() == 0: print("Loading Vlasov solver %s" % (self.vlasov_module)) print("Loading Poisson solver %s" % (self.poisson_module)) print("") if not self.cfg.is_preconditioner_none(): print("Using Preconditioner %s" % (self.cfg.get_preconditioner())) if not self.cfg.is_dissipation_none(): if self.cfg.is_dissipation_collisions(): print("Using Collision Operator %s" % (self.cfg.get_collision_operator())) if self.cfg.is_dissipation_double_bracket(): print("Using Double Bracket %s" % (self.cfg.get_double_bracket())) print("") self.vlasov_object = __import__(self.vlasov_module, globals(), locals(), ['PETScVlasovSolver'], 0) self.poisson_object = __import__(self.poisson_module, globals(), locals(), ['PETScPoissonSolver'], 0) # timestep setup ht = self.cfg['grid']['ht'] # timestep size nt = self.cfg['grid']['nt'] # number of timesteps self.nsave = self.cfg['io'][ 'nsave'] # save only every nsave'th timestep # grid setup nx = self.cfg['grid']['nx'] # number of points in x nv = self.cfg['grid']['nv'] # number of points in v L = self.cfg['grid']['L'] vMax = self.cfg['grid']['vmax'] vMin = self.cfg['grid']['vmin'] if vMin == None: vMin = -vMax Lx = L Lv = vMax - vMin hx = Lx / nx # gridstep size in x hv = Lv / (nv - 1) # gridstep size in v self.time = PETSc.Vec().createMPI(1, 1, comm=PETSc.COMM_WORLD) self.time.setName('t') if PETSc.COMM_WORLD.getRank() == 0: self.time.setValue(0, 0.0) self.solver_package = self.cfg['solver']['lu_package'] self.nInitial = self.cfg['solver'][ 'initial_iter'] # number of iterations for initial guess self.coll_freq = self.cfg['solver']['coll_freq'] # collision frequency self.coll_drag = self.cfg['solver']['coll_drag'] # drag factor self.coll_diff = self.cfg['solver']['coll_diff'] # diff factor self.charge = self.cfg['initial_data']['charge'] # particle charge self.mass = self.cfg['initial_data']['mass'] # particle mass output_directory = self.cfg['io']['output_dir'] if output_directory == None or output_directory == "": output_directory = "." tindex = cfgfile.rfind('/') run_filename = cfgfile[tindex:].replace('.cfg', '.') + runid hdf_out_filename = output_directory + '/' + run_filename + ".hdf5" cfg_out_filename = output_directory + '/' + run_filename + ".cfg" # hdf_in_filename = self.cfg['io']['hdf5_input'] # hdf_out_filename = self.cfg['io']['hdf5_output'] self.cfg.write_current_config(cfg_out_filename) # set initial guess method initial_guess_options = { None: self.initial_guess_none, "None": self.initial_guess_none, "": self.initial_guess_none, "rk4": self.initial_guess_rk4, "gear": self.initial_guess_gear, "symplectic2": self.initial_guess_symplectic2, "symplectic4": self.initial_guess_symplectic4, } self.initial_guess_method = initial_guess_options[self.cfg['solver'] ['initial_guess']] # set some PETSc options OptDB = PETSc.Options() OptDB.setValue('ksp_rtol', self.cfg['solver']['petsc_ksp_rtol']) OptDB.setValue('ksp_atol', self.cfg['solver']['petsc_ksp_atol']) OptDB.setValue('ksp_max_it', self.cfg['solver']['petsc_ksp_max_iter']) OptDB.setValue('snes_rtol', self.cfg['solver']['petsc_snes_rtol']) OptDB.setValue('snes_atol', self.cfg['solver']['petsc_snes_atol']) OptDB.setValue('snes_stol', self.cfg['solver']['petsc_snes_stol']) OptDB.setValue('snes_max_it', self.cfg['solver']['petsc_snes_max_iter']) if PETSc.COMM_WORLD.getRank() == 0: print("Initialising Distributed Arrays.") # create DA for 2d grid (f only) # self.da1 = PETSc.DMDA().create(dim=2, dof=1, # sizes=[nx, nv], # proc_sizes=[1, PETSc.COMM_WORLD.getSize()], # boundary_type=['periodic', 'periodic'], # stencil_width=stencil, # stencil_type='box') # self.da1 = PETSc.DMDA().create(dim=2, dof=1, # sizes=[nx, nv], # proc_sizes=[PETSc.COMM_WORLD.getSize(), 1], # boundary_type=['periodic', 'ghosted'], # stencil_width=stencil, # stencil_type='box') self.da1 = PETSc.DMDA().create( dim=2, dof=1, sizes=[nx, nv], proc_sizes=[1, PETSc.COMM_WORLD.getSize()], boundary_type=['periodic', 'ghosted'], stencil_width=stencil, stencil_type='box') # self.da1 = PETSc.DMDA().create(dim=2, dof=1, # sizes=[nx, nv], # proc_sizes=[PETSc.DECIDE, 2], # boundary_type=['periodic', 'ghosted'], # stencil_width=stencil, # stencil_type='box') # self.da1 = PETSc.DMDA().create(dim=2, dof=1, # sizes=[nx, nv], # proc_sizes=[PETSc.DECIDE, PETSc.DECIDE], # boundary_type=['periodic', 'ghosted'], # stencil_width=stencil, # stencil_type='box') # create VIDA for x grid self.dax = PETSc.DMDA().create(dim=1, dof=1, sizes=[nx], proc_sizes=[PETSc.COMM_WORLD.getSize()], boundary_type=('periodic'), stencil_width=stencil, stencil_type='box') # create VIDA for y grid self.day = PETSc.DMDA().create(dim=1, dof=1, sizes=[nv], proc_sizes=[PETSc.COMM_WORLD.getSize()], boundary_type=('ghosted'), stencil_width=stencil, stencil_type='box') # initialise grid self.da1.setUniformCoordinates(xmin=0.0, xmax=Lx, ymin=vMin, ymax=vMax) self.dax.setUniformCoordinates(xmin=0.0, xmax=Lx) self.day.setUniformCoordinates(xmin=vMin, xmax=vMax) # get local index ranges (xs, xe), (ys, ye) = self.da1.getRanges() (xsx, xex), = self.dax.getRanges() # get coordinate vectors coords_x = self.dax.getCoordinates() coords_v = self.day.getCoordinates() # save x coordinate arrays scatter, xVec = PETSc.Scatter.toAll(coords_x) scatter.begin(coords_x, xVec, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) scatter.end(coords_x, xVec, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) xGrid = xVec.getValues(range(nx)).copy() scatter.destroy() xVec.destroy() # save v coordinate arrays scatter, vVec = PETSc.Scatter.toAll(coords_v) scatter.begin(coords_v, vVec, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) scatter.end(coords_v, vVec, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) vGrid = vVec.getValues(range(nv)).copy() scatter.destroy() vVec.destroy() # create grid object self.grid = Grid().create(xGrid, vGrid, nt, nx, nv, ht, hx, hv, stencil) # create vectors for Hamiltonians self.h0 = self.da1.createGlobalVec() # kinetic Hamiltonian self.h1c = self.da1.createGlobalVec() # current potential Hamiltonian self.h2c = self.da1.createGlobalVec() # current external Hamiltonian self.h1h = self.da1.createGlobalVec() # previous potential Hamiltonian self.h2h = self.da1.createGlobalVec() # previous external Hamiltonian # distribution functions self.fl = self.da1.createGlobalVec() # last (k+1, n ) self.fc = self.da1.createGlobalVec() # current (k+1, n+1) self.fh = self.da1.createGlobalVec() # history (k) # distribution function solver vectors self.fb = self.da1.createGlobalVec() # right hand side # moments self.N = self.dax.createGlobalVec() # density self.U = self.dax.createGlobalVec() # velocity density self.E = self.dax.createGlobalVec() # energy density self.A = self.dax.createGlobalVec() # collision factor # local moments self.nc = PETSc.Vec().createSeq(nx) # current density self.uc = PETSc.Vec().createSeq(nx) # current velocity density self.ec = PETSc.Vec().createSeq(nx) # current energy density self.ac = PETSc.Vec().createSeq(nx) # current collision factor self.nh = PETSc.Vec().createSeq(nx) # history density self.uh = PETSc.Vec().createSeq(nx) # history velocity density self.eh = PETSc.Vec().createSeq(nx) # history energy density self.ah = PETSc.Vec().createSeq(nx) # history collision factor # internal potential self.pc_int = self.dax.createGlobalVec() # current self.ph_int = self.dax.createGlobalVec() # history # external potential self.pc_ext = self.dax.createGlobalVec() # current self.ph_ext = self.dax.createGlobalVec() # history # potential solver vectors self.pb = self.dax.createGlobalVec() # right hand side self.pn = self.dax.createGlobalVec() # null vector # set variable names self.h0.setName('h0') self.h1c.setName('h1') self.h2c.setName('h2') self.fc.setName('f') self.pc_int.setName('phi_int') self.pc_ext.setName('phi_ext') self.N.setName('n') self.U.setName('u') self.E.setName('e') # initialise nullspace basis vectors # the Poisson equation has a null space of all constant vectors # that needs to be removed to avoid jumpy potentials self.pn.set(1.) self.pn.normalize() self.p_nullspace = PETSc.NullSpace().create(constant=False, vectors=(self.pn, )) # create Toolbox self.toolbox = Toolbox(self.da1, self.dax, self.grid) # initialise kinetic hamiltonian if PETSc.COMM_WORLD.getRank() == 0: print("Initialising kinetic Hamiltonian.") self.toolbox.initialise_kinetic_hamiltonian(self.h0, self.mass) # create Arakawa initial guess solver object if PETSc.COMM_WORLD.getRank() == 0: print("Instantiating Initial Guess Objects.") self.arakawa_rk4 = PETScArakawaRungeKutta(self.cfg, self.da1, self.grid, self.h0, self.h1h, self.h2h, self.nInitial) self.arakawa_gear = PETScArakawaGear(self.cfg, self.da1, self.grid, self.h0, self.h1h, self.h2h, self.nInitial) self.arakawa_symplectic = PETScArakawaSymplectic( self.cfg, self.da1, self.grid, self.h0, self.h1h, self.h2h, self.nInitial) # create solver dummies self.vlasov_solver = None self.poisson_solver = None self.poisson_ksp = None if PETSc.COMM_WORLD.getRank() == 0: print() print("Run ID: %s" % runid) print() print("Config File: %s" % cfgfile) print("Output File: %s" % hdf_out_filename) print() print("nt = %i" % (self.grid.nt)) print("nx = %i" % (self.grid.nx)) print("nv = %i" % (self.grid.nv)) print() print("ht = %e" % (self.grid.ht)) print("hx = %e" % (self.grid.hx)) print("hv = %e" % (self.grid.hv)) print() print("xMin = %+12.6e" % (self.grid.xMin())) print("xMax = %+12.6e" % (self.grid.vMax())) print("vMin = %+12.6e" % (self.grid.vMin())) print("vMax = %+12.6e" % (self.grid.vMax())) print() print("nu = %7.1e" % (self.coll_freq)) print() print("CFL = %e" % (self.grid.hx / vMax)) print() print() # set initial data N0 = self.dax.createGlobalVec() T0 = self.dax.createGlobalVec() N0.setName('n0') T0.setName('T0') n0 = PETSc.Vec().createSeq(nx) t0 = PETSc.Vec().createSeq(nx) if self.cfg['initial_data']['distribution_python'] != None: init_data = __import__( "runs." + self.cfg['initial_data']['distribution_python'], globals(), locals(), ['distribution'], 0) if PETSc.COMM_WORLD.getRank() == 0: print( "Initialising distribution function with Python function.") self.toolbox.initialise_distribution_function( self.fc, init_data.distribution) N0.set(0.) T0.set(0.) else: N0_arr = self.dax.getVecArray(N0) T0_arr = self.dax.getVecArray(T0) if self.cfg['initial_data']['density_python'] != None: init_data = __import__( "runs." + self.cfg['initial_data']['density_python'], globals(), locals(), ['density'], 0) if PETSc.COMM_WORLD.getRank() == 0: print("Initialising density with Python function.") for i in range(xsx, xex): N0_arr[i] = init_data.density(self.grid.x[i], self.grid.xLength()) else: N0_arr[xsx:xex] = self.cfg['initial_data']['density'] if self.cfg['initial_data']['temperature_python'] != None: init_data = __import__( "runs." + self.cfg['initial_data']['temperature_python'], globals(), locals(), ['temperature'], 0) if PETSc.COMM_WORLD.getRank() == 0: print("Initialising temperature with Python function.") for i in range(xsx, xex): T0_arr[i] = init_data.temperature(self.grid.x[i]) else: T0_arr[xsx:xex] = self.cfg['initial_data']['temperature'] if PETSc.COMM_WORLD.getRank() == 0: print("Initialising distribution function with Maxwellian.") self.copy_xvec_to_seq(N0, n0) self.copy_xvec_to_seq(T0, t0) self.toolbox.initialise_distribution_nT(self.fc, n0, t0) # Fourier Filtering nfourier = cfg['initial_data']['nfourier'] if nfourier > 0: from scipy.fftpack import rfft, irfft if PETSc.COMM_WORLD.getRank() == 0: print("Fourier Filtering") # obtain whole f vector everywhere scatter, Xglobal = PETSc.Scatter.toAll(self.fc) scatter.begin(self.fc, Xglobal, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) scatter.end(self.fc, Xglobal, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) petsc_indices = self.da1.getAO().app2petsc( np.arange(nx * nv, dtype=np.int32)) fTmp = Xglobal.getValues(petsc_indices).copy().reshape((nv, nx)).T scatter.destroy() Xglobal.destroy() # filter distribution function fFft = rfft(fTmp, axis=0) fFft[nfourier:, :] = 0. fTmp = irfft(fFft, axis=0) # store filtered distribution function f_arr = self.da1.getVecArray(self.fc) f_arr[:, :] = fTmp[xs:xe, ys:ye] # normalise f self.normalise_distribution_function() # calculate potential and moments if PETSc.COMM_WORLD.getRank() == 0: print("Calculate initial potential and moments.") self.calculate_moments(output=False) # initialise Gear History if self.cfg['solver']['initial_guess'] == "gear": self.arakawa_gear.initialise_history(self.fc) # check for external potential if self.cfg['initial_data']['external_python'] != None: if PETSc.COMM_WORLD.getRank() == 0: print("Calculate external potential.") external_data = __import__( "runs." + self.cfg['initial_data']['external_python'], globals(), locals(), ['external'], 0) self.external = external_data.external else: self.external = None # calculate external potential self.calculate_external(0.) # create HDF5 output file if PETSc.COMM_WORLD.getRank() == 0: print("Create HDF5 output file.") # use h5py to store attributes hdf5out = h5py.File(hdf_out_filename, 'w', driver='mpio', comm=PETSc.COMM_WORLD.tompi4py()) hdf5out.attrs['charge'] = self.charge hdf5out.close() # create PETSc HDF5 viewer self.hdf5_viewer = PETSc.ViewerHDF5().create( hdf_out_filename, mode=PETSc.Viewer.Mode.APPEND, comm=PETSc.COMM_WORLD) self.hdf5_viewer.pushGroup("/") if PETSc.COMM_WORLD.getRank() == 0: print("Saving initial data to HDF5.") # write grid data to hdf5 file coords_x.setName('x') coords_v.setName('v') self.hdf5_viewer(coords_x) self.hdf5_viewer(coords_v) # write initial data to hdf5 file # self.hdf5_viewer(N0) # self.hdf5_viewer(T0) # save to hdf5 self.hdf5_viewer.setTimestep(0) self.save_hdf5_vectors() if PETSc.COMM_WORLD.getRank() == 0: print("run_base.py: initialisation done.") print("")
P = S.ExactPrecond(PP, QQ, L, F, [V, Q]) # # bcp.apply(QQ) Mass = CP.Assemble(QQ) # # P = IO.matToSparse(P) # # plt.spy(P) # # plt.savefig("plt2") # # ss # # # P[W.dim()-1,W.dim()-1] += 1 # # P.assemblyBegin() # Make matrices useable. # # P.assemblyEnd() ksp = PETSc.KSP().create() ksp.setTolerances(1e-5) ksp.setOperators( A, P) #.getSubMatrix(u_is,u_is),P.getSubMatrix(u_is,u_is)) nsp = PETSc.NullSpace().create(constant=True) ksp.setNullSpace(nsp) # A.destroy() # P.destroy() reshist = {} def monitor(ksp, its, fgnorm): reshist[its] = fgnorm ksp.setMonitor(monitor) OptDB = PETSc.Options() OptDB['pc_factor_shift_amount'] = "0.1" OptDB['ksp_monitor_residual'] = ' ' OptDB['pc_factor_mat_ordering_type'] = 'rcm' OptDB['pc_factor_mat_solver_package'] = 'umfpack' # kspLAMG.max_it = 1