def get_numpy_array(xr): """ Convert a distributed PETSc Vec into a sequential numpy array. Parameters: ----------- xr: PETSc Vec Returns: -------- Array """ xr_size = xr.getSize() seqx = PETSc.Vec() seqx.createSeq(xr_size, comm=PETSc.COMM_SELF) seqx.setFromOptions() # seqx.set(0.0) fromIS = PETSc.IS().createGeneral(range(xr_size), comm=PETSc.COMM_SELF) toIS = PETSc.IS().createGeneral(range(xr_size), comm=PETSc.COMM_SELF) sctr = PETSc.Scatter().create(xr, fromIS, seqx, toIS) sctr.begin(xr, seqx, addv=PETSc.InsertMode.INSERT, mode=PETSc.ScatterMode.FORWARD) sctr.end(xr, seqx, addv=PETSc.InsertMode.INSERT, mode=PETSc.ScatterMode.FORWARD) return seqx.getArray()
def vecscatter(self, readonly=True): """A context manager scattering the arrays of all components of this :class:`MixedDat` into a contiguous :class:`PETSc.Vec` and reverse scattering to the original arrays when exiting the context. :param readonly: Access the data read-only (use :meth:`Dat.data_ro`) or read-write (use :meth:`Dat.data`). Read-write access requires a halo update. .. note:: The :class:`~PETSc.Vec` obtained from this context is in the correct order to be left multiplied by a compatible :class:`MixedMat`. In parallel it is *not* just a concatenation of the underlying :class:`Dat`\s.""" acc = (lambda d: d.vec_ro) if readonly else (lambda d: d.vec) # Allocate memory for the contiguous vector, create the scatter # contexts and stash them on the object for later reuse if not (hasattr(self, '_vec') and hasattr(self, '_sctxs')): self._vec = PETSc.Vec().create() # Size of flattened vector is product of size and cdim of each dat sz = sum(d.dataset.size * d.dataset.cdim for d in self._dats) self._vec.setSizes((sz, None)) self._vec.setUp() self._sctxs = [] # To be compatible with a MatNest (from a MixedMat) the # ordering of a MixedDat constructed of Dats (x_0, ..., x_k) # on P processes is: # (x_0_0, x_1_0, ..., x_k_0, x_0_1, x_1_1, ..., x_k_1, ..., x_k_P) # That is, all the Dats from rank 0, followed by those of # rank 1, ... # Hence the offset into the global Vec is the exclusive # prefix sum of the local size of the mixed dat. offset = MPI.comm.exscan(sz) if offset is None: offset = 0 for d in self._dats: sz = d.dataset.size * d.dataset.cdim with acc(d) as v: vscat = PETSc.Scatter().create(v, None, self._vec, PETSc.IS().createStride(sz, offset, 1)) offset += sz self._sctxs.append(vscat) # Do the actual forward scatter to fill the full vector with values for d, vscat in zip(self._dats, self._sctxs): with acc(d) as v: vscat.scatterBegin(v, self._vec, addv=PETSc.InsertMode.INSERT_VALUES) vscat.scatterEnd(v, self._vec, addv=PETSc.InsertMode.INSERT_VALUES) yield self._vec if not readonly: # Reverse scatter to get the values back to their original locations for d, vscat in zip(self._dats, self._sctxs): with acc(d) as v: vscat.scatterBegin(self._vec, v, addv=PETSc.InsertMode.INSERT_VALUES, mode=PETSc.ScatterMode.REVERSE) vscat.scatterEnd(self._vec, v, addv=PETSc.InsertMode.INSERT_VALUES, mode=PETSc.ScatterMode.REVERSE) self.needs_halo_update = True
def testVisually(self): '''blocks selected visually.''' # if self.comm.rank == 0: g2z,zvec = PETSc.Scatter().toZero(self.bnd.gindBlockWBand) g2z.scatter(self.bnd.gindBlockWBand,zvec, PETSc.InsertMode.INSERT) x = self.bnd.BlockSub2CenterCarWithoutBand(\ self.bnd.BlockInd2SubWithoutBand(zvec.getArray()) ) lx = self.bnd.BlockSub2CenterCarWithoutBand(\ self.bnd.BlockInd2SubWithoutBand(self.bnd.gindBlockWBand.getArray())) try: try: from mayavi import mlab except ImportError: from enthought.mayavi import mlab if self.comm.rank == 0: mlab.figure() mlab.points3d(x[:,0],x[:,1],x[:,2]) mlab.figure() mlab.points3d(lx[:,0],lx[:,1],lx[:,2]) mlab.show() #fig.add(pts1) #fig.add(pts2) except ImportError: import pylab as pl from mpl_toolkits.mplot3d import Axes3D #@UnusedImport fig = pl.figure() ax = fig.add_subplot(111, projection='3d') ax.scatter3D(x[:,0],x[:,1],x[:,2],c='blue',marker='o') ax.scatter3D(lx[:,0],lx[:,1],lx[:,2],c='red',marker='D') pl.savefig('testVis{0}.png'.format(self.comm.rank)) pl.show()
def create_gather_scatter(pdofs, pvec_i, pvec, comm=None): """ Create the ``gather()`` function for updating a global PETSc vector from local ones and the ``scatter()`` function for updating local PETSc vectors from the global one. """ if comm is None: comm = PETSc.COMM_WORLD isg = PETSc.IS().createGeneral(pdofs, comm=comm) g2l = PETSc.Scatter().create(pvec, isg, pvec_i, None) def scatter(pvec_i, pvec): """ Scatter `pvec` to `pvec_i`. """ g2l.scatter(pvec, pvec_i, PETSc.InsertMode.INSERT) def gather(pvec, pvec_i): """ Gather `pvec_i` to `pvec`. """ g2l.scatter(pvec_i, pvec, PETSc.InsertMode.INSERT, PETSc.ScatterMode.REVERSE) return gather, scatter
def __init__(self, in_vec, out_vec, in_inds, out_inds, comm): """ Initialize all attributes. Parameters ---------- in_vec : <Vector> pointer to the input vector. out_vec : <Vector> pointer to the output vector. in_inds : int ndarray input indices for the transfer. out_inds : int ndarray output indices for the transfer. comm : MPI.Comm or <FakeComm> communicator of the system that owns this transfer. """ super(PETScTransfer, self).__init__(in_vec, out_vec, in_inds, out_inds, comm) in_indexset = PETSc.IS().createGeneral(self._in_inds, comm=self._comm) out_indexset = PETSc.IS().createGeneral(self._out_inds, comm=self._comm) self._scatter = PETSc.Scatter().create(out_vec._petsc, out_indexset, in_vec._petsc, in_indexset).scatter if in_vec._ncol > 1: self._transfer = self._multi_transfer
def __init__(self, param, comm): self.dim = 3 self.comm = comm self.param = param self.numPlies = 3 self.numDataPoints = 12 self.numInterfaces = self.numPlies - 1 self.numLayers = self.numPlies + self.numInterfaces self.t = np.asarray([0.2, 0.02, 0.2, 0.02, 0.2]) self.theta = np.asarray([np.pi/4, -123.0, 0.0, -123.0, 3.*np.pi/4]) self.cutoff = np.cumsum(self.t) nx = 50 ny = 10 Lx = 10. Ly = 2. self.nel_per_layer = np.asarray([2,2,2,2,2]) self.isBnd = lambda x: self.isBoundary(x) self.f = lambda x: self.rhs(x) self.da = LayerCake(nx, ny, Lx, Ly, self.t, self.nel_per_layer) # Build layered composite from uniform mesh # Setup global and local matrices + communicators self.A = self.da.createMatrix() r, _ = self.A.getLGMap() # Get local to global mapping self.is_A = PETSc.IS().createGeneral(r.indices) # Create Index Set for local indices A_local = self.A.createSubMatrices(self.is_A)[0] # Construct local submatrix on domain vglobal = self.da.createGlobalVec() vlocal = self.da.createLocalVec() self.scatter_l2g = PETSc.Scatter().create(vlocal, None, vglobal, self.is_A) self.A_local = A_local # Setup elements self.fe = ElasticityQ1() # Compute Material Tensor given material parameters self.isotropic, self.composite = makeMaterials(self.param) # self.getDataDof()
def _setup_6of7_scatters_create(self, var_inds, arg_inds): """ Concatenates lists of indices and creates a PETSc Scatter """ merge = lambda x: numpy.concatenate(x) if len(x) > 0 else [] var_ind_set = PETSc.IS().createGeneral(merge(var_inds), comm=self.comm) arg_ind_set = PETSc.IS().createGeneral(merge(arg_inds), comm=self.comm) if self.app_ordering is not None: var_ind_set = self.app_ordering.app2petsc(var_ind_set) return PETSc.Scatter().create(self.vec['u'].petsc, var_ind_set, self.vec['p'].petsc, arg_ind_set)
def __init__(self, in_vec, out_vec, in_inds, out_inds, comm): """ Initialize all attributes. """ super().__init__(in_vec, out_vec, in_inds, out_inds, comm) in_indexset = PETSc.IS().createGeneral(self._in_inds, comm=self._comm) out_indexset = PETSc.IS().createGeneral(self._out_inds, comm=self._comm) self._scatter = PETSc.Scatter().create(out_vec._petsc, out_indexset, in_vec._petsc, in_indexset).scatter
def createScatter(self, varInds, argInds): merge = lambda x: numpy.concatenate(x) if len(x) > 0 else [] ISvar = PETSc.IS().createGeneral(merge(varInds), comm=self.comm) ISarg = PETSc.IS().createGeneral(merge(argInds), comm=self.comm) ISvar = self.AOvarPETSc.app2petsc(ISvar) if ISvar.array.shape[0] == 0: return None else: return PETSc.Scatter().create(self.vVarPETSc, ISvar, self.vArgPETSc, ISarg)
def __init__(self, n, L, overlap, comm): # Build Grid self.dim = len(n) self.L = L self.comm = comm self.isBnd = lambda x: self.isBoundary(x) self.da = PETSc.DMDA().create(n, dof=1, stencil_width=overlap) self.da.setUniformCoordinates(xmax=L[0], ymax=L[1], zmax=L[2]) self.da.setMatType(PETSc.Mat.Type.AIJ) self.numSub = comm.Get_size() self.sub2proc = [[] for i in range(self.numSub)] for i in range(self.numSub): self.sub2proc[i] = [i, 0] self.M = 1 # Define Finite Element space self.fe = DarcyQ1(self.dim) # Setup global and local matrices + communicators self.A = self.da.createMatrix() r, _ = self.A.getLGMap() # Get local to global mapping self.is_A = PETSc.IS().createGeneral( r.indices) # Create Index Set for local indices A_local = self.A.createSubMatrices( self.is_A)[0] # Construct local submatrix on domain vglobal = self.da.createGlobalVec() vlocal = self.da.createLocalVec() self.scatter_l2g = PETSc.Scatter().create(vlocal, None, vglobal, self.is_A) self.A_local = A_local # Identify boundary nodes nnodes = int(self.da.getCoordinatesLocal()[:].size / self.dim) coords = np.transpose(self.da.getCoordinatesLocal()[:].reshape( (nnodes, self.dim))) Dirich, Neumann, P2P = checkFaces(self.da, self.isBnd, coords) # Construct Partition of Unity self.cS = coarseSpace(self.da, self.A, self.comm, self.scatter_l2g) self.cS.buildPOU(True)
def _initialize_transfer(self, in_vec, out_vec): """ Set up the transfer; do any necessary pre-computation. Optionally implemented by the subclass. Parameters ---------- in_vec : <Vector> reference to the input vector. out_vec : <Vector> reference to the output vector. """ self._transfers = transfers = {} in_inds = self._in_inds out_inds = self._out_inds lens = np.empty(len(in_inds), dtype=int) for i, key in enumerate(in_inds): lens[i] = len(in_inds[key]) if self._comm.size > 1: lensums = np.empty(len(in_inds), dtype=int) self._comm.Allreduce(lens, lensums, op=MPI.SUM) else: lensums = lens for i, key in enumerate(in_inds): # if lensums[i] > 0, then at least one proc in the comm is transferring # data for the given key. That means that all procs in the comm # must particiipate in the collective Scatter call, so we construct # a Scatter here even if it's empty. if lensums[i] > 0: in_set_name, out_set_name = key in_indexset = PETSc.IS().createGeneral(np.array( in_inds[key], 'i'), comm=self._comm) out_indexset = PETSc.IS().createGeneral(np.array( out_inds[key], 'i'), comm=self._comm) in_petsc = in_vec._petsc[in_set_name] out_petsc = out_vec._petsc[out_set_name] transfer = PETSc.Scatter().create(out_petsc, out_indexset, in_petsc, in_indexset) transfers[key] = transfer if in_vec._ncol > 1: self.transfer = self.multi_transfer
def P2N(self, psc_vector, ngs_vector=None): if ngs_vector is None: ngs_vector = ngs.la.CreateParallelVector(self.pardofs) ngs_vector.SetParallelStatus(ngs.la.PARALLEL_STATUS.CUMULATED) ngs_vector[:] = 0.0 locvec = psc.Vec().createWithArray(ngs_vector.FV().NumPy(), comm=MPI.COMM_SELF) if "p2n_scat" not in self.__dict__: self.p2n_scat = psc.Scatter().create(psc_vector, self.iset, locvec, self.isetlocfree) self.p2n_scat.scatter(psc_vector, locvec, addv=psc.InsertMode.INSERT) return ngs_vector
def _initialize_transfer(self): self._transfers = {} for ip_iset, op_iset in self.ip_inds: key = (ip_iset, op_iset) if len(self.ip_inds[key]) > 0: ip_inds = numpy.array(self.ip_inds[key], 'i') op_inds = numpy.array(self.op_inds[key], 'i') ip_indexset = PETSc.IS().createGeneral(ip_inds, comm=self.comm) op_indexset = PETSc.IS().createGeneral(op_inds, comm=self.comm) ip_petsc = self.ip_vec._global_vector._petsc[ip_iset] op_petsc = self.op_vec._global_vector._petsc[op_iset] transfer = PETSc.Scatter().create(op_petsc, op_indexset, ip_petsc, ip_indexset) self._transfers[key] = transfer
def N2P(self, ngs_vector, psc_vector=None): if psc_vector is None: psc_vector = psc.Vec().createMPI(self.nglob * self.es, bsize=self.es, comm=MPI.COMM_WORLD) ngs_vector.Distribute() locvec = psc.Vec().createWithArray(ngs_vector.FV().NumPy(), comm=MPI.COMM_SELF) if "n2p_scat" not in self.__dict__: self.n2p_scat = psc.Scatter().create(locvec, self.isetlocfree, psc_vector, self.iset) psc_vector.set(0) self.n2p_scat.scatter(locvec, psc_vector, addv=psc.InsertMode.ADD) # 1 max, 2 sum+keep return psc_vector
def _initializePETScScatters(self): """ Defines a scatter for when a variable is its own argument """ n, c = self.name, self.copy args = self.variables[n, c]['args'] if (n, c) in args: varIndices = args[n, c] m1 = numpy.sum(self.argSizes[:self.rank]) m2 = m1 + args[n, c].shape[0] argIndices = numpy.array(numpy.linspace(m1, m2 - 1, m2 - m1), 'i') ISvar = PETSc.IS().createGeneral(varIndices, comm=self.comm) ISarg = PETSc.IS().createGeneral(argIndices, comm=self.comm) self.scatterFull = PETSc.Scatter().create(self.vVarPETSc, ISvar, self.vArgPETSc, ISarg) else: self.scatterFull = None
def __init__(self, comm, N): self.comm = comm self.N = N # global problem size self.h = 1 / N # grid spacing on unit interval self.n = N // comm.size + int( comm.rank < (N % comm.size)) # owned part of global problem self.start = comm.exscan(self.n) if comm.rank == 0: self.start = 0 gindices = numpy.arange(self.start - 1, self.start + self.n + 1, dtype=PETSc.IntType) % N # periodic self.mat = PETSc.Mat().create(comm=comm) size = (self.n, self.N) # local and global sizes self.mat.setSizes((size, size)) self.mat.setFromOptions() self.mat.setPreallocationNNZ( (3, 1) ) # Conservative preallocation for 3 "local" columns and one non-local # Allow matrix insertion using local indices [0:n+2] lgmap = PETSc.LGMap().create(list(gindices), comm=comm) self.mat.setLGMap(lgmap, lgmap) # Global and local vectors self.gvec = self.mat.createVecRight() self.lvec = PETSc.Vec().create(comm=PETSc.COMM_SELF) self.lvec.setSizes(self.n + 2) self.lvec.setUp() # Configure scatter from global to local isg = PETSc.IS().createGeneral(list(gindices), comm=comm) self.g2l = PETSc.Scatter().create(self.gvec, isg, self.lvec, None) self.tozero, self.zvec = PETSc.Scatter.toZero(self.gvec) self.history = [] if False: # Print some diagnostics print('[%d] local size %d, global size %d, starting offset %d' % (comm.rank, self.n, self.N, self.start)) self.gvec.setArray(numpy.arange(self.start, self.start + self.n)) self.gvec.view() self.g2l.scatter(self.gvec, self.lvec, PETSc.InsertMode.INSERT) for rank in range(comm.size): if rank == comm.rank: print('Contents of local Vec on rank %d' % rank) self.lvec.view() comm.barrier()
def create_gather_to_zero(pvec): """ Create the ``gather_to_zero()`` function for collecting the global PETSc vector on the task of rank zero. """ g20, pvec_full = PETSc.Scatter().toZero(pvec) def gather_to_zero(pvec): """ Return the global PETSc vector, corresponding to `pvec`, on the task of rank zero. The vector is reused between calls! """ g20.scatter(pvec, pvec_full, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) return pvec_full return gather_to_zero
def __init__(self, src_vec, tgt_vec, src_idxs, tgt_idxs, vec_conns, byobj_conns, mode, sysdata): src_idxs = src_vec.merge_idxs(src_idxs) tgt_idxs = tgt_vec.merge_idxs(tgt_idxs) self.byobj_conns = byobj_conns self.comm = comm = src_vec.comm self.sysdata = sysdata uvec = src_vec.petsc_vec pvec = tgt_vec.petsc_vec name = src_vec._sysdata.pathname if trace: debug("'%s': creating index sets for '%s' DataTransfer: %s %s" % (name, src_vec._sysdata.pathname, src_idxs, tgt_idxs)) src_idx_set = PETSc.IS().createGeneral(src_idxs, comm=comm) if trace: debug("src_idx_set DONE") tgt_idx_set = PETSc.IS().createGeneral(tgt_idxs, comm=comm) if trace: debug("tgt_idx_set DONE") try: if trace: # pragma: no cover self.src_idxs = src_idxs self.tgt_idxs = tgt_idxs self.vec_conns = vec_conns arrow = '-->' if mode == 'fwd' else '<--' debug( "'%s': new %s scatter (sizes: %d, %d)\n %s %s %s %s %s %s" % (name, mode, len(src_idx_set.indices), len(tgt_idx_set.indices), [v for u, v in vec_conns], arrow, [u for u, v in vec_conns], src_idx_set.indices, arrow, tgt_idx_set.indices)) self.scatter = PETSc.Scatter().create(uvec, src_idx_set, pvec, tgt_idx_set) if trace: debug("scatter creation DONE") except Exception as err: raise RuntimeError( "ERROR in %s (src_idxs=%s, tgt_idxs=%s, usize=%d, psize=%d): %s" % (name, src_idxs, tgt_idxs, src_vec.vec.size, tgt_vec.vec.size, str(err)))
def _initialize_transfer(self, in_vec, out_vec): """ Set up the transfer; do any necessary pre-computation. Optionally implemented by the subclass. Parameters ---------- in_vec : <Vector> reference to the input vector. out_vec : <Vector> reference to the output vector. """ in_indexset = PETSc.IS().createGeneral(self._in_inds, comm=self._comm) out_indexset = PETSc.IS().createGeneral(self._out_inds, comm=self._comm) self._transfer = PETSc.Scatter().create(out_vec._petsc, out_indexset, in_vec._petsc, in_indexset) if in_vec._ncol > 1: self.transfer = self.multi_transfer
def vecscatters(self): """Get the vecscatters from the dof layout of this dataset to a PETSc Vec.""" # To be compatible with a MatNest (from a MixedMat) the # ordering of a MixedDat constructed of Dats (x_0, ..., x_k) # on P processes is: # (x_0_0, x_1_0, ..., x_k_0, x_0_1, x_1_1, ..., x_k_1, ..., x_k_P) # That is, all the Dats from rank 0, followed by those of # rank 1, ... # Hence the offset into the global Vec is the exclusive # prefix sum of the local size of the mixed dat. size = sum(d.size * d.cdim for d in self) offset = self.comm.exscan(size) if offset is None: offset = 0 scatters = [] for d in self: size = d.size * d.cdim vscat = PETSc.Scatter().createWithData( d.layout_vec, None, self.layout_vec, PETSc.IS().createStride(size, offset, 1, comm=d.comm)) offset += size scatters.append(vscat) return tuple(scatters)
def __init__(self, src_vec, tgt_vec, src_idxs, tgt_idxs, vec_conns, byobj_conns, mode): super(PetscDataTransfer, self).__init__(src_idxs, tgt_idxs, vec_conns, byobj_conns, mode) self.comm = comm = src_vec.comm uvec = src_vec.petsc_vec pvec = tgt_vec.petsc_vec name = src_vec.pathname if trace: debug("'%s': creating index sets for '%s' DataTransfer: %s %s" % (name, src_vec.pathname, src_idxs, tgt_idxs)) src_idx_set = PETSc.IS().createGeneral(src_idxs, comm=comm) tgt_idx_set = PETSc.IS().createGeneral(tgt_idxs, comm=comm) try: if trace: self.src_idxs = src_idxs self.tgt_idxs = tgt_idxs arrow = '-->' if mode == 'fwd' else '<--' debug( "'%s': new %s scatter (sizes: %d, %d)\n%s %s %s %s %s %s" % (name, mode, len(src_idx_set.indices), len(tgt_idx_set.indices), [v for u, v in vec_conns], arrow, [u for u, v in vec_conns ], src_idx_set.indices, arrow, tgt_idx_set.indices)) self.scatter = PETSc.Scatter().create(uvec, src_idx_set, pvec, tgt_idx_set) except Exception as err: raise RuntimeError( "ERROR in %s (src_idxs=%s, tgt_idxs=%s, usize=%d, psize=%d): %s" % (name, src_idxs, tgt_idxs, src_vec.vec.size, tgt_vec.vec.size, str(err)))
def solve_stability(self): """ Solve the 2nd-order stability problem """ def global_mask(fun1, fun2, V): # Find the indices where fun1 ~= fun2 at a tolerance diff = fun1.vector() - fun2.vector() diff.abs() diff_glob = PETScVector(mpi_comm_self()) diff.gather(diff_glob, pl.array(range(V.dim()), "intc")) mask = diff_glob < DOLFIN_EPS_LARGE return mask # Locate the elastic part mask = global_mask(self.alpha, self.alpha_prev, self.V_alpha) if pl.all(mask): self.print0("\033[1;36m 2nd stability: elastic phase\033[1;m") self.rq = pl.inf return True else: self._u_alpha_prev.vector()[:] = 0.0 self._u_alpha.vector()[:] = 1.0 assign(self._u_alpha_prev.sub(1), self.alpha_prev) assign(self._u_alpha.sub(1), self.alpha) mask = global_mask(self._u_alpha, self._u_alpha_prev, self._V_u_alpha) self.elas_dofs = set((pl.where(mask == True)[0]).astype(pl.intc)) bc_elas_dofs = self.elas_dofs.union(self.bc_dofs) indices = sorted( set(range(self.ownership[0], self.ownership[1])) - bc_elas_dofs) # Assemble K and M self._K = PETScMatrix() self._M = PETScMatrix() assemble(self._rqP, self._K) assemble(self._rqN, self._M) self._K_mat = self._K.mat() self._M_mat = self._M.mat() # Eliminate the elastic/BC part using PETSc.IS self.IS = PETSc.IS() self.IS.createGeneral(indices) self._K_mat_reduced = self._K_mat.getSubMatrix(self.IS, self.IS) self._K = PETScMatrix(self._K_mat_reduced) self._M_mat_reduced = self._M_mat.getSubMatrix(self.IS, self.IS) self._M = PETScMatrix(self._M_mat_reduced) # Stop if M ~= 0 if self._M.norm("linf") < DOLFIN_EPS_LARGE: self.rq = pl.inf self.print0( "\033[1;36m 2nd stability: Rayleigh quotient: %.3e\033[1;m" % self.rq) return True # Setup the eigenvalue solver self.eigensolver = SLEPcEigenSolver(self._K, self._M) self.set_eigensolver_parameters() # Use last known directions for initial guess assign(self._u_alpha.sub(0), self.V) assign(self._u_alpha.sub(1), self.Beta) _u_alpha_vec = as_backend_type(self._u_alpha.vector()).vec() _u_alpha_vec_reduced = _u_alpha_vec.getSubVector(self.IS) # self.eps.setInitialSpace(_u_alpha_vec_reduced) # Solve the eigenvalue problem self.print0( "\033[1;36m 2nd stability: solving the eigenvalue problem\033[1;m" ) self.eps.solve() r, c, rx, cx = self.eigensolver.get_eigenpair(0) self.print0("\033[1;36m 2nd stability: smallest ev: %.3e\033[1;m" % r) # From reduced vector to full vector self.scatter = PETSc.Scatter() rx_vec = as_backend_type(rx).vec() self.scatter.create(_u_alpha_vec_reduced, None, _u_alpha_vec, self.IS) _u_alpha_vec.zeroEntries() self.scatter.scatter(rx_vec, _u_alpha_vec) _u_alpha_vec.ghostUpdate() # Check the Rayleigh quotient (in theory we should have r == rq) self.rq = assemble(self.rqP) / assemble(self.rqN) if abs(r - self.rq) > DOLFIN_EPS_LARGE: self.print0( "\033[1;36m 2nd stability: Rayleigh quotient: %.3e\033[1;m" % self.rq) # Obtain the perturbation directions to V and Beta assign(self.V, self._u_alpha.sub(0)) assign(self.Beta, self._u_alpha.sub(1)) # Scale V u_mean = self.u.vector().norm("l2") if self.V.vector().norm("l2") > DOLFIN_EPS_LARGE: coeff = u_mean / self.V.vector().norm("l2") self.V.vector()[:] = coeff * self.V.vector() # Scale and project Beta to the admissible space alpha_mean = self.alpha.vector().norm("l2") if self.Beta.vector().norm("l2") > DOLFIN_EPS_LARGE: coeff = alpha_mean / self.Beta.vector().norm("l2") self.Beta.vector()[:] = coeff * self.Beta.vector() self.Beta.vector()[self.Beta.vector() < 0] = 0.0 # Determine if the solution is unique if self.rq > 1: return True
def setUp(self): v1, v2 = PETSc.Vec().createSeq(0), PETSc.Vec().createSeq(0) i1, i2 = PETSc.IS().createGeneral([]), PETSc.IS().createGeneral([]) self.obj = PETSc.Scatter().create(v1, i1, v2, i2) del v1, v2, i1, i2
def createGLVectors(self): self.SelectBlock() self.computeCP() self.larray = np.zeros((self.numBlockWBandAssigned,)+\ (self.m+self.StencilWidth*2,)*self.Dim,order='F') self.lvec = PETSc.Vec().createWithArray(self.larray,comm=self.comm) lsize = self.numBlockWBandAssigned*self.m**self.Dim self.gvec = PETSc.Vec().createMPI((lsize,PETSc.DECIDE)) self.gvec.setUp() self.wvec = self.gvec.copy() # self.createIndicesHelper() tind = np.arange((self.m+self.StencilWidth*2)**self.Dim) tind = tind.reshape((self.m+self.StencilWidth*2,)*self.Dim,order='F') for dim in xrange(self.Dim): tind = np.delete(tind,0,dim) tind = np.delete(tind,np.s_[-1],dim) tind = tind.flatten(order='F') # ISList = [] # c = (self.m+self.StencilWidth*2)**self.Dim # for i in xrange(self.BlockWBandStart,self.BlockWBandEnd): # ti = i*c # ISList.extend(list(tind+ti)) tind = np.tile(tind,self.numBlockWBandAssigned) ttind = np.arange(self.BlockWBandStart,self.BlockWBandEnd) tt = (self.m+2*self.StencilWidth)**self.Dim ttind *= tt ttind = np.repeat(ttind, self.m**self.Dim) ttind = tind + ttind ISFrom = PETSc.IS().createGeneral(ttind,comm=self.comm) self.l2g = PETSc.Scatter().create(self.lvec,ISFrom,self.gvec,None) #generate scatter global2local tind = np.arange(tt) tind = self.Ind2Sub(tind,(self.m+2*self.StencilWidth,)*self.Dim) tind -= self.StencilWidth tind = np.tile(tind,(self.numBlockWBandAssigned,1)) ttind = self.ni2pi.petsc2app(np.arange(self.BlockWBandStart,self.BlockWBandEnd)) ttind = self.BlockInd2SubWithoutBand(ttind) ttind = np.repeat(ttind,tt,axis=0) ttind += tind/self.m tind = np.mod(tind,self.m) tind = self.Sub2Ind(tind, (self.m,)*self.Dim) ttind = self.BlockSub2IndWithoutBand(ttind) ttind = self.ni2pi.app2petsc(ttind) ttind *= self.m**self.Dim tind += ttind (ind,) = np.where(tind>=0) tind = tind[ind] ISTo = ind+self.BlockWBandStart*tt ISFrom = PETSc.IS().createGeneral(tind) ISTo = PETSc.IS().createGeneral(ISTo) self.g2l = PETSc.Scatter().create(self.gvec,ISFrom,self.lvec,ISTo) return self.larray,self.lvec,self.gvec,self.wvec
def SelectBlock(self,surface = None): if surface is None: surface = self.surface comm = self.comm numTotalBlock = self.M**self.Dim numBlockAssigned = numTotalBlock // comm.size + int(comm.rank < (numTotalBlock % comm.size)) Blockstart = comm.exscan(numBlockAssigned) if comm.rank == 0:Blockstart = 0 indBlock = np.arange(Blockstart,Blockstart+numBlockAssigned) subBlock = self.BlockInd2SubWithoutBand(indBlock) BlockCenterCar = self.BlockSub2CenterCarWithoutBand(subBlock) cp,_,_,_ = surface.cp(BlockCenterCar) dBlockCenter = self.norm1(cp-BlockCenterCar) p = self.interpDegree if p % 2 == 1: p = ( p + 1 ) / 2 else: p = ( p + 2 ) / 2 bw = 1.1*((p+2)*self.hGrid+self.hBlock/2)#*np.sqrt(self.Dim) (lindBlockWithinBand,) = np.where(dBlockCenter<bw) lindBlockWithinBand = lindBlockWithinBand+Blockstart lBlockSize = lindBlockWithinBand.size numTotalBlockWBand = comm.allreduce(lBlockSize) numBlockWBandAssigned = numTotalBlockWBand // comm.size + int(comm.rank < (numTotalBlockWBand % comm.size)) lindBlockWBandFrom = PETSc.Vec().createWithArray(lindBlockWithinBand,comm=comm) self.gindBlockWBand = PETSc.Vec().createMPI((numBlockWBandAssigned,PETSc.DECIDE),comm=comm) # gsubBlockWBandFrom = PETSc.Vec().createMPI((self.Dim*lBlockize,PETSc.DECIDE),comm=comm) # gsubBlockWBandFrom.setArray(lsubBlockWBand) # self.gsubBlockWBand = PETSc.Vec().createMPI((self.Dim*self.numBlockWBandAssigned,PETSc.DECIDE),comm=comm) BlockWBandStart = comm.exscan(numBlockWBandAssigned) if comm.rank == 0: BlockWBandStart = 0 self.BlockWBandStart = BlockWBandStart LInd = PETSc.IS().createStride(numBlockWBandAssigned,\ first=BlockWBandStart,\ step=1,comm=comm) self.numTotalBlockWBand = numTotalBlockWBand self.numBlockWBandAssigned = numBlockWBandAssigned BlockWBandEnd = BlockWBandStart + numBlockWBandAssigned self.BlockWBandEnd = BlockWBandEnd scatter = PETSc.Scatter().create(lindBlockWBandFrom,LInd,self.gindBlockWBand,None) scatter.scatter(lindBlockWBandFrom,self.gindBlockWBand,PETSc.InsertMode.INSERT) #Natural order Index To Petsc order Index self.ni2pi = PETSc.AO().createMapping(self.gindBlockWBand.getArray().astype(np.int64))
def _init_approximations(self, system): """ Prepare for later approximations. Parameters ---------- system : System The system having its derivs approximated. """ total = system.pathname == '' abs2meta = system._var_allprocs_abs2meta in_slices = system._inputs.get_slice_dict() out_slices = system._outputs.get_slice_dict() approx_wrt_idx = system._owns_approx_wrt_idx coloring = system._get_static_coloring() self._approx_groups = [] self._nruns_uncolored = 0 if self._during_sparsity_comp: wrt_matches = system._coloring_info['wrt_matches'] else: wrt_matches = None for wrt, start, end, vec, _, _ in system._jac_wrt_iter(wrt_matches): if wrt in self._wrt_meta: meta = self._wrt_meta[wrt] if coloring is not None and 'coloring' in meta: continue if vec is system._inputs: slices = in_slices else: slices = out_slices data = self._get_approx_data(system, wrt, meta) directional = meta['directional'] in_idx = range(start, end) if wrt in approx_wrt_idx: if vec is None: vec_idx = None else: # local index into var vec_idx = approx_wrt_idx[wrt].shaped_array(copy=True) # convert into index into input or output vector vec_idx += slices[wrt].start # Directional derivatives for quick partial checking. # Place the indices in a list so that they are all stepped at the same time. if directional: in_idx = [list(in_idx)] vec_idx = [vec_idx] else: if vec is None: # remote wrt if wrt in abs2meta['input']: vec_idx = range( abs2meta['input'][wrt]['global_size']) else: vec_idx = range( abs2meta['output'][wrt]['global_size']) else: vec_idx = LocalRangeIterable(system, wrt) if directional: vec_idx = [v for v in vec_idx if v is not None] # Directional derivatives for quick partial checking. # Place the indices in a list so that they are all stepped at the same time. if directional: in_idx = [list(in_idx)] vec_idx = [list(vec_idx)] if directional: self._nruns_uncolored += 1 else: self._nruns_uncolored += end - start self._approx_groups.append((wrt, data, in_idx, vec, vec_idx, directional, meta['vector'])) if total: # compute scatter from the results vector into a column of the total jacobian sinds, tinds, colsize, has_dist_data = system._get_jac_col_scatter( ) if has_dist_data: src_vec = PETSc.Vec().createWithArray(np.zeros(len( system._outputs), dtype=float), comm=system.comm) tgt_vec = PETSc.Vec().createWithArray(np.zeros(colsize, dtype=float), comm=system.comm) src_inds = PETSc.IS().createGeneral(sinds, comm=system.comm) tgt_inds = PETSc.IS().createGeneral(tinds, comm=system.comm) self._jac_scatter = (has_dist_data, PETSc.Scatter().create( src_vec, src_inds, tgt_vec, tgt_inds), src_vec, tgt_vec) else: self._jac_scatter = (has_dist_data, sinds, tinds) else: self._jac_scatter = None
def __init__(self, A_IS): """ Initialize the domain decomposition preconditioner, multipreconditioner and coarse space with its operators Parameters ========== A_IS : petsc.Mat The matrix of the problem in IS format. A must be a symmetric positive definite matrix with symmetric positive semi-definite submatrices PETSc.Options ============= PCBNN_switchtoASM :Bool Default is False If True then the domain decomposition preconditioner is the BNN preconditioner. If false then the domain decomposition precondition is the Additive Schwarz preconditioner with minimal overlap. PCBNN_kscaling : Bool Default is True. If true then kscaling (partition of unity that is proportional to the diagonal of the submatrices of A) is used when a partition of unity is required. Otherwise multiplicity scaling is used when a partition of unity is required. This may occur in two occasions: - to scale the local BNN matrices if PCBNN_switchtoASM=True, - in the GenEO eigenvalue problem for eigmin if PCBNN_switchtoASM=False and PCBNN_GenEO=True with PCBNN_GenEO_eigmin > 0 (see projection.__init__ for the meaning of these options). PCBNN_verbose : Bool Default is False. If True, some information about the preconditioners is printed when the code is executed. PCBNN_GenEO : Bool Default is False. If True then the coarse space is enriched by solving local generalized eigenvalue problems. PCBNN_CoarseProjection :True Default is True If False then there is no coarse projection: Two level Additive Schwarz or One-level preconditioner depending on PCBNN_addCoarseSolve If True, the coarse projection is applied: Projected preconditioner of hybrid preconditioner depending on PCBNN_addCoarseSolve PCBNN_addCoarseSolve : False Default is True If True then (R0t A0\R0 r) is added to the preconditioned residual False corresponds to the projected preconditioner (need to choose initial guess accordingly) (or the one level preconditioner if PCBNN_CoarseProjection = False) True corresponds to the hybrid preconditioner (or the fully additive preconditioner if PCBNN_CoarseProjection = False) """ OptDB = PETSc.Options() self.switchtoASM = OptDB.getBool('PCBNN_switchtoASM', False) #use Additive Schwarz as a preconditioner instead of BNN self.kscaling = OptDB.getBool('PCBNN_kscaling', True) #kscaling if true, multiplicity scaling if false self.verbose = OptDB.getBool('PCBNN_verbose', False) self.GenEO = OptDB.getBool('PCBNN_GenEO', True) self.addCS = OptDB.getBool('PCBNN_addCoarseSolve', False) self.projCS = OptDB.getBool('PCBNN_CoarseProjection', True) #extract Neumann matrix from A in IS format Ms = A_IS.copy().getISLocalMat() # convert A_IS from matis to mpiaij A_mpiaij = A_IS.convertISToAIJ() r, _ = A_mpiaij.getLGMap() #r, _ = A_IS.getLGMap() is_A = PETSc.IS().createGeneral(r.indices) # extract exact local solver As = A_mpiaij.createSubMatrices(is_A)[0] vglobal, _ = A_mpiaij.getVecs() vlocal, _ = Ms.getVecs() scatter_l2g = PETSc.Scatter().create(vlocal, None, vglobal, is_A) #compute the multiplicity of each degree vlocal.set(1.) vglobal.set(0.) scatter_l2g(vlocal, vglobal, PETSc.InsertMode.ADD_VALUES) scatter_l2g(vglobal, vlocal, PETSc.InsertMode.INSERT_VALUES, PETSc.ScatterMode.SCATTER_REVERSE) NULL,mult_max = vglobal.max() # k-scaling or multiplicity scaling of the local (non-assembled) matrix if self.kscaling == False: Ms.diagonalScale(vlocal,vlocal) else: v1 = As.getDiagonal() v2 = Ms.getDiagonal() Ms.diagonalScale(v1/v2, v1/v2) # the default local solver is the scaled non assembled local matrix (as in BNN) if self.switchtoASM: Atildes = As if mpi.COMM_WORLD.rank == 0: print('The user has chosen to switch to Additive Schwarz instead of BNN.') else: #(default) Atildes = Ms ksp_Atildes = PETSc.KSP().create(comm=PETSc.COMM_SELF) ksp_Atildes.setOptionsPrefix("ksp_Atildes_") ksp_Atildes.setOperators(Atildes) ksp_Atildes.setType('preonly') pc_Atildes = ksp_Atildes.getPC() pc_Atildes.setType('cholesky') pc_Atildes.setFactorSolverType('mumps') ksp_Atildes.setFromOptions() self.A = A_mpiaij self.Ms = Ms self.As = As self.ksp_Atildes = ksp_Atildes self.works_1 = vlocal.copy() self.works_2 = self.works_1.copy() self.scatter_l2g = scatter_l2g self.mult_max = mult_max self.minV0 = minimal_V0(self.ksp_Atildes) if self.GenEO == True: GenEOV0 = GenEO_V0(self.ksp_Atildes,self.Ms,self.As,self.mult_max,self.minV0.V0s) self.V0s = GenEOV0.V0s else: self.V0s = self.minV0.V0s self.proj = coarse_operators(self.V0s,self.A,self.scatter_l2g,vlocal)
def calc_gradient(self, sim, params): """Calculate the gradient of a figure of merit which depends on the permittivity and permeability. Parameters ---------- sim : FDFD Simulation object. sim = self.sim params : numpy.array or list of floats List of design parameters. Returns ------- numpy.array **(Master node only)** Gradient of figure of merit, i.e. list of derivatives of fom with respect to each design variable """ # Semantically, we would not normally need to override this method, # however it turns out that the operations needed to compute the # gradient of a field-dependent function and a permittivit-dependent # function are very similar (both require the calculation of the # derivative of the materials wrt the design parameters.) For the sake # of performance, we combine the two calculations here. w_pml_l = sim.w_pml_left w_pml_r = sim.w_pml_right w_pml_t = sim.w_pml_top w_pml_b = sim.w_pml_bottom M = sim.M N = sim.N # get the current diagonal elements of A. # only these elements change when the design variables change. Ai = PETSc.Vec() Af = PETSc.Vec() Ai = sim.get_A_diag(Ai) # Get the derivatives w.r.t. eps, mu if (NOT_PARALLEL): dFdeps, dFdeps_conj, dFdmu, dFdmu_conj = self.calc_dFdm( sim, params) x = sim.x.copy() x_adj = sim.x_adj.copy() step = self._step update_boxes = self.get_update_boxes(sim, params) lenp = len(params) grad_full = None if (RANK == 0): grad_full = np.zeros(sim.nunks, dtype=np.double) gradient = np.zeros(lenp) for i in xrange(lenp): #if(NOT_PARALLEL): # print i p0 = params[i] ub = update_boxes[i] # perturb the system params[i] += step self.update_system(params) self.sim.update(ub) # get the updated diagonal elements of A Af = sim.get_A_diag(Af) # calculate dAdp and assemble the full result on the master node dAdp = (Af - Ai) / step product = x_adj * dAdp * x grad_part = -2 * np.real(np.sum(product[...])) # send the partially computed gradient to the master node to finish # up the calculation MPI.COMM_WORLD.Gatherv(grad_part, grad_full, root=0) # We also need dAdp to account for the derivative of eps and mu gatherer, dAdp_full = PETSc.Scatter().toZero(dAdp) gatherer.scatter(dAdp, dAdp_full, False, PETSc.Scatter.Mode.FORWARD) # finish calculating the gradient if (NOT_PARALLEL): # derivative with respect to fields gradient[i] = np.sum(grad_full) # Next we compute the derivative with respect to eps and mu. We # exclude the PML regions because changes to the materials in # the PMLs are generally not something we want to consider. jmin = ub[0] jmax = ub[1] imin = ub[2] imax = ub[3] if (jmin < w_pml_l): jmin = w_pml_l if (jmax > N - w_pml_r): jmax = N - w_pml_r if (imin < w_pml_b): imin = w_pml_b if (imax > M - w_pml_t): imax = M - w_pml_t # note that the extraction of eps and mu from A must be handled # slightly differently in the TE and TM cases since the signs # along the diagonal are swapped and eps and mu are positioned # in different parts if (isinstance(sim, fdfd.FDFD_TM)): dmudp = dAdp_full[0:M * N].reshape([M, N]) * 1j depsdp = dAdp_full[M * N:2 * M * N].reshape([M, N]) / 1j elif (isinstance(sim, fdfd.FDFD_TE)): depsdp = dAdp_full[0:M * N].reshape([M, N]) / 1j dmudp = dAdp_full[M * N:2 * M * N].reshape([M, N]) * 1j gradient[i] += np.real( np.sum(dFdeps[imin:imax, jmin:jmax] * \ depsdp[imin:imax, jmin:jmax]) + \ np.sum(dFdeps_conj[imin:imax, jmin:jmax] * \ np.conj(depsdp[imin:imax, jmin:jmax])) + \ np.sum(dFdmu[imin:imax, jmin:jmax] * \ dmudp[imin:imax, jmin:jmax]) + \ np.sum(dFdmu_conj[imin:imax, jmin:jmax] * \ np.conj(dmudp[imin:imax, jmin:jmax])) \ ) # revert the system to its original state params[i] = p0 self.update_system(params) self.sim.update(ub) if (NOT_PARALLEL): return gradient
def __init__(self, A): """ Initialize the domain decomposition preconditioner, multipreconditioner and coarse space with its operators Parameters ========== A : petsc.Mat The matrix of the problem in IS format. A must be a symmetric positive definite matrix with symmetric positive semi-definite submatrices PETSc.Options ============= PCBNN_switchtoASM :Bool Default is False If True then the domain decomposition preconditioner is the BNN preconditioner. If false then the domain decomposition precondition is the Additive Schwarz preconditioner with minimal overlap. PCBNN_kscaling : Bool Default is True. If true then kscaling (partition of unity that is proportional to the diagonal of the submatrices of A) is used when a partition of unity is required. Otherwise multiplicity scaling is used when a partition of unity is required. This may occur in two occasions: - to scale the local BNN matrices if PCBNN_switchtoASM=True, - in the GenEO eigenvalue problem for eigmin if PCBNN_switchtoASM=False and PCBNN_GenEO=True with PCBNN_GenEO_eigmin > 0 (see projection.__init__ for the meaning of these options). PCBNN_verbose : Bool Default is False. If True, some information about the preconditioners is printed when the code is executed. """ OptDB = PETSc.Options() self.switchtoASM = OptDB.getBool('PCBNN_switchtoASM', False) #use Additive Schwarz as a preconditioner instead of BNN self.kscaling = OptDB.getBool('PCBNN_kscaling', True) #kscaling if true, multiplicity scaling if false self.verbose = OptDB.getBool('PCBNN_verbose', False) # convert matis to mpiaij, extract local matrices r, _ = A.getLGMap() is_A = PETSc.IS().createGeneral(r.indices) A_mpiaij = A.convertISToAIJ() A_mpiaij_local = A_mpiaij.createSubMatrices(is_A)[0] A_scaled = A.copy().getISLocalMat() vglobal, _ = A.getVecs() vlocal, _ = A_scaled.getVecs() scatter_l2g = PETSc.Scatter().create(vlocal, None, vglobal, is_A) #compute the multiplicity of each degree of freedom and max of the multiplicity vlocal.set(1.) vglobal.set(0.) scatter_l2g(vlocal, vglobal, PETSc.InsertMode.ADD_VALUES) scatter_l2g(vglobal, vlocal, PETSc.InsertMode.INSERT_VALUES, PETSc.ScatterMode.SCATTER_REVERSE) NULL,mult_max = vglobal.max() # k-scaling or multiplicity scaling of the local (non-assembled) matrix if self.kscaling == False: A_scaled.diagonalScale(vlocal,vlocal) else: v1 = A_mpiaij_local.getDiagonal() v2 = A_scaled.getDiagonal() A_scaled.diagonalScale(v1/v2, v1/v2) # the default local solver is the scaled non assembled local matrix (as in BNN) if self.switchtoASM: Alocal = A_mpiaij_local if mpi.COMM_WORLD.rank == 0: print('The user has chosen to switch to Additive Schwarz instead of BNN.') else: #(default) Alocal = A_scaled localksp = PETSc.KSP().create(comm=PETSc.COMM_SELF) localksp.setOptionsPrefix("localksp_") localksp.setOperators(Alocal) localksp.setType('preonly') localpc = localksp.getPC() localpc.setType('cholesky') localpc.setFactorSolverType('mumps') localksp.setFromOptions() self.A = A self.A_scaled = A_scaled self.A_mpiaij_local = A_mpiaij_local self.localksp = localksp self.workl_1 = vlocal.copy() self.workl_2 = self.workl_1.copy() self.scatter_l2g = scatter_l2g self.mult_max = mult_max self.proj = projection(self)
def solve(self, x0=None, atol=None, rtol=None, max_it=None): r""" This method solves the sparse linear system, converts the solution vector from a PETSc.Vec instance to a numpy array, and finally destroys all the petsc objects to free memory. Parameters ---------- solver_type : string, optional Default is the iterative solver 'cg' based on the Conjugate Gradient method. preconditioner_type : string, optional Default is the 'jacobi' preconditioner, i.e., diagonal scaling preconditioning. The preconditioner is used with iterative solvers. When a direct solver is used, this parameter is ignored. factorization_type : string, optional The factorization type used with the direct solver. Default is 'lu'. This parameter is ignored when an iterative solver is used. Returns ------- Returns a numpy array corresponding to the solution of the linear sparse system Ax = b. Notes ----- Certain combinations of iterative solvers and precondioners or direct solvers and factorization types are not supported. The summary table of the different possibilities can be found here: https://www.mcs.anl.gov/petsc/documentation/linearsolvertable.html """ self.x0 = np.zeros_like(self.b) if x0 is None else x0 self._initialize_b_x() self._initialize_A() self._create_solver() self._set_tolerances(atol=atol, rtol=rtol, max_it=max_it) self.ksp.setOperators(self.petsc_A) self.ksp.setFromOptions() # Solve the linear system self.ksp.solve(self.petsc_b, self.petsc_x) # Gather the solution to all processors gather_to_0, self.petsc_s = PETSc.Scatter().toAll(self.petsc_x) gather_to_0.scatter(self.petsc_x, self.petsc_s, PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD) # Convert solution vector from PETSc.Vec instance to a numpy array self.solution = PETSc.Vec.getArray(self.petsc_s) # Destroy petsc solver, coefficients matrix, rhs, and solution vectors PETSc.KSP.destroy(self.ksp) PETSc.Mat.destroy(self.petsc_A) PETSc.Vec.destroy(self.petsc_b) PETSc.Vec.destroy(self.petsc_x) PETSc.Vec.destroy(self.petsc_s) return self.solution