def setupPETScmatrix(Vr, Vc, mattype, mpicomm): """ Set up a PETSc matrix partitioned consistently with Fenics mesh Vr, Vc = function spaces for the rows and columns """ # extract local to global map for each fct space VrDM, VcDM = Vr.dofmap(), Vc.dofmap() r_map = PETSc.LGMap().create(VrDM.dofs(), comm=mpicomm) c_map = PETSc.LGMap().create(VcDM.dofs(), comm=mpicomm) # set up matrix petscmatrix = PETSc.Mat() petscmatrix.create(mpicomm) try: localdimVr = VrDM.local_dimension("owned") localdimVc = VcDM.local_dimension("owned") except: VrDMim = VrDM.index_map() localdimVr = VrDMim.size(VrDMim.MapSize_OWNED) VcDMim = VcDM.index_map() localdimVc = VcDMim.size(VcDMim.MapSize_OWNED) petscmatrix.setSizes([ [localdimVr, Vr.dim()], \ [localdimVc, Vc.dim()] ]) petscmatrix.setType(mattype) # 'aij', 'dense' petscmatrix.setUp() petscmatrix.setLGMap(r_map, c_map) # compare PETSc and Fenics local partitions: Istart, Iend = petscmatrix.getOwnershipRange() assert list(VrDM.dofs()) == range(Istart, Iend) return petscmatrix, VrDM, VcDM
def petsc_serial_matrix(test_space, trial_space, nnz=None): ''' PETsc.Mat from trial_space to test_space to be filled in the with block. The spaces can be represented by intergers meaning generic R^n. ''' # Decide local to global map # For our custom case everything is serial if is_number(test_space) and is_number(trial_space): comm = mpi_comm_world().tompi4py() # Local same as global sizes = [[test_space, test_space], [trial_space, trial_space]] row_map = PETSc.IS().createStride(test_space, 0, 1, comm) col_map = PETSc.IS().createStride(trial_space, 0, 1, comm) # With function space this can be extracted else: mesh = test_space.mesh() comm = mesh.mpi_comm().tompi4py() row_map = test_space.dofmap() col_map = trial_space.dofmap() sizes = [[ row_map.index_map().size(IndexMap.MapSize_OWNED), row_map.index_map().size(IndexMap.MapSize_GLOBAL) ], [ col_map.index_map().size(IndexMap.MapSize_OWNED), col_map.index_map().size(IndexMap.MapSize_GLOBAL) ]] row_map = map(int, row_map.tabulate_local_to_global_dofs()) col_map = map(int, col_map.tabulate_local_to_global_dofs()) assert comm.size == 1 lgmap = lambda indices: (PETSc.LGMap().create(indices, comm=comm) if isinstance(indices, list) else PETSc.LGMap(). createIS(indices)) row_lgmap, col_lgmap = map(lgmap, (row_map, col_map)) # Alloc mat = PETSc.Mat().createAIJ(sizes, nnz=nnz, comm=comm) mat.setUp() mat.setLGMap(row_lgmap, col_lgmap) mat.assemblyBegin() # Fill yield mat # Tear down mat.assemblyEnd()
def petsc_identity(N, dofs=None): """ Create an identity matrix using petsc4py. Note: this currently only works in one process. """ from petsc4py import PETSc v = PETSc.Vec() v.create() v.setSizes(N) v.setType('standard') v.setValues(range(N), [1.0] * N) A = PETSc.Mat() A.createAIJ([N, N], nnz=N) if dofs is not None: lgmap = PETSc.LGMap().create(dofs) A.setLGMap(lgmap, lgmap) A.setDiagonal(v) A.assemble() return dlf.PETScMatrix(A)
def identity_matrix(V, d=1.): '''Diagonal matrix''' # Avoiding assembly if this is Real space. diag = Function(V).vector() global_size = diag.size() local_size = diag.local_size() comm = V.mesh().mpi_comm().tompi4py() mat = PETSc.Mat().createAIJ(size=[[local_size, global_size], [local_size, global_size]], nnz=1, comm=comm) diag = as_backend_type(diag).vec() diag.set(d) mat.setDiagonal(diag) lgmap = PETSc.LGMap().create(map( int, V.dofmap().tabulate_local_to_global_dofs()), comm=comm) mat.setLGMap(lgmap, lgmap) mat.assemblyBegin() mat.assemblyEnd() return PETScMatrix(mat)
def lgmap(self): """A PETSc LGMap mapping process-local indices to global indices for this :class:`DataSet`. """ lgmap = PETSc.LGMap() lgmap.create(indices=np.arange(1, dtype=IntType), bsize=self.cdim, comm=self.comm) return lgmap
def unblocked_lgmap(self): """A PETSc LGMap mapping process-local indices to global indices for this :class:`DataSet` with a block size of 1. """ indices = self.lgmap.indices lgmap = PETSc.LGMap().create(indices=indices, bsize=1, comm=self.lgmap.comm) return lgmap
def masked_lgmap(lgmap, mask, block=True): if block: indices = lgmap.block_indices.copy() bsize = lgmap.getBlockSize() else: indices = lgmap.indices.copy() bsize = 1 indices[mask] = -1 return PETSc.LGMap().create(indices=indices, bsize=bsize, comm=lgmap.comm)
def __init__(self, ghosted_csr_mat, par_bs, par_n, par_N, par_nghost, subdomain2global, blockVecType="simple", pde=None): self.pde = pde p4pyPETSc.Mat.__init__(self) self.ghosted_csr_mat = ghosted_csr_mat self.blockVecType = blockVecType assert self.blockVecType == "simple", "petsc4py wrappers require self.blockVecType=simple" self.create(p4pyPETSc.COMM_WORLD) blockSize = max(1, par_bs) if blockSize >= 1 and blockVecType != "simple": ## \todo fix block aij in ParMat_petsc4py self.setType('baij') self.setSizes([[blockSize * par_n, blockSize * par_N], [blockSize * par_n, blockSize * par_N]], bsize=blockSize) self.setBlockSize(blockSize) self.subdomain2global = subdomain2global #no need to include extra block dofs? else: self.setType('aij') self.setSizes([[par_n * blockSize, par_N * blockSize], [par_n * blockSize, par_N * blockSize]], bsize=1) if blockSize > 1: #have to build in block dofs subdomain2globalTotal = numpy.zeros( (blockSize * subdomain2global.shape[0], ), 'i') for j in range(blockSize): subdomain2globalTotal[ j::blockSize] = subdomain2global * blockSize + j self.subdomain2global = subdomain2globalTotal else: self.subdomain2global = subdomain2global import Comm comm = Comm.get() logEvent( "ParMat_petsc4py comm.rank= %s blockSize = %s par_n= %s par_N=%s par_nghost=%s par_jacobian.getSizes()= %s " % (comm.rank(), blockSize, par_n, par_N, par_nghost, self.getSizes())) self.csr_rep = ghosted_csr_mat.getCSRrepresentation() blockOwned = blockSize * par_n self.csr_rep_owned = ghosted_csr_mat.getSubMatCSRrepresentation( 0, blockOwned) self.petsc_l2g = p4pyPETSc.LGMap() self.petsc_l2g.create(self.subdomain2global) self.colind_global = self.petsc_l2g.apply( self.csr_rep_owned[1]) #prealloc needs global indices self.setPreallocationCSR( [self.csr_rep_owned[0], self.colind_global, self.csr_rep_owned[2]]) self.setUp() self.setLGMap(self.petsc_l2g) self.setFromOptions()
def petsc_serial_matrix(test_space, trial_space, nnz=None): ''' PETsc.Mat from trial_space to test_space to be filled in the with block. The spaces can be represented by intergers meaning generic R^n. ''' mesh = test_space.mesh() comm = mesh.mpi_comm() assert comm.size == 1 row_map = test_space.dofmap() col_map = trial_space.dofmap() sizes = [[ row_map.index_map().size(df.IndexMap.MapSize.OWNED), row_map.index_map().size(df.IndexMap.MapSize.GLOBAL) ], [ col_map.index_map().size(df.IndexMap.MapSize.OWNED), col_map.index_map().size(df.IndexMap.MapSize.GLOBAL) ]] row_map = list(map(int, row_map.tabulate_local_to_global_dofs())) col_map = list(map(int, col_map.tabulate_local_to_global_dofs())) lgmap = lambda indices: (PETSc.LGMap().create(indices, comm=comm) if isinstance(indices, list) else PETSc.LGMap(). createIS(indices)) row_lgmap, col_lgmap = list(map(lgmap, (row_map, col_map))) # Alloc mat = PETSc.Mat().createAIJ(sizes, nnz=nnz, comm=comm) mat.setUp() mat.setLGMap(row_lgmap, col_lgmap) mat.assemblyBegin() # Fill yield mat # Tear down mat.assemblyEnd()
def lgmap(self): """A PETSc LGMap mapping process-local indices to global indices for this :class:`DataSet`. """ lgmap = PETSc.LGMap() if self.comm.size == 1: lgmap.create(indices=np.arange(self.size, dtype=IntType), bsize=self.cdim, comm=self.comm) else: lgmap.create(indices=self.halo.local_to_global_numbering, bsize=self.cdim, comm=self.comm) return lgmap
def __init__(self,ghosted_csr_mat=None,par_bs=None,par_n=None,par_N=None,par_nghost=None,subdomain2global=None,blockVecType="simple",pde=None, par_nc=None, par_Nc=None, proteus_jacobian=None, nzval_proteus2petsc=None): p4pyPETSc.Mat.__init__(self) if ghosted_csr_mat is None: return#when duplicating for petsc usage self.pde = pde if par_nc is None: par_nc = par_n if par_Nc is None: par_Nc = par_N self.proteus_jacobian=proteus_jacobian self.nzval_proteus2petsc = nzval_proteus2petsc self.ghosted_csr_mat=ghosted_csr_mat self.blockVecType = blockVecType assert self.blockVecType == "simple", "petsc4py wrappers require self.blockVecType=simple" self.create(p4pyPETSc.COMM_WORLD) self.blockSize = max(1,par_bs) if self.blockSize > 1 and blockVecType != "simple": ## \todo fix block aij in ParMat_petsc4py self.setType('baij') self.setSizes([[self.blockSize*par_n,self.blockSize*par_N],[self.blockSize*par_nc,self.blockSize*par_Nc]],bsize=self.blockSize) self.setBlockSize(self.blockSize) self.subdomain2global = subdomain2global #no need to include extra block dofs? else: self.setType('aij') self.setSizes([[par_n*self.blockSize,par_N*self.blockSize],[par_nc*self.blockSize,par_Nc*self.blockSize]],bsize=1) if self.blockSize > 1: #have to build in block dofs subdomain2globalTotal = numpy.zeros((self.blockSize*subdomain2global.shape[0],),'i') for j in range(self.blockSize): subdomain2globalTotal[j::self.blockSize]=subdomain2global*self.blockSize+j self.subdomain2global=subdomain2globalTotal else: self.subdomain2global=subdomain2global from proteus import Comm comm = Comm.get() logEvent("ParMat_petsc4py comm.rank= %s blockSize = %s par_n= %s par_N=%s par_nghost=%s par_jacobian.getSizes()= %s " % (comm.rank(),self.blockSize,par_n,par_N,par_nghost,self.getSizes())) self.csr_rep = ghosted_csr_mat.getCSRrepresentation() if self.proteus_jacobian is not None: self.proteus_csr_rep = self.proteus_jacobian.getCSRrepresentation() if self.blockSize > 1: blockOwned = self.blockSize*par_n self.csr_rep_local = ghosted_csr_mat.getSubMatCSRrepresentation(0,blockOwned) else: self.csr_rep_local = ghosted_csr_mat.getSubMatCSRrepresentation(0,par_n) self.petsc_l2g = p4pyPETSc.LGMap() self.petsc_l2g.create(self.subdomain2global) self.setUp() self.setLGMap(self.petsc_l2g) # self.colind_global = self.petsc_l2g.apply(self.csr_rep_local[1]) #prealloc needs global indices self.setPreallocationCSR([self.csr_rep_local[0],self.colind_global,self.csr_rep_local[2]]) self.setFromOptions()
def assemble_mtx_to_petsc(pmtx, mtx, pdofs, drange, is_overlap=True, comm=None, verbose=False): """ Assemble a local CSR matrix to a global PETSc matrix. WIP --- Try Mat.setValuesCSR() - no lgmap - filtering vectorized? """ if comm is None: comm = PETSc.COMM_WORLD lgmap = PETSc.LGMap().create(pdofs, comm=comm) if is_overlap: pmtx.setLGMap(lgmap, lgmap) data, prows, cols = mtx.data, mtx.indptr, mtx.indices output('setting matrix values...', verbose=verbose) tt = time.clock() for ir, rdof in enumerate(pdofs): if (rdof < drange[0]) or (rdof >= drange[1]): continue for ic in range(prows[ir], prows[ir + 1]): # output(ir, rdof, cols[ic]) pmtx.setValueLocal(ir, cols[ic], data[ic], PETSc.InsertMode.INSERT_VALUES) output('...done in', time.clock() - tt, verbose=verbose) output('assembling matrix...', verbose=verbose) tt = time.clock() pmtx.assemble() output('...done in', time.clock() - tt, verbose=verbose) else: pmtx.setLGMap(lgmap, lgmap) output('setting matrix values...', verbose=verbose) tt = time.clock() pmtx.setValuesLocalCSR(mtx.indptr, mtx.indices, mtx.data, PETSc.InsertMode.ADD_VALUES) output('...done in', time.clock() - tt, verbose=verbose) output('assembling matrix...', verbose=verbose) tt = time.clock() pmtx.assemble() output('...done in', time.clock() - tt, verbose=verbose)
def set_lg_map(mat): '''Set local-to-global-map on the matrix''' # NOTE; serial only - so we own everything but still sometimes we need # to tell that to petsc (especiialy when bcs are to be applied) if is_number(mat): return mat assert is_petsc_mat(mat) or isinstance(mat, block_mat), (type(mat)) if isinstance(mat, block_mat): blocks = np.array(map(set_lg_map, mat.blocks.flatten())).reshape(mat.blocks.shape) return block_mat(blocks) comm = mpi_comm_world().tompi4py() # Work with matrix rowmap, colmap = range(mat.size(0)), range(mat.size(1)) row_lgmap = PETSc.LGMap().create(rowmap, comm=comm) col_lgmap = PETSc.LGMap().create(colmap, comm=comm) as_petsc(mat).setLGMap(row_lgmap, col_lgmap) return mat
def setUp(self): comm = PETSc.COMM_WORLD comm_size = comm.getSize() comm_rank = comm.getRank() lsize = 10 first = lsize * comm_rank last = first + lsize if comm_rank > 0: first -= 1 if comm_rank < (comm_size - 1): last += 1 self.idx = list(range(first, last)) bs = self.BS self.lgmap = PETSc.LGMap().create(self.idx, bs, comm=PETSc.COMM_WORLD)
def __init__(self, comm, N): self.comm = comm self.N = N # global problem size self.h = 1 / N # grid spacing on unit interval self.n = N // comm.size + int( comm.rank < (N % comm.size)) # owned part of global problem self.start = comm.exscan(self.n) if comm.rank == 0: self.start = 0 gindices = numpy.arange(self.start - 1, self.start + self.n + 1, dtype=PETSc.IntType) % N # periodic self.mat = PETSc.Mat().create(comm=comm) size = (self.n, self.N) # local and global sizes self.mat.setSizes((size, size)) self.mat.setFromOptions() self.mat.setPreallocationNNZ( (3, 1) ) # Conservative preallocation for 3 "local" columns and one non-local # Allow matrix insertion using local indices [0:n+2] lgmap = PETSc.LGMap().create(list(gindices), comm=comm) self.mat.setLGMap(lgmap, lgmap) # Global and local vectors self.gvec = self.mat.createVecRight() self.lvec = PETSc.Vec().create(comm=PETSc.COMM_SELF) self.lvec.setSizes(self.n + 2) self.lvec.setUp() # Configure scatter from global to local isg = PETSc.IS().createGeneral(list(gindices), comm=comm) self.g2l = PETSc.Scatter().create(self.gvec, isg, self.lvec, None) self.tozero, self.zvec = PETSc.Scatter.toZero(self.gvec) self.history = [] if False: # Print some diagnostics print('[%d] local size %d, global size %d, starting offset %d' % (comm.rank, self.n, self.N, self.start)) self.gvec.setArray(numpy.arange(self.start, self.start + self.n)) self.gvec.view() self.g2l.scatter(self.gvec, self.lvec, PETSc.InsertMode.INSERT) for rank in range(comm.size): if rank == comm.rank: print('Contents of local Vec on rank %d' % rank) self.lvec.view() comm.barrier()
def assemble_mtx_to_petsc(pmtx, mtx, pdofs, drange, is_overlap=True, comm=None, verbose=False): """ Assemble a local CSR matrix to a global PETSc matrix. """ if comm is None: comm = PETSc.COMM_WORLD timer = Timer() lgmap = PETSc.LGMap().create(pdofs, comm=comm) pmtx.setLGMap(lgmap, lgmap) if is_overlap: output('setting matrix values...', verbose=verbose) timer.start() mask = (pdofs < drange[0]) | (pdofs >= drange[1]) nnz_per_row = nm.diff(mtx.indptr) mtx2 = mtx.copy() mtx2.data[nm.repeat(mask, nnz_per_row)] = 0 mtx2.eliminate_zeros() pmtx.setValuesLocalCSR(mtx2.indptr, mtx2.indices, mtx2.data, PETSc.InsertMode.INSERT_VALUES) output('...done in', timer.stop(), verbose=verbose) output('assembling matrix...', verbose=verbose) timer.start() pmtx.assemble() output('...done in', timer.stop(), verbose=verbose) else: output('setting matrix values...', verbose=verbose) timer.start() pmtx.setValuesLocalCSR(mtx.indptr, mtx.indices, mtx.data, PETSc.InsertMode.ADD_VALUES) output('...done in', timer.stop(), verbose=verbose) output('assembling matrix...', verbose=verbose) timer.start() pmtx.assemble() output('...done in', timer.stop(), verbose=verbose)
def assemble_rhs_to_petsc(prhs, rhs, pdofs, drange, is_overlap=True, comm=None, verbose=False): """ Assemble a local right-hand side vector to a global PETSc vector. """ if comm is None: comm = PETSc.COMM_WORLD lgmap = PETSc.LGMap().create(pdofs, comm=comm) if is_overlap: prhs.setLGMap(lgmap) output('setting rhs values...', verbose=verbose) tt = time.clock() for ir, rdof in enumerate(pdofs): if (rdof < drange[0]) or (rdof >= drange[1]): continue prhs.setValueLocal(ir, rhs[ir], PETSc.InsertMode.INSERT_VALUES) output('...done in', time.clock() - tt, verbose=verbose) output('assembling rhs...', verbose=verbose) tt = time.clock() prhs.assemble() output('...done in', time.clock() - tt, verbose=verbose) else: prhs.setLGMap(lgmap) output('setting rhs values...', verbose=verbose) tt = time.clock() prhs.setValuesLocal(nm.arange(len(rhs), dtype=nm.int32), rhs, PETSc.InsertMode.ADD_VALUES) output('...done in', time.clock() - tt, verbose=verbose) output('assembling rhs...', verbose=verbose) tt = time.clock() prhs.assemble() output('...done in', time.clock() - tt, verbose=verbose)
def CreatePETScMatrix(ngs_mat): pardofs = ngs_mat.row_pardofs # comm = MPI.COMM_WORLD comm = pardofs.comm.mpi4py globnums, nglob = pardofs.EnumerateGlobally() iset = psc.IS().createGeneral(indices=globnums, comm=comm) lgmap = psc.LGMap().createIS(iset) locmat = ngs_mat.local_mat val, col, ind = locmat.CSR() ind = np.array(ind, dtype='int32') apsc_loc = psc.Mat().createAIJ(size=(locmat.height, locmat.width), csr=(ind, col, val), comm=MPI.COMM_SELF) mat = psc.Mat().createPython(size=nglob, comm=comm) mat.setType(psc.Mat.Type.IS) mat.setLGMap(lgmap) mat.setISLocalMat(apsc_loc) mat.assemble() mat.convert("mpiaij") return mat
def CreatePETScMatrix(ngs_mat, freedofs=None): pardofs = ngs_mat.row_pardofs comm = pardofs.comm.mpi4py locmat = ngs_mat.local_mat eh, ew = locmat.entrysizes if eh != ew: raise Exception("only square entries are allowed") val, col, ind = locmat.CSR() ind = np.array(ind).astype(psc.IntType) col = np.array(col).astype(psc.IntType) apsc_loc = psc.Mat().createBAIJ(size=(eh * locmat.height, eh * locmat.width), bsize=eh, csr=(ind, col, val), comm=MPI.COMM_SELF) if freedofs is not None: locfree = np.flatnonzero(freedofs).astype(psc.IntType) isfree_loc = psc.IS().createBlock(indices=locfree, bsize=eh) apsc_loc = apsc_loc.createSubMatrices(isfree_loc)[0] globnums, nglob = pardofs.EnumerateGlobally(freedofs) if freedofs is not None: globnums = np.array(globnums, dtype=psc.IntType)[freedofs] lgmap = psc.LGMap().create(indices=globnums, bsize=eh, comm=comm) mat = psc.Mat().create(comm=comm) mat.setSizes(size=nglob * eh, bsize=eh) mat.setType(psc.Mat.Type.IS) mat.setLGMap(lgmap) mat.setISLocalMat(apsc_loc) mat.assemble() mat.convert("mpiaij") return mat
def __init__(self, array=None, bs=None, n=None, N=None, nghosts=None, subdomain2global=None, blockVecType="simple"): if array == None: return #cek hack, don't know why init gets called by PETSc.Vec duplicate function p4pyPETSc.Vec.__init__(self) blockSize = max(1, bs) self.dim_proc = n * blockSize self.nghosts = nghosts self.blockVecType = blockVecType assert self.blockVecType == "simple", "petsc4py wrappers require self.blockVecType=simple" self.proteus_array = array if nghosts == None: if blockVecType == "simple": self.createWithArray(array, size=(blockSize * n, blockSize * N), bsize=1) else: self.createWithArray(array, size=(blockSize * n, blockSize * N), bsize=blockSize) self.subdomain2global = subdomain2global self.petsc_l2g = None else: assert nghosts >= 0, "The number of ghostnodes must be non-negative" assert subdomain2global.shape[0] == (n + nghosts), ( "The subdomain2global map is the wrong length n=%i,nghosts=%i,shape=%i \n" % (n, n + nghosts, subdomain2global.shape[0])) assert len(array.flat) == (n + nghosts) * blockSize if blockVecType == "simple": ghosts = numpy.zeros((blockSize * nghosts), 'i') for j in range(blockSize): ghosts[j::blockSize] = subdomain2global[n:] * blockSize + j self.createGhostWithArray(ghosts, array, size=(blockSize * n, blockSize * N), bsize=1) if blockSize > 1: #have to build in block dofs subdomain2globalTotal = numpy.zeros( (blockSize * subdomain2global.shape[0], ), 'i') for j in range(blockSize): subdomain2globalTotal[ j::blockSize] = subdomain2global * blockSize + j self.subdomain2global = subdomain2globalTotal else: self.subdomain2global = subdomain2global self.petsc_l2g = p4pyPETSc.LGMap() self.petsc_l2g.create(self.subdomain2global) self.setLGMap(self.petsc_l2g) else: #TODO need to debug ghosts = subdomain2global[n:] self.createGhostWithArray(ghosts, array, size=(blockSize * n, blockSize * N), bsize=blockSize) self.subdomain2global = subdomain2global self.petsc_l2g = p4pyPETSc.LGMap() self.petsc_l2g.create(self.subdomain2global) self.setLGMap(self.petsc_l2g) self.setFromOptions()
from petsc4py import PETSc from dolfin import MPI, mpi_comm_world #mycomm = mpi_comm_world() mycomm = PETSc.COMM_WORLD mpisize = MPI.size(mycomm) mpirank = MPI.rank(mycomm) mesh = dl.UnitSquareMesh(100, 100) Vr = dl.FunctionSpace(mesh, 'Lagrange', 1) Vc = dl.FunctionSpace(mesh, 'Lagrange', 1) # Vrdofmap, Vcdofmap = Vr.dofmap(), Vc.dofmap() #print 'rank={}, Vr dofmap:'.format(mpirank), Vr.dofmap().dofs() #print 'rank={}, Vc dofmap:'.format(mpirank), Vc.dofmap().dofs() rmap = PETSc.LGMap().create(Vrdofmap.dofs(), mycomm) cmap = PETSc.LGMap().create(Vcdofmap.dofs(), mycomm) """ if mpirank == 0: gindices = PETSc.IS().createGeneral([6,7,8]) #gindices = [6,7,8] else: gindices = PETSc.IS().createGeneral([0,1,2,3,4,5]) #gindices = [0,1,2,3,4,5] rmap = PETSc.LGMap().create(gindices, mycomm) cmap = PETSc.LGMap().create(gindices, mycomm) print 'rank={}, rmap indices:'.format(mpirank), rmap.getIndices() print 'rank={}, cmap indices:'.format(mpirank), cmap.getIndices() """ # MPETSc = PETSc.Mat()
def _create_DMPlex(self, dim, coords, cells, elev): """ Create a PETSc DMPlex object from the mesh attributes Args: dim: mesh dimensions coords: mesh coordinates cells: cell nodes indices elev: nodes elevation """ t0 = clock() self.dm = PETSc.DMPlex().createFromCellList(dim, cells, coords, comm=PETSc.COMM_WORLD) if MPIrank == 0 and self.verbose: print('Create DMPlex (%0.02f seconds)' % (clock() - t0)) # Create boundary labels t0 = clock() label = "boundary" self._set_DMPlex_boundary_points(label) # label coarse DM in case it is ever needed again self.dm.createLabel("coarse") pStart, pEnd = self.dm.getDepthStratum(0) for pt in range(pStart, pEnd): self.dm.setLabelValue("coarse", pt, 1) # Define one DoF on the nodes self.dm.setNumFields(1) origSect = self.dm.createSection(1, [1, 0, 0]) origSect.setFieldName(0, "points") origSect.setUp() self.dm.setDefaultSection(origSect) origVec = self.dm.createGlobalVector() # Distribute to other processors if any if MPIsize > 1: sf = self.dm.distribute(overlap=1) newSect, newVec = self.dm.distributeField(sf, origSect, origVec) self.dm.setDefaultSection(newSect) newSect.destroy() newVec.destroy() sf.destroy() origVec.destroy() origSect.destroy() self.hGlobal = self.dm.createGlobalVector() self.hLocal = self.dm.createLocalVector() self.sizes = self.hGlobal.getSizes(), self.hGlobal.getSizes() # Local/Global mapping self.lgmap_row = self.dm.getLGMap() l2g = self.lgmap_row.indices.copy() offproc = l2g < 0 l2g[offproc] = -(l2g[offproc] + 1) self.lgmap_col = PETSc.LGMap().create(l2g, comm=PETSc.COMM_WORLD) del l2g if MPIrank == 0 and self.verbose: print('Distribute DMPlex (%0.02f seconds)' % (clock() - t0)) # Get natural numbering t0 = clock() coords = MPI.COMM_WORLD.bcast(coords, root=0) elev = MPI.COMM_WORLD.bcast(elev, root=0) self._naturalNumbering(coords) self.hLocal.setArray(elev[self.natural2local]) self.dm.localToGlobal(self.hLocal, self.hGlobal) self.dm.globalToLocal(self.hGlobal, self.hLocal) if MPIrank == 0 and self.verbose: print('Distribute field to DMPlex (%0.02f seconds)' % (clock() - t0)) # Forcing event number self.rainNb = -1 self.tecNb = -1 return
def setUp(self): self.idx = self._mk_idx(PETSc.COMM_WORLD) self.iset = PETSc.IS().createGeneral(self.idx, comm=PETSc.COMM_WORLD) self.lgmap = PETSc.LGMap().create(self.iset)
def _init_block(self): self._blocks = [[self]] mat = PETSc.Mat() row_lg = PETSc.LGMap() col_lg = PETSc.LGMap() rdim, cdim = self.sparsity.dims if MPI.comm.size == 1: # The PETSc local to global mapping is the identity in the sequential case row_lg.create( indices=np.arange(self.sparsity.nrows, dtype=PETSc.IntType), bsize=rdim) col_lg.create( indices=np.arange(self.sparsity.ncols, dtype=PETSc.IntType), bsize=cdim) self._array = np.zeros(self.sparsity.nz, dtype=PETSc.RealType) # We're not currently building a blocked matrix, so need to scale the # number of rows and columns by the sparsity dimensions # FIXME: This needs to change if we want to do blocked sparse # NOTE: using _rowptr and _colidx since we always want the host values mat.createAIJWithArrays( (self.sparsity.nrows * rdim, self.sparsity.ncols * cdim), (self.sparsity._rowptr, self.sparsity._colidx, self._array)) else: # We get the PETSc local to global mapping from the halo. # This gives us "block" indices, we need to splat those # out to dof indices for vector fields since we don't # currently assemble into block matrices. rindices = self.sparsity.rmaps[0].toset.halo.global_to_petsc_numbering cindices = self.sparsity.cmaps[0].toset.halo.global_to_petsc_numbering row_lg.create(indices=rindices, bsize=rdim) col_lg.create(indices=cindices, bsize=cdim) mat.createAIJ(size=((self.sparsity.nrows * rdim, None), (self.sparsity.ncols * cdim, None)), nnz=(self.sparsity.nnz, self.sparsity.onnz), bsize=(rdim, cdim)) mat.setBlockSizes(rdim, cdim) mat.setLGMap(rmap=row_lg, cmap=col_lg) # Do not stash entries destined for other processors, just drop them # (we take care of those in the halo) mat.setOption(mat.Option.IGNORE_OFF_PROC_ENTRIES, True) # Any add or insertion that would generate a new entry that has not # been preallocated will raise an error mat.setOption(mat.Option.NEW_NONZERO_ALLOCATION_ERR, True) # Do not ignore zeros while we fill the initial matrix so that # petsc doesn't compress things out. mat.setOption(mat.Option.IGNORE_ZERO_ENTRIES, False) # When zeroing rows (e.g. for enforcing Dirichlet bcs), keep those in # the nonzero structure of the matrix. Otherwise PETSc would compact # the sparsity and render our sparsity caching useless. mat.setOption(mat.Option.KEEP_NONZERO_PATTERN, True) # We completely fill the allocated matrix when zeroing the # entries, so raise an error if we "missed" one. mat.setOption(mat.Option.UNUSED_NONZERO_LOCATION_ERR, True) # Put zeros in all the places we might eventually put a value. sparsity.fill_with_zeros(mat, self.sparsity.dims, self.sparsity.maps) # Now we've filled up our matrix, so the sparsity is # "complete", we can ignore subsequent zero entries. mat.setOption(mat.Option.IGNORE_ZERO_ENTRIES, True) self._handle = mat # Matrices start zeroed. self._version_set_zero()
from petsc4py import PETSc mesh = dlf.UnitSquareMesh(100, 100) W = dlf.FunctionSpace(mesh, 'CG', 1) def boundary(x, on_boundary): return on_boundary bc = dlf.DirichletBC(W, dlf.Constant(0.0), boundary) N = mesh.num_vertices() v = PETSc.Vec() v.create() v.setSizes(N) v.setType('standard') v.setValues(range(N), [N]*N) B_pet = PETSc.Mat() B_pet.createAIJ([N,N], nnz=N) lgmap = PETSc.LGMap().create(W.dofmap().dofs()) B_pet.setLGMap(lgmap, lgmap) B_pet.setDiagonal(v) B_pet.assemblyBegin() B_pet.assemblyEnd() B = dlf.PETScMatrix(B_pet) bc.apply(B) # error
def lgmap(self): """A PETSc LGMap mapping process-local indices to global indices for this :class:`MixedDataSet`. """ lgmap = PETSc.LGMap() if self.comm.size == 1: size = sum(s.size * s.cdim for s in self) lgmap.create(indices=np.arange(size, dtype=IntType), bsize=1, comm=self.comm) return lgmap # Compute local to global maps for a monolithic mixed system # from the individual local to global maps for each field. # Exposition: # # We have N fields and P processes. The global row # ordering is: # # f_0_p_0, f_1_p_0, ..., f_N_p_0; f_0_p_1, ..., ; f_0_p_P, # ..., f_N_p_P. # # We have per-field local to global numberings, to convert # these into multi-field local to global numberings, we note # the following: # # For each entry in the per-field l2g map, we first determine # the rank that entry belongs to, call this r. # # We know that this must be offset by: # 1. The sum of all field lengths with rank < r # 2. The sum of all lower-numbered field lengths on rank r. # # Finally, we need to shift the field-local entry by the # current field offset. idx_size = sum(s.total_size * s.cdim for s in self) indices = np.full(idx_size, -1, dtype=IntType) owned_sz = np.array([sum(s.size * s.cdim for s in self)], dtype=IntType) field_offset = np.empty_like(owned_sz) self.comm.Scan(owned_sz, field_offset) field_offset -= owned_sz all_field_offsets = np.empty(self.comm.size, dtype=IntType) self.comm.Allgather(field_offset, all_field_offsets) start = 0 all_local_offsets = np.zeros(self.comm.size, dtype=IntType) current_offsets = np.zeros(self.comm.size + 1, dtype=IntType) for s in self: idx = indices[start:start + s.total_size * s.cdim] owned_sz[0] = s.size * s.cdim self.comm.Scan(owned_sz, field_offset) self.comm.Allgather(field_offset, current_offsets[1:]) # Find the ranks each entry in the l2g belongs to l2g = s.halo.local_to_global_numbering # If cdim > 1, we need to unroll the node numbering to dof # numbering if s.cdim > 1: new_l2g = np.empty(l2g.shape[0] * s.cdim, dtype=l2g.dtype) for i in range(s.cdim): new_l2g[i::s.cdim] = l2g * s.cdim + i l2g = new_l2g tmp_indices = np.searchsorted(current_offsets, l2g, side="right") - 1 idx[:] = l2g[:] - current_offsets[tmp_indices] + \ all_field_offsets[tmp_indices] + all_local_offsets[tmp_indices] self.comm.Allgather(owned_sz, current_offsets[1:]) all_local_offsets += current_offsets[1:] start += s.total_size * s.cdim lgmap.create(indices=indices, bsize=1, comm=self.comm) return lgmap
comm = mesh.mpi_comm().tompi4py() mat = PETSc.Mat() mat.create(comm) # Local, global sizes = [ dofmap.index_map().size(IndexMap.MapSize_OWNED), dofmap.index_map().size(IndexMap.MapSize_GLOBAL) ] # Square mat.setSizes([sizes, sizes]) # Sparse mat.setType('aij') mat.setUp() # Map from local rows to gloval rows lgmap = map(int, dofmap.tabulate_local_to_global_dofs()) lgmap = PETSc.LGMap().create(lgmap, comm=comm) mat.setLGMap(lgmap, lgmap) # Fill the values mat.setDiagonal(v) mat.assemblyBegin() mat.assemblyEnd() A = PETScMatrix(mat) bc.apply(A) # Check if we can do matvec. Just no crash x, y = mat.createVecs() x.setRandom() y.setRandom() x, y = PETScVector(x), PETScVector(y)
def scalar_lgmap(self): if self.cdim == 1: return self.lgmap indices = self.lgmap.block_indices return PETSc.LGMap().create(indices=indices, bsize=1, comm=self.comm)
gfu.vec.data = inv * f.vec masterprint("ngs-dot =", InnerProduct(gfu.vec, f.vec)) pardofs = fes.ParallelDofs() globnums, nglob = pardofs.EnumerateGlobally() locmat = a.mat.local_mat val, col, ind = locmat.CSR() ind = np.array(ind, dtype='int32') apsc_loc = psc.Mat().createAIJ(size=(locmat.height, locmat.width), csr=(ind, col, val), comm=MPI.COMM_SELF) IS = psc.IS().createBlock(bsize=1, indices=globnums, comm=comm) lgmap = psc.LGMap().create(bsize=1, indices=globnums, comm=comm) mat = psc.Mat().createPython(size=nglob, comm=comm) mat.setType(psc.Mat.Type.IS) mat.setLGMap(lgmap) mat.setISLocalMat(apsc_loc) mat.assemble() f.vec.Cumulate() v1, v2 = mat.createVecs() v2loc = v2.getSubVector(IS) v2loc.getArray()[:] = f.vec.FV() v2.restoreSubVector(IS, v2loc)