def get_numpy_array(xr):
"""
Convert a distributed PETSc Vec into a sequential numpy array.
Parameters:
-----------
xr: PETSc Vec
Returns:
--------
Array
"""
xr_size = xr.getSize()
seqx = PETSc.Vec()
seqx.createSeq(xr_size, comm=PETSc.COMM_SELF)
seqx.setFromOptions()
# seqx.set(0.0)
fromIS = PETSc.IS().createGeneral(range(xr_size), comm=PETSc.COMM_SELF)
toIS = PETSc.IS().createGeneral(range(xr_size), comm=PETSc.COMM_SELF)
sctr = PETSc.Scatter().create(xr, fromIS, seqx, toIS)
sctr.begin(xr,
seqx,
addv=PETSc.InsertMode.INSERT,
mode=PETSc.ScatterMode.FORWARD)
sctr.end(xr,
seqx,
addv=PETSc.InsertMode.INSERT,
mode=PETSc.ScatterMode.FORWARD)
return seqx.getArray()
def vecscatter(self, readonly=True):
"""A context manager scattering the arrays of all components of this
:class:`MixedDat` into a contiguous :class:`PETSc.Vec` and reverse
scattering to the original arrays when exiting the context.
:param readonly: Access the data read-only (use :meth:`Dat.data_ro`)
or read-write (use :meth:`Dat.data`). Read-write
access requires a halo update.
.. note::
The :class:`~PETSc.Vec` obtained from this context is in
the correct order to be left multiplied by a compatible
:class:`MixedMat`. In parallel it is *not* just a
concatenation of the underlying :class:`Dat`\s."""
acc = (lambda d: d.vec_ro) if readonly else (lambda d: d.vec)
# Allocate memory for the contiguous vector, create the scatter
# contexts and stash them on the object for later reuse
if not (hasattr(self, '_vec') and hasattr(self, '_sctxs')):
self._vec = PETSc.Vec().create()
# Size of flattened vector is product of size and cdim of each dat
sz = sum(d.dataset.size * d.dataset.cdim for d in self._dats)
self._vec.setSizes((sz, None))
self._vec.setUp()
self._sctxs = []
# To be compatible with a MatNest (from a MixedMat) the
# ordering of a MixedDat constructed of Dats (x_0, ..., x_k)
# on P processes is:
# (x_0_0, x_1_0, ..., x_k_0, x_0_1, x_1_1, ..., x_k_1, ..., x_k_P)
# That is, all the Dats from rank 0, followed by those of
# rank 1, ...
# Hence the offset into the global Vec is the exclusive
# prefix sum of the local size of the mixed dat.
offset = MPI.comm.exscan(sz)
if offset is None:
offset = 0
for d in self._dats:
sz = d.dataset.size * d.dataset.cdim
with acc(d) as v:
vscat = PETSc.Scatter().create(v, None, self._vec,
PETSc.IS().createStride(sz, offset, 1))
offset += sz
self._sctxs.append(vscat)
# Do the actual forward scatter to fill the full vector with values
for d, vscat in zip(self._dats, self._sctxs):
with acc(d) as v:
vscat.scatterBegin(v, self._vec, addv=PETSc.InsertMode.INSERT_VALUES)
vscat.scatterEnd(v, self._vec, addv=PETSc.InsertMode.INSERT_VALUES)
yield self._vec
if not readonly:
# Reverse scatter to get the values back to their original locations
for d, vscat in zip(self._dats, self._sctxs):
with acc(d) as v:
vscat.scatterBegin(self._vec, v, addv=PETSc.InsertMode.INSERT_VALUES,
mode=PETSc.ScatterMode.REVERSE)
vscat.scatterEnd(self._vec, v, addv=PETSc.InsertMode.INSERT_VALUES,
mode=PETSc.ScatterMode.REVERSE)
self.needs_halo_update = True
def testVisually(self):
'''blocks selected visually.'''
# if self.comm.rank == 0:
g2z,zvec = PETSc.Scatter().toZero(self.bnd.gindBlockWBand)
g2z.scatter(self.bnd.gindBlockWBand,zvec, PETSc.InsertMode.INSERT)
x = self.bnd.BlockSub2CenterCarWithoutBand(\
self.bnd.BlockInd2SubWithoutBand(zvec.getArray()) )
lx = self.bnd.BlockSub2CenterCarWithoutBand(\
self.bnd.BlockInd2SubWithoutBand(self.bnd.gindBlockWBand.getArray()))
try:
try:
from mayavi import mlab
except ImportError:
from enthought.mayavi import mlab
if self.comm.rank == 0:
mlab.figure()
mlab.points3d(x[:,0],x[:,1],x[:,2])
mlab.figure()
mlab.points3d(lx[:,0],lx[:,1],lx[:,2])
mlab.show()
#fig.add(pts1)
#fig.add(pts2)
except ImportError:
import pylab as pl
from mpl_toolkits.mplot3d import Axes3D #@UnusedImport
fig = pl.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter3D(x[:,0],x[:,1],x[:,2],c='blue',marker='o')
ax.scatter3D(lx[:,0],lx[:,1],lx[:,2],c='red',marker='D')
pl.savefig('testVis{0}.png'.format(self.comm.rank))
pl.show()
def create_gather_scatter(pdofs, pvec_i, pvec, comm=None):
"""
Create the ``gather()`` function for updating a global PETSc vector from
local ones and the ``scatter()`` function for updating local PETSc vectors
from the global one.
"""
if comm is None:
comm = PETSc.COMM_WORLD
isg = PETSc.IS().createGeneral(pdofs, comm=comm)
g2l = PETSc.Scatter().create(pvec, isg, pvec_i, None)
def scatter(pvec_i, pvec):
"""
Scatter `pvec` to `pvec_i`.
"""
g2l.scatter(pvec, pvec_i, PETSc.InsertMode.INSERT)
def gather(pvec, pvec_i):
"""
Gather `pvec_i` to `pvec`.
"""
g2l.scatter(pvec_i, pvec, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.REVERSE)
return gather, scatter
def __init__(self, in_vec, out_vec, in_inds, out_inds, comm):
"""
Initialize all attributes.
Parameters
----------
in_vec : <Vector>
pointer to the input vector.
out_vec : <Vector>
pointer to the output vector.
in_inds : int ndarray
input indices for the transfer.
out_inds : int ndarray
output indices for the transfer.
comm : MPI.Comm or <FakeComm>
communicator of the system that owns this transfer.
"""
super(PETScTransfer, self).__init__(in_vec, out_vec, in_inds, out_inds,
comm)
in_indexset = PETSc.IS().createGeneral(self._in_inds, comm=self._comm)
out_indexset = PETSc.IS().createGeneral(self._out_inds,
comm=self._comm)
self._scatter = PETSc.Scatter().create(out_vec._petsc, out_indexset,
in_vec._petsc,
in_indexset).scatter
if in_vec._ncol > 1:
self._transfer = self._multi_transfer
def __init__(self, param, comm):
self.dim = 3
self.comm = comm
self.param = param
self.numPlies = 3
self.numDataPoints = 12
self.numInterfaces = self.numPlies - 1
self.numLayers = self.numPlies + self.numInterfaces
self.t = np.asarray([0.2, 0.02, 0.2, 0.02, 0.2])
self.theta = np.asarray([np.pi/4, -123.0, 0.0, -123.0, 3.*np.pi/4])
self.cutoff = np.cumsum(self.t)
nx = 50
ny = 10
Lx = 10.
Ly = 2.
self.nel_per_layer = np.asarray([2,2,2,2,2])
self.isBnd = lambda x: self.isBoundary(x)
self.f = lambda x: self.rhs(x)
self.da = LayerCake(nx, ny, Lx, Ly, self.t, self.nel_per_layer) # Build layered composite from uniform mesh
# Setup global and local matrices + communicators
self.A = self.da.createMatrix()
r, _ = self.A.getLGMap() # Get local to global mapping
self.is_A = PETSc.IS().createGeneral(r.indices) # Create Index Set for local indices
A_local = self.A.createSubMatrices(self.is_A)[0] # Construct local submatrix on domain
vglobal = self.da.createGlobalVec()
vlocal = self.da.createLocalVec()
self.scatter_l2g = PETSc.Scatter().create(vlocal, None, vglobal, self.is_A)
self.A_local = A_local
# Setup elements
self.fe = ElasticityQ1()
# Compute Material Tensor given material parameters
self.isotropic, self.composite = makeMaterials(self.param)
#
self.getDataDof()
def _setup_6of7_scatters_create(self, var_inds, arg_inds):
""" Concatenates lists of indices and creates a PETSc Scatter """
merge = lambda x: numpy.concatenate(x) if len(x) > 0 else []
var_ind_set = PETSc.IS().createGeneral(merge(var_inds), comm=self.comm)
arg_ind_set = PETSc.IS().createGeneral(merge(arg_inds), comm=self.comm)
if self.app_ordering is not None:
var_ind_set = self.app_ordering.app2petsc(var_ind_set)
return PETSc.Scatter().create(self.vec['u'].petsc, var_ind_set,
self.vec['p'].petsc, arg_ind_set)
def __init__(self, in_vec, out_vec, in_inds, out_inds, comm):
"""
Initialize all attributes.
"""
super().__init__(in_vec, out_vec, in_inds, out_inds, comm)
in_indexset = PETSc.IS().createGeneral(self._in_inds, comm=self._comm)
out_indexset = PETSc.IS().createGeneral(self._out_inds, comm=self._comm)
self._scatter = PETSc.Scatter().create(out_vec._petsc, out_indexset, in_vec._petsc,
in_indexset).scatter
def createScatter(self, varInds, argInds):
merge = lambda x: numpy.concatenate(x) if len(x) > 0 else []
ISvar = PETSc.IS().createGeneral(merge(varInds), comm=self.comm)
ISarg = PETSc.IS().createGeneral(merge(argInds), comm=self.comm)
ISvar = self.AOvarPETSc.app2petsc(ISvar)
if ISvar.array.shape[0] == 0:
return None
else:
return PETSc.Scatter().create(self.vVarPETSc, ISvar,
self.vArgPETSc, ISarg)
def __init__(self, n, L, overlap, comm):
# Build Grid
self.dim = len(n)
self.L = L
self.comm = comm
self.isBnd = lambda x: self.isBoundary(x)
self.da = PETSc.DMDA().create(n, dof=1, stencil_width=overlap)
self.da.setUniformCoordinates(xmax=L[0], ymax=L[1], zmax=L[2])
self.da.setMatType(PETSc.Mat.Type.AIJ)
self.numSub = comm.Get_size()
self.sub2proc = [[] for i in range(self.numSub)]
for i in range(self.numSub):
self.sub2proc[i] = [i, 0]
self.M = 1
# Define Finite Element space
self.fe = DarcyQ1(self.dim)
# Setup global and local matrices + communicators
self.A = self.da.createMatrix()
r, _ = self.A.getLGMap() # Get local to global mapping
self.is_A = PETSc.IS().createGeneral(
r.indices) # Create Index Set for local indices
A_local = self.A.createSubMatrices(
self.is_A)[0] # Construct local submatrix on domain
vglobal = self.da.createGlobalVec()
vlocal = self.da.createLocalVec()
self.scatter_l2g = PETSc.Scatter().create(vlocal, None, vglobal,
self.is_A)
self.A_local = A_local
# Identify boundary nodes
nnodes = int(self.da.getCoordinatesLocal()[:].size / self.dim)
coords = np.transpose(self.da.getCoordinatesLocal()[:].reshape(
(nnodes, self.dim)))
Dirich, Neumann, P2P = checkFaces(self.da, self.isBnd, coords)
# Construct Partition of Unity
self.cS = coarseSpace(self.da, self.A, self.comm, self.scatter_l2g)
self.cS.buildPOU(True)
def _initialize_transfer(self, in_vec, out_vec):
"""
Set up the transfer; do any necessary pre-computation.
Optionally implemented by the subclass.
Parameters
----------
in_vec : <Vector>
reference to the input vector.
out_vec : <Vector>
reference to the output vector.
"""
self._transfers = transfers = {}
in_inds = self._in_inds
out_inds = self._out_inds
lens = np.empty(len(in_inds), dtype=int)
for i, key in enumerate(in_inds):
lens[i] = len(in_inds[key])
if self._comm.size > 1:
lensums = np.empty(len(in_inds), dtype=int)
self._comm.Allreduce(lens, lensums, op=MPI.SUM)
else:
lensums = lens
for i, key in enumerate(in_inds):
# if lensums[i] > 0, then at least one proc in the comm is transferring
# data for the given key. That means that all procs in the comm
# must particiipate in the collective Scatter call, so we construct
# a Scatter here even if it's empty.
if lensums[i] > 0:
in_set_name, out_set_name = key
in_indexset = PETSc.IS().createGeneral(np.array(
in_inds[key], 'i'),
comm=self._comm)
out_indexset = PETSc.IS().createGeneral(np.array(
out_inds[key], 'i'),
comm=self._comm)
in_petsc = in_vec._petsc[in_set_name]
out_petsc = out_vec._petsc[out_set_name]
transfer = PETSc.Scatter().create(out_petsc, out_indexset,
in_petsc, in_indexset)
transfers[key] = transfer
if in_vec._ncol > 1:
self.transfer = self.multi_transfer
def P2N(self, psc_vector, ngs_vector=None):
if ngs_vector is None:
ngs_vector = ngs.la.CreateParallelVector(self.pardofs)
ngs_vector.SetParallelStatus(ngs.la.PARALLEL_STATUS.CUMULATED)
ngs_vector[:] = 0.0
locvec = psc.Vec().createWithArray(ngs_vector.FV().NumPy(),
comm=MPI.COMM_SELF)
if "p2n_scat" not in self.__dict__:
self.p2n_scat = psc.Scatter().create(psc_vector, self.iset, locvec,
self.isetlocfree)
self.p2n_scat.scatter(psc_vector, locvec, addv=psc.InsertMode.INSERT)
return ngs_vector
def _initialize_transfer(self):
self._transfers = {}
for ip_iset, op_iset in self.ip_inds:
key = (ip_iset, op_iset)
if len(self.ip_inds[key]) > 0:
ip_inds = numpy.array(self.ip_inds[key], 'i')
op_inds = numpy.array(self.op_inds[key], 'i')
ip_indexset = PETSc.IS().createGeneral(ip_inds, comm=self.comm)
op_indexset = PETSc.IS().createGeneral(op_inds, comm=self.comm)
ip_petsc = self.ip_vec._global_vector._petsc[ip_iset]
op_petsc = self.op_vec._global_vector._petsc[op_iset]
transfer = PETSc.Scatter().create(op_petsc, op_indexset,
ip_petsc, ip_indexset)
self._transfers[key] = transfer
def N2P(self, ngs_vector, psc_vector=None):
if psc_vector is None:
psc_vector = psc.Vec().createMPI(self.nglob * self.es,
bsize=self.es,
comm=MPI.COMM_WORLD)
ngs_vector.Distribute()
locvec = psc.Vec().createWithArray(ngs_vector.FV().NumPy(),
comm=MPI.COMM_SELF)
if "n2p_scat" not in self.__dict__:
self.n2p_scat = psc.Scatter().create(locvec, self.isetlocfree,
psc_vector, self.iset)
psc_vector.set(0)
self.n2p_scat.scatter(locvec, psc_vector,
addv=psc.InsertMode.ADD) # 1 max, 2 sum+keep
return psc_vector
def _initializePETScScatters(self):
""" Defines a scatter for when a variable is its own argument """
n, c = self.name, self.copy
args = self.variables[n, c]['args']
if (n, c) in args:
varIndices = args[n, c]
m1 = numpy.sum(self.argSizes[:self.rank])
m2 = m1 + args[n, c].shape[0]
argIndices = numpy.array(numpy.linspace(m1, m2 - 1, m2 - m1), 'i')
ISvar = PETSc.IS().createGeneral(varIndices, comm=self.comm)
ISarg = PETSc.IS().createGeneral(argIndices, comm=self.comm)
self.scatterFull = PETSc.Scatter().create(self.vVarPETSc, ISvar,
self.vArgPETSc, ISarg)
else:
self.scatterFull = None
def __init__(self, comm, N):
self.comm = comm
self.N = N # global problem size
self.h = 1 / N # grid spacing on unit interval
self.n = N // comm.size + int(
comm.rank < (N % comm.size)) # owned part of global problem
self.start = comm.exscan(self.n)
if comm.rank == 0: self.start = 0
gindices = numpy.arange(self.start - 1,
self.start + self.n + 1,
dtype=PETSc.IntType) % N # periodic
self.mat = PETSc.Mat().create(comm=comm)
size = (self.n, self.N) # local and global sizes
self.mat.setSizes((size, size))
self.mat.setFromOptions()
self.mat.setPreallocationNNZ(
(3, 1)
) # Conservative preallocation for 3 "local" columns and one non-local
# Allow matrix insertion using local indices [0:n+2]
lgmap = PETSc.LGMap().create(list(gindices), comm=comm)
self.mat.setLGMap(lgmap, lgmap)
# Global and local vectors
self.gvec = self.mat.createVecRight()
self.lvec = PETSc.Vec().create(comm=PETSc.COMM_SELF)
self.lvec.setSizes(self.n + 2)
self.lvec.setUp()
# Configure scatter from global to local
isg = PETSc.IS().createGeneral(list(gindices), comm=comm)
self.g2l = PETSc.Scatter().create(self.gvec, isg, self.lvec, None)
self.tozero, self.zvec = PETSc.Scatter.toZero(self.gvec)
self.history = []
if False: # Print some diagnostics
print('[%d] local size %d, global size %d, starting offset %d' %
(comm.rank, self.n, self.N, self.start))
self.gvec.setArray(numpy.arange(self.start, self.start + self.n))
self.gvec.view()
self.g2l.scatter(self.gvec, self.lvec, PETSc.InsertMode.INSERT)
for rank in range(comm.size):
if rank == comm.rank:
print('Contents of local Vec on rank %d' % rank)
self.lvec.view()
comm.barrier()
def create_gather_to_zero(pvec):
"""
Create the ``gather_to_zero()`` function for collecting the global PETSc
vector on the task of rank zero.
"""
g20, pvec_full = PETSc.Scatter().toZero(pvec)
def gather_to_zero(pvec):
"""
Return the global PETSc vector, corresponding to `pvec`, on the task of
rank zero. The vector is reused between calls!
"""
g20.scatter(pvec, pvec_full, PETSc.InsertMode.INSERT,
PETSc.ScatterMode.FORWARD)
return pvec_full
return gather_to_zero
def __init__(self, src_vec, tgt_vec, src_idxs, tgt_idxs, vec_conns,
byobj_conns, mode, sysdata):
src_idxs = src_vec.merge_idxs(src_idxs)
tgt_idxs = tgt_vec.merge_idxs(tgt_idxs)
self.byobj_conns = byobj_conns
self.comm = comm = src_vec.comm
self.sysdata = sysdata
uvec = src_vec.petsc_vec
pvec = tgt_vec.petsc_vec
name = src_vec._sysdata.pathname
if trace:
debug("'%s': creating index sets for '%s' DataTransfer: %s %s" %
(name, src_vec._sysdata.pathname, src_idxs, tgt_idxs))
src_idx_set = PETSc.IS().createGeneral(src_idxs, comm=comm)
if trace: debug("src_idx_set DONE")
tgt_idx_set = PETSc.IS().createGeneral(tgt_idxs, comm=comm)
if trace: debug("tgt_idx_set DONE")
try:
if trace: # pragma: no cover
self.src_idxs = src_idxs
self.tgt_idxs = tgt_idxs
self.vec_conns = vec_conns
arrow = '-->' if mode == 'fwd' else '<--'
debug(
"'%s': new %s scatter (sizes: %d, %d)\n %s %s %s %s %s %s"
% (name, mode, len(src_idx_set.indices),
len(tgt_idx_set.indices), [v for u, v in vec_conns],
arrow, [u for u, v in vec_conns], src_idx_set.indices,
arrow, tgt_idx_set.indices))
self.scatter = PETSc.Scatter().create(uvec, src_idx_set, pvec,
tgt_idx_set)
if trace: debug("scatter creation DONE")
except Exception as err:
raise RuntimeError(
"ERROR in %s (src_idxs=%s, tgt_idxs=%s, usize=%d, psize=%d): %s"
% (name, src_idxs, tgt_idxs, src_vec.vec.size,
tgt_vec.vec.size, str(err)))
def _initialize_transfer(self, in_vec, out_vec):
"""
Set up the transfer; do any necessary pre-computation.
Optionally implemented by the subclass.
Parameters
----------
in_vec : <Vector>
reference to the input vector.
out_vec : <Vector>
reference to the output vector.
"""
in_indexset = PETSc.IS().createGeneral(self._in_inds, comm=self._comm)
out_indexset = PETSc.IS().createGeneral(self._out_inds, comm=self._comm)
self._transfer = PETSc.Scatter().create(out_vec._petsc, out_indexset, in_vec._petsc,
in_indexset)
if in_vec._ncol > 1:
self.transfer = self.multi_transfer
def vecscatters(self):
"""Get the vecscatters from the dof layout of this dataset to a PETSc Vec."""
# To be compatible with a MatNest (from a MixedMat) the
# ordering of a MixedDat constructed of Dats (x_0, ..., x_k)
# on P processes is:
# (x_0_0, x_1_0, ..., x_k_0, x_0_1, x_1_1, ..., x_k_1, ..., x_k_P)
# That is, all the Dats from rank 0, followed by those of
# rank 1, ...
# Hence the offset into the global Vec is the exclusive
# prefix sum of the local size of the mixed dat.
size = sum(d.size * d.cdim for d in self)
offset = self.comm.exscan(size)
if offset is None:
offset = 0
scatters = []
for d in self:
size = d.size * d.cdim
vscat = PETSc.Scatter().createWithData(
d.layout_vec, None, self.layout_vec,
PETSc.IS().createStride(size, offset, 1, comm=d.comm))
offset += size
scatters.append(vscat)
return tuple(scatters)
def __init__(self, src_vec, tgt_vec, src_idxs, tgt_idxs, vec_conns,
byobj_conns, mode):
super(PetscDataTransfer, self).__init__(src_idxs, tgt_idxs, vec_conns,
byobj_conns, mode)
self.comm = comm = src_vec.comm
uvec = src_vec.petsc_vec
pvec = tgt_vec.petsc_vec
name = src_vec.pathname
if trace:
debug("'%s': creating index sets for '%s' DataTransfer: %s %s" %
(name, src_vec.pathname, src_idxs, tgt_idxs))
src_idx_set = PETSc.IS().createGeneral(src_idxs, comm=comm)
tgt_idx_set = PETSc.IS().createGeneral(tgt_idxs, comm=comm)
try:
if trace:
self.src_idxs = src_idxs
self.tgt_idxs = tgt_idxs
arrow = '-->' if mode == 'fwd' else '<--'
debug(
"'%s': new %s scatter (sizes: %d, %d)\n%s %s %s %s %s %s" %
(name, mode, len(src_idx_set.indices),
len(tgt_idx_set.indices), [v
for u, v in vec_conns], arrow,
[u for u, v in vec_conns
], src_idx_set.indices, arrow, tgt_idx_set.indices))
self.scatter = PETSc.Scatter().create(uvec, src_idx_set, pvec,
tgt_idx_set)
except Exception as err:
raise RuntimeError(
"ERROR in %s (src_idxs=%s, tgt_idxs=%s, usize=%d, psize=%d): %s"
% (name, src_idxs, tgt_idxs, src_vec.vec.size,
tgt_vec.vec.size, str(err)))
def solve_stability(self):
"""
Solve the 2nd-order stability problem
"""
def global_mask(fun1, fun2, V):
# Find the indices where fun1 ~= fun2 at a tolerance
diff = fun1.vector() - fun2.vector()
diff.abs()
diff_glob = PETScVector(mpi_comm_self())
diff.gather(diff_glob, pl.array(range(V.dim()), "intc"))
mask = diff_glob < DOLFIN_EPS_LARGE
return mask
# Locate the elastic part
mask = global_mask(self.alpha, self.alpha_prev, self.V_alpha)
if pl.all(mask):
self.print0("\033[1;36m 2nd stability: elastic phase\033[1;m")
self.rq = pl.inf
return True
else:
self._u_alpha_prev.vector()[:] = 0.0
self._u_alpha.vector()[:] = 1.0
assign(self._u_alpha_prev.sub(1), self.alpha_prev)
assign(self._u_alpha.sub(1), self.alpha)
mask = global_mask(self._u_alpha, self._u_alpha_prev,
self._V_u_alpha)
self.elas_dofs = set((pl.where(mask == True)[0]).astype(pl.intc))
bc_elas_dofs = self.elas_dofs.union(self.bc_dofs)
indices = sorted(
set(range(self.ownership[0], self.ownership[1])) - bc_elas_dofs)
# Assemble K and M
self._K = PETScMatrix()
self._M = PETScMatrix()
assemble(self._rqP, self._K)
assemble(self._rqN, self._M)
self._K_mat = self._K.mat()
self._M_mat = self._M.mat()
# Eliminate the elastic/BC part using PETSc.IS
self.IS = PETSc.IS()
self.IS.createGeneral(indices)
self._K_mat_reduced = self._K_mat.getSubMatrix(self.IS, self.IS)
self._K = PETScMatrix(self._K_mat_reduced)
self._M_mat_reduced = self._M_mat.getSubMatrix(self.IS, self.IS)
self._M = PETScMatrix(self._M_mat_reduced)
# Stop if M ~= 0
if self._M.norm("linf") < DOLFIN_EPS_LARGE:
self.rq = pl.inf
self.print0(
"\033[1;36m 2nd stability: Rayleigh quotient: %.3e\033[1;m"
% self.rq)
return True
# Setup the eigenvalue solver
self.eigensolver = SLEPcEigenSolver(self._K, self._M)
self.set_eigensolver_parameters()
# Use last known directions for initial guess
assign(self._u_alpha.sub(0), self.V)
assign(self._u_alpha.sub(1), self.Beta)
_u_alpha_vec = as_backend_type(self._u_alpha.vector()).vec()
_u_alpha_vec_reduced = _u_alpha_vec.getSubVector(self.IS)
# self.eps.setInitialSpace(_u_alpha_vec_reduced)
# Solve the eigenvalue problem
self.print0(
"\033[1;36m 2nd stability: solving the eigenvalue problem\033[1;m"
)
self.eps.solve()
r, c, rx, cx = self.eigensolver.get_eigenpair(0)
self.print0("\033[1;36m 2nd stability: smallest ev: %.3e\033[1;m" %
r)
# From reduced vector to full vector
self.scatter = PETSc.Scatter()
rx_vec = as_backend_type(rx).vec()
self.scatter.create(_u_alpha_vec_reduced, None, _u_alpha_vec, self.IS)
_u_alpha_vec.zeroEntries()
self.scatter.scatter(rx_vec, _u_alpha_vec)
_u_alpha_vec.ghostUpdate()
# Check the Rayleigh quotient (in theory we should have r == rq)
self.rq = assemble(self.rqP) / assemble(self.rqN)
if abs(r - self.rq) > DOLFIN_EPS_LARGE:
self.print0(
"\033[1;36m 2nd stability: Rayleigh quotient: %.3e\033[1;m"
% self.rq)
# Obtain the perturbation directions to V and Beta
assign(self.V, self._u_alpha.sub(0))
assign(self.Beta, self._u_alpha.sub(1))
# Scale V
u_mean = self.u.vector().norm("l2")
if self.V.vector().norm("l2") > DOLFIN_EPS_LARGE:
coeff = u_mean / self.V.vector().norm("l2")
self.V.vector()[:] = coeff * self.V.vector()
# Scale and project Beta to the admissible space
alpha_mean = self.alpha.vector().norm("l2")
if self.Beta.vector().norm("l2") > DOLFIN_EPS_LARGE:
coeff = alpha_mean / self.Beta.vector().norm("l2")
self.Beta.vector()[:] = coeff * self.Beta.vector()
self.Beta.vector()[self.Beta.vector() < 0] = 0.0
# Determine if the solution is unique
if self.rq > 1:
return True
def setUp(self):
v1, v2 = PETSc.Vec().createSeq(0), PETSc.Vec().createSeq(0)
i1, i2 = PETSc.IS().createGeneral([]), PETSc.IS().createGeneral([])
self.obj = PETSc.Scatter().create(v1, i1, v2, i2)
del v1, v2, i1, i2
def createGLVectors(self):
self.SelectBlock()
self.computeCP()
self.larray = np.zeros((self.numBlockWBandAssigned,)+\
(self.m+self.StencilWidth*2,)*self.Dim,order='F')
self.lvec = PETSc.Vec().createWithArray(self.larray,comm=self.comm)
lsize = self.numBlockWBandAssigned*self.m**self.Dim
self.gvec = PETSc.Vec().createMPI((lsize,PETSc.DECIDE))
self.gvec.setUp()
self.wvec = self.gvec.copy()
# self.createIndicesHelper()
tind = np.arange((self.m+self.StencilWidth*2)**self.Dim)
tind = tind.reshape((self.m+self.StencilWidth*2,)*self.Dim,order='F')
for dim in xrange(self.Dim):
tind = np.delete(tind,0,dim)
tind = np.delete(tind,np.s_[-1],dim)
tind = tind.flatten(order='F')
# ISList = []
# c = (self.m+self.StencilWidth*2)**self.Dim
# for i in xrange(self.BlockWBandStart,self.BlockWBandEnd):
# ti = i*c
# ISList.extend(list(tind+ti))
tind = np.tile(tind,self.numBlockWBandAssigned)
ttind = np.arange(self.BlockWBandStart,self.BlockWBandEnd)
tt = (self.m+2*self.StencilWidth)**self.Dim
ttind *= tt
ttind = np.repeat(ttind, self.m**self.Dim)
ttind = tind + ttind
ISFrom = PETSc.IS().createGeneral(ttind,comm=self.comm)
self.l2g = PETSc.Scatter().create(self.lvec,ISFrom,self.gvec,None)
#generate scatter global2local
tind = np.arange(tt)
tind = self.Ind2Sub(tind,(self.m+2*self.StencilWidth,)*self.Dim)
tind -= self.StencilWidth
tind = np.tile(tind,(self.numBlockWBandAssigned,1))
ttind = self.ni2pi.petsc2app(np.arange(self.BlockWBandStart,self.BlockWBandEnd))
ttind = self.BlockInd2SubWithoutBand(ttind)
ttind = np.repeat(ttind,tt,axis=0)
ttind += tind/self.m
tind = np.mod(tind,self.m)
tind = self.Sub2Ind(tind, (self.m,)*self.Dim)
ttind = self.BlockSub2IndWithoutBand(ttind)
ttind = self.ni2pi.app2petsc(ttind)
ttind *= self.m**self.Dim
tind += ttind
(ind,) = np.where(tind>=0)
tind = tind[ind]
ISTo = ind+self.BlockWBandStart*tt
ISFrom = PETSc.IS().createGeneral(tind)
ISTo = PETSc.IS().createGeneral(ISTo)
self.g2l = PETSc.Scatter().create(self.gvec,ISFrom,self.lvec,ISTo)
return self.larray,self.lvec,self.gvec,self.wvec
def SelectBlock(self,surface = None):
if surface is None:
surface = self.surface
comm = self.comm
numTotalBlock = self.M**self.Dim
numBlockAssigned = numTotalBlock // comm.size + int(comm.rank < (numTotalBlock % comm.size))
Blockstart = comm.exscan(numBlockAssigned)
if comm.rank == 0:Blockstart = 0
indBlock = np.arange(Blockstart,Blockstart+numBlockAssigned)
subBlock = self.BlockInd2SubWithoutBand(indBlock)
BlockCenterCar = self.BlockSub2CenterCarWithoutBand(subBlock)
cp,_,_,_ = surface.cp(BlockCenterCar)
dBlockCenter = self.norm1(cp-BlockCenterCar)
p = self.interpDegree
if p % 2 == 1:
p = ( p + 1 ) / 2
else:
p = ( p + 2 ) / 2
bw = 1.1*((p+2)*self.hGrid+self.hBlock/2)#*np.sqrt(self.Dim)
(lindBlockWithinBand,) = np.where(dBlockCenter<bw)
lindBlockWithinBand = lindBlockWithinBand+Blockstart
lBlockSize = lindBlockWithinBand.size
numTotalBlockWBand = comm.allreduce(lBlockSize)
numBlockWBandAssigned = numTotalBlockWBand // comm.size + int(comm.rank < (numTotalBlockWBand % comm.size))
lindBlockWBandFrom = PETSc.Vec().createWithArray(lindBlockWithinBand,comm=comm)
self.gindBlockWBand = PETSc.Vec().createMPI((numBlockWBandAssigned,PETSc.DECIDE),comm=comm)
# gsubBlockWBandFrom = PETSc.Vec().createMPI((self.Dim*lBlockize,PETSc.DECIDE),comm=comm)
# gsubBlockWBandFrom.setArray(lsubBlockWBand)
# self.gsubBlockWBand = PETSc.Vec().createMPI((self.Dim*self.numBlockWBandAssigned,PETSc.DECIDE),comm=comm)
BlockWBandStart = comm.exscan(numBlockWBandAssigned)
if comm.rank == 0:
BlockWBandStart = 0
self.BlockWBandStart = BlockWBandStart
LInd = PETSc.IS().createStride(numBlockWBandAssigned,\
first=BlockWBandStart,\
step=1,comm=comm)
self.numTotalBlockWBand = numTotalBlockWBand
self.numBlockWBandAssigned = numBlockWBandAssigned
BlockWBandEnd = BlockWBandStart + numBlockWBandAssigned
self.BlockWBandEnd = BlockWBandEnd
scatter = PETSc.Scatter().create(lindBlockWBandFrom,LInd,self.gindBlockWBand,None)
scatter.scatter(lindBlockWBandFrom,self.gindBlockWBand,PETSc.InsertMode.INSERT)
#Natural order Index To Petsc order Index
self.ni2pi = PETSc.AO().createMapping(self.gindBlockWBand.getArray().astype(np.int64))
def _init_approximations(self, system):
"""
Prepare for later approximations.
Parameters
----------
system : System
The system having its derivs approximated.
"""
total = system.pathname == ''
abs2meta = system._var_allprocs_abs2meta
in_slices = system._inputs.get_slice_dict()
out_slices = system._outputs.get_slice_dict()
approx_wrt_idx = system._owns_approx_wrt_idx
coloring = system._get_static_coloring()
self._approx_groups = []
self._nruns_uncolored = 0
if self._during_sparsity_comp:
wrt_matches = system._coloring_info['wrt_matches']
else:
wrt_matches = None
for wrt, start, end, vec, _, _ in system._jac_wrt_iter(wrt_matches):
if wrt in self._wrt_meta:
meta = self._wrt_meta[wrt]
if coloring is not None and 'coloring' in meta:
continue
if vec is system._inputs:
slices = in_slices
else:
slices = out_slices
data = self._get_approx_data(system, wrt, meta)
directional = meta['directional']
in_idx = range(start, end)
if wrt in approx_wrt_idx:
if vec is None:
vec_idx = None
else:
# local index into var
vec_idx = approx_wrt_idx[wrt].shaped_array(copy=True)
# convert into index into input or output vector
vec_idx += slices[wrt].start
# Directional derivatives for quick partial checking.
# Place the indices in a list so that they are all stepped at the same time.
if directional:
in_idx = [list(in_idx)]
vec_idx = [vec_idx]
else:
if vec is None: # remote wrt
if wrt in abs2meta['input']:
vec_idx = range(
abs2meta['input'][wrt]['global_size'])
else:
vec_idx = range(
abs2meta['output'][wrt]['global_size'])
else:
vec_idx = LocalRangeIterable(system, wrt)
if directional:
vec_idx = [v for v in vec_idx if v is not None]
# Directional derivatives for quick partial checking.
# Place the indices in a list so that they are all stepped at the same time.
if directional:
in_idx = [list(in_idx)]
vec_idx = [list(vec_idx)]
if directional:
self._nruns_uncolored += 1
else:
self._nruns_uncolored += end - start
self._approx_groups.append((wrt, data, in_idx, vec, vec_idx,
directional, meta['vector']))
if total:
# compute scatter from the results vector into a column of the total jacobian
sinds, tinds, colsize, has_dist_data = system._get_jac_col_scatter(
)
if has_dist_data:
src_vec = PETSc.Vec().createWithArray(np.zeros(len(
system._outputs),
dtype=float),
comm=system.comm)
tgt_vec = PETSc.Vec().createWithArray(np.zeros(colsize,
dtype=float),
comm=system.comm)
src_inds = PETSc.IS().createGeneral(sinds, comm=system.comm)
tgt_inds = PETSc.IS().createGeneral(tinds, comm=system.comm)
self._jac_scatter = (has_dist_data, PETSc.Scatter().create(
src_vec, src_inds, tgt_vec, tgt_inds), src_vec, tgt_vec)
else:
self._jac_scatter = (has_dist_data, sinds, tinds)
else:
self._jac_scatter = None
def __init__(self, A_IS):
"""
Initialize the domain decomposition preconditioner, multipreconditioner and coarse space with its operators
Parameters
==========
A_IS : petsc.Mat
The matrix of the problem in IS format. A must be a symmetric positive definite matrix
with symmetric positive semi-definite submatrices
PETSc.Options
=============
PCBNN_switchtoASM :Bool
Default is False
If True then the domain decomposition preconditioner is the BNN preconditioner. If false then the domain
decomposition precondition is the Additive Schwarz preconditioner with minimal overlap.
PCBNN_kscaling : Bool
Default is True.
If true then kscaling (partition of unity that is proportional to the diagonal of the submatrices of A)
is used when a partition of unity is required. Otherwise multiplicity scaling is used when a partition
of unity is required. This may occur in two occasions:
- to scale the local BNN matrices if PCBNN_switchtoASM=True,
- in the GenEO eigenvalue problem for eigmin if PCBNN_switchtoASM=False and PCBNN_GenEO=True with
PCBNN_GenEO_eigmin > 0 (see projection.__init__ for the meaning of these options).
PCBNN_verbose : Bool
Default is False.
If True, some information about the preconditioners is printed when the code is executed.
PCBNN_GenEO : Bool
Default is False.
If True then the coarse space is enriched by solving local generalized eigenvalue problems.
PCBNN_CoarseProjection :True
Default is True
If False then there is no coarse projection: Two level Additive Schwarz or One-level preconditioner depending on PCBNN_addCoarseSolve
If True, the coarse projection is applied: Projected preconditioner of hybrid preconditioner depending on PCBNN_addCoarseSolve
PCBNN_addCoarseSolve : False
Default is True
If True then (R0t A0\R0 r) is added to the preconditioned residual
False corresponds to the projected preconditioner (need to choose initial guess accordingly) (or the one level preconditioner if PCBNN_CoarseProjection = False)
True corresponds to the hybrid preconditioner (or the fully additive preconditioner if PCBNN_CoarseProjection = False)
"""
OptDB = PETSc.Options()
self.switchtoASM = OptDB.getBool('PCBNN_switchtoASM', False) #use Additive Schwarz as a preconditioner instead of BNN
self.kscaling = OptDB.getBool('PCBNN_kscaling', True) #kscaling if true, multiplicity scaling if false
self.verbose = OptDB.getBool('PCBNN_verbose', False)
self.GenEO = OptDB.getBool('PCBNN_GenEO', True)
self.addCS = OptDB.getBool('PCBNN_addCoarseSolve', False)
self.projCS = OptDB.getBool('PCBNN_CoarseProjection', True)
#extract Neumann matrix from A in IS format
Ms = A_IS.copy().getISLocalMat()
# convert A_IS from matis to mpiaij
A_mpiaij = A_IS.convertISToAIJ()
r, _ = A_mpiaij.getLGMap() #r, _ = A_IS.getLGMap()
is_A = PETSc.IS().createGeneral(r.indices)
# extract exact local solver
As = A_mpiaij.createSubMatrices(is_A)[0]
vglobal, _ = A_mpiaij.getVecs()
vlocal, _ = Ms.getVecs()
scatter_l2g = PETSc.Scatter().create(vlocal, None, vglobal, is_A)
#compute the multiplicity of each degree
vlocal.set(1.)
vglobal.set(0.)
scatter_l2g(vlocal, vglobal, PETSc.InsertMode.ADD_VALUES)
scatter_l2g(vglobal, vlocal, PETSc.InsertMode.INSERT_VALUES, PETSc.ScatterMode.SCATTER_REVERSE)
NULL,mult_max = vglobal.max()
# k-scaling or multiplicity scaling of the local (non-assembled) matrix
if self.kscaling == False:
Ms.diagonalScale(vlocal,vlocal)
else:
v1 = As.getDiagonal()
v2 = Ms.getDiagonal()
Ms.diagonalScale(v1/v2, v1/v2)
# the default local solver is the scaled non assembled local matrix (as in BNN)
if self.switchtoASM:
Atildes = As
if mpi.COMM_WORLD.rank == 0:
print('The user has chosen to switch to Additive Schwarz instead of BNN.')
else: #(default)
Atildes = Ms
ksp_Atildes = PETSc.KSP().create(comm=PETSc.COMM_SELF)
ksp_Atildes.setOptionsPrefix("ksp_Atildes_")
ksp_Atildes.setOperators(Atildes)
ksp_Atildes.setType('preonly')
pc_Atildes = ksp_Atildes.getPC()
pc_Atildes.setType('cholesky')
pc_Atildes.setFactorSolverType('mumps')
ksp_Atildes.setFromOptions()
self.A = A_mpiaij
self.Ms = Ms
self.As = As
self.ksp_Atildes = ksp_Atildes
self.works_1 = vlocal.copy()
self.works_2 = self.works_1.copy()
self.scatter_l2g = scatter_l2g
self.mult_max = mult_max
self.minV0 = minimal_V0(self.ksp_Atildes)
if self.GenEO == True:
GenEOV0 = GenEO_V0(self.ksp_Atildes,self.Ms,self.As,self.mult_max,self.minV0.V0s)
self.V0s = GenEOV0.V0s
else:
self.V0s = self.minV0.V0s
self.proj = coarse_operators(self.V0s,self.A,self.scatter_l2g,vlocal)
def calc_gradient(self, sim, params):
"""Calculate the gradient of a figure of merit which depends on the
permittivity and permeability.
Parameters
----------
sim : FDFD
Simulation object. sim = self.sim
params : numpy.array or list of floats
List of design parameters.
Returns
-------
numpy.array
**(Master node only)** Gradient of figure of merit, i.e. list of
derivatives of fom with respect to each design variable
"""
# Semantically, we would not normally need to override this method,
# however it turns out that the operations needed to compute the
# gradient of a field-dependent function and a permittivit-dependent
# function are very similar (both require the calculation of the
# derivative of the materials wrt the design parameters.) For the sake
# of performance, we combine the two calculations here.
w_pml_l = sim.w_pml_left
w_pml_r = sim.w_pml_right
w_pml_t = sim.w_pml_top
w_pml_b = sim.w_pml_bottom
M = sim.M
N = sim.N
# get the current diagonal elements of A.
# only these elements change when the design variables change.
Ai = PETSc.Vec()
Af = PETSc.Vec()
Ai = sim.get_A_diag(Ai)
# Get the derivatives w.r.t. eps, mu
if (NOT_PARALLEL):
dFdeps, dFdeps_conj, dFdmu, dFdmu_conj = self.calc_dFdm(
sim, params)
x = sim.x.copy()
x_adj = sim.x_adj.copy()
step = self._step
update_boxes = self.get_update_boxes(sim, params)
lenp = len(params)
grad_full = None
if (RANK == 0):
grad_full = np.zeros(sim.nunks, dtype=np.double)
gradient = np.zeros(lenp)
for i in xrange(lenp):
#if(NOT_PARALLEL):
# print i
p0 = params[i]
ub = update_boxes[i]
# perturb the system
params[i] += step
self.update_system(params)
self.sim.update(ub)
# get the updated diagonal elements of A
Af = sim.get_A_diag(Af)
# calculate dAdp and assemble the full result on the master node
dAdp = (Af - Ai) / step
product = x_adj * dAdp * x
grad_part = -2 * np.real(np.sum(product[...]))
# send the partially computed gradient to the master node to finish
# up the calculation
MPI.COMM_WORLD.Gatherv(grad_part, grad_full, root=0)
# We also need dAdp to account for the derivative of eps and mu
gatherer, dAdp_full = PETSc.Scatter().toZero(dAdp)
gatherer.scatter(dAdp, dAdp_full, False,
PETSc.Scatter.Mode.FORWARD)
# finish calculating the gradient
if (NOT_PARALLEL):
# derivative with respect to fields
gradient[i] = np.sum(grad_full)
# Next we compute the derivative with respect to eps and mu. We
# exclude the PML regions because changes to the materials in
# the PMLs are generally not something we want to consider.
jmin = ub[0]
jmax = ub[1]
imin = ub[2]
imax = ub[3]
if (jmin < w_pml_l): jmin = w_pml_l
if (jmax > N - w_pml_r): jmax = N - w_pml_r
if (imin < w_pml_b): imin = w_pml_b
if (imax > M - w_pml_t): imax = M - w_pml_t
# note that the extraction of eps and mu from A must be handled
# slightly differently in the TE and TM cases since the signs
# along the diagonal are swapped and eps and mu are positioned
# in different parts
if (isinstance(sim, fdfd.FDFD_TM)):
dmudp = dAdp_full[0:M * N].reshape([M, N]) * 1j
depsdp = dAdp_full[M * N:2 * M * N].reshape([M, N]) / 1j
elif (isinstance(sim, fdfd.FDFD_TE)):
depsdp = dAdp_full[0:M * N].reshape([M, N]) / 1j
dmudp = dAdp_full[M * N:2 * M * N].reshape([M, N]) * 1j
gradient[i] += np.real(
np.sum(dFdeps[imin:imax, jmin:jmax] * \
depsdp[imin:imax, jmin:jmax]) + \
np.sum(dFdeps_conj[imin:imax, jmin:jmax] * \
np.conj(depsdp[imin:imax, jmin:jmax])) + \
np.sum(dFdmu[imin:imax, jmin:jmax] * \
dmudp[imin:imax, jmin:jmax]) + \
np.sum(dFdmu_conj[imin:imax, jmin:jmax] * \
np.conj(dmudp[imin:imax, jmin:jmax])) \
)
# revert the system to its original state
params[i] = p0
self.update_system(params)
self.sim.update(ub)
if (NOT_PARALLEL):
return gradient
def __init__(self, A):
"""
Initialize the domain decomposition preconditioner, multipreconditioner and coarse space with its operators
Parameters
==========
A : petsc.Mat
The matrix of the problem in IS format. A must be a symmetric positive definite matrix
with symmetric positive semi-definite submatrices
PETSc.Options
=============
PCBNN_switchtoASM :Bool
Default is False
If True then the domain decomposition preconditioner is the BNN preconditioner. If false then the domain
decomposition precondition is the Additive Schwarz preconditioner with minimal overlap.
PCBNN_kscaling : Bool
Default is True.
If true then kscaling (partition of unity that is proportional to the diagonal of the submatrices of A)
is used when a partition of unity is required. Otherwise multiplicity scaling is used when a partition
of unity is required. This may occur in two occasions:
- to scale the local BNN matrices if PCBNN_switchtoASM=True,
- in the GenEO eigenvalue problem for eigmin if PCBNN_switchtoASM=False and PCBNN_GenEO=True with
PCBNN_GenEO_eigmin > 0 (see projection.__init__ for the meaning of these options).
PCBNN_verbose : Bool
Default is False.
If True, some information about the preconditioners is printed when the code is executed.
"""
OptDB = PETSc.Options()
self.switchtoASM = OptDB.getBool('PCBNN_switchtoASM', False) #use Additive Schwarz as a preconditioner instead of BNN
self.kscaling = OptDB.getBool('PCBNN_kscaling', True) #kscaling if true, multiplicity scaling if false
self.verbose = OptDB.getBool('PCBNN_verbose', False)
# convert matis to mpiaij, extract local matrices
r, _ = A.getLGMap()
is_A = PETSc.IS().createGeneral(r.indices)
A_mpiaij = A.convertISToAIJ()
A_mpiaij_local = A_mpiaij.createSubMatrices(is_A)[0]
A_scaled = A.copy().getISLocalMat()
vglobal, _ = A.getVecs()
vlocal, _ = A_scaled.getVecs()
scatter_l2g = PETSc.Scatter().create(vlocal, None, vglobal, is_A)
#compute the multiplicity of each degree of freedom and max of the multiplicity
vlocal.set(1.)
vglobal.set(0.)
scatter_l2g(vlocal, vglobal, PETSc.InsertMode.ADD_VALUES)
scatter_l2g(vglobal, vlocal, PETSc.InsertMode.INSERT_VALUES, PETSc.ScatterMode.SCATTER_REVERSE)
NULL,mult_max = vglobal.max()
# k-scaling or multiplicity scaling of the local (non-assembled) matrix
if self.kscaling == False:
A_scaled.diagonalScale(vlocal,vlocal)
else:
v1 = A_mpiaij_local.getDiagonal()
v2 = A_scaled.getDiagonal()
A_scaled.diagonalScale(v1/v2, v1/v2)
# the default local solver is the scaled non assembled local matrix (as in BNN)
if self.switchtoASM:
Alocal = A_mpiaij_local
if mpi.COMM_WORLD.rank == 0:
print('The user has chosen to switch to Additive Schwarz instead of BNN.')
else: #(default)
Alocal = A_scaled
localksp = PETSc.KSP().create(comm=PETSc.COMM_SELF)
localksp.setOptionsPrefix("localksp_")
localksp.setOperators(Alocal)
localksp.setType('preonly')
localpc = localksp.getPC()
localpc.setType('cholesky')
localpc.setFactorSolverType('mumps')
localksp.setFromOptions()
self.A = A
self.A_scaled = A_scaled
self.A_mpiaij_local = A_mpiaij_local
self.localksp = localksp
self.workl_1 = vlocal.copy()
self.workl_2 = self.workl_1.copy()
self.scatter_l2g = scatter_l2g
self.mult_max = mult_max
self.proj = projection(self)
def solve(self, x0=None, atol=None, rtol=None, max_it=None):
r"""
This method solves the sparse linear system, converts the
solution vector from a PETSc.Vec instance to a numpy array,
and finally destroys all the petsc objects to free memory.
Parameters
----------
solver_type : string, optional
Default is the iterative solver 'cg' based on the
Conjugate Gradient method.
preconditioner_type : string, optional
Default is the 'jacobi' preconditioner, i.e., diagonal
scaling preconditioning. The preconditioner is used with
iterative solvers. When a direct solver is used, this
parameter is ignored.
factorization_type : string, optional
The factorization type used with the direct solver.
Default is 'lu'. This parameter is ignored when an
iterative solver is used.
Returns
-------
Returns a numpy array corresponding to the solution of
the linear sparse system Ax = b.
Notes
-----
Certain combinations of iterative solvers and precondioners
or direct solvers and factorization types are not supported.
The summary table of the different possibilities
can be found here:
https://www.mcs.anl.gov/petsc/documentation/linearsolvertable.html
"""
self.x0 = np.zeros_like(self.b) if x0 is None else x0
self._initialize_b_x()
self._initialize_A()
self._create_solver()
self._set_tolerances(atol=atol, rtol=rtol, max_it=max_it)
self.ksp.setOperators(self.petsc_A)
self.ksp.setFromOptions()
# Solve the linear system
self.ksp.solve(self.petsc_b, self.petsc_x)
# Gather the solution to all processors
gather_to_0, self.petsc_s = PETSc.Scatter().toAll(self.petsc_x)
gather_to_0.scatter(self.petsc_x, self.petsc_s,
PETSc.InsertMode.INSERT, PETSc.ScatterMode.FORWARD)
# Convert solution vector from PETSc.Vec instance to a numpy array
self.solution = PETSc.Vec.getArray(self.petsc_s)
# Destroy petsc solver, coefficients matrix, rhs, and solution vectors
PETSc.KSP.destroy(self.ksp)
PETSc.Mat.destroy(self.petsc_A)
PETSc.Vec.destroy(self.petsc_b)
PETSc.Vec.destroy(self.petsc_x)
PETSc.Vec.destroy(self.petsc_s)
return self.solution
