def __init__(self, src_vec, tgt_vec,
src_idxs, tgt_idxs, vec_conns, byobj_conns):
super(PetscDataXfer, self).__init__(src_idxs, tgt_idxs,
vec_conns, byobj_conns)
self.comm = comm = src_vec.comm
uvec = src_vec.petsc_vec
pvec = tgt_vec.petsc_vec
name = src_vec.pathname
if trace:
debug("'%s': creating index sets for '%s' DataXfer: %s %s" %
(name, src_vec.pathname, src_idxs, tgt_idxs))
src_idx_set = PETSc.IS().createGeneral(src_idxs, comm=comm)
tgt_idx_set = PETSc.IS().createGeneral(tgt_idxs, comm=comm)
try:
if trace:
self.src_idxs = src_idxs
self.tgt_idxs = tgt_idxs
debug("'%s': creating scatter %s --> %s %s --> %s" %
(name, [v for u, v in vec_conns], [u for u, v in vec_conns],
src_idx_set.indices, tgt_idx_set.indices))
self.scatter = PETSc.Scatter().create(uvec, src_idx_set,
pvec, tgt_idx_set)
except Exception as err:
raise RuntimeError("ERROR in %s (src_idxs=%s, tgt_idxs=%s, usize=%d, psize=%d): %s" %
(name, src_idxs, tgt_idxs,
src_vec.vec.size,
tgt_vec.vec.size, str(err)))
def _get_flattened_sizes(self):
"""
Collect all flattened sizes of vars stored in our internal array.
Returns
-------
list of `OrderedDict`
Contains an entry for each process in this object's communicator.
Each entry is an `OrderedDict` mapping var name to local size for
'pass by vector' variables.
"""
sizes = []
for name, meta in self.get_vecvars():
if meta.get('remote'):
sizes.append((name, 0))
else:
sizes.append((name, meta['size']))
# collect local var sizes from all of the processes that share the same comm
# these sizes will be the same in all processes except in cases
# where a variable belongs to a multiprocessor component. In that
# case, the part of the component that runs in a given process will
# only have a slice of each of the component's variables.
if trace:
debug("'%s': allgathering local unknown sizes: local=%s" % (self.pathname,
sizes))
return self.comm.allgather(sizes)
def _get_flattened_sizes(self):
"""
Collect all flattened sizes of vars stored in our internal array.
Returns
-------
list of `OrderedDict`
Contains an entry for each process in this object's communicator.
Each entry is an `OrderedDict` mapping var name to local size for
'pass by vector' variables.
"""
sizes = []
for name, meta in self.get_vecvars():
if meta.get('remote'):
sizes.append((name, 0))
else:
sizes.append((name, meta['size']))
# collect local var sizes from all of the processes that share the same comm
# these sizes will be the same in all processes except in cases
# where a variable belongs to a multiprocessor component. In that
# case, the part of the component that runs in a given process will
# only have a slice of each of the component's variables.
if trace:
debug("'%s': allgathering local unknown sizes: local=%s" %
(self.pathname, sizes))
return self.comm.allgather(sizes)
def setup(self, unknowns_dict, relevance, var_of_interest=None,
store_byobjs=False):
"""
Create internal data storage for variables in unknowns_dict.
Args
----
unknowns_dict : `OrderedDict`
A dictionary of absolute variable names keyed to an associated
metadata dictionary.
relevant_vars : iter of str
Names of variables that are relevant a particular variable of
interest.
store_byobjs : bool, optional
Indicates that 'pass by object' vars should be stored. This is only true
for the unknowns vecwrapper.
"""
super(PetscSrcVecWrapper, self).setup(unknowns_dict, relevance=relevance,
var_of_interest=var_of_interest,
store_byobjs=store_byobjs)
if trace:
debug("'%s': creating src petsc_vec: size(%d) %s vec=%s" %
(self.pathname, len(self.vec), self.keys(), self.vec))
self.petsc_vec = PETSc.Vec().createWithArray(self.vec, comm=self.comm)
def get_view(self, sys_pathname, comm, varmap):
view = super(PetscSrcVecWrapper, self).get_view(sys_pathname, comm, varmap)
if trace:
debug("'%s': creating src petsc_vec (view): (size %d )%s: vec=%s" %
(sys_pathname, len(view.vec), view.keys(), view.vec))
view.petsc_vec = PETSc.Vec().createWithArray(view.vec, comm=comm)
return view
def __init__(self, src_vec, tgt_vec,
src_idxs, tgt_idxs, vec_conns, byobj_conns, mode):
super(PetscDataTransfer, self).__init__(src_idxs, tgt_idxs,
vec_conns, byobj_conns, mode)
self.comm = comm = src_vec.comm
uvec = src_vec.petsc_vec
pvec = tgt_vec.petsc_vec
name = src_vec.pathname
if trace:
debug("'%s': creating index sets for '%s' DataTransfer: %s %s" %
(name, src_vec.pathname, src_idxs, tgt_idxs))
src_idx_set = PETSc.IS().createGeneral(src_idxs, comm=comm)
tgt_idx_set = PETSc.IS().createGeneral(tgt_idxs, comm=comm)
try:
if trace:
self.src_idxs = src_idxs
self.tgt_idxs = tgt_idxs
arrow = '-->' if mode == 'fwd' else '<--'
debug("'%s': new %s scatter (sizes: %d, %d)\n%s %s %s %s %s %s" %
(name, mode, len(src_idx_set.indices), len(tgt_idx_set.indices),
[v for u, v in vec_conns], arrow, [u for u, v in vec_conns],
src_idx_set.indices, arrow, tgt_idx_set.indices))
self.scatter = PETSc.Scatter().create(uvec, src_idx_set,
pvec, tgt_idx_set)
except Exception as err:
raise RuntimeError("ERROR in %s (src_idxs=%s, tgt_idxs=%s, usize=%d, psize=%d): %s" %
(name, src_idxs, tgt_idxs,
src_vec.vec.size,
tgt_vec.vec.size, str(err)))
def setup(self,
unknowns_dict,
relevance,
var_of_interest=None,
store_byobjs=False):
"""
Create internal data storage for variables in unknowns_dict.
Args
----
unknowns_dict : `OrderedDict`
A dictionary of absolute variable names keyed to an associated
metadata dictionary.
relevant_vars : iter of str
Names of variables that are relevant a particular variable of
interest.
store_byobjs : bool, optional
Indicates that 'pass by object' vars should be stored. This is only true
for the unknowns vecwrapper.
"""
super(PetscSrcVecWrapper, self).setup(unknowns_dict,
relevance=relevance,
var_of_interest=var_of_interest,
store_byobjs=store_byobjs)
if trace:
debug("'%s': creating src petsc_vec: size(%d) %s vec=%s" %
(self.pathname, len(self.vec), self.keys(), self.vec))
self.petsc_vec = PETSc.Vec().createWithArray(self.vec, comm=self.comm)
def setup(
self,
parent_params_vec,
params_dict,
srcvec,
my_params,
connections,
relevance,
var_of_interest=None,
store_byobjs=False,
):
"""
Configure this vector to store a flattened array of the variables
in params_dict. Variable shape and value are retrieved from srcvec.
Args
----
parent_params_vec : `VecWrapper` or None
`VecWrapper` of parameters from the parent `System`.
params_dict : `OrderedDict`
Dictionary of parameter absolute name mapped to metadata dict.
srcvec : `VecWrapper`
Source `VecWrapper` corresponding to the target `VecWrapper` we're building.
my_params : list of str
A list of absolute names of parameters that the `VecWrapper` we're building
will 'own'.
connections : dict of str : str
A dict of absolute target names mapped to the absolute name of their
source variable.
relevance : `Relevance` object
Object that knows what vars are relevant for each var_of_interest.
var_of_interest : str or None
Name of the current variable of interest.
store_byobjs : bool, optional
If True, store 'pass by object' variables in the `VecWrapper` we're building.
"""
super(PetscTgtVecWrapper, self).setup(
parent_params_vec,
params_dict,
srcvec,
my_params,
connections,
relevance=relevance,
var_of_interest=var_of_interest,
store_byobjs=store_byobjs,
)
if trace:
debug(
"'%s': creating tgt petsc_vec: (size %d) %s: vec=%s"
% (self.pathname, len(self.vec), self.keys(), self.vec)
)
self.petsc_vec = PETSc.Vec().createWithArray(self.vec, comm=self.comm)
def get_view(self, sys_pathname, comm, varmap):
view = super(PetscSrcVecWrapper,
self).get_view(sys_pathname, comm, varmap)
if trace:
debug("'%s': creating src petsc_vec (view): (size %d )%s: vec=%s" %
(sys_pathname, len(view.vec), view.keys(), view.vec))
view.petsc_vec = PETSc.Vec().createWithArray(view.vec, comm=comm)
return view
def get_view(self, sys_pathname, comm, varmap, relevance, var_of_interest):
view = super(PetscSrcVecWrapper, self).get_view(sys_pathname, comm, varmap,
relevance, var_of_interest)
if trace:
debug("'%s': creating src petsc_vec (view): %s: voi=%s, vec=%s" %
(sys_pathname, view.keys(), var_of_interest, view.vec))
view.petsc_vec = PETSc.Vec().createWithArray(view.vec, comm=comm)
return view
def _setup_variables(self, compute_indices=False):
"""
Returns copies of our params and unknowns dictionaries,
re-keyed to use absolute variable names.
Args
----
compute_indices : bool, optional
If True, call setup_distrib_idxs() to set values of
'src_indices' metadata.
"""
self._to_abs_unames = {}
self._to_abs_pnames = {}
if MPI and compute_indices and self.is_active():
self.setup_distrib_idxs()
# now update our distrib_size metadata for any distributed
# unknowns
sizes = []
names = []
for name, meta in iteritems(self._unknowns_dict):
if 'src_indices' in meta:
sizes.append(len(meta['src_indices']))
names.append(name)
if sizes:
if trace:
debug("allgathering src index sizes:")
allsizes = np.zeros((self.comm.size, len(sizes)), dtype=int)
self.comm.Allgather(np.array(sizes, dtype=int), allsizes)
for i, name in enumerate(names):
self._unknowns_dict[name]['distrib_size'] = np.sum(
allsizes[:, i])
# rekey with absolute path names and add promoted names
_new_params = OrderedDict()
for name, meta in iteritems(self._params_dict):
pathname = self._get_var_pathname(name)
_new_params[pathname] = meta
meta['pathname'] = pathname
meta['promoted_name'] = name
self._params_dict[name]['promoted_name'] = name
self._to_abs_pnames[name] = (pathname, )
_new_unknowns = OrderedDict()
for name, meta in iteritems(self._unknowns_dict):
pathname = self._get_var_pathname(name)
_new_unknowns[pathname] = meta
meta['pathname'] = pathname
meta['promoted_name'] = name
self._to_abs_unames[name] = (pathname, )
self._post_setup_vars = True
return _new_params, _new_unknowns
def _setup_variables(self, compute_indices=False):
"""
Returns copies of our params and unknowns dictionaries,
re-keyed to use absolute variable names.
Args
----
compute_indices : bool, optional
If True, call setup_distrib_idxs() to set values of
'src_indices' metadata.
"""
self._to_abs_unames = {}
self._to_abs_pnames = {}
if MPI and compute_indices and self.is_active():
self.setup_distrib_idxs()
# now update our distrib_size metadata for any distributed
# unknowns
sizes = []
names = []
for name, meta in iteritems(self._unknowns_dict):
if 'src_indices' in meta:
sizes.append(len(meta['src_indices']))
names.append(name)
if sizes:
if trace:
debug("allgathering src index sizes:")
allsizes = np.zeros((self.comm.size, len(sizes)), dtype=int)
self.comm.Allgather(np.array(sizes, dtype=int), allsizes)
for i, name in enumerate(names):
self._unknowns_dict[name]['distrib_size'] = np.sum(allsizes[:, i])
# rekey with absolute path names and add promoted names
_new_params = OrderedDict()
for name, meta in iteritems(self._params_dict):
pathname = self._get_var_pathname(name)
_new_params[pathname] = meta
meta['pathname'] = pathname
meta['promoted_name'] = name
self._params_dict[name]['promoted_name'] = name
self._to_abs_pnames[name] = (pathname,)
_new_unknowns = OrderedDict()
for name, meta in iteritems(self._unknowns_dict):
pathname = self._get_var_pathname(name)
_new_unknowns[pathname] = meta
meta['pathname'] = pathname
meta['promoted_name'] = name
self._to_abs_unames[name] = (pathname,)
self._post_setup_vars = True
return _new_params, _new_unknowns
def norm(self):
"""
Returns
-------
float
The norm of the distributed vector.
"""
if trace:
debug("%s: norm: petsc_vec.assemble" % self.pathname)
self.petsc_vec.assemble()
return self.petsc_vec.norm()
def norm(self):
"""
Returns
-------
float
The norm of the distributed vector.
"""
if trace:
debug("%s: norm: petsc_vec.assemble" % self.pathname)
self.petsc_vec.assemble()
return self.petsc_vec.norm()
def solve(self, rhs_mat, system, mode):
""" Solves the linear system for the problem in self.system. The
full solution vector is returned.
Args
----
rhs_mat : dict of ndarray
Dictionary containing one ndarry per top level quantity of
interest. Each array contains the right-hand side for the linear
solve.
system : `System`
Parent `System` object.
mode : string
Derivative mode, can be 'fwd' or 'rev'.
Returns
-------
dict of ndarray : Solution vectors
"""
options = self.options
self.mode = mode
self.ksp.setTolerances(max_it=options['maxiter'],
atol=options['atol'],
rtol=options['rtol'])
unknowns_mat = {}
for voi, rhs in iteritems(rhs_mat):
sol_vec = np.zeros(rhs.shape)
# Set these in the system
if trace:
debug("creating sol_buf petsc vec for voi", voi)
self.sol_buf_petsc = PETSc.Vec().createWithArray(sol_vec,
comm=system.comm)
if trace:
debug("creating rhs_buf petsc vec for voi", voi)
self.rhs_buf_petsc = PETSc.Vec().createWithArray(rhs,
comm=system.comm)
# Petsc can only handle one right-hand-side at a time for now
self.voi = voi
self.system = system
self.ksp.solve(self.rhs_buf_petsc, self.sol_buf_petsc)
self.system = None
unknowns_mat[voi] = sol_vec
#print system.name, 'Linear solution vec', d_unknowns
self.system = None
return unknowns_mat
def solve(self, rhs_mat, system, mode):
""" Solves the linear system for the problem in self.system. The
full solution vector is returned.
Args
----
rhs_mat : dict of ndarray
Dictionary containing one ndarry per top level quantity of
interest. Each array contains the right-hand side for the linear
solve.
system : `System`
Parent `System` object.
mode : string
Derivative mode, can be 'fwd' or 'rev'.
Returns
-------
dict of ndarray : Solution vectors
"""
options = self.options
self.mode = mode
self.ksp.setTolerances(max_it=options['maxiter'],
atol=options['atol'],
rtol=options['rtol'])
unknowns_mat = {}
for voi, rhs in iteritems(rhs_mat):
sol_vec = np.zeros(rhs.shape)
# Set these in the system
if trace:
debug("creating sol_buf petsc vec for voi", voi)
self.sol_buf_petsc = PETSc.Vec().createWithArray(sol_vec,
comm=system.comm)
if trace:
debug("creating rhs_buf petsc vec for voi", voi)
self.rhs_buf_petsc = PETSc.Vec().createWithArray(rhs,
comm=system.comm)
# Petsc can only handle one right-hand-side at a time for now
self.voi = voi
self.system = system
self.ksp.solve(self.rhs_buf_petsc, self.sol_buf_petsc)
self.system = None
unknowns_mat[voi] = sol_vec
#print system.name, 'Linear solution vec', d_unknowns
self.system = None
return unknowns_mat
def transfer(self, srcvec, tgtvec, mode='fwd', deriv=False):
"""Performs data transfer between a distributed source vector and
a distributed target vector.
Args
----
srcvec : `VecWrapper`
Variables that are the source of the transfer in fwd mode and
the destination of the transfer in rev mode.
tgtvec : `VecWrapper`
Variables that are the destination of the transfer in fwd mode and
the source of the transfer in rev mode.
mode : 'fwd' or 'rev', optional
Direction of the data transfer, source to target ('fwd', the default)
or target to source ('rev').
deriv : bool, optional
If True, this is a derivative data transfer, so no pass_by_obj
variables will be transferred.
"""
if mode == 'rev':
# in reverse mode, srcvec and tgtvec are switched. Note, we only
# run in reverse for derivatives, and derivatives accumulate from
# all targets. This does not involve pass_by_object.
if trace:
conns = ['%s <-- %s' % (u, v) for v, u in self.vec_conns]
debug("'%s': rev scatter %s %s <-- %s" %
(srcvec.pathname, conns, self.src_idxs, self.tgt_idxs))
debug("%s: srcvec = %s\ntgtvec = %s" %
(srcvec.pathname, srcvec.petsc_vec.array,
tgtvec.petsc_vec.array))
self.scatter.scatter(tgtvec.petsc_vec, srcvec.petsc_vec, True,
True)
else:
# forward mode, source to target including pass_by_object
if trace:
conns = ['%s --> %s' % (u, v) for v, u in self.vec_conns]
debug("'%s': fwd scatter %s %s --> %s" %
(srcvec.pathname, conns, self.src_idxs, self.tgt_idxs))
debug("%s: srcvec = %s\n%s: tgtvec = %s" %
(srcvec.pathname, srcvec.petsc_vec.array,
srcvec.pathname, tgtvec.petsc_vec.array))
self.scatter.scatter(srcvec.petsc_vec, tgtvec.petsc_vec, False,
False)
if trace:
debug("scatter done")
if not deriv:
for tgt, src in self.byobj_conns:
raise NotImplementedError("can't transfer '%s' to '%s'" %
(src, tgt))
def transfer(self, srcvec, tgtvec, mode='fwd', deriv=False):
"""Performs data transfer between a distributed source vector and
a distributed target vector.
Args
----
srcvec : `VecWrapper`
Variables that are the source of the transfer in fwd mode and
the destination of the transfer in rev mode.
tgtvec : `VecWrapper`
Variables that are the destination of the transfer in fwd mode and
the source of the transfer in rev mode.
mode : 'fwd' or 'rev', optional
Direction of the data transfer, source to target ('fwd', the default)
or target to source ('rev').
deriv : bool, optional
If True, this is a derivative data transfer, so no pass_by_obj
variables will be transferred.
"""
if mode == 'rev':
# in reverse mode, srcvec and tgtvec are switched. Note, we only
# run in reverse for derivatives, and derivatives accumulate from
# all targets. This does not involve pass_by_object.
if trace:
conns = ['%s <-- %s' % (u, v) for v, u in self.vec_conns]
debug("'%s': rev scatter %s %s <-- %s" %
(srcvec.pathname, conns, self.src_idxs, self.tgt_idxs))
debug("%s: srcvec = %s\ntgtvec = %s" % (srcvec.pathname,
srcvec.petsc_vec.array,
tgtvec.petsc_vec.array))
self.scatter.scatter(tgtvec.petsc_vec, srcvec.petsc_vec, True, True)
else:
# forward mode, source to target including pass_by_object
if trace:
conns = ['%s --> %s' % (u, v) for v, u in self.vec_conns]
debug("'%s': fwd scatter %s %s --> %s" %
(srcvec.pathname, conns, self.src_idxs, self.tgt_idxs))
debug("%s: srcvec = %s\n%s: tgtvec = %s" % (srcvec.pathname,
srcvec.petsc_vec.array,
srcvec.pathname,
tgtvec.petsc_vec.array))
self.scatter.scatter(srcvec.petsc_vec, tgtvec.petsc_vec, False, False)
if trace:
debug("scatter done")
if not deriv:
for tgt, src in self.byobj_conns:
raise NotImplementedError("can't transfer '%s' to '%s'" %
(src, tgt))
def setup(self, system):
""" Setup petsc problem just once."""
lsize = np.sum(system._local_unknown_sizes[None][system.comm.rank, :])
size = np.sum(system._local_unknown_sizes[None])
jac_mat = PETSc.Mat().createPython([(lsize, size), (lsize, size)],
comm=system.comm)
jac_mat.setPythonContext(self)
jac_mat.setUp()
if trace:
debug("creating KSP object for system", system.pathname)
self.ksp = PETSc.KSP().create(comm=system.comm)
self.ksp.setOperators(jac_mat)
self.ksp.setType('fgmres')
self.ksp.setGMRESRestart(1000)
self.ksp.setPCSide(PETSc.PC.Side.RIGHT)
self.ksp.setMonitor(Monitor(self))
if trace:
debug("ksp.getPC()")
debug("rhs_buf, sol_buf size: %d" % lsize)
pc_mat = self.ksp.getPC()
pc_mat.setType('python')
pc_mat.setPythonContext(self)
if trace:
debug("ksp setup done")
self.rhs_buf = np.zeros((lsize, ))
self.sol_buf = np.zeros((lsize, ))
def setup(self, system):
""" Setup petsc problem just once."""
lsize = np.sum(system._local_unknown_sizes[None][system.comm.rank, :])
size = np.sum(system._local_unknown_sizes[None])
jac_mat = PETSc.Mat().createPython([(lsize, size), (lsize, size)],
comm=system.comm)
jac_mat.setPythonContext(self)
jac_mat.setUp()
if trace:
debug("creating KSP object for system",system.pathname)
self.ksp = PETSc.KSP().create(comm=system.comm)
self.ksp.setOperators(jac_mat)
self.ksp.setType('fgmres')
self.ksp.setGMRESRestart(1000)
self.ksp.setPCSide(PETSc.PC.Side.RIGHT)
self.ksp.setMonitor(Monitor(self))
if trace:
debug("ksp.getPC()")
debug("rhs_buf, sol_buf size: %d" % lsize)
pc_mat = self.ksp.getPC()
pc_mat.setType('python')
pc_mat.setPythonContext(self)
if trace:
debug("ksp setup done")
self.rhs_buf = np.zeros((lsize, ))
self.sol_buf = np.zeros((lsize, ))
def _get_flattened_sizes(self):
"""
Returns
-------
list of `OrderedDict`
Contains an entry for each process in this object's communicator.
Each entry is an `OrderedDict` mapping var name to local size for
'pass by vector' params.
"""
psizes = super(PetscTgtVecWrapper, self)._get_flattened_sizes()[0]
if trace:
debug("'%s': allgathering param sizes. local param sizes = %s" % (self.pathname,
psizes))
return self.comm.allgather(psizes)
def setup(self,
parent_params_vec,
params_dict,
srcvec,
my_params,
connections,
relevance,
var_of_interest=None,
store_byobjs=False):
"""
Configure this vector to store a flattened array of the variables
in params_dict. Variable shape and value are retrieved from srcvec.
Args
----
parent_params_vec : `VecWrapper` or None
`VecWrapper` of parameters from the parent `System`.
params_dict : `OrderedDict`
Dictionary of parameter absolute name mapped to metadata dict.
srcvec : `VecWrapper`
Source `VecWrapper` corresponding to the target `VecWrapper` we're building.
my_params : list of str
A list of absolute names of parameters that the `VecWrapper` we're building
will 'own'.
connections : dict of str : str
A dict of absolute target names mapped to the absolute name of their
source variable.
store_byobjs : bool, optional
If True, store 'pass by object' variables in the `VecWrapper` we're building.
"""
super(PetscTgtVecWrapper, self).setup(parent_params_vec,
params_dict,
srcvec,
my_params,
connections,
relevance=relevance,
var_of_interest=var_of_interest,
store_byobjs=store_byobjs)
if trace:
debug("'%s': creating tgt petsc_vec: (size %d) %s: vec=%s" %
(self.pathname, len(self.vec), self.keys(), self.vec))
self.petsc_vec = PETSc.Vec().createWithArray(self.vec, comm=self.comm)
def _get_flattened_sizes(self):
"""
Returns
-------
list of `OrderedDict`
Contains an entry for each process in this object's communicator.
Each entry is an `OrderedDict` mapping var name to local size for
'pass by vector' params.
"""
psizes = []
for name, m in iteritems(self._vardict):
if m.get('owned') and not m.get('pass_by_obj'):
if m.get('remote'):
psizes.append((name, 0))
else:
psizes.append((name, m['size']))
if trace:
msg = "'%s': allgathering param sizes. local param sizes = %s"
debug(msg % (self.pathname, psizes))
return self.comm.allgather(psizes)
def _get_flattened_sizes(self):
"""
Returns
-------
list of `OrderedDict`
Contains an entry for each process in this object's communicator.
Each entry is an `OrderedDict` mapping var name to local size for
'pass by vector' params.
"""
psizes = []
for name, m in iteritems(self._vardict):
if m.get('owned') and not m.get('pass_by_obj'):
if m.get('remote'):
psizes.append((name, 0))
else:
psizes.append((name, m['size']))
if trace:
msg = "'%s': allgathering param sizes. local param sizes = %s"
debug(msg % (self.pathname, psizes))
return self.comm.allgather(psizes)
def _get_flattened_sizes(self):
"""
Returns
-------
list of lists of (name, size) tuples
Contains an entry for each process in this object's communicator.
Each entry is an `OrderedDict` mapping var name to local size for
'pass by vector' params.
"""
psizes = []
for name, m in self._get_vecvars():
if m.get("owned"):
if m.get("remote"):
psizes.append((name, 0))
else:
psizes.append((name, m["size"]))
if trace:
msg = "'%s': allgathering param sizes. local param sizes = %s"
debug(msg % (self.pathname, psizes))
return self.comm.allgather(psizes)
def __init__(self, src_vec, tgt_vec, src_idxs, tgt_idxs, vec_conns,
byobj_conns, mode):
super(PetscDataTransfer, self).__init__(src_idxs, tgt_idxs, vec_conns,
byobj_conns, mode)
self.comm = comm = src_vec.comm
uvec = src_vec.petsc_vec
pvec = tgt_vec.petsc_vec
name = src_vec.pathname
if trace:
debug("'%s': creating index sets for '%s' DataTransfer: %s %s" %
(name, src_vec.pathname, src_idxs, tgt_idxs))
src_idx_set = PETSc.IS().createGeneral(src_idxs, comm=comm)
tgt_idx_set = PETSc.IS().createGeneral(tgt_idxs, comm=comm)
try:
if trace:
self.src_idxs = src_idxs
self.tgt_idxs = tgt_idxs
arrow = '-->' if mode == 'fwd' else '<--'
debug(
"'%s': new %s scatter (sizes: %d, %d)\n%s %s %s %s %s %s" %
(name, mode, len(src_idx_set.indices),
len(tgt_idx_set.indices), [v
for u, v in vec_conns], arrow,
[u for u, v in vec_conns
], src_idx_set.indices, arrow, tgt_idx_set.indices))
self.scatter = PETSc.Scatter().create(uvec, src_idx_set, pvec,
tgt_idx_set)
except Exception as err:
raise RuntimeError(
"ERROR in %s (src_idxs=%s, tgt_idxs=%s, usize=%d, psize=%d): %s"
% (name, src_idxs, tgt_idxs, src_vec.vec.size,
tgt_vec.vec.size, str(err)))
