def test_convert_neg(self):
a = np.arange(16).reshape((4, 4))
a[2, 3] = -5
a[0, 1] = -7
b = convert_neg(a.copy(), a.size)
self.assertEqual(b[2, 3], 11)
self.assertEqual(b[0, 1], 9)
inds = np.where(a != b)
self.assertTrue(np.all(inds[0] == np.array([0, 2])))
self.assertTrue(np.all(inds[1] == np.array([1, 3])))
def _setup_transfers(group, recurse=True):
"""
Compute all transfers that are owned by our parent group.
Parameters
----------
group : <Group>
Parent group.
recurse : bool
Whether to call this method in subsystems.
"""
group._transfers = {}
iproc = group.comm.rank
rev = group._mode == 'rev' or group._mode == 'auto'
def merge(indices_list):
if len(indices_list) > 0:
return np.concatenate(indices_list)
else:
return _empty_idx_array
if recurse:
for subsys in group._subgroups_myproc:
subsys._setup_transfers(recurse)
# Pre-compute map from abs_names to the index of the containing subsystem
abs2isub = {}
for subsys, isub in zip(group._subsystems_myproc,
group._subsystems_myproc_inds):
for type_ in ['input', 'output']:
for abs_name in subsys._var_allprocs_abs_names[type_]:
abs2isub[abs_name] = isub
abs2meta = group._var_abs2meta
allprocs_abs2meta = group._var_allprocs_abs2meta
transfers = group._transfers
vectors = group._vectors
offsets = _global2local_offsets(group._get_var_offsets())
for vec_name in group._lin_rel_vec_name_list:
relvars, _ = group._relevant[vec_name]['@all']
relvars_in = relvars['input']
relvars_out = relvars['output']
# Initialize empty lists for the transfer indices
nsub_allprocs = len(group._subsystems_allprocs)
xfer_in = []
xfer_out = []
fwd_xfer_in = [[] for i in range(nsub_allprocs)]
fwd_xfer_out = [[] for i in range(nsub_allprocs)]
if rev:
rev_xfer_in = [[] for i in range(nsub_allprocs)]
rev_xfer_out = [[] for i in range(nsub_allprocs)]
allprocs_abs2idx = group._var_allprocs_abs2idx[vec_name]
sizes_in = group._var_sizes[vec_name]['input']
sizes_out = group._var_sizes[vec_name]['output']
offsets_in = offsets[vec_name]['input']
offsets_out = offsets[vec_name]['output']
# Loop through all explicit / implicit connections owned by this system
for abs_in, abs_out in iteritems(group._conn_abs_in2out):
if abs_out not in relvars_out or abs_in not in relvars_in:
continue
# Only continue if the input exists on this processor
if abs_in in abs2meta and abs_in in relvars['input']:
# Get meta
meta_in = abs2meta[abs_in]
meta_out = allprocs_abs2meta[abs_out]
idx_in = allprocs_abs2idx[abs_in]
idx_out = allprocs_abs2idx[abs_out]
# Read in and process src_indices
shape_in = meta_in['shape']
shape_out = meta_out['shape']
global_size_out = meta_out['global_size']
src_indices = meta_in['src_indices']
if src_indices is None:
pass
elif src_indices.ndim == 1:
src_indices = convert_neg(src_indices, global_size_out)
else:
if len(shape_out
) == 1 or shape_in == src_indices.shape:
src_indices = src_indices.flatten()
src_indices = convert_neg(src_indices,
global_size_out)
else:
# TODO: this duplicates code found
# in System._setup_scaling.
entries = [list(range(x)) for x in shape_in]
cols = np.vstack(src_indices[i]
for i in product(*entries))
dimidxs = [
convert_neg(cols[:, i], shape_out[i])
for i in range(cols.shape[1])
]
src_indices = np.ravel_multi_index(
dimidxs, shape_out)
# 1. Compute the output indices
offset = offsets_out[iproc, idx_out]
if src_indices is None:
output_inds = np.arange(offset,
offset + meta_in['size'],
dtype=INT_DTYPE)
else:
output_inds = src_indices + offset
# 2. Compute the input indices
input_inds = np.arange(offsets_in[iproc, idx_in],
offsets_in[iproc, idx_in] +
sizes_in[iproc, idx_in],
dtype=INT_DTYPE)
# Now the indices are ready - input_inds, output_inds
xfer_in.append(input_inds)
xfer_out.append(output_inds)
isub = abs2isub[abs_in]
fwd_xfer_in[isub].append(input_inds)
fwd_xfer_out[isub].append(output_inds)
if rev and abs_out in abs2isub:
isub = abs2isub[abs_out]
rev_xfer_in[isub].append(input_inds)
rev_xfer_out[isub].append(output_inds)
xfer_in = merge(xfer_in)
xfer_out = merge(xfer_out)
for isub in range(nsub_allprocs):
fwd_xfer_in[isub] = merge(fwd_xfer_in[isub])
fwd_xfer_out[isub] = merge(fwd_xfer_out[isub])
if rev:
rev_xfer_in[isub] = merge(rev_xfer_in[isub])
rev_xfer_out[isub] = merge(rev_xfer_out[isub])
out_vec = vectors['output'][vec_name]
transfers[vec_name] = {}
xfer_all = DefaultTransfer(vectors['input'][vec_name], out_vec,
xfer_in, xfer_out, group.comm)
transfers[vec_name]['fwd', None] = xfer_all
if rev:
transfers[vec_name]['rev', None] = xfer_all
for isub in range(nsub_allprocs):
transfers[vec_name]['fwd', isub] = DefaultTransfer(
vectors['input'][vec_name], vectors['output'][vec_name],
fwd_xfer_in[isub], fwd_xfer_out[isub], group.comm)
if rev:
transfers[vec_name]['rev', isub] = DefaultTransfer(
vectors['input'][vec_name],
vectors['output'][vec_name], rev_xfer_in[isub],
rev_xfer_out[isub], group.comm)
transfers['nonlinear'] = transfers['linear']
def _setup_driver(self, problem):
"""
Prepare the driver for execution.
This is the final thing to run during setup.
Parameters
----------
problem : <Problem>
Pointer to the containing problem.
"""
self._problem = weakref.ref(problem)
model = problem.model
self._total_jac = None
self._has_scaling = (np.any(
[r['total_scaler'] is not None for r in self._responses.values()])
or np.any([
dv['total_scaler'] is not None
for dv in self._designvars.values()
]))
# Determine if any design variables are discrete.
self._designvars_discrete = [
name for name, meta in self._designvars.items()
if meta['ivc_source'] in model._discrete_outputs
]
if not self.supports['integer_design_vars'] and len(
self._designvars_discrete) > 0:
msg = "Discrete design variables are not supported by this driver: "
msg += '.'.join(self._designvars_discrete)
raise RuntimeError(msg)
con_set = set()
obj_set = set()
dv_set = set()
self._remote_dvs = remote_dv_dict = {}
self._remote_cons = remote_con_dict = {}
self._dist_driver_vars = dist_dict = {}
self._remote_objs = remote_obj_dict = {}
src_design_vars = prom2ivc_src_dict(self._designvars)
src_cons = prom2ivc_src_dict(self._cons)
src_objs = prom2ivc_src_dict(self._objs)
responses = prom2ivc_src_dict(self._responses)
# Now determine if later we'll need to allgather cons, objs, or desvars.
if model.comm.size > 1 and model._subsystems_allprocs:
local_out_vars = set(model._outputs._abs_iter())
remote_dvs = set(src_design_vars) - local_out_vars
remote_cons = set(src_cons) - local_out_vars
remote_objs = set(src_objs) - local_out_vars
all_remote_vois = model.comm.allgather(
(remote_dvs, remote_cons, remote_objs))
for rem_dvs, rem_cons, rem_objs in all_remote_vois:
con_set.update(rem_cons)
obj_set.update(rem_objs)
dv_set.update(rem_dvs)
# If we have remote VOIs, pick an owning rank for each and use that
# to bcast to others later
owning_ranks = model._owning_rank
sizes = model._var_sizes['nonlinear']['output']
abs2meta = model._var_allprocs_abs2meta
rank = model.comm.rank
nprocs = model.comm.size
for i, vname in enumerate(model._var_allprocs_abs_names['output']):
if vname in responses:
indices = responses[vname].get('indices')
elif vname in src_design_vars:
indices = src_design_vars[vname].get('indices')
else:
continue
if abs2meta[vname]['distributed']:
idx = model._var_allprocs_abs2idx['nonlinear'][vname]
dist_sizes = model._var_sizes['nonlinear']['output'][:,
idx]
total_dist_size = np.sum(dist_sizes)
# Determine which indices are on our proc.
offsets = sizes2offsets(dist_sizes)
if indices is not None:
indices = convert_neg(indices, total_dist_size)
true_sizes = np.zeros(nprocs, dtype=INT_DTYPE)
for irank in range(nprocs):
dist_inds = indices[np.logical_and(
indices >= offsets[irank], indices <
(offsets[irank] + dist_sizes[irank]))]
if irank == rank:
local_indices = dist_inds - offsets[rank]
distrib_indices = dist_inds
true_sizes[irank] = dist_inds.size
dist_dict[vname] = (local_indices, true_sizes,
distrib_indices)
else:
dist_dict[vname] = (_full_slice, dist_sizes,
slice(
offsets[rank], offsets[rank] +
dist_sizes[rank]))
else:
owner = owning_ranks[vname]
sz = sizes[owner, i]
if vname in dv_set:
remote_dv_dict[vname] = (owner, sz)
if vname in con_set:
remote_con_dict[vname] = (owner, sz)
if vname in obj_set:
remote_obj_dict[vname] = (owner, sz)
self._remote_responses = self._remote_cons.copy()
self._remote_responses.update(self._remote_objs)
# set up simultaneous deriv coloring
if coloring_mod._use_total_sparsity:
# reset the coloring
if self._coloring_info['dynamic'] or self._coloring_info[
'static'] is not None:
self._coloring_info['coloring'] = None
coloring = self._get_static_coloring()
if coloring is not None and self.supports[
'simultaneous_derivatives']:
if model._owns_approx_jac:
coloring._check_config_partial(model)
else:
coloring._check_config_total(self)
self._setup_simul_coloring()
def _setup_transfers(group):
"""
Compute all transfers that are owned by our parent group.
Parameters
----------
group : <Group>
Parent group.
"""
rev = group._mode != 'fwd'
for subsys in group._subgroups_myproc:
subsys._setup_transfers()
abs2meta = group._var_abs2meta
allprocs_abs2meta = group._var_allprocs_abs2meta
myproc = group.comm.rank
transfers = group._transfers = {}
vectors = group._vectors
offsets = group._get_var_offsets()
vec_names = group._lin_rel_vec_name_list if group._use_derivatives else group._vec_names
mypathlen = len(group.pathname + '.' if group.pathname else '')
sub_inds = group._subsystems_inds
for vec_name in vec_names:
relvars, _ = group._relevant[vec_name]['@all']
# Initialize empty lists for the transfer indices
nsub_allprocs = len(group._subsystems_allprocs)
xfer_in = []
xfer_out = []
fwd_xfer_in = [[] for s in group._subsystems_allprocs]
fwd_xfer_out = [[] for s in group._subsystems_allprocs]
if rev:
rev_xfer_in = [[] for s in group._subsystems_allprocs]
rev_xfer_out = [[] for s in group._subsystems_allprocs]
allprocs_abs2idx = group._var_allprocs_abs2idx[vec_name]
sizes_in = group._var_sizes[vec_name]['input']
sizes_out = group._var_sizes[vec_name]['output']
offsets_in = offsets[vec_name]['input']
offsets_out = offsets[vec_name]['output']
# Loop through all connections owned by this system
for abs_in, abs_out in group._conn_abs_in2out.items():
if abs_out not in relvars['output'] or abs_in not in relvars[
'input']:
continue
# Only continue if the input exists on this processor
if abs_in in abs2meta:
# Get meta
meta_in = abs2meta[abs_in]
meta_out = allprocs_abs2meta[abs_out]
idx_in = allprocs_abs2idx[abs_in]
idx_out = allprocs_abs2idx[abs_out]
# Read in and process src_indices
src_indices = meta_in['src_indices']
if src_indices is None:
owner = group._owning_rank[abs_out]
# if the input is larger than the output on a single proc, we have
# to just loop over the procs in the same way we do when src_indices
# is defined.
if meta_in['size'] > sizes_out[owner, idx_out]:
src_indices = np.arange(meta_in['size'],
dtype=INT_DTYPE)
elif src_indices.ndim == 1:
if _is_slice(src_indices):
indices = _slice_indices(src_indices,
meta_out['global_size'],
meta_out['global_shape'])
src_indices = convert_neg(indices,
meta_out['global_size'])
else:
src_indices = _flatten_src_indices(
src_indices, meta_in['shape'],
meta_out['global_shape'], meta_out['global_size'])
# 1. Compute the output indices
# NOTE: src_indices are relative to a single, possibly distributed variable,
# while the output_inds that we compute are relative to the full distributed
# array that contains all local variables from each rank stacked in rank order.
if src_indices is None:
if meta_out['distributed']:
# input in this case is non-distributed (else src_indices would be
# defined by now). The input size must match the full
# distributed size of the output.
for rank, sz in enumerate(sizes_out[:, idx_out]):
if sz > 0:
out_offset = offsets_out[rank, idx_out]
break
output_inds = np.arange(out_offset,
out_offset +
meta_out['global_size'],
dtype=INT_DTYPE)
else:
rank = myproc if abs_out in abs2meta else owner
offset = offsets_out[rank, idx_out]
output_inds = np.arange(offset,
offset + meta_in['size'],
dtype=INT_DTYPE)
else:
output_inds = np.zeros(src_indices.size, INT_DTYPE)
start = end = 0
for iproc in range(group.comm.size):
end += sizes_out[iproc, idx_out]
if start == end:
continue
# The part of src on iproc
on_iproc = np.logical_and(start <= src_indices,
src_indices < end)
if np.any(on_iproc):
# This converts from iproc-then-ivar to ivar-then-iproc ordering
# Subtract off part of previous procs
# Then add all variables on previous procs
# Then all previous variables on this proc
# - np.sum(out_sizes[:iproc, idx_out])
# + np.sum(out_sizes[:iproc, :])
# + np.sum(out_sizes[iproc, :idx_out])
# + inds
offset = offsets_out[iproc, idx_out] - start
output_inds[
on_iproc] = src_indices[on_iproc] + offset
start = end
# 2. Compute the input indices
input_inds = np.arange(offsets_in[myproc, idx_in],
offsets_in[myproc, idx_in] +
sizes_in[myproc, idx_in],
dtype=INT_DTYPE)
# Now the indices are ready - input_inds, output_inds
sub_in = abs_in[mypathlen:].split('.', 1)[0]
isub = sub_inds[sub_in]
fwd_xfer_in[isub].append(input_inds)
fwd_xfer_out[isub].append(output_inds)
if rev:
sub_out = abs_out[mypathlen:].split('.', 1)[0]
isub = sub_inds[sub_out]
rev_xfer_in[isub].append(input_inds)
rev_xfer_out[isub].append(output_inds)
transfers[vec_name] = {}
for isub in range(nsub_allprocs):
fwd_xfer_in[isub] = _merge(fwd_xfer_in[isub])
fwd_xfer_out[isub] = _merge(fwd_xfer_out[isub])
if rev:
rev_xfer_in[isub] = _merge(rev_xfer_in[isub])
rev_xfer_out[isub] = _merge(rev_xfer_out[isub])
xfer_in = np.concatenate(fwd_xfer_in)
xfer_out = np.concatenate(fwd_xfer_out)
out_vec = vectors['output'][vec_name]
xfer_all = PETScTransfer(vectors['input'][vec_name], out_vec,
xfer_in, xfer_out, group.comm)
transfers[vec_name]['fwd', None] = xfer_all
if rev:
transfers[vec_name]['rev', None] = xfer_all
for isub in range(nsub_allprocs):
transfers[vec_name]['fwd', isub] = PETScTransfer(
vectors['input'][vec_name], vectors['output'][vec_name],
fwd_xfer_in[isub], fwd_xfer_out[isub], group.comm)
if rev:
transfers[vec_name]['rev', isub] = PETScTransfer(
vectors['input'][vec_name],
vectors['output'][vec_name], rev_xfer_in[isub],
rev_xfer_out[isub], group.comm)
if group._use_derivatives:
transfers['nonlinear'] = transfers['linear']
def _setup_transfers(group, recurse=True):
"""
Compute all transfers that are owned by our parent group.
Parameters
----------
group : <Group>
Parent group.
recurse : bool
Whether to call this method in subsystems.
"""
group._transfers = {}
iproc = group.comm.rank
rev = group._mode == 'rev' or group._mode == 'auto'
def merge(indices_list):
if len(indices_list) > 0:
return np.concatenate(indices_list)
else:
return _empty_idx_array
if recurse:
for subsys in group._subgroups_myproc:
subsys._setup_transfers(recurse)
# Pre-compute map from abs_names to the index of the containing subsystem
abs2isub = {}
for subsys, isub in zip(group._subsystems_myproc, group._subsystems_myproc_inds):
for type_ in ['input', 'output']:
for abs_name in subsys._var_allprocs_abs_names[type_]:
abs2isub[abs_name] = isub
abs2meta = group._var_abs2meta
allprocs_abs2meta = group._var_allprocs_abs2meta
transfers = group._transfers
vectors = group._vectors
offsets = _global2local_offsets(group._get_var_offsets())
vec_names = group._lin_rel_vec_name_list if group._use_derivatives else group._vec_names
for vec_name in vec_names:
relvars, _ = group._relevant[vec_name]['@all']
relvars_in = relvars['input']
relvars_out = relvars['output']
# Initialize empty lists for the transfer indices
nsub_allprocs = len(group._subsystems_allprocs)
xfer_in = []
xfer_out = []
fwd_xfer_in = [[] for i in range(nsub_allprocs)]
fwd_xfer_out = [[] for i in range(nsub_allprocs)]
if rev:
rev_xfer_in = [[] for i in range(nsub_allprocs)]
rev_xfer_out = [[] for i in range(nsub_allprocs)]
allprocs_abs2idx = group._var_allprocs_abs2idx[vec_name]
sizes_in = group._var_sizes[vec_name]['input']
sizes_out = group._var_sizes[vec_name]['output']
offsets_in = offsets[vec_name]['input']
offsets_out = offsets[vec_name]['output']
# Loop through all explicit / implicit connections owned by this system
for abs_in, abs_out in iteritems(group._conn_abs_in2out):
if abs_out not in relvars_out or abs_in not in relvars_in:
continue
# Only continue if the input exists on this processor
if abs_in in abs2meta and abs_in in relvars['input']:
# Get meta
meta_in = abs2meta[abs_in]
meta_out = allprocs_abs2meta[abs_out]
idx_in = allprocs_abs2idx[abs_in]
idx_out = allprocs_abs2idx[abs_out]
# Read in and process src_indices
src_indices = meta_in['src_indices']
if src_indices is None:
pass
elif src_indices.ndim == 1:
src_indices = convert_neg(src_indices, meta_out['global_size'])
else:
src_indices = _flatten_src_indices(src_indices, meta_in['shape'],
meta_out['global_shape'],
meta_out['global_size'])
# 1. Compute the output indices
offset = offsets_out[iproc, idx_out]
if src_indices is None:
output_inds = np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE)
else:
output_inds = src_indices + offset
# 2. Compute the input indices
input_inds = np.arange(offsets_in[iproc, idx_in],
offsets_in[iproc, idx_in] +
sizes_in[iproc, idx_in], dtype=INT_DTYPE)
# Now the indices are ready - input_inds, output_inds
xfer_in.append(input_inds)
xfer_out.append(output_inds)
isub = abs2isub[abs_in]
fwd_xfer_in[isub].append(input_inds)
fwd_xfer_out[isub].append(output_inds)
if rev and abs_out in abs2isub:
isub = abs2isub[abs_out]
rev_xfer_in[isub].append(input_inds)
rev_xfer_out[isub].append(output_inds)
xfer_in = merge(xfer_in)
xfer_out = merge(xfer_out)
for isub in range(nsub_allprocs):
fwd_xfer_in[isub] = merge(fwd_xfer_in[isub])
fwd_xfer_out[isub] = merge(fwd_xfer_out[isub])
if rev:
rev_xfer_in[isub] = merge(rev_xfer_in[isub])
rev_xfer_out[isub] = merge(rev_xfer_out[isub])
out_vec = vectors['output'][vec_name]
transfers[vec_name] = {}
xfer_all = DefaultTransfer(vectors['input'][vec_name], out_vec,
xfer_in, xfer_out, group.comm)
transfers[vec_name]['fwd', None] = xfer_all
if rev:
transfers[vec_name]['rev', None] = xfer_all
for isub in range(nsub_allprocs):
transfers[vec_name]['fwd', isub] = DefaultTransfer(
vectors['input'][vec_name], vectors['output'][vec_name],
fwd_xfer_in[isub], fwd_xfer_out[isub], group.comm)
if rev:
transfers[vec_name]['rev', isub] = DefaultTransfer(
vectors['input'][vec_name], vectors['output'][vec_name],
rev_xfer_in[isub], rev_xfer_out[isub], group.comm)
if group._use_derivatives:
transfers['nonlinear'] = transfers['linear']
def _setup_transfers(group, recurse=True):
"""
Compute all transfers that are owned by our parent group.
Parameters
----------
group : <Group>
Parent group.
recurse : bool
Whether to call this method in subsystems.
"""
rev = group._mode == 'rev' or group._mode == 'auto'
def merge(indices_list):
if len(indices_list) > 0:
return np.concatenate(indices_list)
else:
return _empty_idx_array
if recurse:
for subsys in group._subgroups_myproc:
subsys._setup_transfers(recurse)
# Pre-compute map from abs_names to the index of the containing subsystem
abs2isub = {}
for subsys, isub in zip(group._subsystems_myproc,
group._subsystems_myproc_inds):
for type_ in ['input', 'output']:
for abs_name in subsys._var_allprocs_abs_names[type_]:
abs2isub[abs_name] = isub
abs2meta = group._var_abs2meta
allprocs_abs2meta = group._var_allprocs_abs2meta
myproc = group.comm.rank
transfers = group._transfers = {}
vectors = group._vectors
offsets = group._get_var_offsets()
vec_names = group._lin_rel_vec_name_list if group._use_derivatives else group._vec_names
for vec_name in vec_names:
relvars, _ = group._relevant[vec_name]['@all']
# Initialize empty lists for the transfer indices
nsub_allprocs = len(group._subsystems_allprocs)
xfer_in = []
xfer_out = []
fwd_xfer_in = [[] for i in range(nsub_allprocs)]
fwd_xfer_out = [[] for i in range(nsub_allprocs)]
if rev:
rev_xfer_in = [[] for i in range(nsub_allprocs)]
rev_xfer_out = [[] for i in range(nsub_allprocs)]
allprocs_abs2idx = group._var_allprocs_abs2idx[vec_name]
sizes_in = group._var_sizes[vec_name]['input']
sizes_out = group._var_sizes[vec_name]['output']
offsets_in = offsets[vec_name]['input']
offsets_out = offsets[vec_name]['output']
# Loop through all connections owned by this system
for abs_in, abs_out in iteritems(group._conn_abs_in2out):
if abs_out not in relvars['output']:
continue
# Only continue if the input exists on this processor
if abs_in in abs2meta and abs_in in relvars['input']:
# Get meta
meta_in = abs2meta[abs_in]
meta_out = allprocs_abs2meta[abs_out]
idx_in = allprocs_abs2idx[abs_in]
idx_out = allprocs_abs2idx[abs_out]
# Read in and process src_indices
src_indices = meta_in['src_indices']
if src_indices is None:
owner = group._owning_rank[abs_out]
# if the input is larger than the output on a single proc, we have
# to just loop over the procs in the same way we do when src_indices
# is defined.
if meta_in['size'] > sizes_out[owner, idx_out]:
src_indices = np.arange(meta_in['size'],
dtype=INT_DTYPE)
elif src_indices.ndim == 1:
src_indices = convert_neg(src_indices,
meta_out['global_size'])
else:
src_indices = _flatten_src_indices(
src_indices, meta_in['shape'],
meta_out['global_shape'], meta_out['global_size'])
# 1. Compute the output indices
if src_indices is None:
start = 0 if owner == 0 else np.sum(sizes_out[:owner,
idx_out])
offset = offsets_out[owner, idx_out] - start
output_inds = np.arange(offset,
offset + meta_in['size'],
dtype=INT_DTYPE)
else:
output_inds = np.zeros(src_indices.size, INT_DTYPE)
start = end = 0
for iproc in range(group.comm.size):
end += sizes_out[iproc, idx_out]
# The part of src on iproc
on_iproc = np.logical_and(start <= src_indices,
src_indices < end)
if np.any(on_iproc):
# This converts from iproc-then-ivar to ivar-then-iproc ordering
# Subtract off part of previous procs
# Then add all variables on previous procs
# Then all previous variables on this proc
# - np.sum(out_sizes[:iproc, idx_out])
# + np.sum(out_sizes[:iproc, :])
# + np.sum(out_sizes[iproc, :idx_out])
# + inds
offset = offsets_out[iproc, idx_out] - start
output_inds[
on_iproc] = src_indices[on_iproc] + offset
start = end
# 2. Compute the input indices
input_inds = np.arange(offsets_in[myproc, idx_in],
offsets_in[myproc, idx_in] +
sizes_in[myproc, idx_in],
dtype=INT_DTYPE)
# Now the indices are ready - input_inds, output_inds
xfer_in.append(input_inds)
xfer_out.append(output_inds)
isub = abs2isub[abs_in]
fwd_xfer_in[isub].append(input_inds)
fwd_xfer_out[isub].append(output_inds)
if rev and abs_out in abs2isub:
isub = abs2isub[abs_out]
rev_xfer_in[isub].append(input_inds)
rev_xfer_out[isub].append(output_inds)
xfer_in = merge(xfer_in)
xfer_out = merge(xfer_out)
for isub in range(nsub_allprocs):
fwd_xfer_in[isub] = merge(fwd_xfer_in[isub])
fwd_xfer_out[isub] = merge(fwd_xfer_out[isub])
if rev:
rev_xfer_in[isub] = merge(rev_xfer_in[isub])
rev_xfer_out[isub] = merge(rev_xfer_out[isub])
out_vec = vectors['output'][vec_name]
transfers[vec_name] = {}
xfer_all = PETScTransfer(vectors['input'][vec_name], out_vec,
xfer_in, xfer_out, group.comm)
transfers[vec_name]['fwd', None] = xfer_all
if rev:
transfers[vec_name]['rev', None] = xfer_all
for isub in range(nsub_allprocs):
transfers[vec_name]['fwd', isub] = PETScTransfer(
vectors['input'][vec_name], vectors['output'][vec_name],
fwd_xfer_in[isub], fwd_xfer_out[isub], group.comm)
if rev:
transfers[vec_name]['rev', isub] = PETScTransfer(
vectors['input'][vec_name],
vectors['output'][vec_name], rev_xfer_in[isub],
rev_xfer_out[isub], group.comm)
if group._use_derivatives:
transfers['nonlinear'] = transfers['linear']
def _setup_transfers(group, recurse=True):
"""
Compute all transfers that are owned by our parent group.
Parameters
----------
group : <Group>
Parent group.
recurse : bool
Whether to call this method in subsystems.
"""
iproc = group.comm.rank
rev = group._mode == 'rev' or group._mode == 'auto'
if recurse:
for subsys in group._subgroups_myproc:
subsys._setup_transfers(recurse)
abs2meta = group._var_abs2meta
allprocs_abs2meta = group._var_allprocs_abs2meta
group._transfers = transfers = {}
vectors = group._vectors
offsets = _global2local_offsets(group._get_var_offsets())
vec_names = group._lin_rel_vec_name_list if group._use_derivatives else group._vec_names
mypathlen = len(group.pathname + '.' if group.pathname else '')
sub_inds = group._subsystems_inds
for vec_name in vec_names:
relvars, _ = group._relevant[vec_name]['@all']
relvars_in = relvars['input']
relvars_out = relvars['output']
# Initialize empty lists for the transfer indices
nsub_allprocs = len(group._subsystems_allprocs)
xfer_in = []
xfer_out = []
fwd_xfer_in = [[] for s in group._subsystems_allprocs]
fwd_xfer_out = [[] for s in group._subsystems_allprocs]
if rev:
rev_xfer_in = [[] for s in group._subsystems_allprocs]
rev_xfer_out = [[] for s in group._subsystems_allprocs]
allprocs_abs2idx = group._var_allprocs_abs2idx[vec_name]
sizes_in = group._var_sizes[vec_name]['input']
sizes_out = group._var_sizes[vec_name]['output']
offsets_in = offsets[vec_name]['input']
offsets_out = offsets[vec_name]['output']
# Loop through all connections owned by this group
for abs_in, abs_out in group._conn_abs_in2out.items():
if abs_out not in relvars_out or abs_in not in relvars_in:
continue
# Only continue if the input exists on this processor
if abs_in in abs2meta:
# Get meta
meta_in = abs2meta[abs_in]
meta_out = allprocs_abs2meta[abs_out]
idx_in = allprocs_abs2idx[abs_in]
idx_out = allprocs_abs2idx[abs_out]
# Read in and process src_indices
src_indices = meta_in['src_indices']
if src_indices is None:
pass
elif src_indices.ndim == 1:
src_indices = convert_neg(src_indices,
meta_out['global_size'])
else:
src_indices = _flatten_src_indices(
src_indices, meta_in['shape'],
meta_out['global_shape'], meta_out['global_size'])
# 1. Compute the output indices
offset = offsets_out[iproc, idx_out]
if src_indices is None:
output_inds = np.arange(offset,
offset + meta_in['size'],
dtype=INT_DTYPE)
else:
output_inds = src_indices + offset
# 2. Compute the input indices
input_inds = np.arange(offsets_in[iproc, idx_in],
offsets_in[iproc, idx_in] +
sizes_in[iproc, idx_in],
dtype=INT_DTYPE)
# Now the indices are ready - input_inds, output_inds
sub_in = abs_in[mypathlen:].split('.', 1)[0]
isub = sub_inds[sub_in]
fwd_xfer_in[isub].append(input_inds)
fwd_xfer_out[isub].append(output_inds)
if rev and abs_out in abs2meta:
sub_out = abs_out[mypathlen:].split('.', 1)[0]
isub = sub_inds[sub_out]
rev_xfer_in[isub].append(input_inds)
rev_xfer_out[isub].append(output_inds)
tot_size = 0
for isub in range(nsub_allprocs):
fwd_xfer_in[isub] = _merge(fwd_xfer_in[isub])
fwd_xfer_out[isub] = _merge(fwd_xfer_out[isub])
tot_size += fwd_xfer_in[isub].size
if rev:
rev_xfer_in[isub] = _merge(rev_xfer_in[isub])
rev_xfer_out[isub] = _merge(rev_xfer_out[isub])
transfers[vec_name] = {}
if tot_size > 0:
xfer_in = np.concatenate(fwd_xfer_in)
xfer_out = np.concatenate(fwd_xfer_out)
out_vec = vectors['output'][vec_name]
xfer_all = DefaultTransfer(vectors['input'][vec_name], out_vec,
xfer_in, xfer_out, group.comm)
else:
xfer_all = None
transfers[vec_name]['fwd', None] = xfer_all
if rev:
transfers[vec_name]['rev', None] = xfer_all
for isub in range(nsub_allprocs):
if fwd_xfer_in[isub].size > 0:
transfers[vec_name]['fwd', isub] = DefaultTransfer(
vectors['input'][vec_name],
vectors['output'][vec_name], fwd_xfer_in[isub],
fwd_xfer_out[isub], group.comm)
else:
transfers[vec_name]['fwd', isub] = None
if rev:
if rev_xfer_out[isub].size > 0:
transfers[vec_name]['rev', isub] = DefaultTransfer(
vectors['input'][vec_name],
vectors['output'][vec_name], rev_xfer_in[isub],
rev_xfer_out[isub], group.comm)
else:
transfers[vec_name]['rev', isub] = None
if group._use_derivatives:
transfers['nonlinear'] = transfers['linear']
def _setup_transfers(group):
"""
Compute all transfers that are owned by our parent group.
Parameters
----------
group : <Group>
Parent group.
"""
iproc = group.comm.rank
rev = group._mode == 'rev' or group._mode == 'auto'
for subsys in group._subgroups_myproc:
subsys._setup_transfers()
abs2meta = group._var_abs2meta
allprocs_abs2meta_out = group._var_allprocs_abs2meta['output']
group._transfers = transfers = {}
vectors = group._vectors
offsets = _global2local_offsets(group._get_var_offsets())
vec_names = group._lin_rel_vec_name_list if group._use_derivatives else group._vec_names
allsubs = group._subsystems_allprocs
mypathlen = len(group.pathname + '.' if group.pathname else '')
for vec_name in vec_names:
relvars, _ = group._relevant[vec_name]['@all']
relvars_in = relvars['input']
relvars_out = relvars['output']
# Initialize empty lists for the transfer indices
nsub_allprocs = len(allsubs)
xfer_in = []
xfer_out = []
fwd_xfer_in = defaultdict(list)
fwd_xfer_out = defaultdict(list)
if rev:
rev_xfer_in = defaultdict(list)
rev_xfer_out = defaultdict(list)
allprocs_abs2idx = group._var_allprocs_abs2idx[vec_name]
sizes_in = group._var_sizes[vec_name]['input']
sizes_out = group._var_sizes[vec_name]['output']
offsets_in = offsets[vec_name]['input']
offsets_out = offsets[vec_name]['output']
# Loop through all connections owned by this group
for abs_in, abs_out in group._conn_abs_in2out.items():
if abs_out not in relvars_out or abs_in not in relvars_in:
continue
# Only continue if the input exists on this processor
if abs_in in abs2meta['input']:
indices = None
# Get meta
meta_in = abs2meta['input'][abs_in]
meta_out = allprocs_abs2meta_out[abs_out]
idx_in = allprocs_abs2idx[abs_in]
idx_out = allprocs_abs2idx[abs_out]
# Read in and process src_indices
src_indices = meta_in['src_indices']
if src_indices is not None:
if src_indices.ndim == 1:
src_indices = convert_neg(src_indices, meta_out['global_size'])
# 1. Compute the output indices
offset = offsets_out[iproc, idx_out]
if src_indices is None:
output_inds = np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE)
else:
output_inds = src_indices + offset
# 2. Compute the input indices
input_inds = np.arange(offsets_in[iproc, idx_in],
offsets_in[iproc, idx_in] +
sizes_in[iproc, idx_in], dtype=INT_DTYPE)
if indices is not None:
input_inds = input_inds.reshape(indices.shape)
# Now the indices are ready - input_inds, output_inds
sub_in = abs_in[mypathlen:].split('.', 1)[0]
fwd_xfer_in[sub_in].append(input_inds)
fwd_xfer_out[sub_in].append(output_inds)
if rev and abs_out in abs2meta['output']:
sub_out = abs_out[mypathlen:].split('.', 1)[0]
rev_xfer_in[sub_out].append(input_inds)
rev_xfer_out[sub_out].append(output_inds)
tot_size = 0
for sname, inds in fwd_xfer_in.items():
fwd_xfer_in[sname] = arr = _merge(inds)
fwd_xfer_out[sname] = _merge(fwd_xfer_out[sname])
tot_size += arr.size
if rev:
for sname, inds in rev_xfer_in.items():
rev_xfer_in[sname] = _merge(inds)
rev_xfer_out[sname] = _merge(rev_xfer_out[sname])
transfers[vec_name] = {}
if tot_size > 0:
try:
xfer_in = np.concatenate(list(fwd_xfer_in.values()))
xfer_out = np.concatenate(list(fwd_xfer_out.values()))
except ValueError:
xfer_in = xfer_out = np.zeros(0, dtype=INT_DTYPE)
out_vec = vectors['output'][vec_name]
xfer_all = DefaultTransfer(vectors['input'][vec_name], out_vec,
xfer_in, xfer_out, group.comm)
else:
xfer_all = None
transfers[vec_name]['fwd'] = xfwd = {}
xfwd[None] = xfer_all
if rev:
transfers[vec_name]['rev'] = xrev = {}
xrev[None] = xfer_all
for sname, inds in fwd_xfer_in.items():
if inds.size > 0:
xfwd[sname] = DefaultTransfer(vectors['input'][vec_name],
vectors['output'][vec_name],
inds, fwd_xfer_out[sname], group.comm)
else:
xfwd[sname] = None
if rev:
for sname, inds in rev_xfer_out.items():
if inds.size > 0:
xrev[sname] = DefaultTransfer(vectors['input'][vec_name],
vectors['output'][vec_name],
rev_xfer_in[sname], inds, group.comm)
else:
xrev[sname] = None
if group._use_derivatives:
transfers['nonlinear'] = transfers['linear']
def _setup_transfers(group, recurse=True):
"""
Compute all transfers that are owned by our parent group.
Parameters
----------
group : <Group>
Parent group.
recurse : bool
Whether to call this method in subsystems.
"""
group._transfers = {}
rev = group._mode == 'rev'
def merge(indices_list):
if len(indices_list) > 0:
return np.concatenate(indices_list)
else:
return _empty_idx_array
if recurse:
for subsys in group._subgroups_myproc:
subsys._setup_transfers(recurse)
# Pre-compute map from abs_names to the index of the containing subsystem
abs2isub = {}
for subsys, isub in zip(group._subsystems_myproc,
group._subsystems_myproc_inds):
for type_ in ['input', 'output']:
for abs_name in subsys._var_allprocs_abs_names[type_]:
abs2isub[abs_name] = isub
abs2meta = group._var_abs2meta
allprocs_abs2meta = group._var_allprocs_abs2meta
myproc = group.comm.rank
transfers = group._transfers
vectors = group._vectors
for vec_name in group._lin_rel_vec_name_list:
relvars, _ = group._relevant[vec_name]['@all']
# Initialize empty lists for the transfer indices
nsub_allprocs = len(group._subsystems_allprocs)
xfer_in = {}
xfer_out = {}
fwd_xfer_in = [{} for i in range(nsub_allprocs)]
fwd_xfer_out = [{} for i in range(nsub_allprocs)]
if rev:
rev_xfer_in = [{} for i in range(nsub_allprocs)]
rev_xfer_out = [{} for i in range(nsub_allprocs)]
for set_name_in in group._num_var_byset[vec_name]['input']:
for set_name_out in group._num_var_byset[vec_name]['output']:
key = (set_name_in, set_name_out)
xfer_in[key] = []
xfer_out[key] = []
for isub in range(nsub_allprocs):
fwd_xfer_in[isub][key] = []
fwd_xfer_out[isub][key] = []
if rev:
rev_xfer_in[isub][key] = []
rev_xfer_out[isub][key] = []
allprocs_abs2idx_byset = group._var_allprocs_abs2idx_byset[
vec_name]
sizes_byset_in = group._var_sizes_byset[vec_name]['input']
sizes_byset_out = group._var_sizes_byset[vec_name]['output']
offsets = group._get_varset_offsets()
offsets_byset_in = offsets[vec_name]['input']
offsets_byset_out = offsets[vec_name]['output']
# Loop through all explicit / implicit connections owned by this system
for abs_in, abs_out in iteritems(group._conn_abs_in2out):
if abs_out not in relvars['output']:
continue
# Only continue if the input exists on this processor
if abs_in in abs2meta and abs_in in relvars['input']:
# Get meta
meta_in = abs2meta[abs_in]
meta_out = allprocs_abs2meta[abs_out]
# Get varset info
set_name_in = meta_in['var_set']
set_name_out = meta_out['var_set']
idx_byset_in = allprocs_abs2idx_byset[abs_in]
idx_byset_out = allprocs_abs2idx_byset[abs_out]
# Get the sizes (byset) array
sizes_in = sizes_byset_in[set_name_in]
sizes_out = sizes_byset_out[set_name_out]
offsets_in = offsets_byset_in[set_name_in]
offsets_out = offsets_byset_out[set_name_out]
# Read in and process src_indices
shape_in = meta_in['shape']
shape_out = meta_out['shape']
global_shape_out = meta_out['global_shape']
global_size_out = meta_out['global_size']
src_indices = meta_in['src_indices']
if src_indices is None:
owner = group._owning_rank[abs_out]
# if the input is larger than the output on a single proc, we have
# to just loop over the procs in the same way we do when src_indices
# is defined.
if meta_in['size'] > sizes_out[owner, idx_byset_out]:
src_indices = np.arange(meta_in['size'],
dtype=INT_DTYPE)
elif src_indices.ndim == 1:
src_indices = convert_neg(src_indices, global_size_out)
else:
if len(shape_out
) == 1 or shape_in == src_indices.shape:
src_indices = src_indices.flatten()
src_indices = convert_neg(src_indices,
global_size_out)
else:
# TODO: this duplicates code found
# in System._setup_scaling.
entries = [list(range(x)) for x in shape_in]
cols = np.vstack(src_indices[i]
for i in product(*entries))
dimidxs = [
convert_neg(cols[:, i], global_shape_out[i])
for i in range(cols.shape[1])
]
src_indices = np.ravel_multi_index(
dimidxs, global_shape_out)
# 1. Compute the output indices
if src_indices is None:
start = 0 if owner == 0 else np.sum(
sizes_out[:owner, idx_byset_out])
offset = offsets_out[owner, idx_byset_out] - start
output_inds = np.arange(offset,
offset + meta_in['size'],
dtype=INT_DTYPE)
else:
output_inds = np.zeros(src_indices.size, INT_DTYPE)
start = end = 0
for iproc in range(group.comm.size):
end += sizes_out[iproc, idx_byset_out]
# The part of src on iproc
on_iproc = np.logical_and(start <= src_indices,
src_indices < end)
if np.any(on_iproc):
# This converts from iproc-then-ivar to ivar-then-iproc ordering
# Subtract off part of previous procs
# Then add all variables on previous procs
# Then all previous variables on this proc
# - np.sum(out_sizes[:iproc, idx_byset_out])
# + np.sum(out_sizes[:iproc, :])
# + np.sum(out_sizes[iproc, :idx_byset_out])
# + inds
offset = offsets_out[iproc,
idx_byset_out] - start
output_inds[
on_iproc] = src_indices[on_iproc] + offset
start = end
# 2. Compute the input indices
input_inds = np.arange(offsets_in[myproc, idx_byset_in],
offsets_in[myproc, idx_byset_in] +
sizes_in[myproc, idx_byset_in],
dtype=INT_DTYPE)
# Now the indices are ready - input_inds, output_inds
key = (set_name_in, set_name_out)
xfer_in[key].append(input_inds)
xfer_out[key].append(output_inds)
isub = abs2isub[abs_in]
fwd_xfer_in[isub][key].append(input_inds)
fwd_xfer_out[isub][key].append(output_inds)
if rev and abs_out in abs2isub:
isub = abs2isub[abs_out]
rev_xfer_in[isub][key].append(input_inds)
rev_xfer_out[isub][key].append(output_inds)
for set_name_in in group._num_var_byset[vec_name]['input']:
for set_name_out in group._num_var_byset[vec_name]['output']:
key = (set_name_in, set_name_out)
xfer_in[key] = merge(xfer_in[key])
xfer_out[key] = merge(xfer_out[key])
for isub in range(nsub_allprocs):
fwd_xfer_in[isub][key] = merge(fwd_xfer_in[isub][key])
fwd_xfer_out[isub][key] = merge(
fwd_xfer_out[isub][key])
if rev:
rev_xfer_in[isub][key] = merge(
rev_xfer_in[isub][key])
rev_xfer_out[isub][key] = merge(
rev_xfer_out[isub][key])
out_vec = vectors['output'][vec_name]
transfers[vec_name] = {}
xfer_all = PETScTransfer(vectors['input'][vec_name], out_vec,
xfer_in, xfer_out, group.comm)
transfers[vec_name]['fwd', None] = xfer_all
if rev:
transfers[vec_name]['rev', None] = xfer_all
for isub in range(nsub_allprocs):
transfers[vec_name]['fwd', isub] = PETScTransfer(
vectors['input'][vec_name], vectors['output'][vec_name],
fwd_xfer_in[isub], fwd_xfer_out[isub], group.comm)
if rev:
transfers[vec_name]['rev', isub] = PETScTransfer(
vectors['input'][vec_name],
vectors['output'][vec_name], rev_xfer_in[isub],
rev_xfer_out[isub], group.comm)
transfers['nonlinear'] = transfers['linear']
def _setup_transfers(group, recurse=True):
"""
Compute all transfers that are owned by our parent group.
Parameters
----------
group : <Group>
Parent group.
recurse : bool
Whether to call this method in subsystems.
"""
group._transfers = {}
rev = group._mode == 'rev' or group._mode == 'auto'
def merge(indices_list):
if len(indices_list) > 0:
return np.concatenate(indices_list)
else:
return _empty_idx_array
if recurse:
for subsys in group._subgroups_myproc:
subsys._setup_transfers(recurse)
# Pre-compute map from abs_names to the index of the containing subsystem
abs2isub = {}
for subsys, isub in zip(group._subsystems_myproc, group._subsystems_myproc_inds):
for type_ in ['input', 'output']:
for abs_name in subsys._var_allprocs_abs_names[type_]:
abs2isub[abs_name] = isub
abs2meta = group._var_abs2meta
allprocs_abs2meta = group._var_allprocs_abs2meta
myproc = group.comm.rank
transfers = group._transfers
vectors = group._vectors
offsets = group._get_var_offsets()
vec_names = group._lin_rel_vec_name_list if group._use_derivatives else group._vec_names
for vec_name in vec_names:
relvars, _ = group._relevant[vec_name]['@all']
# Initialize empty lists for the transfer indices
nsub_allprocs = len(group._subsystems_allprocs)
xfer_in = []
xfer_out = []
fwd_xfer_in = [[] for i in range(nsub_allprocs)]
fwd_xfer_out = [[] for i in range(nsub_allprocs)]
if rev:
rev_xfer_in = [[] for i in range(nsub_allprocs)]
rev_xfer_out = [[] for i in range(nsub_allprocs)]
allprocs_abs2idx = group._var_allprocs_abs2idx[vec_name]
sizes_in = group._var_sizes[vec_name]['input']
sizes_out = group._var_sizes[vec_name]['output']
offsets_in = offsets[vec_name]['input']
offsets_out = offsets[vec_name]['output']
# Loop through all connections owned by this system
for abs_in, abs_out in iteritems(group._conn_abs_in2out):
if abs_out not in relvars['output']:
continue
# Only continue if the input exists on this processor
if abs_in in abs2meta and abs_in in relvars['input']:
# Get meta
meta_in = abs2meta[abs_in]
meta_out = allprocs_abs2meta[abs_out]
idx_in = allprocs_abs2idx[abs_in]
idx_out = allprocs_abs2idx[abs_out]
# Read in and process src_indices
src_indices = meta_in['src_indices']
if src_indices is None:
owner = group._owning_rank[abs_out]
# if the input is larger than the output on a single proc, we have
# to just loop over the procs in the same way we do when src_indices
# is defined.
if meta_in['size'] > sizes_out[owner, idx_out]:
src_indices = np.arange(meta_in['size'], dtype=INT_DTYPE)
elif src_indices.ndim == 1:
src_indices = convert_neg(src_indices, meta_out['global_size'])
else:
src_indices = _flatten_src_indices(src_indices, meta_in['shape'],
meta_out['global_shape'],
meta_out['global_size'])
# 1. Compute the output indices
if src_indices is None:
start = 0 if owner == 0 else np.sum(sizes_out[:owner, idx_out])
offset = offsets_out[owner, idx_out] - start
output_inds = np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE)
else:
output_inds = np.zeros(src_indices.size, INT_DTYPE)
start = end = 0
for iproc in range(group.comm.size):
end += sizes_out[iproc, idx_out]
# The part of src on iproc
on_iproc = np.logical_and(start <= src_indices, src_indices < end)
if np.any(on_iproc):
# This converts from iproc-then-ivar to ivar-then-iproc ordering
# Subtract off part of previous procs
# Then add all variables on previous procs
# Then all previous variables on this proc
# - np.sum(out_sizes[:iproc, idx_out])
# + np.sum(out_sizes[:iproc, :])
# + np.sum(out_sizes[iproc, :idx_out])
# + inds
offset = offsets_out[iproc, idx_out] - start
output_inds[on_iproc] = src_indices[on_iproc] + offset
start = end
# 2. Compute the input indices
input_inds = np.arange(offsets_in[myproc, idx_in],
offsets_in[myproc, idx_in] +
sizes_in[myproc, idx_in], dtype=INT_DTYPE)
# Now the indices are ready - input_inds, output_inds
xfer_in.append(input_inds)
xfer_out.append(output_inds)
isub = abs2isub[abs_in]
fwd_xfer_in[isub].append(input_inds)
fwd_xfer_out[isub].append(output_inds)
if rev and abs_out in abs2isub:
isub = abs2isub[abs_out]
rev_xfer_in[isub].append(input_inds)
rev_xfer_out[isub].append(output_inds)
xfer_in = merge(xfer_in)
xfer_out = merge(xfer_out)
for isub in range(nsub_allprocs):
fwd_xfer_in[isub] = merge(fwd_xfer_in[isub])
fwd_xfer_out[isub] = merge(fwd_xfer_out[isub])
if rev:
rev_xfer_in[isub] = merge(rev_xfer_in[isub])
rev_xfer_out[isub] = merge(rev_xfer_out[isub])
out_vec = vectors['output'][vec_name]
transfers[vec_name] = {}
xfer_all = PETScTransfer(vectors['input'][vec_name], out_vec,
xfer_in, xfer_out, group.comm)
transfers[vec_name]['fwd', None] = xfer_all
if rev:
transfers[vec_name]['rev', None] = xfer_all
for isub in range(nsub_allprocs):
transfers[vec_name]['fwd', isub] = PETScTransfer(
vectors['input'][vec_name], vectors['output'][vec_name],
fwd_xfer_in[isub], fwd_xfer_out[isub], group.comm)
if rev:
transfers[vec_name]['rev', isub] = PETScTransfer(
vectors['input'][vec_name], vectors['output'][vec_name],
rev_xfer_in[isub], rev_xfer_out[isub], group.comm)
if group._use_derivatives:
transfers['nonlinear'] = transfers['linear']
