def _setup_driver(self, problem):
"""
Prepare the driver for execution.
This is the final thing to run during setup.
Parameters
----------
problem : <Problem>
Pointer to the containing problem.
"""
self._problem = weakref.ref(problem)
model = problem.model
self._total_jac = None
self._has_scaling = (np.any(
[r['total_scaler'] is not None for r in self._responses.values()])
or np.any([
dv['total_scaler'] is not None
for dv in self._designvars.values()
]))
# Determine if any design variables are discrete.
self._designvars_discrete = [
name for name, meta in self._designvars.items()
if meta['ivc_source'] in model._discrete_outputs
]
if not self.supports['integer_design_vars'] and len(
self._designvars_discrete) > 0:
msg = "Discrete design variables are not supported by this driver: "
msg += '.'.join(self._designvars_discrete)
raise RuntimeError(msg)
con_set = set()
obj_set = set()
dv_set = set()
self._remote_dvs = remote_dv_dict = {}
self._remote_cons = remote_con_dict = {}
self._dist_driver_vars = dist_dict = {}
self._remote_objs = remote_obj_dict = {}
src_design_vars = prom2ivc_src_dict(self._designvars)
src_cons = prom2ivc_src_dict(self._cons)
src_objs = prom2ivc_src_dict(self._objs)
responses = prom2ivc_src_dict(self._responses)
# Now determine if later we'll need to allgather cons, objs, or desvars.
if model.comm.size > 1 and model._subsystems_allprocs:
local_out_vars = set(model._outputs._abs_iter())
remote_dvs = set(src_design_vars) - local_out_vars
remote_cons = set(src_cons) - local_out_vars
remote_objs = set(src_objs) - local_out_vars
all_remote_vois = model.comm.allgather(
(remote_dvs, remote_cons, remote_objs))
for rem_dvs, rem_cons, rem_objs in all_remote_vois:
con_set.update(rem_cons)
obj_set.update(rem_objs)
dv_set.update(rem_dvs)
# If we have remote VOIs, pick an owning rank for each and use that
# to bcast to others later
owning_ranks = model._owning_rank
sizes = model._var_sizes['nonlinear']['output']
abs2meta = model._var_allprocs_abs2meta
rank = model.comm.rank
nprocs = model.comm.size
for i, vname in enumerate(model._var_allprocs_abs_names['output']):
if vname in responses:
indices = responses[vname].get('indices')
elif vname in src_design_vars:
indices = src_design_vars[vname].get('indices')
else:
continue
if abs2meta[vname]['distributed']:
idx = model._var_allprocs_abs2idx['nonlinear'][vname]
dist_sizes = model._var_sizes['nonlinear']['output'][:,
idx]
total_dist_size = np.sum(dist_sizes)
# Determine which indices are on our proc.
offsets = sizes2offsets(dist_sizes)
if indices is not None:
indices = convert_neg(indices, total_dist_size)
true_sizes = np.zeros(nprocs, dtype=INT_DTYPE)
for irank in range(nprocs):
dist_inds = indices[np.logical_and(
indices >= offsets[irank], indices <
(offsets[irank] + dist_sizes[irank]))]
if irank == rank:
local_indices = dist_inds - offsets[rank]
distrib_indices = dist_inds
true_sizes[irank] = dist_inds.size
dist_dict[vname] = (local_indices, true_sizes,
distrib_indices)
else:
dist_dict[vname] = (_full_slice, dist_sizes,
slice(
offsets[rank], offsets[rank] +
dist_sizes[rank]))
else:
owner = owning_ranks[vname]
sz = sizes[owner, i]
if vname in dv_set:
remote_dv_dict[vname] = (owner, sz)
if vname in con_set:
remote_con_dict[vname] = (owner, sz)
if vname in obj_set:
remote_obj_dict[vname] = (owner, sz)
self._remote_responses = self._remote_cons.copy()
self._remote_responses.update(self._remote_objs)
# set up simultaneous deriv coloring
if coloring_mod._use_total_sparsity:
# reset the coloring
if self._coloring_info['dynamic'] or self._coloring_info[
'static'] is not None:
self._coloring_info['coloring'] = None
coloring = self._get_static_coloring()
if coloring is not None and self.supports[
'simultaneous_derivatives']:
if model._owns_approx_jac:
coloring._check_config_partial(model)
else:
coloring._check_config_total(self)
self._setup_simul_coloring()
def _setup_solvers(self, system, depth):
"""
Assign system instance, set depth, and optionally perform setup.
Parameters
----------
system : <System>
Pointer to the owning system.
depth : int
Depth of the current system (already incremented).
"""
super(BroydenSolver, self)._setup_solvers(system, depth)
self._recompute_jacobian = True
self._computed_jacobians = 0
iproc = system.comm.rank
self._disallow_discrete_outputs()
if self.linear_solver is not None:
self.linear_solver._setup_solvers(system, self._depth + 1)
else:
self.linear_solver = system.linear_solver
if self.linesearch is not None:
self.linesearch._setup_solvers(system, self._depth + 1)
self.linesearch._do_subsolve = True
else:
# In OpenMDAO 3.x, we will be making BoundsEnforceLS the default line search.
# This deprecation warning is to prepare users for the change.
pathname = system.pathname
if pathname:
pathname += ': '
msg = 'Deprecation warning: In V 3.0, the default Broyden solver setup will change ' + \
'to use the BoundsEnforceLS line search.'
warn_deprecation(pathname + msg)
states = self.options['state_vars']
prom2abs = system._var_allprocs_prom2abs_list['output']
# Check names of states.
bad_names = [name for name in states if name not in prom2abs]
if len(bad_names) > 0:
msg = "{}: The following variable names were not found: {}"
raise ValueError(msg.format(self.msginfo, ', '.join(bad_names)))
use_owned = system._use_owned_sizes()
# Size linear system
if len(states) > 0:
# User has specified states, so we must size them.
n = 0
sizes = system._var_allprocs_abs2meta
self._distributed_idx = {}
for i, name in enumerate(states):
size = sizes[prom2abs[name][0]]['global_size']
self._idx[name] = (n, n + size)
n += size
if use_owned:
# To handle true distributed variables, each rank where they reside must
# know the index range that it owns.
vsizes = system._var_sizes['nonlinear']['output']
gstart = np.sum(vsizes[:iproc, i])
gend = gstart + vsizes[iproc, i]
self._distributed_idx[name] = (gstart, gend)
if use_owned:
abs2idx = system._var_allprocs_abs2idx['nonlinear']
local_idx = [abs2idx[prom2abs[name][0]] for name in states]
local_idx = np.array(local_idx)
owned_size_totals = np.sum(system._owned_sizes[:, local_idx],
axis=1)
disps = sizes2offsets(owned_size_totals, dtype=INT_DTYPE)
self._sendcounts = (owned_size_totals, disps)
else:
# Full system size.
self._full_inverse = True
n = np.sum(system._owned_sizes)
if use_owned:
owned_size_totals = np.sum(system._owned_sizes, axis=1)
disps = sizes2offsets(owned_size_totals, dtype=INT_DTYPE)
self._sendcounts = (owned_size_totals, disps)
self.size = n
self.Gm = np.empty((n, n))
self.xm = np.empty((n, ))
self.fxm = np.empty((n, ))
self.delta_xm = None
self.delta_fxm = None
if self._full_inverse:
# Can only use DirectSolver here.
from openmdao.solvers.linear.direct import DirectSolver
if not isinstance(self.linear_solver, DirectSolver):
msg = "{}: Linear solver must be DirectSolver when solving the full model."
raise ValueError(msg.format(self.msginfo,
', '.join(bad_names)))
return
# Always look for states that aren't being solved so we can warn the user.
def sys_recurse(system, all_states):
subs = system._subsystems_myproc
if len(subs) == 0:
# Skip implicit components that appear to solve themselves.
from openmdao.core.implicitcomponent import ImplicitComponent
if overrides_method('solve_nonlinear', system,
ImplicitComponent):
return
all_states.extend(system._list_states())
else:
for subsys in subs:
sub_nl = subsys.nonlinear_solver
if sub_nl and sub_nl.supports['implicit_components']:
continue
sys_recurse(subsys, all_states)
all_states = []
sys_recurse(system, all_states)
all_states = [
system._var_abs2prom['output'][name] for name in all_states
]
missing = set(all_states).difference(states)
if len(missing) > 0:
msg = "The following states are not covered by a solver, and may have been " + \
"omitted from the BroydenSolver 'state_vars': "
msg += ', '.join(sorted(missing))
simple_warning(msg)
def solve(self, vec_names, mode, rel_systems=None):
"""
Run the solver.
Parameters
----------
vec_names : [str, ...]
list of names of the right-hand-side vectors.
mode : str
'fwd' or 'rev'.
rel_systems : set of str
Names of systems relevant to the current solve.
"""
if len(vec_names) > 1 or vec_names[0] != 'linear':
raise RuntimeError(
"DirectSolvers with multiple right-hand-sides are not supported."
)
self._vec_names = vec_names
system = self._system
iproc = system.comm.rank
nproc = system.comm.size
d_residuals = system._vectors['residual']['linear']
d_outputs = system._vectors['output']['linear']
# assign x and b vectors based on mode
if mode == 'fwd':
x_vec = d_outputs._data
b_vec = d_residuals._data
trans_lu = 0
trans_splu = 'N'
else: # rev
x_vec = d_residuals._data
b_vec = d_outputs._data
trans_lu = 1
trans_splu = 'T'
# AssembledJacobians are unscaled.
if self._assembled_jac is not None:
if nproc > 1:
_, nodup2local_inds, local2owned_inds, noncontig_dist_inds = \
system._get_nodup_out_ranges()
# gather the 'owned' parts of b_vec from each process
tmp = np.empty(self._nodup_size, dtype=b_vec.dtype)
mpi_typ = MPI.C_DOUBLE_COMPLEX if np.iscomplex(
b_vec[0]) else MPI.DOUBLE
disps = sizes2offsets(self._owned_size_totals, dtype=INT_DTYPE)
system.comm.Gatherv(
(b_vec[local2owned_inds], local2owned_inds.size, mpi_typ),
(tmp, (self._owned_size_totals, disps), mpi_typ),
root=0)
else:
full_b = tmp = b_vec
with system._unscaled_context(outputs=[d_outputs],
residuals=[d_residuals]):
if iproc == 0:
# convert full_b to the same ordering that the matrix expects, where
# dist vars are contiguous and other vars appear in 'execution' order.
if nproc > 1:
full_b = tmp[noncontig_dist_inds]
if isinstance(self._assembled_jac._int_mtx, DenseMatrix):
arr = scipy.linalg.lu_solve(self._lup,
full_b,
trans=trans_lu)
else:
arr = self._lu.solve(full_b, trans_splu)
if nproc > 1:
if iproc > 0:
arr = np.zeros(tmp.size, dtype=tmp.dtype)
# this may send more data than necessary, but the alternative is to use a lot
# of memory on rank 0 to store the chunk that each proc needs and then do a
# Scatterv.
system.comm.Bcast((arr, mpi_typ), root=0)
x_vec[:] = arr[nodup2local_inds]
else:
x_vec[:] = arr
# matrix-vector-product generated jacobians are scaled.
else:
x_vec[:] = scipy.linalg.lu_solve(self._lup, b_vec, trans=trans_lu)
def _setup_driver(self, problem):
"""
Prepare the driver for execution.
This is the final thing to run during setup.
Parameters
----------
problem : <Problem>
Pointer to the containing problem.
"""
self._problem = weakref.ref(problem)
model = problem.model
self._total_jac = None
self._has_scaling = (
np.any([r['total_scaler'] is not None for r in self._responses.values()]) or
np.any([dv['total_scaler'] is not None for dv in self._designvars.values()])
)
# Determine if any design variables are discrete.
self._designvars_discrete = [name for name, meta in self._designvars.items()
if meta['ivc_source'] in model._discrete_outputs]
if not self.supports['integer_design_vars'] and len(self._designvars_discrete) > 0:
msg = "Discrete design variables are not supported by this driver: "
msg += '.'.join(self._designvars_discrete)
raise RuntimeError(msg)
self._remote_dvs = remote_dv_dict = {}
self._remote_cons = remote_con_dict = {}
self._dist_driver_vars = dist_dict = {}
self._remote_objs = remote_obj_dict = {}
# Only allow distributed design variables on drivers that support it.
if self.supports['distributed_design_vars'] is False:
dist_vars = []
abs2meta_in = model._var_allprocs_abs2meta['input']
discrete_in = model._var_allprocs_discrete['input']
for dv, meta in self._designvars.items():
# For Auto-ivcs, we need to check the distributed metadata on the target instead.
if meta['ivc_source'].startswith('_auto_ivc.'):
for abs_name in model._var_allprocs_prom2abs_list['input'][dv]:
if abs_name in discrete_in:
# Discrete vars aren't distributed.
break
if abs2meta_in[abs_name]['distributed']:
dist_vars.append(dv)
break
elif meta['distributed']:
dist_vars.append(dv)
if dist_vars:
dstr = ', '.join(dist_vars)
msg = "Distributed design variables are not supported by this driver, but the "
msg += f"following variables are distributed: [{dstr}]"
raise RuntimeError(msg)
# Now determine if later we'll need to allgather cons, objs, or desvars.
if model.comm.size > 1 and model._subsystems_allprocs:
con_set = set()
obj_set = set()
dv_set = set()
src_design_vars = _prom2ivc_src_dict(self._designvars)
responses = _prom2ivc_src_dict(self._responses)
local_out_vars = set(model._outputs._abs_iter())
remote_dvs = set(src_design_vars) - local_out_vars
remote_cons = set(_prom2ivc_src_name_iter(self._cons)) - local_out_vars
remote_objs = set(_prom2ivc_src_name_iter(self._objs)) - local_out_vars
all_remote_vois = model.comm.allgather(
(remote_dvs, remote_cons, remote_objs))
for rem_dvs, rem_cons, rem_objs in all_remote_vois:
con_set.update(rem_cons)
obj_set.update(rem_objs)
dv_set.update(rem_dvs)
# If we have remote VOIs, pick an owning rank for each and use that
# to bcast to others later
owning_ranks = model._owning_rank
sizes = model._var_sizes['output']
rank = model.comm.rank
nprocs = model.comm.size
for i, (vname, meta) in enumerate(model._var_allprocs_abs2meta['output'].items()):
if vname in responses:
indices = responses[vname].get('indices')
elif vname in src_design_vars:
indices = src_design_vars[vname].get('indices')
else:
continue
if meta['distributed']:
dist_sizes = sizes[:, i]
tot_size = np.sum(dist_sizes)
# Determine which indices are on our proc.
offsets = sizes2offsets(dist_sizes)
if indices is not None:
indices = indices.shaped_array()
true_sizes = np.zeros(nprocs, dtype=INT_DTYPE)
for irank in range(nprocs):
dist_inds = indices[np.logical_and(indices >= offsets[irank],
indices < (offsets[irank] +
dist_sizes[irank]))]
true_sizes[irank] = dist_inds.size
if irank == rank:
local_indices = dist_inds - offsets[rank]
distrib_indices = dist_inds
ind = indexer(local_indices, src_shape=(tot_size,), flat_src=True)
dist_dict[vname] = (ind, true_sizes, distrib_indices)
else:
dist_dict[vname] = (_full_slice, dist_sizes,
slice(offsets[rank], offsets[rank] + dist_sizes[rank]))
else:
owner = owning_ranks[vname]
sz = sizes[owner, i]
if vname in dv_set:
remote_dv_dict[vname] = (owner, sz)
if vname in con_set:
remote_con_dict[vname] = (owner, sz)
if vname in obj_set:
remote_obj_dict[vname] = (owner, sz)
self._remote_responses = self._remote_cons.copy()
self._remote_responses.update(self._remote_objs)
# set up simultaneous deriv coloring
if coloring_mod._use_total_sparsity:
# reset the coloring
if self._coloring_info['dynamic'] or self._coloring_info['static'] is not None:
self._coloring_info['coloring'] = None
coloring = self._get_static_coloring()
if coloring is not None and self.supports['simultaneous_derivatives']:
if model._owns_approx_jac:
coloring._check_config_partial(model)
else:
coloring._check_config_total(self)
self._setup_simul_coloring()
