def _setup_driver(self, problem): """ Prepare the driver for execution. This is the final thing to run during setup. Parameters ---------- problem : <Problem> Pointer to the containing problem. """ self._problem = weakref.ref(problem) model = problem.model self._total_jac = None self._has_scaling = (np.any( [r['total_scaler'] is not None for r in self._responses.values()]) or np.any([ dv['total_scaler'] is not None for dv in self._designvars.values() ])) # Determine if any design variables are discrete. self._designvars_discrete = [ name for name, meta in self._designvars.items() if meta['ivc_source'] in model._discrete_outputs ] if not self.supports['integer_design_vars'] and len( self._designvars_discrete) > 0: msg = "Discrete design variables are not supported by this driver: " msg += '.'.join(self._designvars_discrete) raise RuntimeError(msg) con_set = set() obj_set = set() dv_set = set() self._remote_dvs = remote_dv_dict = {} self._remote_cons = remote_con_dict = {} self._dist_driver_vars = dist_dict = {} self._remote_objs = remote_obj_dict = {} src_design_vars = prom2ivc_src_dict(self._designvars) src_cons = prom2ivc_src_dict(self._cons) src_objs = prom2ivc_src_dict(self._objs) responses = prom2ivc_src_dict(self._responses) # Now determine if later we'll need to allgather cons, objs, or desvars. if model.comm.size > 1 and model._subsystems_allprocs: local_out_vars = set(model._outputs._abs_iter()) remote_dvs = set(src_design_vars) - local_out_vars remote_cons = set(src_cons) - local_out_vars remote_objs = set(src_objs) - local_out_vars all_remote_vois = model.comm.allgather( (remote_dvs, remote_cons, remote_objs)) for rem_dvs, rem_cons, rem_objs in all_remote_vois: con_set.update(rem_cons) obj_set.update(rem_objs) dv_set.update(rem_dvs) # If we have remote VOIs, pick an owning rank for each and use that # to bcast to others later owning_ranks = model._owning_rank sizes = model._var_sizes['nonlinear']['output'] abs2meta = model._var_allprocs_abs2meta rank = model.comm.rank nprocs = model.comm.size for i, vname in enumerate(model._var_allprocs_abs_names['output']): if vname in responses: indices = responses[vname].get('indices') elif vname in src_design_vars: indices = src_design_vars[vname].get('indices') else: continue if abs2meta[vname]['distributed']: idx = model._var_allprocs_abs2idx['nonlinear'][vname] dist_sizes = model._var_sizes['nonlinear']['output'][:, idx] total_dist_size = np.sum(dist_sizes) # Determine which indices are on our proc. offsets = sizes2offsets(dist_sizes) if indices is not None: indices = convert_neg(indices, total_dist_size) true_sizes = np.zeros(nprocs, dtype=INT_DTYPE) for irank in range(nprocs): dist_inds = indices[np.logical_and( indices >= offsets[irank], indices < (offsets[irank] + dist_sizes[irank]))] if irank == rank: local_indices = dist_inds - offsets[rank] distrib_indices = dist_inds true_sizes[irank] = dist_inds.size dist_dict[vname] = (local_indices, true_sizes, distrib_indices) else: dist_dict[vname] = (_full_slice, dist_sizes, slice( offsets[rank], offsets[rank] + dist_sizes[rank])) else: owner = owning_ranks[vname] sz = sizes[owner, i] if vname in dv_set: remote_dv_dict[vname] = (owner, sz) if vname in con_set: remote_con_dict[vname] = (owner, sz) if vname in obj_set: remote_obj_dict[vname] = (owner, sz) self._remote_responses = self._remote_cons.copy() self._remote_responses.update(self._remote_objs) # set up simultaneous deriv coloring if coloring_mod._use_total_sparsity: # reset the coloring if self._coloring_info['dynamic'] or self._coloring_info[ 'static'] is not None: self._coloring_info['coloring'] = None coloring = self._get_static_coloring() if coloring is not None and self.supports[ 'simultaneous_derivatives']: if model._owns_approx_jac: coloring._check_config_partial(model) else: coloring._check_config_total(self) self._setup_simul_coloring()
def _setup_solvers(self, system, depth): """ Assign system instance, set depth, and optionally perform setup. Parameters ---------- system : <System> Pointer to the owning system. depth : int Depth of the current system (already incremented). """ super(BroydenSolver, self)._setup_solvers(system, depth) self._recompute_jacobian = True self._computed_jacobians = 0 iproc = system.comm.rank self._disallow_discrete_outputs() if self.linear_solver is not None: self.linear_solver._setup_solvers(system, self._depth + 1) else: self.linear_solver = system.linear_solver if self.linesearch is not None: self.linesearch._setup_solvers(system, self._depth + 1) self.linesearch._do_subsolve = True else: # In OpenMDAO 3.x, we will be making BoundsEnforceLS the default line search. # This deprecation warning is to prepare users for the change. pathname = system.pathname if pathname: pathname += ': ' msg = 'Deprecation warning: In V 3.0, the default Broyden solver setup will change ' + \ 'to use the BoundsEnforceLS line search.' warn_deprecation(pathname + msg) states = self.options['state_vars'] prom2abs = system._var_allprocs_prom2abs_list['output'] # Check names of states. bad_names = [name for name in states if name not in prom2abs] if len(bad_names) > 0: msg = "{}: The following variable names were not found: {}" raise ValueError(msg.format(self.msginfo, ', '.join(bad_names))) use_owned = system._use_owned_sizes() # Size linear system if len(states) > 0: # User has specified states, so we must size them. n = 0 sizes = system._var_allprocs_abs2meta self._distributed_idx = {} for i, name in enumerate(states): size = sizes[prom2abs[name][0]]['global_size'] self._idx[name] = (n, n + size) n += size if use_owned: # To handle true distributed variables, each rank where they reside must # know the index range that it owns. vsizes = system._var_sizes['nonlinear']['output'] gstart = np.sum(vsizes[:iproc, i]) gend = gstart + vsizes[iproc, i] self._distributed_idx[name] = (gstart, gend) if use_owned: abs2idx = system._var_allprocs_abs2idx['nonlinear'] local_idx = [abs2idx[prom2abs[name][0]] for name in states] local_idx = np.array(local_idx) owned_size_totals = np.sum(system._owned_sizes[:, local_idx], axis=1) disps = sizes2offsets(owned_size_totals, dtype=INT_DTYPE) self._sendcounts = (owned_size_totals, disps) else: # Full system size. self._full_inverse = True n = np.sum(system._owned_sizes) if use_owned: owned_size_totals = np.sum(system._owned_sizes, axis=1) disps = sizes2offsets(owned_size_totals, dtype=INT_DTYPE) self._sendcounts = (owned_size_totals, disps) self.size = n self.Gm = np.empty((n, n)) self.xm = np.empty((n, )) self.fxm = np.empty((n, )) self.delta_xm = None self.delta_fxm = None if self._full_inverse: # Can only use DirectSolver here. from openmdao.solvers.linear.direct import DirectSolver if not isinstance(self.linear_solver, DirectSolver): msg = "{}: Linear solver must be DirectSolver when solving the full model." raise ValueError(msg.format(self.msginfo, ', '.join(bad_names))) return # Always look for states that aren't being solved so we can warn the user. def sys_recurse(system, all_states): subs = system._subsystems_myproc if len(subs) == 0: # Skip implicit components that appear to solve themselves. from openmdao.core.implicitcomponent import ImplicitComponent if overrides_method('solve_nonlinear', system, ImplicitComponent): return all_states.extend(system._list_states()) else: for subsys in subs: sub_nl = subsys.nonlinear_solver if sub_nl and sub_nl.supports['implicit_components']: continue sys_recurse(subsys, all_states) all_states = [] sys_recurse(system, all_states) all_states = [ system._var_abs2prom['output'][name] for name in all_states ] missing = set(all_states).difference(states) if len(missing) > 0: msg = "The following states are not covered by a solver, and may have been " + \ "omitted from the BroydenSolver 'state_vars': " msg += ', '.join(sorted(missing)) simple_warning(msg)
def solve(self, vec_names, mode, rel_systems=None): """ Run the solver. Parameters ---------- vec_names : [str, ...] list of names of the right-hand-side vectors. mode : str 'fwd' or 'rev'. rel_systems : set of str Names of systems relevant to the current solve. """ if len(vec_names) > 1 or vec_names[0] != 'linear': raise RuntimeError( "DirectSolvers with multiple right-hand-sides are not supported." ) self._vec_names = vec_names system = self._system iproc = system.comm.rank nproc = system.comm.size d_residuals = system._vectors['residual']['linear'] d_outputs = system._vectors['output']['linear'] # assign x and b vectors based on mode if mode == 'fwd': x_vec = d_outputs._data b_vec = d_residuals._data trans_lu = 0 trans_splu = 'N' else: # rev x_vec = d_residuals._data b_vec = d_outputs._data trans_lu = 1 trans_splu = 'T' # AssembledJacobians are unscaled. if self._assembled_jac is not None: if nproc > 1: _, nodup2local_inds, local2owned_inds, noncontig_dist_inds = \ system._get_nodup_out_ranges() # gather the 'owned' parts of b_vec from each process tmp = np.empty(self._nodup_size, dtype=b_vec.dtype) mpi_typ = MPI.C_DOUBLE_COMPLEX if np.iscomplex( b_vec[0]) else MPI.DOUBLE disps = sizes2offsets(self._owned_size_totals, dtype=INT_DTYPE) system.comm.Gatherv( (b_vec[local2owned_inds], local2owned_inds.size, mpi_typ), (tmp, (self._owned_size_totals, disps), mpi_typ), root=0) else: full_b = tmp = b_vec with system._unscaled_context(outputs=[d_outputs], residuals=[d_residuals]): if iproc == 0: # convert full_b to the same ordering that the matrix expects, where # dist vars are contiguous and other vars appear in 'execution' order. if nproc > 1: full_b = tmp[noncontig_dist_inds] if isinstance(self._assembled_jac._int_mtx, DenseMatrix): arr = scipy.linalg.lu_solve(self._lup, full_b, trans=trans_lu) else: arr = self._lu.solve(full_b, trans_splu) if nproc > 1: if iproc > 0: arr = np.zeros(tmp.size, dtype=tmp.dtype) # this may send more data than necessary, but the alternative is to use a lot # of memory on rank 0 to store the chunk that each proc needs and then do a # Scatterv. system.comm.Bcast((arr, mpi_typ), root=0) x_vec[:] = arr[nodup2local_inds] else: x_vec[:] = arr # matrix-vector-product generated jacobians are scaled. else: x_vec[:] = scipy.linalg.lu_solve(self._lup, b_vec, trans=trans_lu)
def _setup_driver(self, problem): """ Prepare the driver for execution. This is the final thing to run during setup. Parameters ---------- problem : <Problem> Pointer to the containing problem. """ self._problem = weakref.ref(problem) model = problem.model self._total_jac = None self._has_scaling = ( np.any([r['total_scaler'] is not None for r in self._responses.values()]) or np.any([dv['total_scaler'] is not None for dv in self._designvars.values()]) ) # Determine if any design variables are discrete. self._designvars_discrete = [name for name, meta in self._designvars.items() if meta['ivc_source'] in model._discrete_outputs] if not self.supports['integer_design_vars'] and len(self._designvars_discrete) > 0: msg = "Discrete design variables are not supported by this driver: " msg += '.'.join(self._designvars_discrete) raise RuntimeError(msg) self._remote_dvs = remote_dv_dict = {} self._remote_cons = remote_con_dict = {} self._dist_driver_vars = dist_dict = {} self._remote_objs = remote_obj_dict = {} # Only allow distributed design variables on drivers that support it. if self.supports['distributed_design_vars'] is False: dist_vars = [] abs2meta_in = model._var_allprocs_abs2meta['input'] discrete_in = model._var_allprocs_discrete['input'] for dv, meta in self._designvars.items(): # For Auto-ivcs, we need to check the distributed metadata on the target instead. if meta['ivc_source'].startswith('_auto_ivc.'): for abs_name in model._var_allprocs_prom2abs_list['input'][dv]: if abs_name in discrete_in: # Discrete vars aren't distributed. break if abs2meta_in[abs_name]['distributed']: dist_vars.append(dv) break elif meta['distributed']: dist_vars.append(dv) if dist_vars: dstr = ', '.join(dist_vars) msg = "Distributed design variables are not supported by this driver, but the " msg += f"following variables are distributed: [{dstr}]" raise RuntimeError(msg) # Now determine if later we'll need to allgather cons, objs, or desvars. if model.comm.size > 1 and model._subsystems_allprocs: con_set = set() obj_set = set() dv_set = set() src_design_vars = _prom2ivc_src_dict(self._designvars) responses = _prom2ivc_src_dict(self._responses) local_out_vars = set(model._outputs._abs_iter()) remote_dvs = set(src_design_vars) - local_out_vars remote_cons = set(_prom2ivc_src_name_iter(self._cons)) - local_out_vars remote_objs = set(_prom2ivc_src_name_iter(self._objs)) - local_out_vars all_remote_vois = model.comm.allgather( (remote_dvs, remote_cons, remote_objs)) for rem_dvs, rem_cons, rem_objs in all_remote_vois: con_set.update(rem_cons) obj_set.update(rem_objs) dv_set.update(rem_dvs) # If we have remote VOIs, pick an owning rank for each and use that # to bcast to others later owning_ranks = model._owning_rank sizes = model._var_sizes['output'] rank = model.comm.rank nprocs = model.comm.size for i, (vname, meta) in enumerate(model._var_allprocs_abs2meta['output'].items()): if vname in responses: indices = responses[vname].get('indices') elif vname in src_design_vars: indices = src_design_vars[vname].get('indices') else: continue if meta['distributed']: dist_sizes = sizes[:, i] tot_size = np.sum(dist_sizes) # Determine which indices are on our proc. offsets = sizes2offsets(dist_sizes) if indices is not None: indices = indices.shaped_array() true_sizes = np.zeros(nprocs, dtype=INT_DTYPE) for irank in range(nprocs): dist_inds = indices[np.logical_and(indices >= offsets[irank], indices < (offsets[irank] + dist_sizes[irank]))] true_sizes[irank] = dist_inds.size if irank == rank: local_indices = dist_inds - offsets[rank] distrib_indices = dist_inds ind = indexer(local_indices, src_shape=(tot_size,), flat_src=True) dist_dict[vname] = (ind, true_sizes, distrib_indices) else: dist_dict[vname] = (_full_slice, dist_sizes, slice(offsets[rank], offsets[rank] + dist_sizes[rank])) else: owner = owning_ranks[vname] sz = sizes[owner, i] if vname in dv_set: remote_dv_dict[vname] = (owner, sz) if vname in con_set: remote_con_dict[vname] = (owner, sz) if vname in obj_set: remote_obj_dict[vname] = (owner, sz) self._remote_responses = self._remote_cons.copy() self._remote_responses.update(self._remote_objs) # set up simultaneous deriv coloring if coloring_mod._use_total_sparsity: # reset the coloring if self._coloring_info['dynamic'] or self._coloring_info['static'] is not None: self._coloring_info['coloring'] = None coloring = self._get_static_coloring() if coloring is not None and self.supports['simultaneous_derivatives']: if model._owns_approx_jac: coloring._check_config_partial(model) else: coloring._check_config_total(self) self._setup_simul_coloring()