class Driver(object): """ Top-level container for the systems and drivers. Attributes ---------- fail : bool Reports whether the driver ran successfully. iter_count : int Keep track of iterations for case recording. options : <OptionsDictionary> Dictionary with general pyoptsparse options. recording_options : <OptionsDictionary> Dictionary with driver recording options. debug_print : <OptionsDictionary> Dictionary with debugging printing options. cite : str Listing of relevant citataions that should be referenced when publishing work that uses this class. _problem : <Problem> Pointer to the containing problem. supports : <OptionsDictionary> Provides a consistant way for drivers to declare what features they support. _designvars : dict Contains all design variable info. _cons : dict Contains all constraint info. _objs : dict Contains all objective info. _responses : dict Contains all response info. _rec_mgr : <RecordingManager> Object that manages all recorders added to this driver. _vars_to_record: dict Dict of lists of var names indicating what to record _model_viewer_data : dict Structure of model, used to make n2 diagram. _remote_dvs : dict Dict of design variables that are remote on at least one proc. Values are (owning rank, size). _remote_cons : dict Dict of constraints that are remote on at least one proc. Values are (owning rank, size). _remote_objs : dict Dict of objectives that are remote on at least one proc. Values are (owning rank, size). _remote_responses : dict A combined dict containing entries from _remote_cons and _remote_objs. _simul_coloring_info : tuple of dicts A data structure describing coloring for simultaneous derivs. _total_jac_sparsity : dict, str, or None Specifies sparsity of sub-jacobians of the total jacobian. Only used by pyOptSparseDriver. _res_jacs : dict Dict of sparse subjacobians for use with certain optimizers, e.g. pyOptSparseDriver. _total_jac : _TotalJacInfo or None Cached total jacobian handling object. """ def __init__(self, **kwargs): """ Initialize the driver. Parameters ---------- **kwargs : dict of keyword arguments Keyword arguments that will be mapped into the Driver options. """ self._rec_mgr = RecordingManager() self._vars_to_record = { 'desvarnames': set(), 'responsenames': set(), 'objectivenames': set(), 'constraintnames': set(), 'sysinclnames': set(), } self._problem = None self._designvars = None self._cons = None self._objs = None self._responses = None # Driver options self.options = OptionsDictionary() self.options.declare( 'debug_print', types=list, is_valid=_is_debug_print_opts_valid, desc="List of what type of Driver variables to print at each " "iteration. Valid items in list are 'desvars', 'ln_cons', " "'nl_cons', 'objs'", default=[]) # Case recording options self.recording_options = OptionsDictionary() self.recording_options.declare('record_metadata', types=bool, default=True, desc='Record metadata') self.recording_options.declare( 'record_desvars', types=bool, default=True, desc='Set to True to record design variables at the ' 'driver level') self.recording_options.declare( 'record_responses', types=bool, default=False, desc='Set to True to record responses at the driver level') self.recording_options.declare( 'record_objectives', types=bool, default=True, desc='Set to True to record objectives at the driver level') self.recording_options.declare( 'record_constraints', types=bool, default=True, desc='Set to True to record constraints at the ' 'driver level') self.recording_options.declare( 'includes', types=list, default=['*'], desc='Patterns for variables to include in recording') self.recording_options.declare( 'excludes', types=list, default=[], desc='Patterns for vars to exclude in recording ' '(processed post-includes)') self.recording_options.declare( 'record_derivatives', types=bool, default=False, desc='Set to True to record derivatives at the driver ' 'level') self.recording_options.declare( 'record_inputs', types=bool, default=True, desc='Set to True to record inputs at the driver level') # What the driver supports. self.supports = OptionsDictionary() self.supports.declare('inequality_constraints', types=bool, default=False) self.supports.declare('equality_constraints', types=bool, default=False) self.supports.declare('linear_constraints', types=bool, default=False) self.supports.declare('two_sided_constraints', types=bool, default=False) self.supports.declare('multiple_objectives', types=bool, default=False) self.supports.declare('integer_design_vars', types=bool, default=False) self.supports.declare('gradients', types=bool, default=False) self.supports.declare('active_set', types=bool, default=False) self.supports.declare('simultaneous_derivatives', types=bool, default=False) self.supports.declare('total_jac_sparsity', types=bool, default=False) # TODO, support these in OpenMDAO self.supports.declare('integer_design_vars', types=bool, default=False) self.iter_count = 0 self._model_viewer_data = None self.cite = "" self._simul_coloring_info = None self._total_jac_sparsity = None self._res_jacs = {} self._total_jac = None self.fail = False self._declare_options() self.options.update(kwargs) def add_recorder(self, recorder): """ Add a recorder to the driver. Parameters ---------- recorder : BaseRecorder A recorder instance. """ self._rec_mgr.append(recorder) def cleanup(self): """ Clean up resources prior to exit. """ self._rec_mgr.close() def _declare_options(self): """ Declare options before kwargs are processed in the init method. This is optionally implemented by subclasses of Driver. """ pass def _setup_comm(self, comm): """ Perform any driver-specific setup of communicators for the model. Parameters ---------- comm : MPI.Comm or <FakeComm> or None The communicator for the Problem. Returns ------- MPI.Comm or <FakeComm> or None The communicator for the Problem model. """ return comm def _setup_driver(self, problem): """ Prepare the driver for execution. This is the final thing to run during setup. Parameters ---------- problem : <Problem> Pointer to the containing problem. """ self._problem = problem model = problem.model self._total_jac = None self._objs = objs = OrderedDict() self._cons = cons = OrderedDict() self._responses = model.get_responses(recurse=True) response_size = 0 for name, data in iteritems(self._responses): if data['type'] == 'con': cons[name] = data else: objs[name] = data response_size += data['size'] # Gather up the information for design vars. self._designvars = model.get_design_vars(recurse=True) self._has_scaling = ( np.any([r['scaler'] is not None for r in self._responses.values()]) or np.any( [dv['scaler'] is not None for dv in self._designvars.values()])) con_set = set() obj_set = set() dv_set = set() self._remote_dvs = dv_dict = {} self._remote_cons = con_dict = {} self._remote_objs = obj_dict = {} # Now determine if later we'll need to allgather cons, objs, or desvars. if model.comm.size > 1 and model._subsystems_allprocs: local_out_vars = set(model._outputs._views) remote_dvs = set(self._designvars) - local_out_vars remote_cons = set(self._cons) - local_out_vars remote_objs = set(self._objs) - local_out_vars all_remote_vois = model.comm.allgather( (remote_dvs, remote_cons, remote_objs)) for rem_dvs, rem_cons, rem_objs in all_remote_vois: con_set.update(rem_cons) obj_set.update(rem_objs) dv_set.update(rem_dvs) # If we have remote VOIs, pick an owning rank for each and use that # to bcast to others later owning_ranks = model._owning_rank sizes = model._var_sizes['nonlinear']['output'] for i, vname in enumerate(model._var_allprocs_abs_names['output']): owner = owning_ranks[vname] if vname in dv_set: dv_dict[vname] = (owner, sizes[owner, i]) if vname in con_set: con_dict[vname] = (owner, sizes[owner, i]) if vname in obj_set: obj_dict[vname] = (owner, sizes[owner, i]) self._remote_responses = self._remote_cons.copy() self._remote_responses.update(self._remote_objs) # set up case recording self._setup_recording() desvar_size = np.sum(data['size'] for data in itervalues(self._designvars)) # set up simultaneous deriv coloring if (coloring_mod._use_sparsity and self._simul_coloring_info and self.supports['simultaneous_derivatives']): if problem._mode == 'fwd': self._setup_simul_coloring(problem._mode) else: raise RuntimeError( "simultaneous derivs are currently not supported in rev mode." ) # if we're using simultaneous derivatives then our effective design var size is less # than the full design var size if self._simul_coloring_info: col_lists = self._simul_coloring_info[0] if col_lists: desvar_size = len(col_lists[0]) desvar_size += len(col_lists) - 1 if ((problem._mode == 'fwd' and desvar_size > response_size) or (problem._mode == 'rev' and response_size > desvar_size)): warnings.warn( "Inefficient choice of derivative mode. You chose '%s' for a " "problem with %d design variables and %d response variables " "(objectives and constraints)." % (problem._mode, desvar_size, response_size), RuntimeWarning) def _setup_recording(self): """ Set up case recording. """ problem = self._problem model = problem.model mydesvars = myobjectives = myconstraints = myresponses = set() myinputs = set() mysystem_outputs = set() incl = self.recording_options['includes'] excl = self.recording_options['excludes'] rec_desvars = self.recording_options['record_desvars'] rec_objectives = self.recording_options['record_objectives'] rec_constraints = self.recording_options['record_constraints'] rec_responses = self.recording_options['record_responses'] rec_inputs = self.recording_options['record_inputs'] all_desvars = { n for n in self._designvars if check_path(n, incl, excl, True) } all_objectives = { n for n in self._objs if check_path(n, incl, excl, True) } all_constraints = { n for n in self._cons if check_path(n, incl, excl, True) } if rec_desvars: mydesvars = all_desvars if rec_objectives: myobjectives = all_objectives if rec_constraints: myconstraints = all_constraints if rec_responses: myresponses = { n for n in self._responses if check_path(n, incl, excl, True) } # get the includes that were requested for this Driver recording if incl: # The my* variables are sets # First gather all of the desired outputs # The following might only be the local vars if MPI mysystem_outputs = { n for n in model._outputs if check_path(n, incl, excl) } # If MPI, and on rank 0, need to gather up all the variables # even those not local to rank 0 if MPI: all_vars = model.comm.gather(mysystem_outputs, root=0) if MPI.COMM_WORLD.rank == 0: mysystem_outputs = all_vars[-1] for d in all_vars[:-1]: mysystem_outputs.update(d) # de-duplicate mysystem_outputs mysystem_outputs = mysystem_outputs.difference( all_desvars, all_objectives, all_constraints) if rec_inputs: prob = self._problem root = prob.model myinputs = {n for n in root._inputs if check_path(n, incl, excl)} if MPI: all_vars = root.comm.gather(myinputs, root=0) if MPI.COMM_WORLD.rank == 0: myinputs = all_vars[-1] for d in all_vars[:-1]: myinputs.update(d) if MPI: # filter based on who owns the variables # TODO Eventually, we think we can get rid of this next check. But to be safe, # we are leaving it in there. if not model.is_active(): raise RuntimeError( "RecordingManager.startup should never be called when " "running in parallel on an inactive System") rrank = problem.comm.rank rowned = model._owning_rank mydesvars = [n for n in mydesvars if rrank == rowned[n]] myresponses = [n for n in myresponses if rrank == rowned[n]] myobjectives = [n for n in myobjectives if rrank == rowned[n]] myconstraints = [n for n in myconstraints if rrank == rowned[n]] mysystem_outputs = [ n for n in mysystem_outputs if rrank == rowned[n] ] myinputs = [n for n in myinputs if rrank == rowned[n]] self._filtered_vars_to_record = { 'des': mydesvars, 'obj': myobjectives, 'con': myconstraints, 'res': myresponses, 'sys': mysystem_outputs, 'in': myinputs } self._rec_mgr.startup(self) if self._rec_mgr._recorders: from openmdao.devtools.problem_viewer.problem_viewer import _get_viewer_data self._model_viewer_data = _get_viewer_data(problem) if self.recording_options['record_metadata']: self._rec_mgr.record_metadata(self) def _get_voi_val(self, name, meta, remote_vois, unscaled=False, ignore_indices=False): """ Get the value of a variable of interest (objective, constraint, or design var). This will retrieve the value if the VOI is remote. Parameters ---------- name : str Name of the variable of interest. meta : dict Metadata for the variable of interest. remote_vois : dict Dict containing (owning_rank, size) for all remote vois of a particular type (design var, constraint, or objective). unscaled : bool Set to True if unscaled (physical) design variables are desired. ignore_indices : bool Set to True if the full array is desired, not just those indicated by indices. Returns ------- float or ndarray The value of the named variable of interest. """ model = self._problem.model comm = model.comm vec = model._outputs._views_flat indices = meta['indices'] if name in remote_vois: owner, size = remote_vois[name] if owner == comm.rank: if indices is None or ignore_indices: val = vec[name].copy() else: val = vec[name][indices] else: if indices is not None: size = len(indices) val = np.empty(size) comm.Bcast(val, root=owner) else: if indices is None or ignore_indices: val = vec[name].copy() else: val = vec[name][indices] if self._has_scaling and not unscaled: # Scale design variable values adder = meta['adder'] if adder is not None: val += adder scaler = meta['scaler'] if scaler is not None: val *= scaler return val def get_design_var_values(self, filter=None, unscaled=False, ignore_indices=False): """ Return the design variable values. This is called to gather the initial design variable state. Parameters ---------- filter : list List of desvar names used by recorders. unscaled : bool Set to True if unscaled (physical) design variables are desired. ignore_indices : bool Set to True if the full array is desired, not just those indicated by indices. Returns ------- dict Dictionary containing values of each design variable. """ if filter: dvs = filter else: # use all the designvars dvs = self._designvars return { n: self._get_voi_val(n, self._designvars[n], self._remote_dvs, unscaled=unscaled, ignore_indices=ignore_indices) for n in dvs } def set_design_var(self, name, value): """ Set the value of a design variable. Parameters ---------- name : str Global pathname of the design variable. value : float or ndarray Value for the design variable. """ if (name in self._remote_dvs and self._problem.model._owning_rank[name] != self._problem.comm.rank): return meta = self._designvars[name] indices = meta['indices'] if indices is None: indices = slice(None) desvar = self._problem.model._outputs._views_flat[name] desvar[indices] = value if self._has_scaling: # Scale design variable values scaler = meta['scaler'] if scaler is not None: desvar[indices] *= 1.0 / scaler adder = meta['adder'] if adder is not None: desvar[indices] -= adder def get_response_values(self, filter=None): """ Return response values. Parameters ---------- filter : list List of response names used by recorders. Returns ------- dict Dictionary containing values of each response. """ if filter: resps = filter else: resps = self._responses return { n: self._get_voi_val(n, self._responses[n], self._remote_objs) for n in resps } def get_objective_values(self, unscaled=False, filter=None): """ Return objective values. Parameters ---------- unscaled : bool Set to True if unscaled (physical) design variables are desired. filter : list List of objective names used by recorders. Returns ------- dict Dictionary containing values of each objective. """ if filter: objs = filter else: objs = self._objs return { n: self._get_voi_val(n, self._objs[n], self._remote_objs, unscaled=unscaled) for n in objs } def get_constraint_values(self, ctype='all', lintype='all', unscaled=False, filter=None): """ Return constraint values. Parameters ---------- ctype : string Default is 'all'. Optionally return just the inequality constraints with 'ineq' or the equality constraints with 'eq'. lintype : string Default is 'all'. Optionally return just the linear constraints with 'linear' or the nonlinear constraints with 'nonlinear'. unscaled : bool Set to True if unscaled (physical) design variables are desired. filter : list List of constraint names used by recorders. Returns ------- dict Dictionary containing values of each constraint. """ if filter is not None: cons = filter else: cons = self._cons con_dict = {} for name in cons: meta = self._cons[name] if lintype == 'linear' and not meta['linear']: continue if lintype == 'nonlinear' and meta['linear']: continue if ctype == 'eq' and meta['equals'] is None: continue if ctype == 'ineq' and meta['equals'] is not None: continue con_dict[name] = self._get_voi_val(name, meta, self._remote_cons, unscaled=unscaled) return con_dict def _get_ordered_nl_responses(self): """ Return the names of nonlinear responses in the order used by the driver. Default order is objectives followed by nonlinear constraints. This is used for simultaneous derivative coloring and sparsity determination. Returns ------- list of str The nonlinear response names in order. """ order = list(self._objs) order.extend(n for n, meta in iteritems(self._cons) if not ('linear' in meta and meta['linear'])) return order def run(self): """ Execute this driver. The base `Driver` just runs the model. All other drivers overload this method. Returns ------- boolean Failure flag; True if failed to converge, False is successful. """ with RecordingDebugging(self._get_name(), self.iter_count, self) as rec: failure_flag, _, _ = self._problem.model._solve_nonlinear() self.iter_count += 1 return failure_flag def _compute_totals(self, of=None, wrt=None, return_format='flat_dict', global_names=True): """ Compute derivatives of desired quantities with respect to desired inputs. All derivatives are returned using driver scaling. Parameters ---------- of : list of variable name strings or None Variables whose derivatives will be computed. Default is None, which uses the driver's objectives and constraints. wrt : list of variable name strings or None Variables with respect to which the derivatives will be computed. Default is None, which uses the driver's desvars. return_format : string Format to return the derivatives. Default is a 'flat_dict', which returns them in a dictionary whose keys are tuples of form (of, wrt). For the scipy optimizer, 'array' is also supported. global_names : bool Set to True when passing in global names to skip some translation steps. Returns ------- derivs : object Derivatives in form requested by 'return_format'. """ total_jac = self._total_jac debug_print = 'totals' in self.options['debug_print'] and ( not MPI or MPI.COMM_WORLD.rank == 0) if debug_print: header = 'Driver total derivatives for iteration: ' + str( self.iter_count) print(header) print(len(header) * '-' + '\n') if self._problem.model._owns_approx_jac: if total_jac is None: self._total_jac = total_jac = _TotalJacInfo( self._problem, of, wrt, global_names, return_format, approx=True, debug_print=debug_print) return total_jac.compute_totals_approx() else: if total_jac is None: total_jac = _TotalJacInfo(self._problem, of, wrt, global_names, return_format, debug_print=debug_print) # don't cache linear constraint jacobian if not total_jac.has_lin_cons: self._total_jac = total_jac return total_jac.compute_totals() def record_iteration(self): """ Record an iteration of the current Driver. """ if not self._rec_mgr._recorders: return # Get the data to record (collective calls that get across all ranks) opts = self.recording_options filt = self._filtered_vars_to_record if opts['record_desvars']: des_vars = self.get_design_var_values() else: des_vars = {} if opts['record_objectives']: obj_vars = self.get_objective_values() else: obj_vars = {} if opts['record_constraints']: con_vars = self.get_constraint_values() else: con_vars = {} if opts['record_responses']: # res_vars = self.get_response_values() # not really working yet res_vars = {} else: res_vars = {} des_vars = {name: des_vars[name] for name in filt['des']} obj_vars = {name: obj_vars[name] for name in filt['obj']} con_vars = {name: con_vars[name] for name in filt['con']} # res_vars = {name: res_vars[name] for name in filt['res']} model = self._problem.model sys_vars = {} in_vars = {} outputs = model._outputs inputs = model._inputs views = outputs._views views_in = inputs._views sys_vars = { name: views[name] for name in outputs._names if name in filt['sys'] } if self.recording_options['record_inputs']: in_vars = { name: views_in[name] for name in inputs._names if name in filt['in'] } if MPI: des_vars = self._gather_vars(model, des_vars) res_vars = self._gather_vars(model, res_vars) obj_vars = self._gather_vars(model, obj_vars) con_vars = self._gather_vars(model, con_vars) sys_vars = self._gather_vars(model, sys_vars) in_vars = self._gather_vars(model, in_vars) outs = {} if not MPI or model.comm.rank == 0: outs.update(des_vars) outs.update(res_vars) outs.update(obj_vars) outs.update(con_vars) outs.update(sys_vars) data = {'out': outs, 'in': in_vars} metadata = create_local_meta(self._get_name()) self._rec_mgr.record_iteration(self, data, metadata) def _gather_vars(self, root, local_vars): """ Gather and return only variables listed in `local_vars` from the `root` System. Parameters ---------- root : <System> the root System for the Problem local_vars : dict local variable names and values Returns ------- dct : dict variable names and values. """ # if trace: # debug("gathering vars for recording in %s" % root.pathname) all_vars = root.comm.gather(local_vars, root=0) # if trace: # debug("DONE gathering rec vars for %s" % root.pathname) if root.comm.rank == 0: dct = all_vars[-1] for d in all_vars[:-1]: dct.update(d) return dct def _get_name(self): """ Get name of current Driver. Returns ------- str Name of current Driver. """ return "Driver" def set_simul_deriv_color(self, simul_info): """ Set the coloring (and possibly the sub-jac sparsity) for simultaneous total derivatives. Parameters ---------- simul_info : str or tuple :: # Information about simultaneous coloring for design vars and responses. If a # string, then simul_info is assumed to be the name of a file that contains the # coloring information in JSON format. If a tuple, the structure looks like this: ( # First, a list of column index lists, each index list representing columns # having the same color, except for the very first index list, which contains # indices of all columns that are not colored. [ [i1, i2, i3, ...] # list of non-colored columns [ia, ib, ...] # list of columns in first color [ic, id, ...] # list of columns in second color ... # remaining color lists, one list of columns per color ], # Next is a list of lists, one for each column, containing the nonzero rows for # that column. If a column is not colored, then it will have a None entry # instead of a list. [ [r1, rn, ...] # list of nonzero rows for column 0 None, # column 1 is not colored [ra, rb, ...] # list of nonzero rows for column 2 ... ], # The last tuple entry can be None, indicating that no sparsity structure is # specified, or it can be a nested dictionary where the outer keys are response # names, the inner keys are design variable names, and the value is a tuple of # the form (row_list, col_list, shape). { resp1_name: { dv1_name: (rows, cols, shape), # for sub-jac d_resp1/d_dv1 dv2_name: (rows, cols, shape), ... }, resp2_name: { ... } ... } ) """ if self.supports['simultaneous_derivatives']: self._simul_coloring_info = simul_info else: raise RuntimeError( "Driver '%s' does not support simultaneous derivatives." % self._get_name()) def set_total_jac_sparsity(self, sparsity): """ Set the sparsity of sub-jacobians of the total jacobian. Note: This currently will have no effect if you are not using the pyOptSparseDriver. Parameters ---------- sparsity : str or dict :: # Sparsity is a nested dictionary where the outer keys are response # names, the inner keys are design variable names, and the value is a tuple of # the form (row_list, col_list, shape). { resp1: { dv1: (rows, cols, shape), # for sub-jac d_resp1/d_dv1 dv2: (rows, cols, shape), ... }, resp2: { ... } ... } """ if self.supports['total_jac_sparsity']: self._total_jac_sparsity = sparsity else: raise RuntimeError( "Driver '%s' does not support setting of total jacobian sparsity." % self._get_name()) def _setup_simul_coloring(self, mode='fwd'): """ Set up metadata for simultaneous derivative solution. Parameters ---------- mode : str Derivative direction, either 'fwd' or 'rev'. """ if mode == 'rev': raise NotImplementedError( "Simultaneous derivatives are currently not supported " "in 'rev' mode") # command line simul_coloring uses this env var to turn pre-existing coloring off if not coloring_mod._use_sparsity: return if isinstance(self._simul_coloring_info, string_types): with open(self._simul_coloring_info, 'r') as f: self._simul_coloring_info = json.load(f) tup = self._simul_coloring_info column_lists, row_map = tup[:2] if len(tup) > 2: sparsity = tup[2] if self._total_jac_sparsity is not None: raise RuntimeError( "Total jac sparsity was set in both _simul_coloring_info" " and _total_jac_sparsity.") self._total_jac_sparsity = sparsity self._simul_coloring_info = column_lists, row_map def _pre_run_model_debug_print(self): """ Optionally print some debugging information before the model runs. """ debug_opt = self.options['debug_print'] if not debug_opt or debug_opt == ['totals']: return if not MPI or MPI.COMM_WORLD.rank == 0: header = 'Driver debug print for iter coord: {}'.format( get_formatted_iteration_coordinate()) print(header) print(len(header) * '-') if 'desvars' in debug_opt: desvar_vals = self.get_design_var_values(unscaled=True, ignore_indices=True) if not MPI or MPI.COMM_WORLD.rank == 0: print("Design Vars") if desvar_vals: pprint.pprint(desvar_vals) else: print("None") print() sys.stdout.flush() def _post_run_model_debug_print(self): """ Optionally print some debugging information after the model runs. """ if 'nl_cons' in self.options['debug_print']: cons = self.get_constraint_values(lintype='nonlinear', unscaled=True) if not MPI or MPI.COMM_WORLD.rank == 0: print("Nonlinear constraints") if cons: pprint.pprint(cons) else: print("None") print() if 'ln_cons' in self.options['debug_print']: cons = self.get_constraint_values(lintype='linear', unscaled=True) if not MPI or MPI.COMM_WORLD.rank == 0: print("Linear constraints") if cons: pprint.pprint(cons) else: print("None") print() if 'objs' in self.options['debug_print']: objs = self.get_objective_values(unscaled=True) if not MPI or MPI.COMM_WORLD.rank == 0: print("Objectives") if objs: pprint.pprint(objs) else: print("None") print() sys.stdout.flush()
class ExperimentalDriver(object): """ A fake driver class used for doc generation testing. Attributes ---------- fail : bool Reports whether the driver ran successfully. iter_count : int Keep track of iterations for case recording. options : list List of options options : <OptionsDictionary> Dictionary with general pyoptsparse options. recording_options : <OptionsDictionary> Dictionary with driver recording options. cite : str Listing of relevant citations that should be referenced when publishing work that uses this class. _problem : <Problem> Pointer to the containing problem. supports : <OptionsDictionary> Provides a consistant way for drivers to declare what features they support. _designvars : dict Contains all design variable info. _cons : dict Contains all constraint info. _objs : dict Contains all objective info. _responses : dict Contains all response info. _rec_mgr : <RecordingManager> Object that manages all recorders added to this driver. _vars_to_record : dict Dict of lists of var names indicating what to record _model_viewer_data : dict Structure of model, used to make n2 diagram. _remote_dvs : dict Dict of design variables that are remote on at least one proc. Values are (owning rank, size). _remote_cons : dict Dict of constraints that are remote on at least one proc. Values are (owning rank, size). _remote_objs : dict Dict of objectives that are remote on at least one proc. Values are (owning rank, size). _remote_responses : dict A combined dict containing entries from _remote_cons and _remote_objs. _total_coloring : tuple of dicts A data structure describing coloring for simultaneous derivs. _res_jacs : dict Dict of sparse subjacobians for use with certain optimizers, e.g. pyOptSparseDriver. """ def __init__(self): """ Initialize the driver. """ self._rec_mgr = RecordingManager() self._vars_to_record = { 'desvarnames': set(), 'responsenames': set(), 'objectivenames': set(), 'constraintnames': set(), 'sysinclnames': set(), } self._problem = None self._designvars = None self._cons = None self._objs = None self._responses = None self.options = OptionsDictionary() self.recording_options = OptionsDictionary() ########################### self.recording_options.declare('record_desvars', types=bool, default=True, desc='Set to True to record design variables at the \ driver level') self.recording_options.declare('record_responses', types=bool, default=False, desc='Set to True to record responses at the driver level') self.recording_options.declare('record_objectives', types=bool, default=True, desc='Set to True to record objectives at the \ driver level') self.recording_options.declare('record_constraints', types=bool, default=True, desc='Set to True to record constraints at the \ driver level') self.recording_options.declare('includes', types=list, default=[], desc='Patterns for variables to include in recording. \ Uses fnmatch wildcards') self.recording_options.declare('excludes', types=list, default=[], desc='Patterns for vars to exclude in recording ' '(processed post-includes). Uses fnmatch wildcards') self.recording_options.declare('record_derivatives', types=bool, default=False, desc='Set to True to record derivatives at the driver \ level') ########################### # What the driver supports. self.supports = OptionsDictionary() self.supports.declare('inequality_constraints', types=bool, default=False) self.supports.declare('equality_constraints', types=bool, default=False) self.supports.declare('linear_constraints', types=bool, default=False) self.supports.declare('two_sided_constraints', types=bool, default=False) self.supports.declare('multiple_objectives', types=bool, default=False) self.supports.declare('integer_design_vars', types=bool, default=False) self.supports.declare('gradients', types=bool, default=False) self.supports.declare('active_set', types=bool, default=False) self.supports.declare('simultaneous_derivatives', types=bool, default=False) self.supports.declare('distributed_design_vars', types=bool, default=False) self.iter_count = 0 self.options = None self._model_viewer_data = None self.cite = "" # TODO, support these in OpenMDAO self.supports.declare('integer_design_vars', types=bool, default=False) self._res_jacs = {} self.fail = False def add_recorder(self, recorder): """ Add a recorder to the driver. Parameters ---------- recorder : CaseRecorder A recorder instance. """ self._rec_mgr.append(recorder) def cleanup(self): """ Clean up resources prior to exit. """ self._rec_mgr.close() def _setup_driver(self, problem): """ Prepare the driver for execution. This is the final thing to run during setup. Parameters ---------- problem : <Problem> Pointer to the containing problem. """ pass def _get_voi_val(self, name, meta, remote_vois): """ Get the value of a variable of interest (objective, constraint, or design var). This will retrieve the value if the VOI is remote. Parameters ---------- name : str Name of the variable of interest. meta : dict Metadata for the variable of interest. remote_vois : dict Dict containing (owning_rank, size) for all remote vois of a particular type (design var, constraint, or objective). Returns ------- float or ndarray The value of the named variable of interest. """ model = self._problem.model comm = model.comm vec = model._outputs._views_flat indices = meta['indices'] if name in remote_vois: owner, size = remote_vois[name] if owner == comm.rank: if indices is None: val = vec[name].copy() else: val = vec[name][indices] else: if indices is not None: size = indices.indexed_src_size val = np.empty(size) comm.Bcast(val, root=owner) else: if indices is None: val = vec[name].copy() else: val = vec[name][indices] if self._has_scaling: # Scale design variable values adder = meta['adder'] if adder is not None: val += adder scaler = meta['scaler'] if scaler is not None: val *= scaler return val def get_design_var_values(self, filter=None): """ Return the design variable values. This is called to gather the initial design variable state. Parameters ---------- filter : list List of desvar names used by recorders. Returns ------- dict Dictionary containing values of each design variable. """ if filter: dvs = filter else: # use all the designvars dvs = self._designvars return {n: self._get_voi_val(n, self._designvars[n], self._remote_dvs) for n in dvs} def set_design_var(self, name, value): """ Set the value of a design variable. Parameters ---------- name : str Global pathname of the design variable. value : float or ndarray Value for the design variable. """ if (name in self._remote_dvs and self._problem.model._owning_rank['output'][name] != self._problem.comm.rank): return meta = self._designvars[name] indices = meta['indices'] if indices is None: indices = slice(None) desvar = self._problem.model._outputs._views_flat[name] desvar[indices] = value if self._has_scaling: # Scale design variable values scaler = meta['scaler'] if scaler is not None: desvar[indices] *= 1.0 / scaler adder = meta['adder'] if adder is not None: desvar[indices] -= adder def get_response_values(self, filter=None): """ Return response values. Parameters ---------- filter : list List of response names used by recorders. Returns ------- dict Dictionary containing values of each response. """ if filter: resps = filter else: resps = self._responses return {n: self._get_voi_val(n, self._responses[n], self._remote_objs) for n in resps} def get_objective_values(self, filter=None): """ Return objective values. Parameters ---------- filter : list List of objective names used by recorders. Returns ------- dict Dictionary containing values of each objective. """ if filter: objs = filter else: objs = self._objs return {n: self._get_voi_val(n, self._objs[n], self._remote_objs) for n in objs} def get_constraint_values(self, ctype='all', lintype='all', filter=None): """ Return constraint values. Parameters ---------- ctype : str Default is 'all'. Optionally return just the inequality constraints with 'ineq' or the equality constraints with 'eq'. lintype : str Default is 'all'. Optionally return just the linear constraints with 'linear' or the nonlinear constraints with 'nonlinear'. filter : list List of constraint names used by recorders. Returns ------- dict Dictionary containing values of each constraint. """ if filter is not None: cons = filter else: cons = self._cons con_dict = {} for name in cons: meta = self._cons[name] if lintype == 'linear' and not meta['linear']: continue if lintype == 'nonlinear' and meta['linear']: continue if ctype == 'eq' and meta['equals'] is None: continue if ctype == 'ineq' and meta['equals'] is not None: continue con_dict[name] = self._get_voi_val(name, meta, self._remote_cons) return con_dict def run(self): """ Execute this driver. The base `Driver` just runs the model. All other drivers overload this method. Returns ------- bool Failure flag; True if failed to converge, False is successful. """ with Recording(self._get_name(), self.iter_count, self) as rec: self._problem.model.run_solve_nonlinear() self.iter_count += 1 return False def _dict2array_jac(self, derivs): osize = 0 isize = 0 do_wrt = True islices = {} oslices = {} for okey, oval in derivs.items(): if do_wrt: for ikey, val in oval.items(): istart = isize isize += val.shape[1] islices[ikey] = slice(istart, isize) do_wrt = False ostart = osize osize += oval[ikey].shape[0] oslices[okey] = slice(ostart, osize) new_derivs = np.zeros((osize, isize)) relevant = self._problem.model._relevant for okey, odict in derivs.items(): for ikey, val in odict.items(): if okey in relevant[ikey] or ikey in relevant[okey]: new_derivs[oslices[okey], islices[ikey]] = val return new_derivs def _compute_totals(self, of=None, wrt=None, return_format='flat_dict', use_abs_names=True): """ Compute derivatives of desired quantities with respect to desired inputs. All derivatives are returned using driver scaling. Parameters ---------- of : list of variable name str or None Variables whose derivatives will be computed. Default is None, which uses the driver's objectives and constraints. wrt : list of variable name str or None Variables with respect to which the derivatives will be computed. Default is None, which uses the driver's desvars. return_format : str Format to return the derivatives. Default is a 'flat_dict', which returns them in a dictionary whose keys are tuples of form (of, wrt). For the scipy optimizer, 'array' is also supported. use_abs_names : bool Set to True when passing in global names to skip some translation steps. Returns ------- derivs : object Derivatives in form requested by 'return_format'. """ prob = self._problem # Compute the derivatives in dict format... if prob.model._owns_approx_jac: derivs = prob._compute_totals_approx(of=of, wrt=wrt, return_format='dict', use_abs_names=use_abs_names) else: derivs = prob._compute_totals(of=of, wrt=wrt, return_format='dict', use_abs_names=use_abs_names) # ... then convert to whatever the driver needs. if return_format in ('dict', 'array'): if self._has_scaling: for okey, odict in derivs.items(): for ikey, val in odict.items(): iscaler = self._designvars[ikey]['scaler'] oscaler = self._responses[okey]['scaler'] # Scale response side if oscaler is not None: val[:] = (oscaler * val.T).T # Scale design var side if iscaler is not None: val *= 1.0 / iscaler else: raise RuntimeError("Derivative scaling by the driver only supports the 'dict' and " "'array' formats at present.") if return_format == 'array': derivs = self._dict2array_jac(derivs) return derivs def record_iteration(self): """ Record an iteration of the current Driver. """ if not self._rec_mgr._recorders: return metadata = create_local_meta(self._get_name()) # Get the data to record data = {} if self.recording_options['record_desvars']: # collective call that gets across all ranks desvars = self.get_design_var_values() else: desvars = {} if self.recording_options['record_responses']: # responses = self.get_response_values() # not really working yet responses = {} else: responses = {} if self.recording_options['record_objectives']: objectives = self.get_objective_values() else: objectives = {} if self.recording_options['record_constraints']: constraints = self.get_constraint_values() else: constraints = {} desvars = {name: desvars[name] for name in self._filtered_vars_to_record['des']} # responses not working yet # responses = {name: responses[name] for name in self._filtered_vars_to_record['res']} objectives = {name: objectives[name] for name in self._filtered_vars_to_record['obj']} constraints = {name: constraints[name] for name in self._filtered_vars_to_record['con']} if self.recording_options['includes']: root = self._problem.model outputs = root._outputs # outputsinputs, outputs, residuals = root.get_nonlinear_vectors() sysvars = {} for name, value in outputs._names.items(): if name in self._filtered_vars_to_record['sys']: sysvars[name] = value else: sysvars = {} if MPI: root = self._problem.model desvars = self._gather_vars(root, desvars) responses = self._gather_vars(root, responses) objectives = self._gather_vars(root, objectives) constraints = self._gather_vars(root, constraints) sysvars = self._gather_vars(root, sysvars) data['des'] = desvars data['res'] = responses data['obj'] = objectives data['con'] = constraints data['sys'] = sysvars self._rec_mgr.record_iteration(self, data, metadata) def _gather_vars(self, root, local_vars): """ Gather and return only variables listed in `local_vars` from the `root` System. Parameters ---------- root : <System> the root System for the Problem local_vars : dict local variable names and values Returns ------- dct : dict variable names and values. """ # if trace: # debug("gathering vars for recording in %s" % root.pathname) all_vars = root.comm.gather(local_vars, root=0) # if trace: # debug("DONE gathering rec vars for %s" % root.pathname) if root.comm.rank == 0: dct = all_vars[-1] for d in all_vars[:-1]: dct.update(d) return dct def _get_name(self): """ Get name of current Driver. Returns ------- str Name of current Driver. """ return "Driver"
class Driver(object): """ Base class for drivers in OpenMDAO. Drivers can only be placed in a Problem, and every problem has a Driver. Driver is the simplest driver that runs (solves using solve_nonlinear) a problem once. """ def __init__(self): super(Driver, self).__init__() self.recorders = RecordingManager() # What this driver supports self.supports = OptionsDictionary(read_only=True) self.supports.add_option('inequality_constraints', True) self.supports.add_option('equality_constraints', True) self.supports.add_option('linear_constraints', True) self.supports.add_option('multiple_objectives', True) self.supports.add_option('two_sided_constraints', True) self.supports.add_option('integer_design_vars', True) # inheriting Drivers should override this setting and set it to False # if they don't use gradients. self.supports.add_option('gradients', True) # This driver's options self.options = OptionsDictionary() self._desvars = OrderedDict() self._objs = OrderedDict() self._cons = OrderedDict() self._voi_sets = [] self._vars_to_record = None # We take root during setup self.root = None self.iter_count = 0 self.dv_conversions = {} self.fn_conversions = {} def _setup(self): """ Updates metadata for params, constraints and objectives, and check for errors. Also determines all variables that need to be gathered for case recording. """ root = self.root desvars = OrderedDict() objs = OrderedDict() cons = OrderedDict() if self.__class__ is Driver: has_gradients = False else: has_gradients = self.supports['gradients'] item_tups = [ ('Parameter', self._desvars, desvars), ('Objective', self._objs, objs), ('Constraint', self._cons, cons) ] for item_name, item, newitem in item_tups: for name, meta in iteritems(item): # Check validity of variable if name not in root.unknowns: msg = "{} '{}' not found in unknowns." msg = msg.format(item_name, name) raise ValueError(msg) rootmeta = root.unknowns.metadata(name) if name in self._desvars: rootmeta['is_desvar'] = True if name in self._objs: rootmeta['is_objective'] = True if name in self._cons: rootmeta['is_constraint'] = True if MPI and 'src_indices' in rootmeta: raise ValueError("'%s' is a distributed variable and may " "not be used as a design var, objective, " "or constraint." % name) if has_gradients and rootmeta.get('pass_by_obj'): if 'optimizer' in self.options: oname = self.options['optimizer'] else: oname = self.__class__.__name__ raise RuntimeError("%s '%s' is a 'pass_by_obj' variable " "and can't be used with a gradient " "based driver of type '%s'." % (item_name, name, oname)) # Size is useful metadata to save if 'indices' in meta: meta['size'] = len(meta['indices']) else: meta['size'] = rootmeta['size'] newitem[name] = meta self._desvars = desvars self._objs = objs self._cons = cons # Cache scalers for derivative calculation self.dv_conversions = OrderedDict() for name, meta in iteritems(desvars): scaler = meta.get('scaler') if isinstance(scaler, np.ndarray): if all(scaler == 1.0): continue elif scaler == 1.0: continue self.dv_conversions[name] = np.reciprocal(scaler) self.fn_conversions = OrderedDict() for name, meta in chain(iteritems(objs), iteritems(cons)): scaler = meta.get('scaler') if isinstance(scaler, np.ndarray): if all(scaler == 1.0): continue elif scaler == 1.0: continue self.fn_conversions[name] = scaler def _setup_communicators(self, comm, parent_dir): """ Assign a communicator to the root `System`. Args ---- comm : an MPI communicator (real or fake) The communicator being offered by the Problem. parent_dir : str Absolute directory of the Problem. """ self.root._setup_communicators(comm, parent_dir) def get_req_procs(self): """ Returns ------- tuple A tuple of the form (min_procs, max_procs), indicating the min and max processors usable by this `Driver`. """ return self.root.get_req_procs() def cleanup(self): """ Clean up resources prior to exit. """ self.recorders.close() def _map_voi_indices(self): poi_indices = OrderedDict() qoi_indices = OrderedDict() for name, meta in chain(iteritems(self._cons), iteritems(self._objs)): # set indices of interest if 'indices' in meta: qoi_indices[name] = meta['indices'] for name, meta in iteritems(self._desvars): # set indices of interest if 'indices' in meta: poi_indices[name] = meta['indices'] return poi_indices, qoi_indices def _of_interest(self, voi_list): """Return a list of tuples, with the given voi_list organized into tuples based on the previously defined grouping of VOIs. """ vois = [] remaining = set(voi_list) for voi_set in self._voi_sets: vois.append([]) for i, voi_set in enumerate(self._voi_sets): for v in voi_list: if v in voi_set: vois[i].append(v) remaining.remove(v) vois = [tuple(x) for x in vois if x] for v in voi_list: if v in remaining: vois.append((v,)) return vois def desvars_of_interest(self): """ Returns ------- list of tuples of str The list of design vars, organized into tuples according to previously defined VOI groups. """ return self._of_interest(self._desvars) def outputs_of_interest(self): """ Returns ------- list of tuples of str The list of constraints and objectives, organized into tuples according to previously defined VOI groups. """ return self._of_interest(list(chain(self._objs, self._cons))) def parallel_derivs(self, vnames): """ Specifies that the named variables of interest are to be grouped together so that their derivatives can be solved for concurrently. Args ---- vnames : iter of str The names of variables of interest that are to be grouped. """ #make sure all vnames are desvars, constraints, or objectives for n in vnames: if not (n in self._desvars or n in self._objs or n in self._cons): raise RuntimeError("'%s' is not a param, objective, or " "constraint" % n) for grp in self._voi_sets: for vname in vnames: if vname in grp: msg = "'%s' cannot be added to VOI set %s because it " + \ "already exists in VOI set: %s" raise RuntimeError(msg % (vname, tuple(vnames), grp)) param_intsect = set(vnames).intersection(self._desvars.keys()) if param_intsect and len(param_intsect) != len(vnames): raise RuntimeError("%s cannot be grouped because %s are design " "vars and %s are not." % (vnames, list(param_intsect), list(set(vnames).difference(param_intsect)))) if MPI: self._voi_sets.append(tuple(vnames)) else: warnings.warn("parallel derivs %s specified but not running under MPI") def add_recorder(self, recorder): """ Adds a recorder to the driver. Args ---- recorder : BaseRecorder A recorder instance. """ self.recorders.append(recorder) def add_desvar(self, name, lower=None, upper=None, low=None, high=None, indices=None, adder=0.0, scaler=1.0): """ Adds a design variable to this driver. Args ---- name : string Name of the design variable in the root system. lower : float or ndarray, optional Lower boundary for the param upper : upper or ndarray, optional Upper boundary for the param indices : iter of int, optional If a param is an array, these indicate which entries are of interest for derivatives. adder : float or ndarray, optional Value to add to the model value to get the scaled value. Adder is first in precedence. scaler : float or ndarray, optional value to multiply the model value to get the scaled value. Scaler is second in precedence. """ if name in self._desvars: msg = "Desvar '{}' already exists." raise RuntimeError(msg.format(name)) if low is not None or high is not None: warnings.simplefilter('always', DeprecationWarning) warnings.warn("'low' and 'high' are deprecated. " "Use 'lower' and 'upper' instead.", DeprecationWarning,stacklevel=2) warnings.simplefilter('ignore', DeprecationWarning) if low is not None and lower is None: lower = low if high is not None and upper is None: upper = high if isinstance(lower, np.ndarray): lower = lower.flatten() elif lower is None or lower == -float('inf'): lower = -sys.float_info.max if isinstance(upper, np.ndarray): upper = upper.flatten() elif upper is None or upper == float('inf'): upper = sys.float_info.max if isinstance(adder, np.ndarray): adder = adder.flatten().astype('float') else: adder = float(adder) if isinstance(scaler, np.ndarray): scaler = scaler.flatten().astype('float') else: scaler = float(scaler) # Scale the lower and upper values lower = (lower + adder)*scaler upper = (upper + adder)*scaler param = OrderedDict() param['lower'] = lower param['upper'] = upper param['adder'] = adder param['scaler'] = scaler if indices: param['indices'] = np.array(indices, dtype=int) self._desvars[name] = param def add_param(self, name, lower=None, upper=None, indices=None, adder=0.0, scaler=1.0): """ Deprecated. Use ``add_desvar`` instead. """ warnings.simplefilter('always', DeprecationWarning) warnings.warn("Driver.add_param() is deprecated. Use add_desvar() instead.", DeprecationWarning,stacklevel=2) warnings.simplefilter('ignore', DeprecationWarning) self.add_desvar(name, lower=lower, upper=upper, indices=indices, adder=adder, scaler=scaler) def get_desvars(self): """ Returns a dict of possibly distributed design variables. Returns ------- dict Keys are the param object names, and the values are the param values. """ desvars = OrderedDict() for key, meta in iteritems(self._desvars): desvars[key] = self._get_distrib_var(key, meta, 'design var') return desvars def _get_distrib_var(self, name, meta, voi_type): uvec = self.root.unknowns comm = self.root.comm nproc = comm.size iproc = comm.rank if nproc > 1: owner = self.root._owning_ranks[name] if iproc == owner: flatval = uvec._dat[name].val else: flatval = None else: owner = 0 flatval = uvec._dat[name].val if 'indices' in meta and not (nproc > 1 and owner != iproc): # Make sure our indices are valid try: flatval = flatval[meta['indices']] except IndexError: msg = "Index for {} '{}' is out of bounds. " msg += "Requested index: {}, " msg += "shape: {}." raise IndexError(msg.format(voi_type, name, meta['indices'], uvec.metadata(name)['shape'])) if nproc > 1: # TODO: use Bcast for improved performance if trace: debug("%s.driver._get_distrib_var bcast: val=%s" % (self.root.pathname, flatval)) flatval = comm.bcast(flatval, root=owner) if trace: debug("%s.driver._get_distrib_var bcast DONE" % self.root.pathname) scaler = meta['scaler'] adder = meta['adder'] if isinstance(scaler, np.ndarray) or isinstance(adder, np.ndarray) \ or scaler != 1.0 or adder != 0.0: return (flatval + adder)*scaler else: return flatval def get_desvar_metadata(self): """ Returns a dict of design variable metadata. Returns ------- dict Keys are the param object names, and the values are the param values. """ return self._desvars def set_desvar(self, name, value): """ Sets a design variable. Args ---- name : string Name of the design variable in the root system. val : ndarray or float value to assign to the design variable. """ val = self.root.unknowns._dat[name].val if not isinstance(val, _ByObjWrapper) and \ self.root.unknowns._dat[name].val.size == 0: return meta = self._desvars[name] scaler = meta['scaler'] adder = meta['adder'] if isinstance(scaler, np.ndarray) or isinstance(adder, np.ndarray) \ or scaler != 1.0 or adder != 0.0: value = value/scaler - adder # Only set the indices we requested when we set the design variable. idx = meta.get('indices') if idx is not None: self.root.unknowns[name][idx] = value else: self.root.unknowns[name] = value def add_objective(self, name, indices=None, adder=0.0, scaler=1.0): """ Adds an objective to this driver. Args ---- name : string Promoted pathname of the output that will serve as the objective. indices : iter of int, optional If an objective is an array, these indicate which entries are of interest for derivatives. adder : float or ndarray, optional Value to add to the model value to get the scaled value. Adder is first in precedence. scaler : float or ndarray, optional value to multiply the model value to get the scaled value. Scaler is second in precedence. """ if len(self._objs) > 0 and not self.supports["multiple_objectives"]: raise RuntimeError("Attempted to add multiple objectives to a " "driver that does not support multiple " "objectives.") if name in self._objs: msg = "Objective '{}' already exists." raise RuntimeError(msg.format(name)) if isinstance(adder, np.ndarray): adder = adder.flatten().astype('float') else: adder = float(adder) if isinstance(scaler, np.ndarray): scaler = scaler.flatten().astype('float') else: scaler = float(scaler) obj = OrderedDict() obj['adder'] = adder obj['scaler'] = scaler if indices: obj['indices'] = indices if len(indices) > 1 and not self.supports['multiple_objectives']: raise RuntimeError("Multiple objective indices specified for " "variable '%s', but driver '%s' doesn't " "support multiple objectives." % (name, self.pathname)) self._objs[name] = obj def get_objectives(self, return_type='dict'): """ Gets all objectives of this driver. Args ---- return_type : string Set to 'dict' to return a dictionary, or set to 'array' to return a flat ndarray. Returns ------- dict (for return_type 'dict') Key is the objective name string, value is an ndarray with the values. ndarray (for return_type 'array') Array containing all objective values in the order they were added. """ objs = OrderedDict() for key, meta in iteritems(self._objs): objs[key] = self._get_distrib_var(key, meta, 'objective') return objs def add_constraint(self, name, lower=None, upper=None, equals=None, linear=False, jacs=None, indices=None, adder=0.0, scaler=1.0): """ Adds a constraint to this driver. For inequality constraints, `lower` or `upper` must be specified. For equality constraints, `equals` must be specified. Args ---- name : string Promoted pathname of the output that will serve as the quantity to constrain. lower : float or ndarray, optional Constrain the quantity to be greater than or equal to this value. upper : float or ndarray, optional Constrain the quantity to be less than or equal to this value. equals : float or ndarray, optional Constrain the quantity to be equal to this value. linear : bool, optional Set to True if this constraint is linear with respect to all design variables so that it can be calculated once and cached. jacs : dict of functions, optional Dictionary of user-defined functions that return the flattened Jacobian of this constraint with repsect to the design vars of this driver, as indicated by the dictionary keys. Default is None to let OpenMDAO calculate all derivatives. Note, this is currently unsupported indices : iter of int, optional If a constraint is an array, these indicate which entries are of interest for derivatives. adder : float or ndarray, optional Value to add to the model value to get the scaled value. Adder is first in precedence. scaler : float or ndarray, optional value to multiply the model value to get the scaled value. Scaler is second in precedence. """ if name in self._cons: msg = "Constraint '{}' already exists." raise RuntimeError(msg.format(name)) if equals is not None and (lower is not None or upper is not None): msg = "Constraint '{}' cannot be both equality and inequality." raise RuntimeError(msg.format(name)) if equals is not None and self.supports['equality_constraints'] is False: msg = "Driver does not support equality constraint '{}'." raise RuntimeError(msg.format(name)) if equals is None and self.supports['inequality_constraints'] is False: msg = "Driver does not support inequality constraint '{}'." raise RuntimeError(msg.format(name)) if lower is not None and upper is not None and self.supports['two_sided_constraints'] is False: msg = "Driver does not support 2-sided constraint '{}'." raise RuntimeError(msg.format(name)) if lower is None and upper is None and equals is None: msg = "Constraint '{}' needs to define lower, upper, or equals." raise RuntimeError(msg.format(name)) if isinstance(scaler, np.ndarray): scaler = scaler.flatten().astype('float') else: scaler = float(scaler) if isinstance(adder, np.ndarray): adder = adder.flatten().astype('float') else: adder = float(adder) if isinstance(lower, np.ndarray): lower = lower.flatten() if isinstance(upper, np.ndarray): upper = upper.flatten() if isinstance(equals, np.ndarray): equals = equals.flatten() # Scale the lower and upper values if lower is not None: lower = (lower + adder)*scaler if upper is not None: upper = (upper + adder)*scaler if equals is not None: equals = (equals + adder)*scaler con = OrderedDict() con['lower'] = lower con['upper'] = upper con['equals'] = equals con['linear'] = linear con['adder'] = adder con['scaler'] = scaler con['jacs'] = jacs if indices: con['indices'] = indices self._cons[name] = con def get_constraints(self, ctype='all', lintype='all'): """ Gets all constraints for this driver. Args ---- ctype : string Default is 'all'. Optionally return just the inequality constraints with 'ineq' or the equality constraints with 'eq'. lintype : string Default is 'all'. Optionally return just the linear constraints with 'linear' or the nonlinear constraints with 'nonlinear'. Returns ------- dict Key is the constraint name string, value is an ndarray with the values. """ cons = OrderedDict() for key, meta in iteritems(self._cons): if lintype == 'linear' and meta['linear'] is False: continue if lintype == 'nonlinear' and meta['linear']: continue if ctype == 'eq' and meta['equals'] is None: continue if ctype == 'ineq' and meta['equals'] is not None: continue cons[key] = self._get_distrib_var(key, meta, 'constraint') return cons def get_constraint_metadata(self): """ Returns a dict of constraint metadata. Returns ------- dict Keys are the constraint object names, and the values are the param values. """ return self._cons def run(self, problem): """ Runs the driver. This function should be overridden when inheriting. Args ---- problem : `Problem` Our parent `Problem`. """ self.run_once(problem) def run_once(self, problem): """ Runs root's solve_nonlinear one time Args ---- problem : `Problem` Our parent `Problem`. """ system = problem.root # Metadata Setup self.iter_count += 1 metadata = self.metadata = create_local_meta(None, 'Driver') system.ln_solver.local_meta = metadata update_local_meta(metadata, (self.iter_count,)) # Solve the system once and record results. with system._dircontext: system.solve_nonlinear(metadata=metadata) self.recorders.record_iteration(system, metadata) def calc_gradient(self, indep_list, unknown_list, mode='auto', return_format='array', sparsity=None): """ Returns the scaled gradient for the system that is contained in self.root, scaled by all scalers that were specified when the desvars and constraints were added. Args ---- indep_list : list of strings List of independent variable names that derivatives are to be calculated with respect to. All params must have a IndepVarComp. unknown_list : list of strings List of output or state names that derivatives are to be calculated for. All must be valid unknowns in OpenMDAO. mode : string, optional Deriviative direction, can be 'fwd', 'rev', 'fd', or 'auto'. Default is 'auto', which uses mode specified on the linear solver in root. return_format : string, optional Format for the derivatives, can be 'array' or 'dict'. sparsity : dict, optional Dictionary that gives the relevant design variables for each constraint. This option is only supported in the `dict` return format. Returns ------- ndarray or dict Jacobian of unknowns with respect to params. """ J = self._problem.calc_gradient(indep_list, unknown_list, mode=mode, return_format=return_format, dv_scale=self.dv_conversions, cn_scale=self.fn_conversions, sparsity=sparsity) self.recorders.record_derivatives(J, self.metadata) return J def generate_docstring(self): """ Generates a numpy-style docstring for a user-created Driver class. Returns ------- docstring : str string that contains a basic numpy docstring. """ #start the docstring off docstring = ' \"\"\"\n' #Put options into docstring firstTime = 1 for key, value in sorted(vars(self).items()): if type(value)==OptionsDictionary: if key == "supports": continue if firstTime: #start of Options docstring docstring += '\n Options\n -------\n' firstTime = 0 docstring += value._generate_docstring(key) #finish up docstring docstring += '\n \"\"\"\n' return docstring
class SolverBase(object): """ Common base class for Linear and Nonlinear solver. Should not be used by users. Always inherit from `LinearSolver` or `NonlinearSolver`.""" def __init__(self): self.iter_count = 0 self.options = OptionsDictionary() desc = 'Set to 0 to disable printing, set to 1 to print the ' \ 'residual to stdout each iteration, set to 2 to print ' \ 'subiteration residuals as well.' self.options.add_option('iprint', 0, values=[0, 1, 2], desc=desc) self.options.add_option('err_on_maxiter', False, desc='If True, raise an AnalysisError if not converged at maxiter.') self.recorders = RecordingManager() self.local_meta = None def setup(self, sub): """ Solvers override to define post-setup initiailzation. Args ---- sub: `System` System that owns this solver. """ pass def cleanup(self): """ Clean up resources prior to exit. """ self.recorders.close() def print_norm(self, solver_string, pathname, iteration, res, res0, msg=None, indent=0, solver='NL', u_norm=None): """ Prints out the norm of the residual in a neat readable format. Args ---- solver_string: string Unique string to identify your solver type (e.g., 'LN_GS' or 'NEWTON'). pathname: dict Parent system pathname. iteration: int Current iteration number res: float Norm of the absolute residual value. res0: float Norm of the baseline initial residual for relative comparison. msg: string, optional Message that indicates convergence. ident: int, optional Additional indentation levels for subiterations. solver: string, optional Solver type if not LN or NL (mostly for line search operations.) u_norm: float, optional Norm of the u vector, if applicable. """ if pathname=='': name = 'root' else: name = 'root.' + pathname # Find indentation level level = pathname.count('.') # No indentation for driver; top solver is no indentation. level = level + indent indent = ' ' * level if msg is not None: form = indent + '[%s] %s: %s %d | %s' if u_norm: form += ' (%s)' % u_norm print(form % (name, solver, solver_string, iteration, msg)) return form = indent + '[%s] %s: %s %d | %.9g %.9g' if u_norm: form += ' (%s)' % u_norm print(form % (name, solver, solver_string, iteration, res, res/res0)) def print_all_convergence(self): """ Turns on iprint for this solver and all subsolvers. Override if your solver has subsolvers.""" self.options['iprint'] = 1 def generate_docstring(self): """ Generates a numpy-style docstring for a user-created System class. Returns ------- docstring : str string that contains a basic numpy docstring. """ #start the docstring off docstrings = [' \"\"\"'] #Put options into docstring firstTime = 1 for key, value in sorted(vars(self).items()): if type(value)==OptionsDictionary: if firstTime: #start of Options docstring docstrings.extend(['',' Options',' -------']) firstTime = 0 docstrings.append(value._generate_docstring(key)) #finish up docstring docstrings.extend([' \"\"\"','']) return '\n'.join(docstrings)
class ExperimentalDriver(object): """ A fake driver class used for doc generation testing. Attributes ---------- fail : bool Reports whether the driver ran successfully. iter_count : int Keep track of iterations for case recording. options : list List of options options : <OptionsDictionary> Dictionary with general pyoptsparse options. recording_options : <OptionsDictionary> Dictionary with driver recording options. cite : str Listing of relevant citations that should be referenced when publishing work that uses this class. _problem : <Problem> Pointer to the containing problem. supports : <OptionsDictionary> Provides a consistant way for drivers to declare what features they support. _designvars : dict Contains all design variable info. _cons : dict Contains all constraint info. _objs : dict Contains all objective info. _responses : dict Contains all response info. _rec_mgr : <RecordingManager> Object that manages all recorders added to this driver. _vars_to_record: dict Dict of lists of var names indicating what to record _model_viewer_data : dict Structure of model, used to make n2 diagram. _remote_dvs : dict Dict of design variables that are remote on at least one proc. Values are (owning rank, size). _remote_cons : dict Dict of constraints that are remote on at least one proc. Values are (owning rank, size). _remote_objs : dict Dict of objectives that are remote on at least one proc. Values are (owning rank, size). _remote_responses : dict A combined dict containing entries from _remote_cons and _remote_objs. _total_coloring : tuple of dicts A data structure describing coloring for simultaneous derivs. _res_jacs : dict Dict of sparse subjacobians for use with certain optimizers, e.g. pyOptSparseDriver. """ def __init__(self): """ Initialize the driver. """ self._rec_mgr = RecordingManager() self._vars_to_record = { 'desvarnames': set(), 'responsenames': set(), 'objectivenames': set(), 'constraintnames': set(), 'sysinclnames': set(), } self._problem = None self._designvars = None self._cons = None self._objs = None self._responses = None self.options = OptionsDictionary() self.recording_options = OptionsDictionary() ########################### self.recording_options.declare('record_metadata', types=bool, desc='Record metadata', default=True) self.recording_options.declare( 'record_desvars', types=bool, default=True, desc='Set to True to record design variables at the \ driver level') self.recording_options.declare( 'record_responses', types=bool, default=False, desc='Set to True to record responses at the driver level') self.recording_options.declare( 'record_objectives', types=bool, default=True, desc='Set to True to record objectives at the \ driver level') self.recording_options.declare( 'record_constraints', types=bool, default=True, desc='Set to True to record constraints at the \ driver level') self.recording_options.declare( 'includes', types=list, default=[], desc='Patterns for variables to include in recording') self.recording_options.declare( 'excludes', types=list, default=[], desc='Patterns for vars to exclude in recording ' '(processed post-includes)') self.recording_options.declare( 'record_derivatives', types=bool, default=False, desc='Set to True to record derivatives at the driver \ level') ########################### # What the driver supports. self.supports = OptionsDictionary() self.supports.declare('inequality_constraints', types=bool, default=False) self.supports.declare('equality_constraints', types=bool, default=False) self.supports.declare('linear_constraints', types=bool, default=False) self.supports.declare('two_sided_constraints', types=bool, default=False) self.supports.declare('multiple_objectives', types=bool, default=False) self.supports.declare('integer_design_vars', types=bool, default=False) self.supports.declare('gradients', types=bool, default=False) self.supports.declare('active_set', types=bool, default=False) self.supports.declare('simultaneous_derivatives', types=bool, default=False) self.iter_count = 0 self.options = None self._model_viewer_data = None self.cite = "" # TODO, support these in OpenMDAO self.supports.declare('integer_design_vars', types=bool, default=False) self._res_jacs = {} self.fail = False def add_recorder(self, recorder): """ Add a recorder to the driver. Parameters ---------- recorder : CaseRecorder A recorder instance. """ self._rec_mgr.append(recorder) def cleanup(self): """ Clean up resources prior to exit. """ self._rec_mgr.close() def _setup_driver(self, problem): """ Prepare the driver for execution. This is the final thing to run during setup. Parameters ---------- problem : <Problem> Pointer to the containing problem. """ self._problem = problem model = problem.model self._objs = objs = OrderedDict() self._cons = cons = OrderedDict() self._responses = model.get_responses(recurse=True) response_size = 0 for name, data in iteritems(self._responses): if data['type'] == 'con': cons[name] = data else: objs[name] = data response_size += data['size'] # Gather up the information for design vars. self._designvars = model.get_design_vars(recurse=True) desvar_size = np.sum(data['size'] for data in itervalues(self._designvars)) if ((problem._mode == 'fwd' and desvar_size > response_size) or (problem._mode == 'rev' and response_size > desvar_size)): warnings.warn( "Inefficient choice of derivative mode. You chose '%s' for a " "problem with %d design variables and %d response variables " "(objectives and constraints)." % (problem._mode, desvar_size, response_size), RuntimeWarning) self._has_scaling = ( np.any([r['scaler'] is not None for r in self._responses.values()]) or np.any( [dv['scaler'] is not None for dv in self._designvars.values()])) con_set = set() obj_set = set() dv_set = set() self._remote_dvs = dv_dict = {} self._remote_cons = con_dict = {} self._remote_objs = obj_dict = {} # Now determine if later we'll need to allgather cons, objs, or desvars. if model.comm.size > 1 and model._subsystems_allprocs: local_out_vars = set(model._outputs._views) remote_dvs = set(self._designvars) - local_out_vars remote_cons = set(self._cons) - local_out_vars remote_objs = set(self._objs) - local_out_vars all_remote_vois = model.comm.allgather( (remote_dvs, remote_cons, remote_objs)) for rem_dvs, rem_cons, rem_objs in all_remote_vois: con_set.update(rem_cons) obj_set.update(rem_objs) dv_set.update(rem_dvs) # If we have remote VOIs, pick an owning rank for each and use that # to bcast to others later owning_ranks = model._owning_rank['output'] sizes = model._var_sizes['nonlinear']['output'] for i, vname in enumerate(model._var_allprocs_abs_names['output']): owner = owning_ranks[vname] if vname in dv_set: dv_dict[vname] = (owner, sizes[owner, i]) if vname in con_set: con_dict[vname] = (owner, sizes[owner, i]) if vname in obj_set: obj_dict[vname] = (owner, sizes[owner, i]) self._remote_responses = self._remote_cons.copy() self._remote_responses.update(self._remote_objs) # Case recording setup mydesvars = myobjectives = myconstraints = myresponses = set() mysystem_outputs = set() incl = self.recording_options['includes'] excl = self.recording_options['excludes'] rec_desvars = self.recording_options['record_desvars'] rec_objectives = self.recording_options['record_objectives'] rec_constraints = self.recording_options['record_constraints'] rec_responses = self.recording_options['record_responses'] # includes and excludes for outputs are specified using promoted names # NOTE: only local var names are in abs2prom, all will be gathered later abs2prom = model._var_abs2prom['output'] all_desvars = { n for n in self._designvars if n in abs2prom and check_path(abs2prom[n], incl, excl, True) } all_objectives = { n for n in self._objs if n in abs2prom and check_path(abs2prom[n], incl, excl, True) } all_constraints = { n for n in self._cons if n in abs2prom and check_path(abs2prom[n], incl, excl, True) } if rec_desvars: mydesvars = all_desvars if rec_objectives: myobjectives = all_objectives if rec_constraints: myconstraints = all_constraints if rec_responses: myresponses = { n for n in self._responses if n in abs2prom and check_path(abs2prom[n], incl, excl, True) } # get the includes that were requested for this Driver recording if incl: prob = self._problem root = prob.model # The my* variables are sets # First gather all of the desired outputs # The following might only be the local vars if MPI mysystem_outputs = { n for n in root._outputs if n in abs2prom and check_path(abs2prom[n], incl, excl) } # If MPI, and on rank 0, need to gather up all the variables # even those not local to rank 0 if MPI: all_vars = root.comm.gather(mysystem_outputs, root=0) if MPI.COMM_WORLD.rank == 0: mysystem_outputs = all_vars[-1] for d in all_vars[:-1]: mysystem_outputs.update(d) # de-duplicate mysystem_outputs mysystem_outputs = mysystem_outputs.difference( all_desvars, all_objectives, all_constraints) if MPI: # filter based on who owns the variables # TODO Eventually, we think we can get rid of this next check. But to be safe, # we are leaving it in there. if not model.is_active(): raise RuntimeError( "RecordingManager.startup should never be called when " "running in parallel on an inactive System") rrank = self._problem.comm.rank # root ( aka model ) rank. rowned = model._owning_rank['output'] mydesvars = [n for n in mydesvars if rrank == rowned[n]] myresponses = [n for n in myresponses if rrank == rowned[n]] myobjectives = [n for n in myobjectives if rrank == rowned[n]] myconstraints = [n for n in myconstraints if rrank == rowned[n]] mysystem_outputs = [ n for n in mysystem_outputs if rrank == rowned[n] ] self._filtered_vars_to_record = { 'des': mydesvars, 'obj': myobjectives, 'con': myconstraints, 'res': myresponses, 'sys': mysystem_outputs, } self._rec_mgr.startup(self) def _get_voi_val(self, name, meta, remote_vois): """ Get the value of a variable of interest (objective, constraint, or design var). This will retrieve the value if the VOI is remote. Parameters ---------- name : str Name of the variable of interest. meta : dict Metadata for the variable of interest. remote_vois : dict Dict containing (owning_rank, size) for all remote vois of a particular type (design var, constraint, or objective). Returns ------- float or ndarray The value of the named variable of interest. """ model = self._problem.model comm = model.comm vec = model._outputs._views_flat indices = meta['indices'] if name in remote_vois: owner, size = remote_vois[name] if owner == comm.rank: if indices is None: val = vec[name].copy() else: val = vec[name][indices] else: if indices is not None: size = len(indices) val = np.empty(size) comm.Bcast(val, root=owner) else: if indices is None: val = vec[name].copy() else: val = vec[name][indices] if self._has_scaling: # Scale design variable values adder = meta['adder'] if adder is not None: val += adder scaler = meta['scaler'] if scaler is not None: val *= scaler return val def get_design_var_values(self, filter=None): """ Return the design variable values. This is called to gather the initial design variable state. Parameters ---------- filter : list List of desvar names used by recorders. Returns ------- dict Dictionary containing values of each design variable. """ if filter: dvs = filter else: # use all the designvars dvs = self._designvars return { n: self._get_voi_val(n, self._designvars[n], self._remote_dvs) for n in dvs } def set_design_var(self, name, value): """ Set the value of a design variable. Parameters ---------- name : str Global pathname of the design variable. value : float or ndarray Value for the design variable. """ if (name in self._remote_dvs and self._problem.model._owning_rank['output'][name] != self._problem.comm.rank): return meta = self._designvars[name] indices = meta['indices'] if indices is None: indices = slice(None) desvar = self._problem.model._outputs._views_flat[name] desvar[indices] = value if self._has_scaling: # Scale design variable values scaler = meta['scaler'] if scaler is not None: desvar[indices] *= 1.0 / scaler adder = meta['adder'] if adder is not None: desvar[indices] -= adder def get_response_values(self, filter=None): """ Return response values. Parameters ---------- filter : list List of response names used by recorders. Returns ------- dict Dictionary containing values of each response. """ if filter: resps = filter else: resps = self._responses return { n: self._get_voi_val(n, self._responses[n], self._remote_objs) for n in resps } def get_objective_values(self, filter=None): """ Return objective values. Parameters ---------- filter : list List of objective names used by recorders. Returns ------- dict Dictionary containing values of each objective. """ if filter: objs = filter else: objs = self._objs return { n: self._get_voi_val(n, self._objs[n], self._remote_objs) for n in objs } def get_constraint_values(self, ctype='all', lintype='all', filter=None): """ Return constraint values. Parameters ---------- ctype : string Default is 'all'. Optionally return just the inequality constraints with 'ineq' or the equality constraints with 'eq'. lintype : string Default is 'all'. Optionally return just the linear constraints with 'linear' or the nonlinear constraints with 'nonlinear'. filter : list List of constraint names used by recorders. Returns ------- dict Dictionary containing values of each constraint. """ if filter is not None: cons = filter else: cons = self._cons con_dict = {} for name in cons: meta = self._cons[name] if lintype == 'linear' and not meta['linear']: continue if lintype == 'nonlinear' and meta['linear']: continue if ctype == 'eq' and meta['equals'] is None: continue if ctype == 'ineq' and meta['equals'] is not None: continue con_dict[name] = self._get_voi_val(name, meta, self._remote_cons) return con_dict def run(self): """ Execute this driver. The base `Driver` just runs the model. All other drivers overload this method. Returns ------- boolean Failure flag; True if failed to converge, False is successful. """ with Recording(self._get_name(), self.iter_count, self) as rec: self._problem.model.run_solve_nonlinear() self.iter_count += 1 return False def _dict2array_jac(self, derivs): osize = 0 isize = 0 do_wrt = True islices = {} oslices = {} for okey, oval in iteritems(derivs): if do_wrt: for ikey, val in iteritems(oval): istart = isize isize += val.shape[1] islices[ikey] = slice(istart, isize) do_wrt = False ostart = osize osize += oval[ikey].shape[0] oslices[okey] = slice(ostart, osize) new_derivs = np.zeros((osize, isize)) relevant = self._problem.model._relevant for okey, odict in iteritems(derivs): for ikey, val in iteritems(odict): if okey in relevant[ikey] or ikey in relevant[okey]: new_derivs[oslices[okey], islices[ikey]] = val return new_derivs def _compute_totals(self, of=None, wrt=None, return_format='flat_dict', global_names=True): """ Compute derivatives of desired quantities with respect to desired inputs. All derivatives are returned using driver scaling. Parameters ---------- of : list of variable name strings or None Variables whose derivatives will be computed. Default is None, which uses the driver's objectives and constraints. wrt : list of variable name strings or None Variables with respect to which the derivatives will be computed. Default is None, which uses the driver's desvars. return_format : string Format to return the derivatives. Default is a 'flat_dict', which returns them in a dictionary whose keys are tuples of form (of, wrt). For the scipy optimizer, 'array' is also supported. global_names : bool Set to True when passing in global names to skip some translation steps. Returns ------- derivs : object Derivatives in form requested by 'return_format'. """ prob = self._problem # Compute the derivatives in dict format... if prob.model._owns_approx_jac: derivs = prob._compute_totals_approx(of=of, wrt=wrt, return_format='dict', global_names=global_names) else: derivs = prob._compute_totals(of=of, wrt=wrt, return_format='dict', global_names=global_names) # ... then convert to whatever the driver needs. if return_format in ('dict', 'array'): if self._has_scaling: for okey, odict in iteritems(derivs): for ikey, val in iteritems(odict): iscaler = self._designvars[ikey]['scaler'] oscaler = self._responses[okey]['scaler'] # Scale response side if oscaler is not None: val[:] = (oscaler * val.T).T # Scale design var side if iscaler is not None: val *= 1.0 / iscaler else: raise RuntimeError( "Derivative scaling by the driver only supports the 'dict' and " "'array' formats at present.") if return_format == 'array': derivs = self._dict2array_jac(derivs) return derivs def record_iteration(self): """ Record an iteration of the current Driver. """ if not self._rec_mgr._recorders: return metadata = create_local_meta(self._get_name()) # Get the data to record data = {} if self.recording_options['record_desvars']: # collective call that gets across all ranks desvars = self.get_design_var_values() else: desvars = {} if self.recording_options['record_responses']: # responses = self.get_response_values() # not really working yet responses = {} else: responses = {} if self.recording_options['record_objectives']: objectives = self.get_objective_values() else: objectives = {} if self.recording_options['record_constraints']: constraints = self.get_constraint_values() else: constraints = {} desvars = { name: desvars[name] for name in self._filtered_vars_to_record['des'] } # responses not working yet # responses = {name: responses[name] for name in self._filtered_vars_to_record['res']} objectives = { name: objectives[name] for name in self._filtered_vars_to_record['obj'] } constraints = { name: constraints[name] for name in self._filtered_vars_to_record['con'] } if self.recording_options['includes']: root = self._problem.model outputs = root._outputs # outputsinputs, outputs, residuals = root.get_nonlinear_vectors() sysvars = {} for name, value in iteritems(outputs._names): if name in self._filtered_vars_to_record['sys']: sysvars[name] = value else: sysvars = {} if MPI: root = self._problem.model desvars = self._gather_vars(root, desvars) responses = self._gather_vars(root, responses) objectives = self._gather_vars(root, objectives) constraints = self._gather_vars(root, constraints) sysvars = self._gather_vars(root, sysvars) data['des'] = desvars data['res'] = responses data['obj'] = objectives data['con'] = constraints data['sys'] = sysvars self._rec_mgr.record_iteration(self, data, metadata) def _gather_vars(self, root, local_vars): """ Gather and return only variables listed in `local_vars` from the `root` System. Parameters ---------- root : <System> the root System for the Problem local_vars : dict local variable names and values Returns ------- dct : dict variable names and values. """ # if trace: # debug("gathering vars for recording in %s" % root.pathname) all_vars = root.comm.gather(local_vars, root=0) # if trace: # debug("DONE gathering rec vars for %s" % root.pathname) if root.comm.rank == 0: dct = all_vars[-1] for d in all_vars[:-1]: dct.update(d) return dct def _get_name(self): """ Get name of current Driver. Returns ------- str Name of current Driver. """ return "Driver"
class Driver(object): """ Top-level container for the systems and drivers. Attributes ---------- fail : bool Reports whether the driver ran successfully. iter_count : int Keep track of iterations for case recording. metadata : list List of metadata options : <OptionsDictionary> Dictionary with general pyoptsparse options. _problem : <Problem> Pointer to the containing problem. supports : <OptionsDictionary> Provides a consistant way for drivers to declare what features they support. _designvars : dict Contains all design variable info. _cons : dict Contains all constraint info. _objs : dict Contains all objective info. _responses : dict Contains all response info. _rec_mgr : <RecordingManager> Object that manages all recorders added to this driver. _model_viewer_data : dict Structure of model, used to make n2 diagram. _remote_dvs : dict Dict of design variables that are remote on at least one proc. Values are (owning rank, size). _remote_cons : dict Dict of constraints that are remote on at least one proc. Values are (owning rank, size). _remote_objs : dict Dict of objectives that are remote on at least one proc. Values are (owning rank, size). _remote_responses : dict A combined dict containing entries from _remote_cons and _remote_objs. """ def __init__(self): """ Initialize the driver. """ self._rec_mgr = RecordingManager() self._problem = None self._designvars = None self._cons = None self._objs = None self._responses = None self.options = OptionsDictionary() # What the driver supports. self.supports = OptionsDictionary() self.supports.declare('inequality_constraints', type_=bool, default=False) self.supports.declare('equality_constraints', type_=bool, default=False) self.supports.declare('linear_constraints', type_=bool, default=False) self.supports.declare('two_sided_constraints', type_=bool, default=False) self.supports.declare('multiple_objectives', type_=bool, default=False) self.supports.declare('integer_design_vars', type_=bool, default=False) self.supports.declare('gradients', type_=bool, default=False) self.supports.declare('active_set', type_=bool, default=False) self.iter_count = 0 self.metadata = None self._model_viewer_data = None # TODO, support these in Openmdao blue self.supports.declare('integer_design_vars', type_=bool, default=False) self.fail = False def add_recorder(self, recorder): """ Add a recorder to the driver. Parameters ---------- recorder : BaseRecorder A recorder instance. """ self._rec_mgr.append(recorder) def cleanup(self): """ Clean up resources prior to exit. """ self._rec_mgr.close() def _setup_driver(self, problem): """ Prepare the driver for execution. This is the final thing to run during setup. Parameters ---------- problem : <Problem> Pointer to the containing problem. """ self._problem = problem model = problem.model self._objs = objs = OrderedDict() self._cons = cons = OrderedDict() self._responses = model.get_responses(recurse=True) for name, data in iteritems(self._responses): if data['type'] == 'con': cons[name] = data else: objs[name] = data # Gather up the information for design vars. self._designvars = model.get_design_vars(recurse=True) con_set = set() obj_set = set() dv_set = set() self._remote_dvs = dv_dict = {} self._remote_cons = con_dict = {} self._remote_objs = obj_dict = {} # Now determine if later we'll need to allgather cons, objs, or desvars. if model.comm.size > 1 and model._subsystems_allprocs: local_out_vars = set(model._outputs._views) remote_dvs = set(self._designvars) - local_out_vars remote_cons = set(self._cons) - local_out_vars remote_objs = set(self._objs) - local_out_vars all_remote_vois = model.comm.allgather( (remote_dvs, remote_cons, remote_objs)) for rem_dvs, rem_cons, rem_objs in all_remote_vois: con_set.update(rem_cons) obj_set.update(rem_objs) dv_set.update(rem_dvs) # If we have remote VOIs, pick an owning rank for each and use that # to bcast to others later owning_ranks = model._owning_rank['output'] sizes = model._var_sizes['nonlinear']['output'] for i, vname in enumerate(model._var_allprocs_abs_names['output']): owner = owning_ranks[vname] if vname in dv_set: dv_dict[vname] = (owner, sizes[owner, i]) if vname in con_set: con_dict[vname] = (owner, sizes[owner, i]) if vname in obj_set: obj_dict[vname] = (owner, sizes[owner, i]) self._remote_responses = self._remote_cons.copy() self._remote_responses.update(self._remote_objs) self._rec_mgr.startup(self) if (self._rec_mgr._recorders): from openmdao.devtools.problem_viewer.problem_viewer import _get_viewer_data self._model_viewer_data = _get_viewer_data(problem) self._rec_mgr.record_metadata(self) def _get_voi_val(self, name, meta, remote_vois): """ Get the value of a variable of interest (objective, constraint, or design var). This will retrieve the value if the VOI is remote. Parameters ---------- name : str Name of the variable of interest. meta : dict Metadata for the variable of interest. remote_vois : dict Dict containing (owning_rank, size) for all remote vois of a particular type (design var, constraint, or objective). Returns ------- float or ndarray The value of the named variable of interest. """ model = self._problem.model comm = model.comm vec = model._outputs._views_flat indices = meta['indices'] if name in remote_vois: owner, size = remote_vois[name] if owner == comm.rank: if indices is None: val = vec[name].copy() else: val = vec[name][indices] else: if indices is not None: size = len(indices) val = np.empty(size) comm.Bcast(val, root=owner) else: if indices is None: val = vec[name].copy() else: val = vec[name][indices] # Scale design variable values adder = meta['adder'] if adder is not None: val += adder scaler = meta['scaler'] if scaler is not None: val *= scaler return val def get_design_var_values(self, filter=None): """ Return the design variable values. This is called to gather the initial design variable state. Parameters ---------- filter : list List of desvar names used by recorders. Returns ------- dict Dictionary containing values of each design variable. """ if filter: dvs = filter else: # use all the designvars dvs = self._designvars return { n: self._get_voi_val(n, self._designvars[n], self._remote_dvs) for n in dvs } def set_design_var(self, name, value): """ Set the value of a design variable. Parameters ---------- name : str Global pathname of the design variable. value : float or ndarray Value for the design variable. """ if (name in self._remote_dvs and self._problem.model._owning_rank['output'][name] != self._problem.comm.rank): return meta = self._designvars[name] indices = meta['indices'] if indices is None: indices = slice(None) desvar = self._problem.model._outputs._views_flat[name] desvar[indices] = value # Scale design variable values scaler = meta['scaler'] if scaler is not None: desvar[indices] *= 1.0 / scaler adder = meta['adder'] if adder is not None: desvar[indices] -= adder def get_response_values(self, filter=None): """ Return response values. Parameters ---------- filter : list List of response names used by recorders. Returns ------- dict Dictionary containing values of each response. """ # TODO: finish this method when we have a driver that requires it. return {} def get_objective_values(self, filter=None): """ Return objective values. Parameters ---------- filter : list List of objective names used by recorders. Returns ------- dict Dictionary containing values of each objective. """ if filter: objs = filter else: objs = self._objs return { n: self._get_voi_val(n, self._objs[n], self._remote_objs) for n in objs } def get_constraint_values(self, ctype='all', lintype='all', filter=None): """ Return constraint values. Parameters ---------- ctype : string Default is 'all'. Optionally return just the inequality constraints with 'ineq' or the equality constraints with 'eq'. lintype : string Default is 'all'. Optionally return just the linear constraints with 'linear' or the nonlinear constraints with 'nonlinear'. filter : list List of constraint names used by recorders. Returns ------- dict Dictionary containing values of each constraint. """ if filter is not None: cons = filter else: cons = self._cons con_dict = {} for name in cons: meta = self._cons[name] if lintype == 'linear' and not meta['linear']: continue if lintype == 'nonlinear' and meta['linear']: continue if ctype == 'eq' and meta['equals'] is None: continue if ctype == 'ineq' and meta['equals'] is not None: continue con_dict[name] = self._get_voi_val(name, meta, self._remote_cons) return con_dict def run(self): """ Execute this driver. The base `Driver` just runs the model. All other drivers overload this method. Returns ------- boolean Failure flag; True if failed to converge, False is successful. """ with Recording(self._get_name(), self.iter_count, self) as rec: failure_flag = self._problem.model._solve_nonlinear() self.iter_count += 1 return failure_flag def _compute_totals(self, of=None, wrt=None, return_format='flat_dict', global_names=True): """ Compute derivatives of desired quantities with respect to desired inputs. All derivatives are returned using driver scaling. Parameters ---------- of : list of variable name strings or None Variables whose derivatives will be computed. Default is None, which uses the driver's objectives and constraints. wrt : list of variable name strings or None Variables with respect to which the derivatives will be computed. Default is None, which uses the driver's desvars. return_format : string Format to return the derivatives. Default is a 'flat_dict', which returns them in a dictionary whose keys are tuples of form (of, wrt). For the scipy optimizer, 'array' is also supported. global_names : bool Set to True when passing in global names to skip some translation steps. Returns ------- derivs : object Derivatives in form requested by 'return_format'. """ prob = self._problem # Compute the derivatives in dict format... if prob.model._owns_approx_jac: derivs = prob._compute_totals_approx(of=of, wrt=wrt, return_format='dict', global_names=global_names) else: derivs = prob._compute_totals(of=of, wrt=wrt, return_format='dict', global_names=global_names) # ... then convert to whatever the driver needs. if return_format == 'dict': for okey, oval in iteritems(derivs): for ikey, val in iteritems(oval): imeta = self._designvars[ikey] ometa = self._responses[okey] iscaler = imeta['scaler'] oscaler = ometa['scaler'] # Scale response side if oscaler is not None: val[:] = (oscaler * val.T).T # Scale design var side if iscaler is not None: val *= 1.0 / iscaler elif return_format == 'array': # Use sizes pre-computed in derivs for ease osize = 0 isize = 0 do_wrt = True islices = {} oslices = {} for okey, oval in iteritems(derivs): if do_wrt: for ikey, val in iteritems(oval): istart = isize isize += val.shape[1] islices[ikey] = slice(istart, isize) do_wrt = False ostart = osize osize += oval[ikey].shape[0] oslices[okey] = slice(ostart, osize) new_derivs = np.zeros((osize, isize)) relevant = prob.model._relevant # Apply driver ref/ref0 and position subjac into array jacobian. for okey, oval in iteritems(derivs): oscaler = self._responses[okey]['scaler'] for ikey, val in iteritems(oval): if okey in relevant[ikey] or ikey in relevant[okey]: iscaler = self._designvars[ikey]['scaler'] # Scale response side if oscaler is not None: val[:] = (oscaler * val.T).T # Scale design var side if iscaler is not None: val *= 1.0 / iscaler new_derivs[oslices[okey], islices[ikey]] = val derivs = new_derivs else: msg = "Derivative scaling by the driver only supports the 'dict' format at present." raise RuntimeError(msg) return derivs def record_iteration(self): """ Record an iteration of the current Driver. """ metadata = create_local_meta(self._get_name()) self._rec_mgr.record_iteration(self, metadata) def _get_name(self): """ Get name of current Driver. Returns ------- str Name of current Driver. """ return "Driver"
class SolverBase(object): """ Common base class for Linear and Nonlinear solver. Should not be used by users. Always inherit from `LinearSolver` or `NonlinearSolver`.""" def __init__(self): self.iter_count = 0 self.options = OptionsDictionary() desc = ( "Set to 0 to disable printing, set to 1 to print the " "residual to stdout each iteration, set to 2 to print " "subiteration residuals as well." ) self.options.add_option("iprint", 0, values=[0, 1, 2], desc=desc) self.recorders = RecordingManager() self.local_meta = None def setup(self, sub): """ Solvers override to define post-setup initiailzation. Args ---- sub: `System` System that owns this solver. """ pass def cleanup(self): """ Clean up resources prior to exit. """ self.recorders.close() def print_norm(self, solver_string, pathname, iteration, res, res0, msg=None, indent=0, solver="NL"): """ Prints out the norm of the residual in a neat readable format. Args ---- solver_string: string Unique string to identify your solver type (e.g., 'LN_GS' or 'NEWTON'). pathname: dict Parent system pathname. iteration: int Current iteration number res: float Absolute residual value. res0: float Baseline initial residual for relative comparison. msg: string, optional Message that indicates convergence. ident: int, optional Additional indentation levels for subiterations. solver: string, optional Solver type if not LN or NL (mostly for line search operations.) """ if pathname == "": name = "root" else: name = "root." + pathname # Find indentation level level = pathname.count(".") # No indentation for driver; top solver is no indentation. level = level + indent indent = " " * level if msg is not None: form = indent + "[%s] %s: %s %d | %s" print(form % (name, solver, solver_string, iteration, msg)) return form = indent + "[%s] %s: %s %d | %.9g %.9g" print(form % (name, solver, solver_string, iteration, res, res / res0)) def print_all_convergence(self): """ Turns on iprint for this solver and all subsolvers. Override if your solver has subsolvers.""" self.options["iprint"] = 1 def generate_docstring(self): """ Generates a numpy-style docstring for a user-created System class. Returns ------- docstring : str string that contains a basic numpy docstring. """ # start the docstring off docstring = ' """\n' # Put options into docstring firstTime = 1 # for py3.4, items from vars must come out in same order. from collections import OrderedDict v = OrderedDict(sorted(vars(self).items())) for key, value in v.items(): if type(value) == OptionsDictionary: if firstTime: # start of Options docstring docstring += "\n Options\n -------\n" firstTime = 0 for (name, val) in sorted(value.items()): docstring += " " + key + "['" docstring += name + "']" docstring += " : " + type(val).__name__ docstring += "(" if type(val).__name__ == "str": docstring += "'" docstring += str(val) if type(val).__name__ == "str": docstring += "'" docstring += ")\n" desc = value._options[name]["desc"] if desc: docstring += " " + desc + "\n" # finish up docstring docstring += '\n """\n' return docstring
class Driver(object): """ Top-level container for the systems and drivers. Options ------- options['debug_print'] : list of strings([]) Indicates what variables to print at each iteration. The valid options are: 'desvars','ln_cons','nl_cons',and 'objs'. recording_options['record_metadata'] : bool(True) Tells recorder whether to record variable attribute metadata. recording_options['record_desvars'] : bool(True) Tells recorder whether to record the desvars of the Driver. recording_options['record_responses'] : bool(False) Tells recorder whether to record the responses of the Driver. recording_options['record_objectives'] : bool(False) Tells recorder whether to record the objectives of the Driver. recording_options['record_constraints'] : bool(False) Tells recorder whether to record the constraints of the Driver. recording_options['includes'] : list of strings("*") Patterns for variables to include in recording. recording_options['excludes'] : list of strings('') Patterns for variables to exclude in recording (processed after includes). Attributes ---------- fail : bool Reports whether the driver ran successfully. iter_count : int Keep track of iterations for case recording. metadata : list List of metadata options : <OptionsDictionary> Dictionary with general pyoptsparse options. recording_options : <OptionsDictionary> Dictionary with driver recording options. debug_print : <OptionsDictionary> Dictionary with debugging printing options. cite : str Listing of relevant citataions that should be referenced when publishing work that uses this class. _problem : <Problem> Pointer to the containing problem. supports : <OptionsDictionary> Provides a consistant way for drivers to declare what features they support. _designvars : dict Contains all design variable info. _cons : dict Contains all constraint info. _objs : dict Contains all objective info. _responses : dict Contains all response info. _rec_mgr : <RecordingManager> Object that manages all recorders added to this driver. _vars_to_record: dict Dict of lists of var names indicating what to record _model_viewer_data : dict Structure of model, used to make n2 diagram. _remote_dvs : dict Dict of design variables that are remote on at least one proc. Values are (owning rank, size). _remote_cons : dict Dict of constraints that are remote on at least one proc. Values are (owning rank, size). _remote_objs : dict Dict of objectives that are remote on at least one proc. Values are (owning rank, size). _remote_responses : dict A combined dict containing entries from _remote_cons and _remote_objs. _simul_coloring_info : tuple of dicts A data structure describing coloring for simultaneous derivs. _res_jacs : dict Dict of sparse subjacobians for use with certain optimizers, e.g. pyOptSparseDriver. """ def __init__(self): """ Initialize the driver. """ self._rec_mgr = RecordingManager() self._vars_to_record = { 'desvarnames': set(), 'responsenames': set(), 'objectivenames': set(), 'constraintnames': set(), 'sysinclnames': set(), } self._problem = None self._designvars = None self._cons = None self._objs = None self._responses = None self.options = OptionsDictionary() self.recording_options = OptionsDictionary() ########################### self.options.declare('debug_print', types=list, is_valid=_is_debug_print_opts_valid, desc="List of what type of Driver variables to print at each " "iteration. Valid items in list are 'desvars','ln_cons'," "'nl_cons','objs'", default=[]) ########################### self.recording_options.declare('record_metadata', types=bool, desc='Record metadata', default=True) self.recording_options.declare('record_desvars', types=bool, default=True, desc='Set to True to record design variables at the \ driver level') self.recording_options.declare('record_responses', types=bool, default=False, desc='Set to True to record responses at the driver level') self.recording_options.declare('record_objectives', types=bool, default=True, desc='Set to True to record objectives at the \ driver level') self.recording_options.declare('record_constraints', types=bool, default=True, desc='Set to True to record constraints at the \ driver level') self.recording_options.declare('includes', types=list, default=['*'], desc='Patterns for variables to include in recording') self.recording_options.declare('excludes', types=list, default=[], desc='Patterns for vars to exclude in recording ' '(processed post-includes)') self.recording_options.declare('record_derivatives', types=bool, default=False, desc='Set to True to record derivatives at the driver \ level') ########################### # What the driver supports. self.supports = OptionsDictionary() self.supports.declare('inequality_constraints', types=bool, default=False) self.supports.declare('equality_constraints', types=bool, default=False) self.supports.declare('linear_constraints', types=bool, default=False) self.supports.declare('two_sided_constraints', types=bool, default=False) self.supports.declare('multiple_objectives', types=bool, default=False) self.supports.declare('integer_design_vars', types=bool, default=False) self.supports.declare('gradients', types=bool, default=False) self.supports.declare('active_set', types=bool, default=False) self.supports.declare('simultaneous_derivatives', types=bool, default=False) # Debug printing. self.debug_print = OptionsDictionary() self.debug_print.declare('debug_print', types=bool, default=False, desc='Overall option to turn on Driver debug printing') self.debug_print.declare('debug_print_desvars', types=bool, default=False, desc='Print design variables') self.debug_print.declare('debug_print_nl_con', types=bool, default=False, desc='Print nonlinear constraints') self.debug_print.declare('debug_print_ln_con', types=bool, default=False, desc='Print linear constraints') self.debug_print.declare('debug_print_objective', types=bool, default=False, desc='Print objectives') self.iter_count = 0 self.metadata = None self._model_viewer_data = None self.cite = "" # TODO, support these in OpenMDAO self.supports.declare('integer_design_vars', types=bool, default=False) self._simul_coloring_info = None self._res_jacs = {} self.fail = False def add_recorder(self, recorder): """ Add a recorder to the driver. Parameters ---------- recorder : BaseRecorder A recorder instance. """ self._rec_mgr.append(recorder) def cleanup(self): """ Clean up resources prior to exit. """ self._rec_mgr.close() def _setup_driver(self, problem): """ Prepare the driver for execution. This is the final thing to run during setup. Parameters ---------- problem : <Problem> Pointer to the containing problem. """ self._problem = problem model = problem.model self._objs = objs = OrderedDict() self._cons = cons = OrderedDict() self._responses = model.get_responses(recurse=True) response_size = 0 for name, data in iteritems(self._responses): if data['type'] == 'con': cons[name] = data else: objs[name] = data response_size += data['size'] # Gather up the information for design vars. self._designvars = model.get_design_vars(recurse=True) desvar_size = np.sum(data['size'] for data in itervalues(self._designvars)) if ((problem._mode == 'fwd' and desvar_size > response_size) or (problem._mode == 'rev' and response_size > desvar_size)): warnings.warn("Inefficient choice of derivative mode. You chose '%s' for a " "problem with %d design variables and %d response variables " "(objectives and constraints)." % (problem._mode, desvar_size, response_size), RuntimeWarning) self._has_scaling = ( np.any([r['scaler'] is not None for r in self._responses.values()]) or np.any([dv['scaler'] is not None for dv in self._designvars.values()]) ) con_set = set() obj_set = set() dv_set = set() self._remote_dvs = dv_dict = {} self._remote_cons = con_dict = {} self._remote_objs = obj_dict = {} # Now determine if later we'll need to allgather cons, objs, or desvars. if model.comm.size > 1 and model._subsystems_allprocs: local_out_vars = set(model._outputs._views) remote_dvs = set(self._designvars) - local_out_vars remote_cons = set(self._cons) - local_out_vars remote_objs = set(self._objs) - local_out_vars all_remote_vois = model.comm.allgather( (remote_dvs, remote_cons, remote_objs)) for rem_dvs, rem_cons, rem_objs in all_remote_vois: con_set.update(rem_cons) obj_set.update(rem_objs) dv_set.update(rem_dvs) # If we have remote VOIs, pick an owning rank for each and use that # to bcast to others later owning_ranks = model._owning_rank['output'] sizes = model._var_sizes['nonlinear']['output'] for i, vname in enumerate(model._var_allprocs_abs_names['output']): owner = owning_ranks[vname] if vname in dv_set: dv_dict[vname] = (owner, sizes[owner, i]) if vname in con_set: con_dict[vname] = (owner, sizes[owner, i]) if vname in obj_set: obj_dict[vname] = (owner, sizes[owner, i]) self._remote_responses = self._remote_cons.copy() self._remote_responses.update(self._remote_objs) # Case recording setup mydesvars = myobjectives = myconstraints = myresponses = set() mysystem_outputs = set() incl = self.recording_options['includes'] excl = self.recording_options['excludes'] rec_desvars = self.recording_options['record_desvars'] rec_objectives = self.recording_options['record_objectives'] rec_constraints = self.recording_options['record_constraints'] rec_responses = self.recording_options['record_responses'] all_desvars = {n for n in self._designvars if check_path(n, incl, excl, True)} all_objectives = {n for n in self._objs if check_path(n, incl, excl, True)} all_constraints = {n for n in self._cons if check_path(n, incl, excl, True)} if rec_desvars: mydesvars = all_desvars if rec_objectives: myobjectives = all_objectives if rec_constraints: myconstraints = all_constraints if rec_responses: myresponses = {n for n in self._responses if check_path(n, incl, excl, True)} # get the includes that were requested for this Driver recording if incl: prob = self._problem root = prob.model # The my* variables are sets # First gather all of the desired outputs # The following might only be the local vars if MPI mysystem_outputs = {n for n in root._outputs if check_path(n, incl, excl)} # If MPI, and on rank 0, need to gather up all the variables # even those not local to rank 0 if MPI: all_vars = root.comm.gather(mysystem_outputs, root=0) if MPI.COMM_WORLD.rank == 0: mysystem_outputs = all_vars[-1] for d in all_vars[:-1]: mysystem_outputs.update(d) # de-duplicate mysystem_outputs mysystem_outputs = mysystem_outputs.difference(all_desvars, all_objectives, all_constraints) if MPI: # filter based on who owns the variables # TODO Eventually, we think we can get rid of this next check. But to be safe, # we are leaving it in there. if not model.is_active(): raise RuntimeError( "RecordingManager.startup should never be called when " "running in parallel on an inactive System") rrank = self._problem.comm.rank # root ( aka model ) rank. rowned = model._owning_rank['output'] mydesvars = [n for n in mydesvars if rrank == rowned[n]] myresponses = [n for n in myresponses if rrank == rowned[n]] myobjectives = [n for n in myobjectives if rrank == rowned[n]] myconstraints = [n for n in myconstraints if rrank == rowned[n]] mysystem_outputs = [n for n in mysystem_outputs if rrank == rowned[n]] self._filtered_vars_to_record = { 'des': mydesvars, 'obj': myobjectives, 'con': myconstraints, 'res': myresponses, 'sys': mysystem_outputs, } self._rec_mgr.startup(self) if self._rec_mgr._recorders: from openmdao.devtools.problem_viewer.problem_viewer import _get_viewer_data self._model_viewer_data = _get_viewer_data(problem) if self.recording_options['record_metadata']: self._rec_mgr.record_metadata(self) # set up simultaneous deriv coloring if self._simul_coloring_info and self.supports['simultaneous_derivatives']: if problem._mode == 'fwd': self._setup_simul_coloring(problem._mode) else: raise RuntimeError("simultaneous derivs are currently not supported in rev mode.") def _get_voi_val(self, name, meta, remote_vois): """ Get the value of a variable of interest (objective, constraint, or design var). This will retrieve the value if the VOI is remote. Parameters ---------- name : str Name of the variable of interest. meta : dict Metadata for the variable of interest. remote_vois : dict Dict containing (owning_rank, size) for all remote vois of a particular type (design var, constraint, or objective). Returns ------- float or ndarray The value of the named variable of interest. """ model = self._problem.model comm = model.comm vec = model._outputs._views_flat indices = meta['indices'] if name in remote_vois: owner, size = remote_vois[name] if owner == comm.rank: if indices is None: val = vec[name].copy() else: val = vec[name][indices] else: if indices is not None: size = len(indices) val = np.empty(size) comm.Bcast(val, root=owner) else: if indices is None: val = vec[name].copy() else: val = vec[name][indices] if self._has_scaling: # Scale design variable values adder = meta['adder'] if adder is not None: val += adder scaler = meta['scaler'] if scaler is not None: val *= scaler return val def get_design_var_values(self, filter=None): """ Return the design variable values. This is called to gather the initial design variable state. Parameters ---------- filter : list List of desvar names used by recorders. Returns ------- dict Dictionary containing values of each design variable. """ if filter: dvs = filter else: # use all the designvars dvs = self._designvars return {n: self._get_voi_val(n, self._designvars[n], self._remote_dvs) for n in dvs} def set_design_var(self, name, value): """ Set the value of a design variable. Parameters ---------- name : str Global pathname of the design variable. value : float or ndarray Value for the design variable. """ if (name in self._remote_dvs and self._problem.model._owning_rank['output'][name] != self._problem.comm.rank): return meta = self._designvars[name] indices = meta['indices'] if indices is None: indices = slice(None) desvar = self._problem.model._outputs._views_flat[name] desvar[indices] = value if self._has_scaling: # Scale design variable values scaler = meta['scaler'] if scaler is not None: desvar[indices] *= 1.0 / scaler adder = meta['adder'] if adder is not None: desvar[indices] -= adder def get_response_values(self, filter=None): """ Return response values. Parameters ---------- filter : list List of response names used by recorders. Returns ------- dict Dictionary containing values of each response. """ if filter: resps = filter else: resps = self._responses return {n: self._get_voi_val(n, self._responses[n], self._remote_objs) for n in resps} def get_objective_values(self, filter=None): """ Return objective values. Parameters ---------- filter : list List of objective names used by recorders. Returns ------- dict Dictionary containing values of each objective. """ if filter: objs = filter else: objs = self._objs return {n: self._get_voi_val(n, self._objs[n], self._remote_objs) for n in objs} def get_constraint_values(self, ctype='all', lintype='all', filter=None): """ Return constraint values. Parameters ---------- ctype : string Default is 'all'. Optionally return just the inequality constraints with 'ineq' or the equality constraints with 'eq'. lintype : string Default is 'all'. Optionally return just the linear constraints with 'linear' or the nonlinear constraints with 'nonlinear'. filter : list List of constraint names used by recorders. Returns ------- dict Dictionary containing values of each constraint. """ if filter is not None: cons = filter else: cons = self._cons con_dict = {} for name in cons: meta = self._cons[name] if lintype == 'linear' and not meta['linear']: continue if lintype == 'nonlinear' and meta['linear']: continue if ctype == 'eq' and meta['equals'] is None: continue if ctype == 'ineq' and meta['equals'] is not None: continue con_dict[name] = self._get_voi_val(name, meta, self._remote_cons) return con_dict def run(self): """ Execute this driver. The base `Driver` just runs the model. All other drivers overload this method. Returns ------- boolean Failure flag; True if failed to converge, False is successful. """ with RecordingDebugging(self._get_name(), self.iter_count, self) as rec: failure_flag, _, _ = self._problem.model._solve_nonlinear() self.iter_count += 1 return failure_flag def _dict2array_jac(self, derivs): osize = 0 isize = 0 do_wrt = True islices = {} oslices = {} for okey, oval in iteritems(derivs): if do_wrt: for ikey, val in iteritems(oval): istart = isize isize += val.shape[1] islices[ikey] = slice(istart, isize) do_wrt = False ostart = osize osize += oval[ikey].shape[0] oslices[okey] = slice(ostart, osize) new_derivs = np.zeros((osize, isize)) relevant = self._problem.model._relevant for okey, odict in iteritems(derivs): for ikey, val in iteritems(odict): if okey in relevant[ikey] or ikey in relevant[okey]: new_derivs[oslices[okey], islices[ikey]] = val return new_derivs def _compute_totals(self, of=None, wrt=None, return_format='flat_dict', global_names=True): """ Compute derivatives of desired quantities with respect to desired inputs. All derivatives are returned using driver scaling. Parameters ---------- of : list of variable name strings or None Variables whose derivatives will be computed. Default is None, which uses the driver's objectives and constraints. wrt : list of variable name strings or None Variables with respect to which the derivatives will be computed. Default is None, which uses the driver's desvars. return_format : string Format to return the derivatives. Default is a 'flat_dict', which returns them in a dictionary whose keys are tuples of form (of, wrt). For the scipy optimizer, 'array' is also supported. global_names : bool Set to True when passing in global names to skip some translation steps. Returns ------- derivs : object Derivatives in form requested by 'return_format'. """ prob = self._problem # Compute the derivatives in dict format... if prob.model._owns_approx_jac: derivs = prob._compute_totals_approx(of=of, wrt=wrt, return_format='dict', global_names=global_names) else: derivs = prob._compute_totals(of=of, wrt=wrt, return_format='dict', global_names=global_names) # ... then convert to whatever the driver needs. if return_format in ('dict', 'array'): if self._has_scaling: for okey, odict in iteritems(derivs): for ikey, val in iteritems(odict): iscaler = self._designvars[ikey]['scaler'] oscaler = self._responses[okey]['scaler'] # Scale response side if oscaler is not None: val[:] = (oscaler * val.T).T # Scale design var side if iscaler is not None: val *= 1.0 / iscaler else: raise RuntimeError("Derivative scaling by the driver only supports the 'dict' and " "'array' formats at present.") if return_format == 'array': derivs = self._dict2array_jac(derivs) return derivs def record_iteration(self): """ Record an iteration of the current Driver. """ if not self._rec_mgr._recorders: return metadata = create_local_meta(self._get_name()) # Get the data to record data = {} if self.recording_options['record_desvars']: # collective call that gets across all ranks desvars = self.get_design_var_values() else: desvars = {} if self.recording_options['record_responses']: # responses = self.get_response_values() # not really working yet responses = {} else: responses = {} if self.recording_options['record_objectives']: objectives = self.get_objective_values() else: objectives = {} if self.recording_options['record_constraints']: constraints = self.get_constraint_values() else: constraints = {} desvars = {name: desvars[name] for name in self._filtered_vars_to_record['des']} # responses not working yet # responses = {name: responses[name] for name in self._filtered_vars_to_record['res']} objectives = {name: objectives[name] for name in self._filtered_vars_to_record['obj']} constraints = {name: constraints[name] for name in self._filtered_vars_to_record['con']} if self.recording_options['includes']: root = self._problem.model outputs = root._outputs # outputsinputs, outputs, residuals = root.get_nonlinear_vectors() sysvars = {} for name, value in iteritems(outputs._names): if name in self._filtered_vars_to_record['sys']: sysvars[name] = value else: sysvars = {} if MPI: root = self._problem.model desvars = self._gather_vars(root, desvars) responses = self._gather_vars(root, responses) objectives = self._gather_vars(root, objectives) constraints = self._gather_vars(root, constraints) sysvars = self._gather_vars(root, sysvars) data['des'] = desvars data['res'] = responses data['obj'] = objectives data['con'] = constraints data['sys'] = sysvars self._rec_mgr.record_iteration(self, data, metadata) def _gather_vars(self, root, local_vars): """ Gather and return only variables listed in `local_vars` from the `root` System. Parameters ---------- root : <System> the root System for the Problem local_vars : dict local variable names and values Returns ------- dct : dict variable names and values. """ # if trace: # debug("gathering vars for recording in %s" % root.pathname) all_vars = root.comm.gather(local_vars, root=0) # if trace: # debug("DONE gathering rec vars for %s" % root.pathname) if root.comm.rank == 0: dct = all_vars[-1] for d in all_vars[:-1]: dct.update(d) return dct def _get_name(self): """ Get name of current Driver. Returns ------- str Name of current Driver. """ return "Driver" def set_simul_deriv_color(self, simul_info): """ Set the coloring for simultaneous derivatives. Parameters ---------- simul_info : str or ({dv1: colors, ...}, {resp1: {dv1: {0: [res_idxs, dv_idxs]} ...} ...}) Information about simultaneous coloring for design vars and responses. If a string, then simul_info is assumed to be the name of a file that contains the coloring information in JSON format. """ if self.supports['simultaneous_derivatives']: self._simul_coloring_info = simul_info else: raise RuntimeError("Driver '%s' does not support simultaneous derivatives." % self._get_name()) def _setup_simul_coloring(self, mode='fwd'): """ Set up metadata for simultaneous derivative solution. Parameters ---------- mode : str Derivative direction, either 'fwd' or 'rev'. """ if mode == 'rev': raise NotImplementedError("Simultaneous derivatives are currently not supported " "in 'rev' mode") # command line simul_coloring uses this env var to turn pre-existing coloring off if not _use_simul_coloring: return prom2abs = self._problem.model._var_allprocs_prom2abs_list['output'] if isinstance(self._simul_coloring_info, string_types): with open(self._simul_coloring_info, 'r') as f: self._simul_coloring_info = json.load(f) coloring, maps = self._simul_coloring_info for dv, colors in iteritems(coloring): if dv not in self._designvars: # convert name from promoted to absolute dv = prom2abs[dv][0] self._designvars[dv]['simul_deriv_color'] = colors for res, dvdict in iteritems(maps): if res not in self._responses: # convert name from promoted to absolute res = prom2abs[res][0] self._responses[res]['simul_map'] = dvdict for dv, col_dict in dvdict.items(): col_dict = {int(k): v for k, v in iteritems(col_dict)} if dv not in self._designvars: # convert name from promoted to absolute and replace dictionary key del dvdict[dv] dv = prom2abs[dv][0] dvdict[dv] = col_dict def _pre_run_model_debug_print(self): """ Optionally print some debugging information before the model runs. """ if not self.options['debug_print']: return if not MPI or MPI.COMM_WORLD.rank == 0: header = 'Driver debug print for iter coord: {}'.format( get_formatted_iteration_coordinate()) print(header) print(len(header) * '-') if 'desvars' in self.options['debug_print']: desvar_vals = self.get_design_var_values() if not MPI or MPI.COMM_WORLD.rank == 0: print("Design Vars") if desvar_vals: for name, value in iteritems(desvar_vals): print("{}: {}".format(name, repr(value))) else: print("None") print() def _post_run_model_debug_print(self): """ Optionally print some debugging information after the model runs. """ if 'nl_cons' in self.options['debug_print']: cons = self.get_constraint_values(lintype='nonlinear') if not MPI or MPI.COMM_WORLD.rank == 0: print("Nonlinear constraints") if cons: for name, value in iteritems(cons): print("{}: {}".format(name, repr(value))) else: print("None") print() if 'ln_cons' in self.options['debug_print']: cons = self.get_constraint_values(lintype='linear') if not MPI or MPI.COMM_WORLD.rank == 0: print("Linear constraints") if cons: for name, value in iteritems(cons): print("{}: {}".format(name, repr(value))) else: print("None") print() if 'objs' in self.options['debug_print']: objs = self.get_objective_values() if not MPI or MPI.COMM_WORLD.rank == 0: print("Objectives") if objs: for name, value in iteritems(objs): print("{}: {}".format(name, repr(value))) else: print("None") print()
class SolverBase(object): """ Common base class for Linear and Nonlinear solver. Should not be used by users. Always inherit from `LinearSolver` or `NonlinearSolver`.""" def __init__(self): self.iter_count = 0 self.options = OptionsDictionary() desc = "Set to 0 to print only failures, set to 1 to print iteration totals to" + \ "stdout, set to 2 to print the residual each iteration to stdout," + \ "or -1 to suppress all printing." self.options.add_option('iprint', 0, values=[-1, 0, 1, 2], desc=desc) self.options.add_option( 'err_on_maxiter', False, desc='If True, raise an AnalysisError if not converged at maxiter.' ) self.recorders = RecordingManager() self.local_meta = None def setup(self, sub): """ Solvers override to define post-setup initiailzation. Args ---- sub: `System` System that owns this solver. """ pass def cleanup(self): """ Clean up resources prior to exit. """ self.recorders.close() def print_norm(self, solver_string, system, iteration, res, res0, msg=None, indent=0, solver='NL', u_norm=None): """ Prints out the norm of the residual in a neat readable format. Args ---- solver_string: string Unique string to identify your solver type (e.g., 'LN_GS' or 'NEWTON'). system: system Parent system, which contains pathname and the preconditioning flag. iteration: int Current iteration number res: float Norm of the absolute residual value. res0: float Norm of the baseline initial residual for relative comparison. msg: string, optional Message that indicates convergence. ident: int, optional Additional indentation levels for subiterations. solver: string, optional Solver type if not LN or NL (mostly for line search operations.) u_norm: float, optional Norm of the u vector, if applicable. """ pathname = system.pathname if pathname == '': name = 'root' else: name = 'root.' + pathname # Find indentation level level = name.count('.') # No indentation for driver; top solver is no indentation. level = level + indent indent = ' ' * level if system._probdata.precon_level > 0: solver_string = 'PRECON:' + solver_string indent += ' ' * system._probdata.precon_level if msg is not None: form = indent + '[%s] %s: %s %d | %s' if u_norm: form += ' (%s)' % u_norm print(form % (name, solver, solver_string, iteration, msg)) return form = indent + '[%s] %s: %s %d | %.9g %.9g' if u_norm: form += ' (%s)' % u_norm print(form % (name, solver, solver_string, iteration, res, res / res0)) def print_all_convergence(self, level=2): """ Turns on iprint for this solver and all subsolvers. Override if your solver has subsolvers. Args ---- level : int(2) iprint level. Set to 2 to print residuals each iteration; set to 1 to print just the iteration totals. """ self.options['iprint'] = level def generate_docstring(self): """ Generates a numpy-style docstring for a user-created System class. Returns ------- docstring : str string that contains a basic numpy docstring. """ #start the docstring off docstrings = [' \"\"\"'] #Put options into docstring firstTime = 1 for key, value in sorted(vars(self).items()): if type(value) == OptionsDictionary: if firstTime: #start of Options docstring docstrings.extend(['', ' Options', ' -------']) firstTime = 0 docstrings.append(value._generate_docstring(key)) #finish up docstring docstrings.extend([' \"\"\"', '']) return '\n'.join(docstrings)
class SolverBase(object): """ Common base class for Linear and Nonlinear solver. Should not be used by users. Always inherit from `LinearSolver` or `NonlinearSolver`.""" def __init__(self): self.iter_count = 0 self.options = OptionsDictionary() desc = 'Set to 0 to disable printing, set to 1 to print the ' \ 'residual to stdout each iteration, set to 2 to print ' \ 'subiteration residuals as well.' self.options.add_option('iprint', 0, values=[0, 1, 2], desc=desc) self.recorders = RecordingManager() self.local_meta = None def setup(self, sub): """ Solvers override to define post-setup initiailzation. Args ---- sub: `System` System that owns this solver. """ pass def cleanup(self): """ Clean up resources prior to exit. """ self.recorders.close() def print_norm(self, solver_string, pathname, iteration, res, res0, msg=None, indent=0, solver='NL'): """ Prints out the norm of the residual in a neat readable format. Args ---- solver_string: string Unique string to identify your solver type (e.g., 'LN_GS' or 'NEWTON'). pathname: dict Parent system pathname. iteration: int Current iteration number res: float Absolute residual value. res0: float Baseline initial residual for relative comparison. msg: string, optional Message that indicates convergence. ident: int, optional Additional indentation levels for subiterations. solver: string, optional Solver type if not LN or NL (mostly for line search operations.) """ if pathname == '': name = 'root' else: name = 'root.' + pathname # Find indentation level level = pathname.count('.') # No indentation for driver; top solver is no indentation. level = level + indent indent = ' ' * level if msg is not None: form = indent + '[%s] %s: %s %d | %s' print(form % (name, solver, solver_string, iteration, msg)) return form = indent + '[%s] %s: %s %d | %.9g %.9g' print(form % (name, solver, solver_string, iteration, res, res / res0)) def print_all_convergence(self): """ Turns on iprint for this solver and all subsolvers. Override if your solver has subsolvers.""" self.options['iprint'] = 1 def generate_docstring(self): """ Generates a numpy-style docstring for a user-created System class. Returns ------- docstring : str string that contains a basic numpy docstring. """ #start the docstring off docstring = ' \"\"\"\n' #Put options into docstring firstTime = 1 #for py3.4, items from vars must come out in same order. from collections import OrderedDict v = OrderedDict(sorted(vars(self).items())) for key, value in v.items(): if type(value) == OptionsDictionary: if firstTime: #start of Options docstring docstring += '\n Options\n -------\n' firstTime = 0 for (name, val) in sorted(value.items()): docstring += " " + key + "['" docstring += name + "']" docstring += " : " + type(val).__name__ docstring += "(" if type(val).__name__ == 'str': docstring += "'" docstring += str(val) if type(val).__name__ == 'str': docstring += "'" docstring += ")\n" desc = value._options[name]['desc'] if (desc): docstring += " " + desc + "\n" #finish up docstring docstring += '\n \"\"\"\n' return docstring