def _setup_solvers(self, system, depth):
"""
Assign system instance, set depth, and optionally perform setup.
Parameters
----------
system : <System>
pointer to the owning system.
depth : int
depth of the current system (already incremented).
"""
self._system = weakref.ref(system)
self._depth = depth
self._problem_meta = system._problem_meta
if system.pathname:
parent_name = self.msginfo
self.options._parent_name = parent_name
self.recording_options._parent_name = parent_name
self.supports._parent_name = parent_name
if isinstance(self, LinearSolver) and not system._use_derivatives:
return
self._rec_mgr.startup(self)
myoutputs = myresiduals = myinputs = []
incl = self.recording_options['includes']
excl = self.recording_options['excludes']
# doesn't matter if we're a linear or nonlinear solver. The names for
# inputs, outputs, and residuals are the same for both the 'linear' and 'nonlinear'
# vectors.
if system.pathname:
incl = ['.'.join((system.pathname, i)) for i in incl]
excl = ['.'.join((system.pathname, i)) for i in excl]
if self.recording_options['record_solver_residuals']:
myresiduals = [
n for n in system._residuals._abs_iter()
if check_path(n, incl, excl)
]
if self.recording_options['record_outputs']:
myoutputs = [
n for n in system._outputs._abs_iter()
if check_path(n, incl, excl)
]
if self.recording_options['record_inputs']:
myinputs = [
n for n in system._inputs._abs_iter()
if check_path(n, incl, excl)
]
self._filtered_vars_to_record = {
'input': myinputs,
'output': myoutputs,
'residual': myresiduals
}
def _setup_solvers(self, system, depth):
"""
Assign system instance, set depth, and optionally perform setup.
Parameters
----------
system : <System>
pointer to the owning system.
depth : int
depth of the current system (already incremented).
"""
self._system = weakref.ref(system)
self._depth = depth
self._solver_info = system._solver_info
self._recording_iter = system._recording_iter
if system.pathname:
parent_name = self.msginfo
self.options._parent_name = parent_name
self.recording_options._parent_name = parent_name
self.supports._parent_name = parent_name
if isinstance(self, LinearSolver) and not system._use_derivatives:
return
self._rec_mgr.startup(self)
self._rec_mgr.record_metadata(self)
myoutputs = myresiduals = myinputs = []
incl = self.recording_options['includes']
excl = self.recording_options['excludes']
# doesn't matter if we're a linear or nonlinear solver. The names for
# inputs, outputs, and residuals are the same for both the 'linear' and 'nonlinear'
# vectors.
if system.pathname:
incl = ['.'.join((system.pathname, i)) for i in incl]
excl = ['.'.join((system.pathname, i)) for i in excl]
if self.recording_options['record_solver_residuals']:
myresiduals = [n for n in system._residuals._views if check_path(n, incl, excl)]
if self.recording_options['record_outputs']:
myoutputs = [n for n in system._outputs._views if check_path(n, incl, excl)]
if self.recording_options['record_inputs']:
myinputs = [n for n in system._inputs._views if check_path(n, incl, excl)]
self._filtered_vars_to_record = {
'input': myinputs,
'output': myoutputs,
'residual': myresiduals
}
# Raise a deprecation warning for changed option.
if 'err_on_maxiter' in self.options and self.options['err_on_maxiter'] is not None:
self.options['err_on_non_converge'] = self.options['err_on_maxiter']
warn_deprecation("The 'err_on_maxiter' option provides backwards compatibility "
"with earlier version of OpenMDAO; use options['err_on_non_converge'] "
"instead.")
def _setup_solvers(self, system, depth):
"""
Assign system instance, set depth, and optionally perform setup.
Parameters
----------
system : <System>
pointer to the owning system.
depth : int
depth of the current system (already incremented).
"""
self._system = system
self._depth = depth
self._solver_info = system._solver_info
self._recording_iter = system._recording_iter
if isinstance(self, LinearSolver) and not system._use_derivatives:
return
self._rec_mgr.startup(self)
self._rec_mgr.record_metadata(self)
myoutputs = myresiduals = myinputs = set()
incl = self.recording_options['includes']
excl = self.recording_options['excludes']
if self.recording_options['record_solver_residuals']:
if isinstance(self, NonlinearSolver):
residuals = system._residuals
else: # it's a LinearSolver
residuals = system._vectors['residual']['linear']
myresiduals = {n for n in residuals._names if check_path(n, incl, excl)}
if self.recording_options['record_outputs']:
if isinstance(self, NonlinearSolver):
outputs = system._outputs
else: # it's a LinearSolver
outputs = system._vectors['output']['linear']
myoutputs = {n for n in outputs._names if check_path(n, incl, excl)}
if self.recording_options['record_inputs']:
if isinstance(self, NonlinearSolver):
inputs = system._inputs
else:
inputs = system._vectors['input']['linear']
myinputs = {n for n in inputs._names if check_path(n, incl, excl)}
self._filtered_vars_to_record = {
'in': myinputs,
'out': myoutputs,
'res': myresiduals
}
def _get_metadata_system(self, system):
# Cannot handle PETScVector yet
from openmdao.api import PETScVector
if PETScVector and isinstance(system._outputs, PETScVector):
return None, None # Cannot handle PETScVector yet
# collect scaling arrays
scaling_vecs = {}
for kind, odict in system._vectors.items():
scaling_vecs[kind] = scaling = {}
for vecname, vec in odict.items():
scaling[vecname] = vec._scaling
# create a copy of the system's metadata excluding what is in 'options_excludes'
excludes = system.recording_options['options_excludes']
if excludes:
user_options = OptionsDictionary()
user_options._all_recordable = system.options._all_recordable
for key in system.options._dict:
if check_path(key, [], excludes, True):
user_options._dict[key] = system.options._dict[key]
user_options._read_only = system.options._read_only
return scaling_vecs, user_options
else:
return scaling_vecs, system.options
def record_metadata_system(self, recording_requester):
"""
Record system metadata.
Parameters
----------
recording_requester : System
The System that would like to record its metadata.
"""
if self.connection:
# Cannot handle PETScVector yet
from openmdao.api import PETScVector
if PETScVector and isinstance(recording_requester._outputs,
PETScVector):
return # Cannot handle PETScVector yet
# collect scaling arrays
scaling_vecs = {}
for kind, odict in iteritems(recording_requester._vectors):
scaling_vecs[kind] = scaling = {}
for vecname, vec in iteritems(odict):
scaling[vecname] = vec._scaling
scaling_factors = pickle.dumps(scaling_vecs, self._pickle_version)
# create a copy of the system's metadata excluding what is in 'options_excludes'
user_options = OptionsDictionary()
excludes = recording_requester.recording_options[
'options_excludes']
for key in recording_requester.options._dict:
if check_path(key, [], excludes, True):
user_options._dict[
key] = recording_requester.options._dict[key]
user_options._read_only = recording_requester.options._read_only
# try to pickle the metadata, report if it failed
try:
pickled_metadata = pickle.dumps(user_options,
self._pickle_version)
except Exception:
pickled_metadata = pickle.dumps(OptionsDictionary(),
self._pickle_version)
warnings.warn(
"Trying to record options which cannot be pickled "
"on system with name: %s. Use the 'options_excludes' "
"recording option on system objects to avoid attempting "
"to record options which cannot be pickled. Skipping "
"recording options for this system." %
recording_requester.name, RuntimeWarning)
path = recording_requester.pathname
if not path:
path = 'root'
scaling_factors = sqlite3.Binary(scaling_factors)
pickled_metadata = sqlite3.Binary(pickled_metadata)
with self.connection as c:
c.execute(
"INSERT INTO system_metadata(id, scaling_factors, component_metadata) "
"VALUES(?,?,?)", (path, scaling_factors, pickled_metadata))
def record_metadata_system(self, recording_requester):
"""
Record system metadata.
Parameters
----------
recording_requester : System
The System that would like to record its metadata.
"""
if self.con:
# Cannot handle PETScVector yet
from openmdao.api import PETScVector
if PETScVector and isinstance(recording_requester._outputs,
PETScVector):
return # Cannot handle PETScVector yet
# collect scaling arrays
scaling_vecs = {}
for kind, odict in iteritems(recording_requester._vectors):
scaling_vecs[kind] = scaling = {}
for vecname, vec in iteritems(odict):
scaling[vecname] = vec._scaling
scaling_factors = pickle.dumps(scaling_vecs, self._pickle_version)
# create a copy of the system's metadata excluding what is in 'metadata_excludes'
user_metadata = OptionsDictionary()
excludes = recording_requester.recording_options[
'metadata_excludes']
for key in recording_requester.options._dict:
if check_path(key, [], excludes, True):
user_metadata._dict[
key] = recording_requester.options._dict[key]
user_metadata._read_only = recording_requester.options._read_only
pickled_metadata = pickle.dumps(user_metadata,
self._pickle_version)
path = recording_requester.pathname
if not path:
path = 'root'
with self.con:
self.con.execute(
"INSERT INTO system_metadata(id, scaling_factors, component_metadata) \
VALUES(?,?, ?)",
(path, sqlite3.Binary(scaling_factors),
sqlite3.Binary(pickled_metadata)))
def _get_metadata_system(self, recording_requester):
# Cannot handle PETScVector yet
from openmdao.api import PETScVector
if PETScVector and isinstance(recording_requester._outputs, PETScVector):
return None, None # Cannot handle PETScVector yet
# collect scaling arrays
scaling_vecs = {}
for kind, odict in iteritems(recording_requester._vectors):
scaling_vecs[kind] = scaling = {}
for vecname, vec in iteritems(odict):
scaling[vecname] = vec._scaling
# create a copy of the system's metadata excluding what is in 'options_excludes'
user_options = OptionsDictionary()
excludes = recording_requester.recording_options['options_excludes']
for key in recording_requester.options._dict:
if check_path(key, [], excludes, True):
user_options._dict[key] = recording_requester.options._dict[key]
user_options._read_only = recording_requester.options._read_only
return scaling_vecs, user_options
def _get_metadata_system(self, recording_requester):
# Cannot handle PETScVector yet
from openmdao.api import PETScVector
if PETScVector and isinstance(recording_requester._outputs,
PETScVector):
return None, None # Cannot handle PETScVector yet
# collect scaling arrays
scaling_vecs = {}
for kind, odict in iteritems(recording_requester._vectors):
scaling_vecs[kind] = scaling = {}
for vecname, vec in iteritems(odict):
scaling[vecname] = vec._scaling
# create a copy of the system's metadata excluding what is in 'options_excludes'
user_options = OptionsDictionary()
excludes = recording_requester.recording_options['options_excludes']
for key in recording_requester.options._dict:
if check_path(key, [], excludes, True):
user_options._dict[key] = recording_requester.options._dict[
key]
user_options._read_only = recording_requester.options._read_only
return scaling_vecs, user_options
def _setup_recording(self):
"""
Set up case recording.
"""
problem = self._problem
model = problem.model
mydesvars = myobjectives = myconstraints = myresponses = set()
myinputs = set()
mysystem_outputs = set()
incl = self.recording_options['includes']
excl = self.recording_options['excludes']
rec_desvars = self.recording_options['record_desvars']
rec_objectives = self.recording_options['record_objectives']
rec_constraints = self.recording_options['record_constraints']
rec_responses = self.recording_options['record_responses']
rec_inputs = self.recording_options['record_inputs']
all_desvars = {
n
for n in self._designvars if check_path(n, incl, excl, True)
}
all_objectives = {
n
for n in self._objs if check_path(n, incl, excl, True)
}
all_constraints = {
n
for n in self._cons if check_path(n, incl, excl, True)
}
if rec_desvars:
mydesvars = all_desvars
if rec_objectives:
myobjectives = all_objectives
if rec_constraints:
myconstraints = all_constraints
if rec_responses:
myresponses = {
n
for n in self._responses if check_path(n, incl, excl, True)
}
# get the includes that were requested for this Driver recording
if incl:
# The my* variables are sets
# First gather all of the desired outputs
# The following might only be the local vars if MPI
mysystem_outputs = {
n
for n in model._outputs if check_path(n, incl, excl)
}
# If MPI, and on rank 0, need to gather up all the variables
# even those not local to rank 0
if MPI:
all_vars = model.comm.gather(mysystem_outputs, root=0)
if MPI.COMM_WORLD.rank == 0:
mysystem_outputs = all_vars[-1]
for d in all_vars[:-1]:
mysystem_outputs.update(d)
# de-duplicate mysystem_outputs
mysystem_outputs = mysystem_outputs.difference(
all_desvars, all_objectives, all_constraints)
if rec_inputs:
prob = self._problem
root = prob.model
myinputs = {n for n in root._inputs if check_path(n, incl, excl)}
if MPI:
all_vars = root.comm.gather(myinputs, root=0)
if MPI.COMM_WORLD.rank == 0:
myinputs = all_vars[-1]
for d in all_vars[:-1]:
myinputs.update(d)
if MPI: # filter based on who owns the variables
# TODO Eventually, we think we can get rid of this next check. But to be safe,
# we are leaving it in there.
if not model.is_active():
raise RuntimeError(
"RecordingManager.startup should never be called when "
"running in parallel on an inactive System")
rrank = problem.comm.rank
rowned = model._owning_rank
mydesvars = [n for n in mydesvars if rrank == rowned[n]]
myresponses = [n for n in myresponses if rrank == rowned[n]]
myobjectives = [n for n in myobjectives if rrank == rowned[n]]
myconstraints = [n for n in myconstraints if rrank == rowned[n]]
mysystem_outputs = [
n for n in mysystem_outputs if rrank == rowned[n]
]
myinputs = [n for n in myinputs if rrank == rowned[n]]
self._filtered_vars_to_record = {
'des': mydesvars,
'obj': myobjectives,
'con': myconstraints,
'res': myresponses,
'sys': mysystem_outputs,
'in': myinputs
}
self._rec_mgr.startup(self)
if self._rec_mgr._recorders:
from openmdao.devtools.problem_viewer.problem_viewer import _get_viewer_data
self._model_viewer_data = _get_viewer_data(problem)
if self.recording_options['record_metadata']:
self._rec_mgr.record_metadata(self)
def _setup_solvers(self, system, depth):
"""
Assign system instance, set depth, and optionally perform setup.
Parameters
----------
system : <System>
pointer to the owning system.
depth : int
depth of the current system (already incremented).
"""
self._system = weakref.ref(system)
self._depth = depth
self._solver_info = system._solver_info
self._recording_iter = system._recording_iter
if system.pathname:
parent_name = self.msginfo
self.options._parent_name = parent_name
self.recording_options._parent_name = parent_name
self.supports._parent_name = parent_name
if isinstance(self, LinearSolver) and not system._use_derivatives:
return
self._rec_mgr.startup(self)
self._rec_mgr.record_metadata(self)
myoutputs = myresiduals = myinputs = set()
incl = self.recording_options['includes']
excl = self.recording_options['excludes']
if self.recording_options['record_solver_residuals']:
if isinstance(self, NonlinearSolver):
residuals = system._residuals
else: # it's a LinearSolver
residuals = system._vectors['residual']['linear']
myresiduals = {n for n in residuals._names if check_path(n, incl, excl)}
if self.recording_options['record_outputs']:
if isinstance(self, NonlinearSolver):
outputs = system._outputs
else: # it's a LinearSolver
outputs = system._vectors['output']['linear']
myoutputs = {n for n in outputs._names if check_path(n, incl, excl)}
if self.recording_options['record_inputs']:
if isinstance(self, NonlinearSolver):
inputs = system._inputs
else:
inputs = system._vectors['input']['linear']
myinputs = {n for n in inputs._names if check_path(n, incl, excl)}
self._filtered_vars_to_record = {
'in': myinputs,
'out': myoutputs,
'res': myresiduals
}
# Raise a deprecation warning for changed option.
if 'err_on_maxiter' in self.options and self.options['err_on_maxiter'] is not None:
self.options['err_on_non_converge'] = self.options['err_on_maxiter']
warn_deprecation("The 'err_on_maxiter' option provides backwards compatibility "
"with earlier version of OpenMDAO; use options['err_on_non_converge'] "
"instead.")
def _get_vars_to_record(self, recording_options):
"""
Get variables to record based on recording options.
Parameters
----------
recording_options : <OptionsDictionary>
Dictionary with recording options.
Returns
-------
dict
Dictionary containing lists of variables to record.
"""
problem = self._problem()
model = problem.model
incl = recording_options['includes']
excl = recording_options['excludes']
# includes and excludes for outputs are specified using promoted names
abs2prom = model._var_allprocs_abs2prom['output']
# 1. If record_outputs is True, get the set of outputs
# 2. Filter those using includes and excludes to get the baseline set of variables to record
# 3. Add or remove from that set any desvars, objs, and cons based on the recording
# options of those
# includes and excludes for outputs are specified using _promoted_ names
# vectors are keyed on absolute name, discretes on relative/promoted name
myinputs = myoutputs = myresiduals = []
if recording_options['record_outputs']:
myoutputs = sorted([
n for n, prom in abs2prom.items()
if check_path(prom, incl, excl)
])
model_outs = model._outputs
if model._var_discrete['output']:
# if we have discrete outputs then residual name set doesn't match output one
if recording_options['record_residuals']:
myresiduals = [
n for n in myoutputs if model_outs._contains_abs(n)
]
elif recording_options['record_residuals']:
myresiduals = myoutputs
elif recording_options['record_residuals']:
myresiduals = [
n for n in model._residuals._abs_iter()
if check_path(abs2prom[n], incl, excl)
]
myoutputs = set(myoutputs)
if recording_options['record_desvars']:
myoutputs.update(self._designvars)
if recording_options['record_objectives'] or recording_options[
'record_responses']:
myoutputs.update(self._objs)
if recording_options['record_constraints'] or recording_options[
'record_responses']:
myoutputs.update(self._cons)
# inputs (if in options). inputs use _absolute_ names for includes/excludes
if 'record_inputs' in recording_options:
if recording_options['record_inputs']:
# sort the results since _var_allprocs_abs2prom isn't ordered
myinputs = sorted([
n for n in model._var_allprocs_abs2prom['input']
if check_path(n, incl, excl)
])
vars2record = {
'input': myinputs,
'output': list(myoutputs),
'residual': myresiduals
}
return vars2record
def _get_vars_to_record(self, recording_options):
"""
Get variables to record based on recording options.
Parameters
----------
recording_options : <OptionsDictionary>
Dictionary with recording options.
Returns
-------
dict
Dictionary containing lists of variables to record.
"""
problem = self._problem
model = problem.model
if MPI:
# TODO: Eventually, we think we can get rid of this next check.
# But to be safe, we are leaving it in there.
if not model.is_active():
raise RuntimeError(
"RecordingManager.startup should never be called when "
"running in parallel on an inactive System")
rrank = problem.comm.rank
rowned = model._owning_rank
incl = recording_options['includes']
excl = recording_options['excludes']
# includes and excludes for outputs are specified using promoted names
# NOTE: only local var names are in abs2prom, all will be gathered later
abs2prom = model._var_abs2prom['output']
all_desvars = {
n
for n in self._designvars
if n in abs2prom and check_path(abs2prom[n], incl, excl, True)
}
all_objectives = {
n
for n in self._objs
if n in abs2prom and check_path(abs2prom[n], incl, excl, True)
}
all_constraints = {
n
for n in self._cons
if n in abs2prom and check_path(abs2prom[n], incl, excl, True)
}
# design variables, objectives and constraints are always in the options
mydesvars = myobjectives = myconstraints = set()
if recording_options['record_desvars']:
if MPI:
mydesvars = [n for n in all_desvars if rrank == rowned[n]]
else:
mydesvars = list(all_desvars)
if recording_options['record_objectives']:
if MPI:
myobjectives = [
n for n in all_objectives if rrank == rowned[n]
]
else:
myobjectives = list(all_objectives)
if recording_options['record_constraints']:
if MPI:
myconstraints = [
n for n in all_constraints if rrank == rowned[n]
]
else:
myconstraints = list(all_constraints)
filtered_vars_to_record = {
'des': mydesvars,
'obj': myobjectives,
'con': myconstraints
}
# responses (if in options)
if 'record_responses' in recording_options:
myresponses = set()
if recording_options['record_responses']:
myresponses = {
n
for n in self._responses if n in abs2prom
and check_path(abs2prom[n], incl, excl, True)
}
if MPI:
myresponses = [
n for n in myresponses if rrank == rowned[n]
]
filtered_vars_to_record['res'] = list(myresponses)
# inputs (if in options)
if 'record_inputs' in recording_options:
myinputs = set()
if recording_options['record_inputs']:
myinputs = {
n
for n in model._inputs if check_path(n, incl, excl)
}
if MPI:
# gather the variables from all ranks to rank 0
all_vars = model.comm.gather(myinputs, root=0)
if MPI.COMM_WORLD.rank == 0:
myinputs = all_vars[-1]
for d in all_vars[:-1]:
myinputs.update(d)
myinputs = [n for n in myinputs if rrank == rowned[n]]
filtered_vars_to_record['in'] = list(myinputs)
# system outputs
myoutputs = set()
if incl:
myoutputs = {
n
for n in model._outputs
if n in abs2prom and check_path(abs2prom[n], incl, excl)
}
if MPI:
# gather the variables from all ranks to rank 0
all_vars = model.comm.gather(myoutputs, root=0)
if MPI.COMM_WORLD.rank == 0:
myoutputs = all_vars[-1]
for d in all_vars[:-1]:
myoutputs.update(d)
# de-duplicate
myoutputs = myoutputs.difference(all_desvars, all_objectives,
all_constraints)
if MPI:
myoutputs = [n for n in myoutputs if rrank == rowned[n]]
filtered_vars_to_record['sys'] = list(myoutputs)
return filtered_vars_to_record
def _setup_driver(self, problem):
"""
Prepare the driver for execution.
This is the final thing to run during setup.
Parameters
----------
problem : <Problem>
Pointer to the containing problem.
"""
self._problem = problem
model = problem.model
self._objs = objs = OrderedDict()
self._cons = cons = OrderedDict()
self._responses = model.get_responses(recurse=True)
response_size = 0
for name, data in iteritems(self._responses):
if data['type'] == 'con':
cons[name] = data
else:
objs[name] = data
response_size += data['size']
# Gather up the information for design vars.
self._designvars = model.get_design_vars(recurse=True)
desvar_size = np.sum(data['size']
for data in itervalues(self._designvars))
if ((problem._mode == 'fwd' and desvar_size > response_size)
or (problem._mode == 'rev' and response_size > desvar_size)):
warnings.warn(
"Inefficient choice of derivative mode. You chose '%s' for a "
"problem with %d design variables and %d response variables "
"(objectives and constraints)." %
(problem._mode, desvar_size, response_size), RuntimeWarning)
self._has_scaling = (
np.any([r['scaler'] is not None for r in self._responses.values()])
or np.any(
[dv['scaler'] is not None
for dv in self._designvars.values()]))
con_set = set()
obj_set = set()
dv_set = set()
self._remote_dvs = dv_dict = {}
self._remote_cons = con_dict = {}
self._remote_objs = obj_dict = {}
# Now determine if later we'll need to allgather cons, objs, or desvars.
if model.comm.size > 1 and model._subsystems_allprocs:
local_out_vars = set(model._outputs._views)
remote_dvs = set(self._designvars) - local_out_vars
remote_cons = set(self._cons) - local_out_vars
remote_objs = set(self._objs) - local_out_vars
all_remote_vois = model.comm.allgather(
(remote_dvs, remote_cons, remote_objs))
for rem_dvs, rem_cons, rem_objs in all_remote_vois:
con_set.update(rem_cons)
obj_set.update(rem_objs)
dv_set.update(rem_dvs)
# If we have remote VOIs, pick an owning rank for each and use that
# to bcast to others later
owning_ranks = model._owning_rank['output']
sizes = model._var_sizes['nonlinear']['output']
for i, vname in enumerate(model._var_allprocs_abs_names['output']):
owner = owning_ranks[vname]
if vname in dv_set:
dv_dict[vname] = (owner, sizes[owner, i])
if vname in con_set:
con_dict[vname] = (owner, sizes[owner, i])
if vname in obj_set:
obj_dict[vname] = (owner, sizes[owner, i])
self._remote_responses = self._remote_cons.copy()
self._remote_responses.update(self._remote_objs)
# Case recording setup
mydesvars = myobjectives = myconstraints = myresponses = set()
mysystem_outputs = set()
incl = self.recording_options['includes']
excl = self.recording_options['excludes']
rec_desvars = self.recording_options['record_desvars']
rec_objectives = self.recording_options['record_objectives']
rec_constraints = self.recording_options['record_constraints']
rec_responses = self.recording_options['record_responses']
# includes and excludes for outputs are specified using promoted names
# NOTE: only local var names are in abs2prom, all will be gathered later
abs2prom = model._var_abs2prom['output']
all_desvars = {
n
for n in self._designvars
if n in abs2prom and check_path(abs2prom[n], incl, excl, True)
}
all_objectives = {
n
for n in self._objs
if n in abs2prom and check_path(abs2prom[n], incl, excl, True)
}
all_constraints = {
n
for n in self._cons
if n in abs2prom and check_path(abs2prom[n], incl, excl, True)
}
if rec_desvars:
mydesvars = all_desvars
if rec_objectives:
myobjectives = all_objectives
if rec_constraints:
myconstraints = all_constraints
if rec_responses:
myresponses = {
n
for n in self._responses
if n in abs2prom and check_path(abs2prom[n], incl, excl, True)
}
# get the includes that were requested for this Driver recording
if incl:
prob = self._problem
root = prob.model
# The my* variables are sets
# First gather all of the desired outputs
# The following might only be the local vars if MPI
mysystem_outputs = {
n
for n in root._outputs
if n in abs2prom and check_path(abs2prom[n], incl, excl)
}
# If MPI, and on rank 0, need to gather up all the variables
# even those not local to rank 0
if MPI:
all_vars = root.comm.gather(mysystem_outputs, root=0)
if MPI.COMM_WORLD.rank == 0:
mysystem_outputs = all_vars[-1]
for d in all_vars[:-1]:
mysystem_outputs.update(d)
# de-duplicate mysystem_outputs
mysystem_outputs = mysystem_outputs.difference(
all_desvars, all_objectives, all_constraints)
if MPI: # filter based on who owns the variables
# TODO Eventually, we think we can get rid of this next check. But to be safe,
# we are leaving it in there.
if not model.is_active():
raise RuntimeError(
"RecordingManager.startup should never be called when "
"running in parallel on an inactive System")
rrank = self._problem.comm.rank # root ( aka model ) rank.
rowned = model._owning_rank['output']
mydesvars = [n for n in mydesvars if rrank == rowned[n]]
myresponses = [n for n in myresponses if rrank == rowned[n]]
myobjectives = [n for n in myobjectives if rrank == rowned[n]]
myconstraints = [n for n in myconstraints if rrank == rowned[n]]
mysystem_outputs = [
n for n in mysystem_outputs if rrank == rowned[n]
]
self._filtered_vars_to_record = {
'des': mydesvars,
'obj': myobjectives,
'con': myconstraints,
'res': myresponses,
'sys': mysystem_outputs,
}
self._rec_mgr.startup(self)
def _get_vars_to_record(self, recording_options):
"""
Get variables to record based on recording options.
Parameters
----------
recording_options : <OptionsDictionary>
Dictionary with recording options.
Returns
-------
dict
Dictionary containing lists of variables to record.
"""
problem = self._problem()
model = problem.model
rrank = problem.comm.rank
incl = recording_options['includes']
excl = recording_options['excludes']
# includes and excludes for outputs are specified using promoted names
abs2prom = model._var_allprocs_abs2prom['output']
allvars = []
# if desvars, etc. are not wanted, we need to exclude them from the outputs even if
# they match the includes list.
skip = set()
if recording_options['record_desvars']:
allvars.extend(self._designvars)
else:
skip.update(self._designvars)
if recording_options['record_objectives'] or recording_options[
'record_responses']:
allvars.extend(self._objs)
else:
skip.update(self._objs)
if recording_options['record_constraints'] or recording_options[
'record_responses']:
allvars.extend(self._cons)
else:
skip.update(self._cons)
vars2record = {
'output': [
n for n in allvars
if n in abs2prom and check_path(abs2prom[n], incl, excl, True)
]
}
# inputs (if in options). inputs use _absolute_ names for includes/excludes
if 'record_inputs' in recording_options:
if recording_options['record_inputs']:
# sort the results since _var_allprocs_abs2prom isn't ordered
vars2record['input'] = sorted([
n for n in model._var_allprocs_abs2prom['input']
if check_path(n, incl, excl)
])
else:
vars2record['input'] = []
if incl:
# loop over abs2prom (which includes both continuous and discrete outputs) since
# the order doesn't matter (we're sorting it at the end).
vars2record['output'].extend(
n for n, prom in abs2prom.items()
if n not in skip and check_path(prom, incl, excl))
# remove dups and make sure order is the same on all procs
vars2record['output'] = sorted(set(vars2record['output']))
return vars2record
def _setup_driver(self, problem):
"""
Prepare the driver for execution.
This is the final thing to run during setup.
Parameters
----------
problem : <Problem>
Pointer to the containing problem.
"""
self._problem = problem
model = problem.model
self._objs = objs = OrderedDict()
self._cons = cons = OrderedDict()
self._responses = model.get_responses(recurse=True)
response_size = 0
for name, data in iteritems(self._responses):
if data['type'] == 'con':
cons[name] = data
else:
objs[name] = data
response_size += data['size']
# Gather up the information for design vars.
self._designvars = model.get_design_vars(recurse=True)
desvar_size = np.sum(data['size'] for data in itervalues(self._designvars))
if ((problem._mode == 'fwd' and desvar_size > response_size) or
(problem._mode == 'rev' and response_size > desvar_size)):
warnings.warn("Inefficient choice of derivative mode. You chose '%s' for a "
"problem with %d design variables and %d response variables "
"(objectives and constraints)." %
(problem._mode, desvar_size, response_size), RuntimeWarning)
self._has_scaling = (
np.any([r['scaler'] is not None for r in self._responses.values()]) or
np.any([dv['scaler'] is not None for dv in self._designvars.values()])
)
con_set = set()
obj_set = set()
dv_set = set()
self._remote_dvs = dv_dict = {}
self._remote_cons = con_dict = {}
self._remote_objs = obj_dict = {}
# Now determine if later we'll need to allgather cons, objs, or desvars.
if model.comm.size > 1 and model._subsystems_allprocs:
local_out_vars = set(model._outputs._views)
remote_dvs = set(self._designvars) - local_out_vars
remote_cons = set(self._cons) - local_out_vars
remote_objs = set(self._objs) - local_out_vars
all_remote_vois = model.comm.allgather(
(remote_dvs, remote_cons, remote_objs))
for rem_dvs, rem_cons, rem_objs in all_remote_vois:
con_set.update(rem_cons)
obj_set.update(rem_objs)
dv_set.update(rem_dvs)
# If we have remote VOIs, pick an owning rank for each and use that
# to bcast to others later
owning_ranks = model._owning_rank['output']
sizes = model._var_sizes['nonlinear']['output']
for i, vname in enumerate(model._var_allprocs_abs_names['output']):
owner = owning_ranks[vname]
if vname in dv_set:
dv_dict[vname] = (owner, sizes[owner, i])
if vname in con_set:
con_dict[vname] = (owner, sizes[owner, i])
if vname in obj_set:
obj_dict[vname] = (owner, sizes[owner, i])
self._remote_responses = self._remote_cons.copy()
self._remote_responses.update(self._remote_objs)
# Case recording setup
mydesvars = myobjectives = myconstraints = myresponses = set()
mysystem_outputs = set()
incl = self.recording_options['includes']
excl = self.recording_options['excludes']
rec_desvars = self.recording_options['record_desvars']
rec_objectives = self.recording_options['record_objectives']
rec_constraints = self.recording_options['record_constraints']
rec_responses = self.recording_options['record_responses']
all_desvars = {n for n in self._designvars
if check_path(n, incl, excl, True)}
all_objectives = {n for n in self._objs
if check_path(n, incl, excl, True)}
all_constraints = {n for n in self._cons
if check_path(n, incl, excl, True)}
if rec_desvars:
mydesvars = all_desvars
if rec_objectives:
myobjectives = all_objectives
if rec_constraints:
myconstraints = all_constraints
if rec_responses:
myresponses = {n for n in self._responses
if check_path(n, incl, excl, True)}
# get the includes that were requested for this Driver recording
if incl:
prob = self._problem
root = prob.model
# The my* variables are sets
# First gather all of the desired outputs
# The following might only be the local vars if MPI
mysystem_outputs = {n for n in root._outputs
if check_path(n, incl, excl)}
# If MPI, and on rank 0, need to gather up all the variables
# even those not local to rank 0
if MPI:
all_vars = root.comm.gather(mysystem_outputs, root=0)
if MPI.COMM_WORLD.rank == 0:
mysystem_outputs = all_vars[-1]
for d in all_vars[:-1]:
mysystem_outputs.update(d)
# de-duplicate mysystem_outputs
mysystem_outputs = mysystem_outputs.difference(all_desvars, all_objectives,
all_constraints)
if MPI: # filter based on who owns the variables
# TODO Eventually, we think we can get rid of this next check. But to be safe,
# we are leaving it in there.
if not model.is_active():
raise RuntimeError(
"RecordingManager.startup should never be called when "
"running in parallel on an inactive System")
rrank = self._problem.comm.rank # root ( aka model ) rank.
rowned = model._owning_rank['output']
mydesvars = [n for n in mydesvars if rrank == rowned[n]]
myresponses = [n for n in myresponses if rrank == rowned[n]]
myobjectives = [n for n in myobjectives if rrank == rowned[n]]
myconstraints = [n for n in myconstraints if rrank == rowned[n]]
mysystem_outputs = [n for n in mysystem_outputs if rrank == rowned[n]]
self._filtered_vars_to_record = {
'des': mydesvars,
'obj': myobjectives,
'con': myconstraints,
'res': myresponses,
'sys': mysystem_outputs,
}
self._rec_mgr.startup(self)
if self._rec_mgr._recorders:
from openmdao.devtools.problem_viewer.problem_viewer import _get_viewer_data
self._model_viewer_data = _get_viewer_data(problem)
if self.recording_options['record_metadata']:
self._rec_mgr.record_metadata(self)
# set up simultaneous deriv coloring
if self._simul_coloring_info and self.supports['simultaneous_derivatives']:
if problem._mode == 'fwd':
self._setup_simul_coloring(problem._mode)
else:
raise RuntimeError("simultaneous derivs are currently not supported in rev mode.")
def _get_vars_to_record(self, recording_options):
"""
Get variables to record based on recording options.
Parameters
----------
recording_options : <OptionsDictionary>
Dictionary with recording options.
Returns
-------
dict
Dictionary containing lists of variables to record.
"""
problem = self._problem
model = problem.model
if MPI:
# TODO: Eventually, we think we can get rid of this next check.
# But to be safe, we are leaving it in there.
if not model.is_active():
raise RuntimeError("RecordingManager.startup should never be called when "
"running in parallel on an inactive System")
rrank = problem.comm.rank
rowned = model._owning_rank
incl = recording_options['includes']
excl = recording_options['excludes']
# includes and excludes for outputs are specified using promoted names
# NOTE: only local var names are in abs2prom, all will be gathered later
abs2prom = model._var_abs2prom['output']
all_desvars = {n for n in self._designvars
if n in abs2prom and check_path(abs2prom[n], incl, excl, True)}
all_objectives = {n for n in self._objs
if n in abs2prom and check_path(abs2prom[n], incl, excl, True)}
all_constraints = {n for n in self._cons
if n in abs2prom and check_path(abs2prom[n], incl, excl, True)}
# design variables, objectives and constraints are always in the options
mydesvars = myobjectives = myconstraints = set()
if recording_options['record_desvars']:
if MPI:
mydesvars = [n for n in all_desvars if rrank == rowned[n]]
else:
mydesvars = list(all_desvars)
if recording_options['record_objectives']:
if MPI:
myobjectives = [n for n in all_objectives if rrank == rowned[n]]
else:
myobjectives = list(all_objectives)
if recording_options['record_constraints']:
if MPI:
myconstraints = [n for n in all_constraints if rrank == rowned[n]]
else:
myconstraints = list(all_constraints)
filtered_vars_to_record = {
'des': mydesvars,
'obj': myobjectives,
'con': myconstraints
}
# responses (if in options)
if 'record_responses' in recording_options:
myresponses = set()
if recording_options['record_responses']:
myresponses = {n for n in self._responses
if n in abs2prom and check_path(abs2prom[n], incl, excl, True)}
if MPI:
myresponses = [n for n in myresponses if rrank == rowned[n]]
filtered_vars_to_record['res'] = list(myresponses)
# inputs (if in options)
if 'record_inputs' in recording_options:
myinputs = set()
if recording_options['record_inputs']:
myinputs = {n for n in model._inputs if check_path(n, incl, excl)}
if MPI:
# gather the variables from all ranks to rank 0
all_vars = model.comm.gather(myinputs, root=0)
if MPI.COMM_WORLD.rank == 0:
myinputs = all_vars[-1]
for d in all_vars[:-1]:
myinputs.update(d)
myinputs = [n for n in myinputs if rrank == rowned[n]]
filtered_vars_to_record['in'] = list(myinputs)
# system outputs (if the options being processed are for the driver itself)
if recording_options is self.recording_options:
myoutputs = set()
if incl:
myoutputs = {n for n in model._outputs
if n in abs2prom and check_path(abs2prom[n], incl, excl)}
if MPI:
# gather the variables from all ranks to rank 0
all_vars = model.comm.gather(myoutputs, root=0)
if MPI.COMM_WORLD.rank == 0:
myoutputs = all_vars[-1]
for d in all_vars[:-1]:
myoutputs.update(d)
# de-duplicate
myoutputs = myoutputs.difference(all_desvars, all_objectives, all_constraints)
if MPI:
myoutputs = [n for n in myoutputs if rrank == rowned[n]]
filtered_vars_to_record['sys'] = list(myoutputs)
return filtered_vars_to_record