def push_mda(self, problem, options):
name = problem.model.__class__.__name__
data = _get_viewer_data(problem)
#print(name, data)
self.scalar_format = options['--scalar-format']
self.tree = data['tree']
#print("TREE("+name+")=", self.tree)
self.connections = data['connections_list']
#print("CONNECTIONS("+name+")=", self.connections)
# MDA informations
self.vars = {}
self.vardescs = {}
self.discmap = {}
self._collect_disc_infos(problem.model, self.tree)
self._collect_var_infos(problem.model)
mda_attrs = self._get_mda_attributes(problem.model, self.tree)
if options['--dry-run']:
print(json.dumps(mda_attrs, indent=2))
# print(self.discmap)
else:
url = self._endpoint('/api/v1/analyses')
resp = self.session.post(url, headers=self.headers, json={'analysis': mda_attrs})
resp.raise_for_status()
print("Analysis %s pushed" % name)
def record_viewer_data(problem):
"""
Record model viewer data for all recorders that have that option enabled.
We don't want to collect the viewer data if it's not needed though,
so first we'll find all recorders that need the data (if any) and
then record it for those recorders.
Parameters
----------
problem : Problem
The problem for which model viewer data is to be recorded.
"""
# assemble list of all objects that may have recorders
systems = [system for system in problem.model.system_iter(include_self=True, recurse=True)]
nl_solv = [system._nonlinear_solver for system in systems if system._nonlinear_solver]
ls_solv = [nl.linesearch for nl in nl_solv if hasattr(nl, 'linesearch') and nl.linesearch]
rec_obj = [problem, problem.driver] + systems + nl_solv + ls_solv
# get all recorders that need to record the viewer data
recorders = set()
for obj in rec_obj:
for recorder in obj._rec_mgr._recorders:
if recorder._record_viewer_data:
recorders.add(recorder)
# if any recorders were found, get the viewer data and record it
if recorders:
from openmdao.devtools.problem_viewer.problem_viewer import _get_viewer_data
viewer_data = _get_viewer_data(problem)
for recorder in recorders:
recorder.record_viewer_data(viewer_data)
def test_model_viewer_has_correct_data_from_problem(self):
"""
Verify that the correct model structure data exists when stored as compared
to the expected structure, using the SellarStateConnection model.
"""
p = Problem(model=SellarStateConnection())
p.setup(check=False)
model_viewer_data = _get_viewer_data(p)
# check expected model tree
self.assertDictEqual(model_viewer_data['tree'], self.expected_tree)
# check expected system pathnames
pathnames = model_viewer_data['sys_pathnames_list']
self.assertListEqual(sorted(pathnames), self.expected_pathnames)
# check expected connections, after mapping cycle_arrows indices back to pathnames
connections = model_viewer_data['connections_list']
for conn in connections:
if 'cycle_arrows' in conn:
cycle_arrows = []
for src, tgt in conn['cycle_arrows']:
cycle_arrows.append(' '.join([pathnames[src], pathnames[tgt]]))
conn['cycle_arrows'] = sorted(cycle_arrows)
self.assertListEqual(connections, self.expected_conns)
# check expected abs2prom map
self.assertDictEqual(model_viewer_data['abs2prom'], self.expected_abs2prom)
def test_model_viewer_has_correct_data_from_problem(self):
"""
Verify that the correct model structure data exists when stored as compared
to the expected structure, using the SellarStateConnection model.
"""
p = Problem(model=SellarStateConnection())
p.setup(check=False)
model_viewer_data = _get_viewer_data(p)
# check expected model tree
self.assertDictEqual(model_viewer_data['tree'], self.expected_tree)
# check expected system pathnames
pathnames = model_viewer_data['sys_pathnames_list']
self.assertListEqual(sorted(pathnames), self.expected_pathnames)
# check expected connections, after mapping cycle_arrows indices back to pathnames
connections = model_viewer_data['connections_list']
for conn in connections:
if 'cycle_arrows' in conn:
cycle_arrows = []
for src, tgt in conn['cycle_arrows']:
cycle_arrows.append(' '.join(
[pathnames[src], pathnames[tgt]]))
conn['cycle_arrows'] = sorted(cycle_arrows)
self.assertListEqual(connections, self.expected_conns)
# check expected abs2prom map
self.assertDictEqual(model_viewer_data['abs2prom'],
self.expected_abs2prom)
def __init__(self, problem, scalar_format):
data = _get_viewer_data(problem)
self.problem = problem
self.scalar_format = scalar_format
self.tree = data["tree"]
self.connections = data["connections_list"]
self.vars = {}
self.vardescs = {}
self.discmap = {}
def test_model_viewer_has_correct_data_from_problem(self):
"""
Verify that the correct model structure data exists when stored as compared
to the expected structure, using the SellarStateConnection model.
"""
p = Problem()
p.model = SellarStateConnection()
p.setup(check=False)
model_viewer_data = _get_viewer_data(p)
tree_json = json.dumps(model_viewer_data['tree'])
conns_json = json.dumps(model_viewer_data['connections_list'])
self.assertEqual(self.expected_tree_json, tree_json)
self.assertEqual(self.expected_conns_json, conns_json)
def test_model_viewer_has_correct_data_from_problem(self):
"""
Verify that the correct model structure data exists when stored as compared
to the expected structure, using the SellarStateConnection model.
"""
p = Problem()
p.model = SellarStateConnection()
p.setup(check=False)
model_viewer_data = _get_viewer_data(p)
tree_json = json.dumps(model_viewer_data['tree'])
conns_json = json.dumps(model_viewer_data['connections_list'])
self.assertEqual(self.expected_tree_json, tree_json)
self.assertEqual(self.expected_conns_json, conns_json)
def test_model_viewer_has_correct_data_from_problem(self):
"""
Verify that the correct model structure data exists when stored as compared
to the expected structure, using the SellarStateConnection model.
"""
p = Problem()
p.model = SellarStateConnection()
p.setup(check=False)
model_viewer_data = _get_viewer_data(p)
self.assertDictEqual(model_viewer_data['tree'], self.expected_tree)
self.assertListEqual(model_viewer_data['connections_list'],
self.expected_conns)
self.assertDictEqual(model_viewer_data['abs2prom'],
self.expected_abs2prom)
def test_model_viewer_has_correct_data_from_sqlite(self):
"""
Verify that the correct data exists when a model structure is recorded
and then pulled out of a sqlite db file and compared to the expected
structure. Uses the SellarStateConnection model.
"""
p = Problem(model=SellarStateConnection())
r = SqliteRecorder(self.sqlite_db_filename)
p.driver.add_recorder(r)
p.setup()
p.final_setup()
r.shutdown()
model_viewer_data = _get_viewer_data(self.sqlite_db_filename)
# check expected model tree
self.assertDictEqual(model_viewer_data['tree'], self.expected_tree)
# check expected system pathnames
pathnames = model_viewer_data['sys_pathnames_list']
self.assertListEqual(sorted(pathnames), self.expected_pathnames)
# check expected connections, after mapping cycle_arrows indices back to pathnames
connections = sorted(model_viewer_data['connections_list'],
key=lambda x: (x['tgt'], x['src']))
for conn in connections:
if 'cycle_arrows' in conn:
cycle_arrows = []
for src, tgt in conn['cycle_arrows']:
cycle_arrows.append(' '.join(
[pathnames[src], pathnames[tgt]]))
conn['cycle_arrows'] = sorted(cycle_arrows)
self.assertEqual(len(connections), len(self.expected_conns))
for c, ex in zip(connections, self.expected_conns):
self.assertEqual(c['src'], ex['src'])
self.assertEqual(c['tgt'], ex['tgt'])
self.assertEqual(c.get('cycle_arrows', []),
ex.get('cycle_arrows', []))
# check expected abs2prom map
self.assertDictEqual(model_viewer_data['abs2prom'],
self.expected_abs2prom)
def test_model_viewer_has_correct_data_from_sqlite(self):
"""
Verify that the correct data exists when a model structure is recorded
and then pulled out of a sqlite db file and compared to the expected
structure. Uses the SellarStateConnection model.
"""
p = Problem()
p.model = SellarStateConnection()
r = SqliteRecorder(self.sqlite_db_filename)
p.driver.add_recorder(r)
p.setup(check=False)
p.final_setup()
r.close()
model_viewer_data = _get_viewer_data(self.sqlite_db_filename)
tree_json = json.dumps(model_viewer_data['tree'])
conns_json = json.dumps(model_viewer_data['connections_list'])
self.assertEqual(self.expected_tree_json, tree_json)
self.assertEqual(self.expected_conns_json, conns_json)
def test_model_viewer_has_correct_data_from_sqlite(self):
"""
Verify that the correct data exists when a model structure is recorded
and then pulled out of a sqlite db file and compared to the expected
structure. Uses the SellarStateConnection model.
"""
p = Problem()
p.model = SellarStateConnection()
r = SqliteRecorder(self.sqlite_db_filename)
p.driver.add_recorder(r)
p.setup(check=False)
p.final_setup()
r.close()
model_viewer_data = _get_viewer_data(self.sqlite_db_filename)
tree_json = json.dumps(model_viewer_data['tree'])
conns_json = json.dumps(model_viewer_data['connections_list'])
self.assertEqual(self.expected_tree_json, tree_json)
self.assertEqual(self.expected_conns_json, conns_json)
def record_viewer_data(problem):
"""
Record model viewer data for all recorders that have that option enabled.
We don't want to collect the viewer data if it's not needed though,
so first we'll find all recorders that need the data (if any) and
then record it for those recorders.
Parameters
----------
problem : Problem
The problem for which model viewer data is to be recorded.
"""
# assemble list of all objects that may have recorders
systems = [
system for system in problem.model.system_iter(include_self=True,
recurse=True)
]
nl_solv = [
system._nonlinear_solver for system in systems
if system._nonlinear_solver
]
ls_solv = [
nl.linesearch for nl in nl_solv
if hasattr(nl, 'linesearch') and nl.linesearch
]
rec_obj = [problem, problem.driver] + systems + nl_solv + ls_solv
# get all recorders that need to record the viewer data
recorders = set()
for obj in rec_obj:
for recorder in obj._rec_mgr._recorders:
if recorder._record_viewer_data:
recorders.add(recorder)
# if any recorders were found, get the viewer data and record it
if recorders:
from openmdao.devtools.problem_viewer.problem_viewer import _get_viewer_data
viewer_data = _get_viewer_data(problem)
for recorder in recorders:
recorder.record_viewer_data(viewer_data)
def test_model_viewer_has_correct_data_from_sqlite(self):
"""
Verify that the correct data exists when a model structure is recorded
and then pulled out of a sqlite db file and compared to the expected
structure. Uses the SellarStateConnection model.
"""
p = Problem(model=SellarStateConnection())
r = SqliteRecorder(self.sqlite_db_filename)
p.driver.add_recorder(r)
p.setup(check=False)
p.final_setup()
r.shutdown()
model_viewer_data = _get_viewer_data(self.sqlite_db_filename)
# check expected model tree
self.assertDictEqual(model_viewer_data['tree'], self.expected_tree)
# check expected system pathnames
pathnames = model_viewer_data['sys_pathnames_list']
self.assertListEqual(sorted(pathnames), self.expected_pathnames)
# check expected connections, after mapping cycle_arrows indices back to pathnames
connections = model_viewer_data['connections_list']
for conn in connections:
if 'cycle_arrows' in conn:
cycle_arrows = []
for src, tgt in conn['cycle_arrows']:
cycle_arrows.append(' '.join([pathnames[src], pathnames[tgt]]))
conn['cycle_arrows'] = sorted(cycle_arrows)
self.assertListEqual(connections, self.expected_conns)
# check expected abs2prom map
self.assertDictEqual(model_viewer_data['abs2prom'], self.expected_abs2prom)
def _setup_driver(self, problem):
"""
Prepare the driver for execution.
This is the final thing to run during setup.
Parameters
----------
problem : <Problem>
Pointer to the containing problem.
"""
self._problem = problem
model = problem.model
self._total_jac = None
self._objs = objs = OrderedDict()
self._cons = cons = OrderedDict()
self._responses = model.get_responses(recurse=True)
response_size = 0
for name, data in iteritems(self._responses):
if data['type'] == 'con':
cons[name] = data
else:
objs[name] = data
response_size += data['size']
# Gather up the information for design vars.
self._designvars = model.get_design_vars(recurse=True)
self._has_scaling = (
np.any([r['scaler'] is not None for r in self._responses.values()]) or
np.any([dv['scaler'] is not None for dv in self._designvars.values()])
)
con_set = set()
obj_set = set()
dv_set = set()
self._remote_dvs = dv_dict = {}
self._remote_cons = con_dict = {}
self._remote_objs = obj_dict = {}
# Now determine if later we'll need to allgather cons, objs, or desvars.
if model.comm.size > 1 and model._subsystems_allprocs:
local_out_vars = set(model._outputs._views)
remote_dvs = set(self._designvars) - local_out_vars
remote_cons = set(self._cons) - local_out_vars
remote_objs = set(self._objs) - local_out_vars
all_remote_vois = model.comm.allgather(
(remote_dvs, remote_cons, remote_objs))
for rem_dvs, rem_cons, rem_objs in all_remote_vois:
con_set.update(rem_cons)
obj_set.update(rem_objs)
dv_set.update(rem_dvs)
# If we have remote VOIs, pick an owning rank for each and use that
# to bcast to others later
owning_ranks = model._owning_rank
sizes = model._var_sizes['nonlinear']['output']
for i, vname in enumerate(model._var_allprocs_abs_names['output']):
owner = owning_ranks[vname]
if vname in dv_set:
dv_dict[vname] = (owner, sizes[owner, i])
if vname in con_set:
con_dict[vname] = (owner, sizes[owner, i])
if vname in obj_set:
obj_dict[vname] = (owner, sizes[owner, i])
self._remote_responses = self._remote_cons.copy()
self._remote_responses.update(self._remote_objs)
# Case recording setup
mydesvars = myobjectives = myconstraints = myresponses = set()
mysystem_outputs = set()
incl = self.recording_options['includes']
excl = self.recording_options['excludes']
rec_desvars = self.recording_options['record_desvars']
rec_objectives = self.recording_options['record_objectives']
rec_constraints = self.recording_options['record_constraints']
rec_responses = self.recording_options['record_responses']
all_desvars = {n for n in self._designvars
if check_path(n, incl, excl, True)}
all_objectives = {n for n in self._objs
if check_path(n, incl, excl, True)}
all_constraints = {n for n in self._cons
if check_path(n, incl, excl, True)}
if rec_desvars:
mydesvars = all_desvars
if rec_objectives:
myobjectives = all_objectives
if rec_constraints:
myconstraints = all_constraints
if rec_responses:
myresponses = {n for n in self._responses
if check_path(n, incl, excl, True)}
# get the includes that were requested for this Driver recording
if incl:
prob = self._problem
root = prob.model
# The my* variables are sets
# First gather all of the desired outputs
# The following might only be the local vars if MPI
mysystem_outputs = {n for n in root._outputs
if check_path(n, incl, excl)}
# If MPI, and on rank 0, need to gather up all the variables
# even those not local to rank 0
if MPI:
all_vars = root.comm.gather(mysystem_outputs, root=0)
if MPI.COMM_WORLD.rank == 0:
mysystem_outputs = all_vars[-1]
for d in all_vars[:-1]:
mysystem_outputs.update(d)
# de-duplicate mysystem_outputs
mysystem_outputs = mysystem_outputs.difference(all_desvars, all_objectives,
all_constraints)
if MPI: # filter based on who owns the variables
# TODO Eventually, we think we can get rid of this next check. But to be safe,
# we are leaving it in there.
if not model.is_active():
raise RuntimeError(
"RecordingManager.startup should never be called when "
"running in parallel on an inactive System")
rrank = self._problem.comm.rank # root ( aka model ) rank.
rowned = model._owning_rank
mydesvars = [n for n in mydesvars if rrank == rowned[n]]
myresponses = [n for n in myresponses if rrank == rowned[n]]
myobjectives = [n for n in myobjectives if rrank == rowned[n]]
myconstraints = [n for n in myconstraints if rrank == rowned[n]]
mysystem_outputs = [n for n in mysystem_outputs if rrank == rowned[n]]
self._filtered_vars_to_record = {
'des': mydesvars,
'obj': myobjectives,
'con': myconstraints,
'res': myresponses,
'sys': mysystem_outputs,
}
self._rec_mgr.startup(self)
if self._rec_mgr._recorders:
from openmdao.devtools.problem_viewer.problem_viewer import _get_viewer_data
self._model_viewer_data = _get_viewer_data(problem)
if self.recording_options['record_metadata']:
self._rec_mgr.record_metadata(self)
# set up simultaneous deriv coloring
if (coloring_mod._use_sparsity and self._simul_coloring_info and
self.supports['simultaneous_derivatives']):
if problem._mode == 'fwd':
self._setup_simul_coloring(problem._mode)
else:
raise RuntimeError("simultaneous derivs are currently not supported in rev mode.")
desvar_size = np.sum(data['size'] for data in itervalues(self._designvars))
# if we're using simultaneous derivatives then our effective design var size is less
# than the full design var size
if self._simul_coloring_info:
col_lists = self._simul_coloring_info[0]
if col_lists:
desvar_size = len(col_lists[0])
desvar_size += len(col_lists) - 1
if ((problem._mode == 'fwd' and desvar_size > response_size) or
(problem._mode == 'rev' and response_size > desvar_size)):
warnings.warn("Inefficient choice of derivative mode. You chose '%s' for a "
"problem with %d design variables and %d response variables "
"(objectives and constraints)." %
(problem._mode, desvar_size, response_size), RuntimeWarning)
def test_obj_pass(self):
prob = Problem()
model = prob.model
indep = model.add_subsystem('indep', IndepVarComp(), promotes_outputs=['x'])
indep.add_discrete_output('x', _DiscreteVal(19))
G = model.add_subsystem('G', ParallelGroup(), promotes_inputs=['x'])
G1 = G.add_subsystem('G1', Group(), promotes_inputs=['x'], promotes_outputs=['y'])
G1.add_subsystem('C1_1', ObjAdderCompEx(_DiscreteVal(5)), promotes_inputs=['x'])
G1.add_subsystem('C1_2', ObjAdderCompEx(_DiscreteVal(7)), promotes_outputs=['y'])
G1.connect('C1_1.y', 'C1_2.x')
G2 = G.add_subsystem('G2', Group(), promotes_inputs=['x'])
G2.add_subsystem('C2_1', ObjAdderCompEx(_DiscreteVal(1)), promotes_inputs=['x'])
G2.add_subsystem('C2_2', ObjAdderCompEx(_DiscreteVal(11)), promotes_outputs=['y'])
G2.connect('C2_1.y', 'C2_2.x')
model.add_subsystem('C3', ObjAdderCompEx(_DiscreteVal(9)))
model.add_subsystem('C4', ObjAdderCompEx(_DiscreteVal(21)))
model.connect('G.y', 'C3.x')
model.connect('G.G2.y', 'C4.x')
prob.setup()
prob.run_model()
self.assertEqual(prob['C3.y'].getval(), 40)
self.assertEqual(prob['C4.y'].getval(), 52)
def _var_iter(obj):
name = obj['name']
if 'children' in obj:
for c in obj['children']:
for vname in _var_iter(c):
if name:
yield '.'.join((name, vname))
else:
yield vname
else:
yield name
# add a test to see if discrete vars show up in view_model
data = _get_viewer_data(prob)
findvars = [
'indep.x',
'G.G1.C1_1.x',
'G.G1.C1_1.y',
'G.G1.C1_2.x',
'G.G1.C1_2.y',
'G.G2.C2_1.x',
'G.G2.C2_1.y',
'G.G2.C2_2.x',
'G.G2.C2_2.y',
'C3.x',
'C3.y',
'C4.x',
'C4.y',
]
vnames = list(_var_iter(data['tree']))
self.assertTrue(sorted(findvars), sorted(vnames))
def _setup_driver(self, problem):
"""
Prepare the driver for execution.
This is the final thing to run during setup.
Parameters
----------
problem : <Problem>
Pointer to the containing problem.
"""
self._problem = problem
model = problem.model
self._objs = objs = OrderedDict()
self._cons = cons = OrderedDict()
self._responses = model.get_responses(recurse=True)
response_size = 0
for name, data in iteritems(self._responses):
if data['type'] == 'con':
cons[name] = data
else:
objs[name] = data
response_size += data['size']
# Gather up the information for design vars.
self._designvars = model.get_design_vars(recurse=True)
desvar_size = np.sum(data['size'] for data in itervalues(self._designvars))
if ((problem._mode == 'fwd' and desvar_size > response_size) or
(problem._mode == 'rev' and response_size > desvar_size)):
warnings.warn("Inefficient choice of derivative mode. You chose '%s' for a "
"problem with %d design variables and %d response variables "
"(objectives and constraints)." %
(problem._mode, desvar_size, response_size), RuntimeWarning)
self._has_scaling = (
np.any([r['scaler'] is not None for r in self._responses.values()]) or
np.any([dv['scaler'] is not None for dv in self._designvars.values()])
)
con_set = set()
obj_set = set()
dv_set = set()
self._remote_dvs = dv_dict = {}
self._remote_cons = con_dict = {}
self._remote_objs = obj_dict = {}
# Now determine if later we'll need to allgather cons, objs, or desvars.
if model.comm.size > 1 and model._subsystems_allprocs:
local_out_vars = set(model._outputs._views)
remote_dvs = set(self._designvars) - local_out_vars
remote_cons = set(self._cons) - local_out_vars
remote_objs = set(self._objs) - local_out_vars
all_remote_vois = model.comm.allgather(
(remote_dvs, remote_cons, remote_objs))
for rem_dvs, rem_cons, rem_objs in all_remote_vois:
con_set.update(rem_cons)
obj_set.update(rem_objs)
dv_set.update(rem_dvs)
# If we have remote VOIs, pick an owning rank for each and use that
# to bcast to others later
owning_ranks = model._owning_rank['output']
sizes = model._var_sizes['nonlinear']['output']
for i, vname in enumerate(model._var_allprocs_abs_names['output']):
owner = owning_ranks[vname]
if vname in dv_set:
dv_dict[vname] = (owner, sizes[owner, i])
if vname in con_set:
con_dict[vname] = (owner, sizes[owner, i])
if vname in obj_set:
obj_dict[vname] = (owner, sizes[owner, i])
self._remote_responses = self._remote_cons.copy()
self._remote_responses.update(self._remote_objs)
# Case recording setup
mydesvars = myobjectives = myconstraints = myresponses = set()
mysystem_outputs = set()
incl = self.recording_options['includes']
excl = self.recording_options['excludes']
rec_desvars = self.recording_options['record_desvars']
rec_objectives = self.recording_options['record_objectives']
rec_constraints = self.recording_options['record_constraints']
rec_responses = self.recording_options['record_responses']
all_desvars = {n for n in self._designvars
if check_path(n, incl, excl, True)}
all_objectives = {n for n in self._objs
if check_path(n, incl, excl, True)}
all_constraints = {n for n in self._cons
if check_path(n, incl, excl, True)}
if rec_desvars:
mydesvars = all_desvars
if rec_objectives:
myobjectives = all_objectives
if rec_constraints:
myconstraints = all_constraints
if rec_responses:
myresponses = {n for n in self._responses
if check_path(n, incl, excl, True)}
# get the includes that were requested for this Driver recording
if incl:
prob = self._problem
root = prob.model
# The my* variables are sets
# First gather all of the desired outputs
# The following might only be the local vars if MPI
mysystem_outputs = {n for n in root._outputs
if check_path(n, incl, excl)}
# If MPI, and on rank 0, need to gather up all the variables
# even those not local to rank 0
if MPI:
all_vars = root.comm.gather(mysystem_outputs, root=0)
if MPI.COMM_WORLD.rank == 0:
mysystem_outputs = all_vars[-1]
for d in all_vars[:-1]:
mysystem_outputs.update(d)
# de-duplicate mysystem_outputs
mysystem_outputs = mysystem_outputs.difference(all_desvars, all_objectives,
all_constraints)
if MPI: # filter based on who owns the variables
# TODO Eventually, we think we can get rid of this next check. But to be safe,
# we are leaving it in there.
if not model.is_active():
raise RuntimeError(
"RecordingManager.startup should never be called when "
"running in parallel on an inactive System")
rrank = self._problem.comm.rank # root ( aka model ) rank.
rowned = model._owning_rank['output']
mydesvars = [n for n in mydesvars if rrank == rowned[n]]
myresponses = [n for n in myresponses if rrank == rowned[n]]
myobjectives = [n for n in myobjectives if rrank == rowned[n]]
myconstraints = [n for n in myconstraints if rrank == rowned[n]]
mysystem_outputs = [n for n in mysystem_outputs if rrank == rowned[n]]
self._filtered_vars_to_record = {
'des': mydesvars,
'obj': myobjectives,
'con': myconstraints,
'res': myresponses,
'sys': mysystem_outputs,
}
self._rec_mgr.startup(self)
if self._rec_mgr._recorders:
from openmdao.devtools.problem_viewer.problem_viewer import _get_viewer_data
self._model_viewer_data = _get_viewer_data(problem)
if self.recording_options['record_metadata']:
self._rec_mgr.record_metadata(self)
# set up simultaneous deriv coloring
if self._simul_coloring_info and self.supports['simultaneous_derivatives']:
if problem._mode == 'fwd':
self._setup_simul_coloring(problem._mode)
else:
raise RuntimeError("simultaneous derivs are currently not supported in rev mode.")
def _setup_driver(self, problem):
"""
Prepare the driver for execution.
This is the final thing to run during setup.
Parameters
----------
problem : <Problem>
Pointer to the containing problem.
"""
self._problem = problem
model = problem.model
self._objs = objs = OrderedDict()
self._cons = cons = OrderedDict()
self._responses = model.get_responses(recurse=True)
for name, data in iteritems(self._responses):
if data['type'] == 'con':
cons[name] = data
else:
objs[name] = data
# Gather up the information for design vars.
self._designvars = model.get_design_vars(recurse=True)
con_set = set()
obj_set = set()
dv_set = set()
self._remote_dvs = dv_dict = {}
self._remote_cons = con_dict = {}
self._remote_objs = obj_dict = {}
# Now determine if later we'll need to allgather cons, objs, or desvars.
if model.comm.size > 1 and model._subsystems_allprocs:
local_out_vars = set(model._outputs._views)
remote_dvs = set(self._designvars) - local_out_vars
remote_cons = set(self._cons) - local_out_vars
remote_objs = set(self._objs) - local_out_vars
all_remote_vois = model.comm.allgather(
(remote_dvs, remote_cons, remote_objs))
for rem_dvs, rem_cons, rem_objs in all_remote_vois:
con_set.update(rem_cons)
obj_set.update(rem_objs)
dv_set.update(rem_dvs)
# If we have remote VOIs, pick an owning rank for each and use that
# to bcast to others later
owning_ranks = model._owning_rank['output']
sizes = model._var_sizes['nonlinear']['output']
for i, vname in enumerate(model._var_allprocs_abs_names['output']):
owner = owning_ranks[vname]
if vname in dv_set:
dv_dict[vname] = (owner, sizes[owner, i])
if vname in con_set:
con_dict[vname] = (owner, sizes[owner, i])
if vname in obj_set:
obj_dict[vname] = (owner, sizes[owner, i])
self._remote_responses = self._remote_cons.copy()
self._remote_responses.update(self._remote_objs)
self._rec_mgr.startup(self)
if (self._rec_mgr._recorders):
from openmdao.devtools.problem_viewer.problem_viewer import _get_viewer_data
self._model_viewer_data = _get_viewer_data(problem)
self._rec_mgr.record_metadata(self)
def _setup_recording(self):
"""
Set up case recording.
"""
problem = self._problem
model = problem.model
mydesvars = myobjectives = myconstraints = myresponses = set()
myinputs = set()
mysystem_outputs = set()
incl = self.recording_options['includes']
excl = self.recording_options['excludes']
rec_desvars = self.recording_options['record_desvars']
rec_objectives = self.recording_options['record_objectives']
rec_constraints = self.recording_options['record_constraints']
rec_responses = self.recording_options['record_responses']
rec_inputs = self.recording_options['record_inputs']
all_desvars = {
n
for n in self._designvars if check_path(n, incl, excl, True)
}
all_objectives = {
n
for n in self._objs if check_path(n, incl, excl, True)
}
all_constraints = {
n
for n in self._cons if check_path(n, incl, excl, True)
}
if rec_desvars:
mydesvars = all_desvars
if rec_objectives:
myobjectives = all_objectives
if rec_constraints:
myconstraints = all_constraints
if rec_responses:
myresponses = {
n
for n in self._responses if check_path(n, incl, excl, True)
}
# get the includes that were requested for this Driver recording
if incl:
# The my* variables are sets
# First gather all of the desired outputs
# The following might only be the local vars if MPI
mysystem_outputs = {
n
for n in model._outputs if check_path(n, incl, excl)
}
# If MPI, and on rank 0, need to gather up all the variables
# even those not local to rank 0
if MPI:
all_vars = model.comm.gather(mysystem_outputs, root=0)
if MPI.COMM_WORLD.rank == 0:
mysystem_outputs = all_vars[-1]
for d in all_vars[:-1]:
mysystem_outputs.update(d)
# de-duplicate mysystem_outputs
mysystem_outputs = mysystem_outputs.difference(
all_desvars, all_objectives, all_constraints)
if rec_inputs:
prob = self._problem
root = prob.model
myinputs = {n for n in root._inputs if check_path(n, incl, excl)}
if MPI:
all_vars = root.comm.gather(myinputs, root=0)
if MPI.COMM_WORLD.rank == 0:
myinputs = all_vars[-1]
for d in all_vars[:-1]:
myinputs.update(d)
if MPI: # filter based on who owns the variables
# TODO Eventually, we think we can get rid of this next check. But to be safe,
# we are leaving it in there.
if not model.is_active():
raise RuntimeError(
"RecordingManager.startup should never be called when "
"running in parallel on an inactive System")
rrank = problem.comm.rank
rowned = model._owning_rank
mydesvars = [n for n in mydesvars if rrank == rowned[n]]
myresponses = [n for n in myresponses if rrank == rowned[n]]
myobjectives = [n for n in myobjectives if rrank == rowned[n]]
myconstraints = [n for n in myconstraints if rrank == rowned[n]]
mysystem_outputs = [
n for n in mysystem_outputs if rrank == rowned[n]
]
myinputs = [n for n in myinputs if rrank == rowned[n]]
self._filtered_vars_to_record = {
'des': mydesvars,
'obj': myobjectives,
'con': myconstraints,
'res': myresponses,
'sys': mysystem_outputs,
'in': myinputs
}
self._rec_mgr.startup(self)
if self._rec_mgr._recorders:
from openmdao.devtools.problem_viewer.problem_viewer import _get_viewer_data
self._model_viewer_data = _get_viewer_data(problem)
if self.recording_options['record_metadata']:
self._rec_mgr.record_metadata(self)
def write_xdsm(problem,
filename,
model_path=None,
recurse=True,
include_external_outputs=True,
out_format='tex',
include_solver=False,
subs=_CHAR_SUBS,
show_browser=True,
add_process_conns=True,
**kwargs):
"""
Writes XDSM diagram of an optimization problem.
With the 'tex' or 'pdf' output format it uses the pyXDSM package, with 'json' or 'HTML'
output format it uses XDSMjs.
If a component (or group) name is not unique in the diagram, the systems absolute path is
used as a label. If the component (or group) name is unique, the relative name of the
system is the label.
In the diagram the connections are marked with the source name.
Writer specific settings and default:
pyXDSM
~~~~~~
* The appearance of the boxes can be controlled with "box_stacking" and "box_width" arguments.
The box stacking can be:
* "horizontal" - All variables in one line
* "vertical" - All variables in one column
* "cut_chars" - The text in the box will be one line with the maximum number of characters
limited by "box_width".
* "max_chars" - The "box_width" argument is used to determine
the maximum allowed width of boxes (in characters).
* "empty" - There are no variable names in the data block. Good for large diagrams.
A default value is taken, if not specified.
* By default the part of variable names following underscores (_)
are not converted to subscripts.
To write in subscripts wrap that part of the name into a round bracket.
Example: To write :math:`x_12` the variable name should be "x(12)"
* "box_lines" can be used to limit the number of lines, if the box stacking is vertical
* "numbered_comps": bool, If True, components are numbered. Defaults to True.
* "number_alignment": str, Horizontal or vertical. Defaults to horizontal. If "numbered_comps"
is True, it positions the number either above or in front of the component label.
XDSMjs
~~~~~~
* If "embed_data" is true, a single standalone HTML file will be generated, which includes
the data of the XDSM diagram.
* variable names with exactly one underscore have a subscript.
Example: "x_12" will be :math:`x_12`
* If "embeddable" is True, gives a single HTML file that doesn't have the <html>, <DOCTYPE>,
<body> and <head> tags. If False, gives a single, standalone HTML file for viewing.
Parameters
----------
problem : Problem
Problem
filename : str
Name of the output files (do not provide file extension)
model_path : str or None
Path to the subsystem to be transcribed to XDSM. If None, use the model root.
recurse : bool
If False, treat the top level of each name as the source/target component.
include_external_outputs : bool
If True, show externally connected outputs when transcribing a subsystem.
Defaults to True.
out_format : str, optional
Output format, one of "tex" (pyXDSM) or "json"/"html" (XDSMjs)
Defaults to "tex".
include_solver : bool
Include or not the problem model's nonlinear solver in the XDSM.
subs : dict(str, tuple), tuple(str, str), optional
Characters to be replaced. Dictionary with writer names and character pairs or just the
character pairs.
show_browser : bool, optional
If True, pop up a browser to view the generated html file.
Defaults to True.
add_process_conns: bool
Add process connections (thin black lines)
Defaults to True
kwargs : dict
Keyword arguments
Returns
-------
XDSM
"""
viewer_data = _get_viewer_data(problem)
driver = problem.driver
if model_path is None:
_model = problem.model
else:
_model = problem.model._get_subsystem(model_path)
# Name is None if the driver is not specified
driver_name = _get_cls_name(driver) if driver else None
design_vars = _model.get_design_vars()
responses = _model.get_responses()
filename = filename.replace('\\', '/') # Needed for LaTeX
out_formats = _OUT_FORMATS
try:
writer = out_formats[out_format]
except KeyError:
msg = 'Invalid output format "{}", choose from: {}'
raise ValueError(msg.format(out_format, out_formats.keys()))
writer_name = writer.lower() # making it case insensitive
if isinstance(subs, dict):
subs = subs[
writer_name] # Getting the character substitutes of the chosen writer
return _write_xdsm(filename,
viewer_data=viewer_data,
optimizer=driver_name,
include_solver=include_solver,
model_path=model_path,
design_vars=design_vars,
responses=responses,
writer=writer,
recurse=recurse,
subs=subs,
include_external_outputs=include_external_outputs,
show_browser=show_browser,
add_process_conns=add_process_conns,
**kwargs)
