def _setup_partials(self):
"""
Check that all partials are declared.
"""
if not self._manual_decl_partials:
meta = self._var_rel2meta
decl_partials = super().declare_partials
for i, (outs, tup) in enumerate(self._exprs_info):
vs, funcs = tup
ins = sorted(set(vs).difference(outs))
for out in sorted(outs):
for inp in ins:
if self.options['has_diag_partials']:
ival = meta[inp]['value']
iarray = isinstance(ival,
ndarray) and ival.size > 1
oval = meta[out]['value']
if iarray and isinstance(
oval, ndarray) and oval.size > 1:
if oval.size != ival.size:
raise RuntimeError(
"%s: has_diag_partials is True but partial(%s, %s) "
"is not square (shape=(%d, %d))." %
(self.msginfo, out, inp, oval.size,
ival.size))
# partial will be declared as diagonal
inds = np.arange(oval.size, dtype=int)
else:
inds = None
decl_partials(of=out,
wrt=inp,
rows=inds,
cols=inds)
else:
decl_partials(of=out, wrt=inp)
super()._setup_partials()
if self._manual_decl_partials:
undeclared = []
for i, (outs, tup) in enumerate(self._exprs_info):
vs, funcs = tup
ins = sorted(set(vs).difference(outs))
for out in sorted(outs):
out = '.'.join(
(self.pathname, out)) if self.pathname else out
for inp in ins:
inp = '.'.join(
(self.pathname, inp)) if self.pathname else inp
if (out, inp) not in self._subjacs_info:
undeclared.append((out, inp))
if undeclared:
idx = len(self.pathname) + 1 if self.pathname else 0
undeclared = ', '.join([
' wrt '.join((f"'{of[idx:]}'", f"'{wrt[idx:]}'"))
for of, wrt in undeclared
])
simple_warning(
f"{self.msginfo}: The following partial derivatives have not been "
f"declared so they are assumed to be zero: [{undeclared}]."
)
def _print_violations(outputs, lower, upper):
"""
Print out which variables exceed their bounds.
Parameters
----------
outputs : <Vector>
Vector containing the outputs.
lower : <Vector>
Vector containing the lower bounds.
upper : <Vector>
Vector containing the upper bounds.
"""
start = end = 0
for name, val in outputs._abs_item_iter():
end += val.size
if upper is not None and any(val > upper[start:end]):
msg = (
f"'{name}' exceeds upper bounds\n Val: {val}\n Upper: {upper[start:end]}\n"
)
simple_warning(msg)
if lower is not None and any(val < lower[start:end]):
msg = (
f"'{name}' exceeds lower bounds\n Val: {val}\n Lower: {lower[start:end]}\n"
)
simple_warning(msg)
start = end
def initialize(self):
"""
Initialize the component.
"""
if not make_interp_spline:
msg = "'MetaModelStructuredComp' requires scipy>=0.19, but the currently" \
" installed version is %s." % scipy_version
simple_warning(msg)
self.options.declare(
'extrapolate',
types=bool,
default=False,
desc='Sets whether extrapolation should be performed '
'when an input is out of bounds.')
self.options.declare(
'training_data_gradients',
types=bool,
default=False,
desc='Sets whether gradients with respect to output '
'training data should be computed.')
self.options.declare('vec_size',
types=int,
default=1,
desc='Number of points to evaluate at once.')
self.options.declare('method',
values=('cubic', 'slinear', 'quintic'),
default="cubic",
desc='Spline interpolation order.')
def dynamic_simul_coloring(driver, run_model=True, do_sparsity=False, show_jac=False):
"""
Compute simultaneous deriv coloring during runtime.
Parameters
----------
driver : <Driver>
The driver performing the optimization.
run_model : bool
If True, call run_model before computing coloring.
do_sparsity : bool
If True, setup the total jacobian sparsity (needed by pyOptSparseDriver).
show_jac : bool
If True, display a visualization of the colored jacobian.
"""
problem = driver._problem
if not problem.model._use_derivatives:
simple_warning("Derivatives have been turned off. Skipping dynamic simul coloring.")
return
driver._total_jac = None
# save the coloring.json file for later inspection
with open("coloring.json", "w") as f:
coloring = get_simul_meta(problem,
repeats=driver.options['dynamic_derivs_repeats'],
tol=1.e-15, include_sparsity=do_sparsity,
setup=False, run_model=run_model, show_jac=show_jac, stream=f)
driver.set_simul_deriv_color(coloring)
driver._setup_simul_coloring()
if do_sparsity:
driver._setup_tot_jac_sparsity()
simul_coloring_summary(coloring, stream=sys.stdout)
def get_code(reference, hide_doc_string=False):
"""
Return the source code of the given reference path to a function.
Parameters
----------
reference : str
Dot path of desired function.
hide_doc_string : bool
Option to hide the docstring.
Returns
-------
IPython.display.Code
Source code of the given class or function.
"""
obj = inspect.getsource(_get_object_from_reference(reference))
if hide_doc_string:
obj = obj.split('"""')
del obj[1]
obj = ''.join(obj)
if ipy:
return Code(obj, language='python')
else:
simple_warning(
"IPython is not installed. Run `pip install openmdao[notebooks]` or "
"`pip install openmdao[docs]` to upgrade.")
def record_system_options(problem):
"""
Record the system options for all systems in the model.
Parameters
----------
problem : Problem
The problem for which all its systems' options are to be recorded.
"""
# get all recorders in the problem
recorders = set(_get_all_recorders(problem))
if recorders:
if problem._system_options_recorded:
simple_warning(
"The model is being run again, if the options or scaling of any "
"components has changed then only their new values will be recorded."
)
else:
problem._system_options_recorded = True
for recorder in recorders:
for sub in problem.model.system_iter(recurse=True,
include_self=True):
if problem._run_counter >= 1:
recorder.record_metadata_system(sub, problem._run_counter)
else:
recorder.record_metadata_system(sub)
def dynamic_simul_coloring(driver, run_model=True, do_sparsity=False, show_jac=False):
"""
Compute simultaneous deriv coloring during runtime.
Parameters
----------
driver : <Driver>
The driver performing the optimization.
run_model : bool
If True, call run_model before computing coloring.
do_sparsity : bool
If True, setup the total jacobian sparsity (needed by pyOptSparseDriver).
show_jac : bool
If True, display a visualization of the colored jacobian.
"""
problem = driver._problem
if not problem.model._use_derivatives:
simple_warning("Derivatives have been turned off. Skipping dynamic simul coloring.")
return
driver._total_jac = None
# save the coloring.json file for later inspection
with open("coloring.json", "w") as f:
coloring = get_simul_meta(problem,
repeats=driver.options['dynamic_derivs_repeats'],
tol=1.e-15, include_sparsity=do_sparsity,
setup=False, run_model=run_model, show_jac=show_jac, stream=f)
driver.set_simul_deriv_color(coloring)
driver._setup_simul_coloring()
if do_sparsity:
driver._setup_tot_jac_sparsity()
simul_coloring_summary(coloring, stream=sys.stdout)
def _setup_simul_coloring(self):
"""
Set up metadata for coloring of total derivative solution.
If set_coloring was called with a filename, load the coloring file.
"""
# command line simul_coloring uses this env var to turn pre-existing coloring off
if not coloring_mod._use_total_sparsity:
return
problem = self._problem()
if not problem.model._use_derivatives:
simple_warning(
"Derivatives are turned off. Skipping simul deriv coloring.")
return
total_coloring = self._get_static_coloring()
if total_coloring._rev and problem._orig_mode not in ('rev', 'auto'):
revcol = total_coloring._rev[0][0]
if revcol:
raise RuntimeError(
"Simultaneous coloring does reverse solves but mode has "
"been set to '%s'" % problem._orig_mode)
if total_coloring._fwd and problem._orig_mode not in ('fwd', 'auto'):
fwdcol = total_coloring._fwd[0][0]
if fwdcol:
raise RuntimeError(
"Simultaneous coloring does forward solves but mode has "
"been set to '%s'" % problem._orig_mode)
def dynamic_sparsity(driver):
"""
Compute deriv sparsity during runtime.
Parameters
----------
driver : <Driver>
The driver performing the optimization.
"""
problem = driver._problem
if not problem.model._use_derivatives:
simple_warning(
"Derivatives have been turned off. Skipping dynamic sparsity computation."
)
return
driver._total_jac = None
repeats = driver.options['dynamic_derivs_repeats']
# save the sparsity.json file for later inspection
with open("sparsity.json", "w") as f:
sparsity = get_sparsity(problem,
mode=problem._mode,
repeats=repeats,
stream=f)
driver.set_total_jac_sparsity(sparsity)
driver._setup_tot_jac_sparsity()
def add_approximation(self, abs_key, system, kwargs, vector=None):
"""
Use this approximation scheme to approximate the derivative d(of)/d(wrt).
Parameters
----------
abs_key : tuple(str,str)
Absolute name pairing of (of, wrt) for the derivative.
system : System
Containing System.
vector : ndarray or None
Direction for difference when using directional derivatives.
kwargs : dict
Additional keyword arguments, to be interpreted by sub-classes.
"""
options = self.DEFAULT_OPTIONS.copy()
options.update(kwargs)
options['vector'] = vector
wrt = abs_key[1]
if wrt in self._wrt_meta:
simple_warning(
f"{system.msginfo}: overriding previous approximation defined for "
f"'{wrt}.")
self._wrt_meta[wrt] = options
self._reset() # force later regen of approx_groups
def show_options_table(reference, recording_options=False):
"""
Return the options table of the given reference path.
Parameters
----------
reference : str
Dot path of desired class or function.
recording_options : bool
If True, display recording options instead of options.
Returns
-------
IPython.display
Options table of the given class or function.
"""
obj = _get_object_from_reference(reference)()
if ipy:
if not hasattr(obj, "options"):
return display(HTML(obj.to_table(fmt='html')))
elif not recording_options:
return display(HTML(obj.options.to_table(fmt='html')))
else:
return display(HTML(obj.recording_options.to_table(fmt='html')))
else:
simple_warning(
"IPython is not installed. Run `pip install openmdao[notebooks]` or "
"`pip install openmdao[docs]` to upgrade.")
def get_case(self, case_id, cache=False):
"""
Get a case from the database.
Parameters
----------
case_id : str or int
The string-identifier of the case to be retrieved or the index of the case.
cache : bool
If True, case will be cached for faster access by key.
Returns
-------
Case
The specified case from the table.
"""
# check to see if we've already cached this case
if isinstance(case_id, int):
case_id = self._get_iteration_coordinate(case_id)
# if we've already cached this case, return the cached instance
if case_id in self._cases:
return self._cases[case_id]
# we don't have it, so fetch it
with sqlite3.connect(self._filename) as con:
con.row_factory = sqlite3.Row
cur = con.cursor()
cur.execute("SELECT * FROM %s WHERE %s='%s'" %
(self._table_name, self._index_name, case_id))
row = cur.fetchone()
con.close()
# if found, extract the data and optionally cache the Case
if row is not None:
if self._format_version >= 5:
source = self._get_row_source(row['id'])
# check for situations where parsing the iter coord doesn't work correctly
iter_source = self._get_source(row[self._index_name])
if iter_source != source:
simple_warning('Mismatched source for %d: %s = %s vs %s' %
(row['id'], row[self._index_name],
iter_source, source))
else:
source = self._get_source(row[self._index_name])
case = Case(source, row, self._prom2abs, self._abs2prom,
self._abs2meta, self._var_info, self._format_version)
# cache it if requested
if cache:
self._cases[case_id] = case
return case
else:
return None
def _setup_solvers(self, system, depth):
"""
Assign system instance, set depth, and optionally perform setup.
Parameters
----------
system : System
pointer to the owning system.
depth : int
depth of the current system (already incremented).
"""
super(LinesearchSolver, self)._setup_solvers(system, depth)
if system._has_bounds:
abs2meta = system._var_abs2meta
start = end = 0
for abs_name, val in system._outputs._abs_val_iter():
end += val.size
meta = abs2meta[abs_name]
var_lower = meta['lower']
var_upper = meta['upper']
if var_lower is None and var_upper is None:
start = end
continue
ref0 = meta['ref0']
ref = meta['ref']
if not np.isscalar(ref0):
ref0 = ref0.ravel()
if not np.isscalar(ref):
ref = ref.ravel()
if var_lower is not None:
if self._lower_bounds is None:
self._lower_bounds = np.full(len(system._outputs),
-np.inf)
if not np.isscalar(var_lower):
var_lower = var_lower.ravel()
self._lower_bounds[start:end] = (var_lower - ref0) / (ref -
ref0)
if var_upper is not None:
if self._upper_bounds is None:
self._upper_bounds = np.full(len(system._outputs),
np.inf)
if not np.isscalar(var_upper):
var_upper = var_upper.ravel()
self._upper_bounds[start:end] = (var_upper - ref0) / (ref -
ref0)
start = end
else:
simple_warning(
f"{self.msginfo}: linesearch is active but no bounds have been set."
)
self._lower_bounds = self._upper_bounds = None
def record_metadata_system(self, recording_requester, run_counter=None):
"""
Record system metadata.
Parameters
----------
recording_requester : System
The System that would like to record its metadata.
run_counter : int or None
The number of times run_driver or run_model has been called.
"""
if self.connection:
scaling_vecs, user_options = self._get_metadata_system(
recording_requester)
if scaling_vecs is None:
return
scaling_factors = pickle.dumps(scaling_vecs, self._pickle_version)
# try to pickle the metadata, report if it failed
try:
pickled_metadata = pickle.dumps(user_options,
self._pickle_version)
except Exception:
try:
for key, values in user_options._dict.items():
pickle.dumps(values, self._pickle_version)
except Exception:
pickled_metadata = pickle.dumps(OptionsDictionary(),
self._pickle_version)
simple_warning(
"Trying to record option '%s' which cannot be pickled on system "
"%s. Set 'recordable' to False. Skipping recording options for "
"this system." % (key, recording_requester.msginfo))
path = recording_requester.pathname
if not path:
path = 'root'
scaling_factors = sqlite3.Binary(scaling_factors)
pickled_metadata = sqlite3.Binary(pickled_metadata)
# Need to use OR IGNORE in here because if the user does run_driver more than once
# the current OpenMDAO code will call this function each time and there will be
# SQL errors for "UNIQUE constraint failed: system_metadata.id"
# Future versions of OpenMDAO will handle this better.
if run_counter is None:
name = path
else:
name = "{}_{}".format(path, str(run_counter))
with self.connection as c:
c.execute(
"INSERT OR IGNORE INTO system_metadata"
"(id, scaling_factors, component_metadata) "
"VALUES(?,?,?)", (name, scaling_factors, pickled_metadata))
def _get_used_before_calc_subs(group, input_srcs):
"""
Return Systems that are executed out of dataflow order.
Parameters
----------
group : <Group>
The Group where we're checking subsystem order.
input_srcs : {}
dict containing variable abs names for sources of the inputs.
This describes all variable connections, either explicit or implicit,
in the entire model.
Returns
-------
dict
A dict mapping names of target Systems to a set of names of their
source Systems that execute after them.
"""
sub2i = {}
parallel_solver = {}
for i, sub in enumerate(group._subsystems_allprocs):
if hasattr(sub,
'_mpi_proc_allocator') and sub._mpi_proc_allocator.parallel:
parallel_solver[sub.name] = sub.nonlinear_solver.SOLVER
sub2i[sub.name] = i
glen = len(group.pathname.split('.')) if group.pathname else 0
ubcs = defaultdict(set)
for tgt_abs, src_abs in input_srcs.items():
if src_abs is not None:
iparts = tgt_abs.split('.')
oparts = src_abs.split('.')
src_sys = oparts[glen]
tgt_sys = iparts[glen]
hierarchy_check = True if oparts[glen + 1] == iparts[glen +
1] else False
if (src_sys in parallel_solver and tgt_sys in parallel_solver
and (parallel_solver[src_sys]
not in ["NL: NLBJ", "NL: Newton", "BROYDEN"])
and src_sys == tgt_sys and not hierarchy_check):
simple_warning(
"Need to attach NonlinearBlockJac, NewtonSolver, or BroydenSolver "
"to '%s' when connecting components inside parallel "
"groups" % (src_sys))
ubcs[tgt_abs.rsplit('.', 1)[0]].add(src_abs.rsplit('.', 1)[0])
if (src_sys in sub2i and tgt_sys in sub2i
and (sub2i[src_sys] > sub2i[tgt_sys])):
ubcs[tgt_sys].add(src_sys)
return ubcs
def record_metadata_system(self, system, run_number=None):
"""
Record system metadata.
Parameters
----------
system : System
The System for which to record metadata.
run_number : int or None
Number indicating which run the metadata is associated with.
None for the first run, 1 for the second, etc.
"""
if self._record_metadata and self.metadata_connection:
scaling_vecs, user_options = self._get_metadata_system(system)
if scaling_vecs is None:
return
scaling_factors = pickle.dumps(scaling_vecs, self._pickle_version)
# try to pickle the metadata, report if it failed
try:
pickled_metadata = pickle.dumps(user_options,
self._pickle_version)
except Exception:
try:
for key, values in user_options._dict.items():
pickle.dumps(values, self._pickle_version)
except Exception:
pickled_metadata = pickle.dumps(OptionsDictionary(),
self._pickle_version)
simple_warning(
"Trying to record option '%s' which cannot be pickled on system "
"%s. Set 'recordable' to False. Skipping recording options for "
"this system." % (key, system.msginfo))
path = system.pathname
if not path:
path = 'root'
scaling_factors = sqlite3.Binary(zlib.compress(scaling_factors))
pickled_metadata = sqlite3.Binary(zlib.compress(pickled_metadata))
if run_number is None:
name = path
else:
name = META_KEY_SEP.join([path, str(run_number)])
with self.metadata_connection as m:
m.execute(
"INSERT INTO system_metadata"
"(id, scaling_factors, component_metadata) "
"VALUES(?,?,?)", (name, scaling_factors, pickled_metadata))
def compute_approximations(self, system, jac, total=False):
"""
Execute the system to compute the approximate sub-Jacobians.
Parameters
----------
system : System
System on which the execution is run.
jac : dict-like
Approximations are stored in the given dict-like object.
total : bool
If True total derivatives are being approximated, else partials.
"""
if not self._wrt_meta:
return
if system.under_complex_step:
# If we are nested under another complex step, then warn and swap to FD.
if not self._fd:
from openmdao.approximation_schemes.finite_difference import FiniteDifference
msg = "Nested complex step detected. Finite difference will be used for '%s'."
simple_warning(msg % system.pathname)
fd = self._fd = FiniteDifference()
empty = {}
for wrt in self._wrt_meta:
fd.add_approximation(wrt, system, empty)
self._fd.compute_approximations(system, jac, total=total)
return
saved_inputs = system._inputs._get_data().copy()
system._inputs._data.imag[:] = 0.0
saved_outputs = system._outputs.asarray(copy=True)
system._outputs._data.imag[:] = 0.0
saved_resids = system._residuals.asarray(copy=True)
system._residuals._data.imag[:] = 0.0
# Turn on complex step.
system._set_complex_step_mode(True)
try:
self._compute_approximations(system, jac, total, under_cs=True)
finally:
# Turn off complex step.
system._set_complex_step_mode(False)
system._inputs.set_val(saved_inputs)
system._outputs.set_val(saved_outputs)
system._residuals.set_val(saved_resids)
def notebook_mode():
"""
Check if the environment is interactive and if tabulate is installed.
Returns
-------
bool
True if the environment is an interactive notebook.
"""
if ipy and tabulate is None:
simple_warning("Tabulate is not installed. Run `pip install openmdao[notebooks]` to "
"install required dependencies. Using ASCII for outputs.")
return ipy
def record_metadata_system(self, recording_requester):
"""
Record system metadata.
Parameters
----------
recording_requester : System
The System that would like to record its metadata.
"""
if self.connection:
scaling_vecs, user_options = self._get_metadata_system(
recording_requester)
if scaling_vecs is None:
return
scaling_factors = pickle.dumps(scaling_vecs, self._pickle_version)
# try to pickle the metadata, report if it failed
try:
pickled_metadata = pickle.dumps(user_options,
self._pickle_version)
except Exception:
try:
for key, values in user_options._dict.items():
pickle.dumps(values, self._pickle_version)
except Exception:
pickled_metadata = pickle.dumps(OptionsDictionary(),
self._pickle_version)
simple_warning(
"Trying to record option '%s' which cannot be pickled on system "
"%s. Set 'recordable' to False. Skipping recording options for "
"this system." % (key, recording_requester.msginfo))
path = recording_requester.pathname
if not path:
path = 'root'
scaling_factors = sqlite3.Binary(scaling_factors)
pickled_metadata = sqlite3.Binary(pickled_metadata)
with self.connection as c:
# Because we can have a recorder attached to multiple Systems,
# and because we are now recording System metadata recursively,
# we can store System metadata multiple times. Need to ignore when that happens
# so we don't get database errors. So use OR IGNORE
c.execute(
"INSERT OR IGNORE INTO system_metadata"
"(id, scaling_factors, component_metadata) "
"VALUES(?,?,?)", (path, scaling_factors, pickled_metadata))
def record_metadata_system(self, recording_requester):
"""
Record system metadata.
Parameters
----------
recording_requester : System
The System that would like to record its metadata.
"""
if self.connection:
scaling_vecs, user_options = self._get_metadata_system(
recording_requester)
if scaling_vecs is None:
return
scaling_factors = pickle.dumps(scaling_vecs, self._pickle_version)
# try to pickle the metadata, report if it failed
try:
pickled_metadata = pickle.dumps(user_options,
self._pickle_version)
except Exception:
pickled_metadata = pickle.dumps(OptionsDictionary(),
self._pickle_version)
simple_warning(
"Trying to record options which cannot be pickled "
"on system with name: %s. Use the 'options_excludes' "
"recording option on system objects to avoid attempting "
"to record options which cannot be pickled. Skipping "
"recording options for this system." %
recording_requester.name, RuntimeWarning)
path = recording_requester.pathname
if not path:
path = 'root'
scaling_factors = sqlite3.Binary(scaling_factors)
pickled_metadata = sqlite3.Binary(pickled_metadata)
with self.connection as c:
# Because we can have a recorder attached to multiple Systems,
# and because we are now recording System metadata recursively,
# we can store System metadata multiple times. Need to ignore when that happens
# so we don't get database errors. So use OR IGNORE
c.execute(
"INSERT OR IGNORE INTO system_metadata"
"(id, scaling_factors, component_metadata) "
"VALUES(?,?,?)", (path, scaling_factors, pickled_metadata))
def _setup_simul_coloring(self):
"""
Set up metadata for simultaneous derivative solution.
"""
# command line simul_coloring uses this env var to turn pre-existing coloring off
if not coloring_mod._use_sparsity:
return
problem = self._problem
if not problem.model._use_derivatives:
simple_warning(
"Derivatives are turned off. Skipping simul deriv coloring.")
return
if isinstance(self._simul_coloring_info, string_types):
with open(self._simul_coloring_info, 'r') as f:
self._simul_coloring_info = coloring_mod._json2coloring(
json.load(f))
if 'rev' in self._simul_coloring_info and problem._orig_mode not in (
'rev', 'auto'):
revcol = self._simul_coloring_info['rev'][0][0]
if revcol:
raise RuntimeError(
"Simultaneous coloring does reverse solves but mode has "
"been set to '%s'" % problem._orig_mode)
if 'fwd' in self._simul_coloring_info and problem._orig_mode not in (
'fwd', 'auto'):
fwdcol = self._simul_coloring_info['fwd'][0][0]
if fwdcol:
raise RuntimeError(
"Simultaneous coloring does forward solves but mode has "
"been set to '%s'" % problem._orig_mode)
# simul_coloring_info can contain data for either fwd, rev, or both, along with optional
# sparsity patterns
if 'sparsity' in self._simul_coloring_info:
sparsity = self._simul_coloring_info['sparsity']
del self._simul_coloring_info['sparsity']
else:
sparsity = None
if sparsity is not None and self._total_jac_sparsity is not None:
raise RuntimeError(
"Total jac sparsity was set in both _simul_coloring_info"
" and _total_jac_sparsity.")
self._total_jac_sparsity = sparsity
def show_options_table(reference, recording_options=False):
"""
Return the options table of the given reference path.
Parameters
----------
reference : str or object
Dot path of desired class or function or an instance.
recording_options : bool
If True, display recording options instead of options.
Returns
-------
IPython.display
Options table of the given class or function.
"""
if isinstance(reference, str):
obj = _get_object_from_reference(reference)()
else:
obj = reference
if ipy:
if not hasattr(obj, "options"):
html = obj.to_table(fmt='html')
elif not recording_options:
html = obj.options.to_table(fmt='html')
else:
html = obj.recording_options.to_table(fmt='html')
# Jupyter notebook imposes right justification, so we have to enforce what we want:
# - Center table headers
# - Left justify table columns
# - Limit column width so there is adequate width left for the deprecation message
style = '<{tag} style="text-align:{align}; max-width:{width}; overflow-wrap:break-word;">'
cols = html.count('<th>') # there could be 5 or 6 columns
width = '300px' if cols > 5 else '600px' # limit width depending on number of columns
html = html.replace('<th>', style.format(tag='th', align='center', width=width))
html = html.replace('<td>', style.format(tag='td', align='left', width=width))
return display(HTML(html))
else:
simple_warning("IPython is not installed. Run `pip install openmdao[notebooks]` or "
"`pip install openmdao[docs]` to upgrade.")
def record_metadata_system(self, recording_requester):
"""
Record system metadata.
Parameters
----------
recording_requester : System
The System that would like to record its metadata.
"""
if self.connection:
scaling_vecs, user_options = self._get_metadata_system(recording_requester)
if scaling_vecs is None:
return
scaling_factors = pickle.dumps(scaling_vecs, self._pickle_version)
# try to pickle the metadata, report if it failed
try:
pickled_metadata = pickle.dumps(user_options, self._pickle_version)
except Exception:
pickled_metadata = pickle.dumps(OptionsDictionary(), self._pickle_version)
simple_warning("Trying to record options which cannot be pickled "
"on system with name: %s. Use the 'options_excludes' "
"recording option on system objects to avoid attempting "
"to record options which cannot be pickled. Skipping "
"recording options for this system." % recording_requester.name,
RuntimeWarning)
path = recording_requester.pathname
if not path:
path = 'root'
scaling_factors = sqlite3.Binary(scaling_factors)
pickled_metadata = sqlite3.Binary(pickled_metadata)
with self.connection as c:
# Because we can have a recorder attached to multiple Systems,
# and because we are now recording System metadata recursively,
# we can store System metadata multiple times. Need to ignore when that happens
# so we don't get database errors. So use OR IGNORE
c.execute("INSERT OR IGNORE INTO system_metadata"
"(id, scaling_factors, component_metadata) "
"VALUES(?,?,?)", (path, scaling_factors, pickled_metadata))
def compute_approximations(self, system, jac, total=False):
"""
Execute the system to compute the approximate sub-Jacobians.
Parameters
----------
system : System
System on which the execution is run.
jac : dict-like
Approximations are stored in the given dict-like object.
total : bool
If True total derivatives are being approximated, else partials.
"""
if not self._exec_dict:
return
if system.under_complex_step:
# If we are nested under another complex step, then warn and swap to FD.
if not self._fd:
from openmdao.approximation_schemes.finite_difference import FiniteDifference
msg = "Nested complex step detected. Finite difference will be used for '%s'."
simple_warning(msg % system.pathname)
fd = self._fd = FiniteDifference()
empty = {}
for lst in itervalues(self._exec_dict):
for apprx in lst:
fd.add_approximation(apprx[0], empty)
self._fd.compute_approximations(system, jac, total=total)
return
# Turn on complex step.
system._set_complex_step_mode(True)
self._compute_approximations(system, jac, total, under_cs=True)
# Turn off complex step.
system._set_complex_step_mode(False)
def notebook_mode():
"""
Check if the environment is interactive and if tabulate is installed.
Returns
-------
bool
True if the environment is an interactive notebook.
"""
ipy = False
try:
from IPython import get_ipython
ipy = get_ipython() is not None
except ImportError:
pass
if ipy and tabulate is None:
simple_warning(
"Tabulate is not installed run `pip install openmdao[notebooks]` to "
"install required dependencies. Using ASCII for outputs.")
return ipy
def _setup_simul_coloring(self):
"""
Set up metadata for simultaneous derivative solution.
"""
# command line simul_coloring uses this env var to turn pre-existing coloring off
if not coloring_mod._use_sparsity:
return
problem = self._problem
if not problem.model._use_derivatives:
simple_warning("Derivatives are turned off. Skipping simul deriv coloring.")
return
if isinstance(self._simul_coloring_info, string_types):
with open(self._simul_coloring_info, 'r') as f:
self._simul_coloring_info = coloring_mod._json2coloring(json.load(f))
if 'rev' in self._simul_coloring_info and problem._orig_mode not in ('rev', 'auto'):
revcol = self._simul_coloring_info['rev'][0][0]
if revcol:
raise RuntimeError("Simultaneous coloring does reverse solves but mode has "
"been set to '%s'" % problem._orig_mode)
if 'fwd' in self._simul_coloring_info and problem._orig_mode not in ('fwd', 'auto'):
fwdcol = self._simul_coloring_info['fwd'][0][0]
if fwdcol:
raise RuntimeError("Simultaneous coloring does forward solves but mode has "
"been set to '%s'" % problem._orig_mode)
# simul_coloring_info can contain data for either fwd, rev, or both, along with optional
# sparsity patterns
if 'sparsity' in self._simul_coloring_info:
sparsity = self._simul_coloring_info['sparsity']
del self._simul_coloring_info['sparsity']
else:
sparsity = None
if sparsity is not None and self._total_jac_sparsity is not None:
raise RuntimeError("Total jac sparsity was set in both _simul_coloring_info"
" and _total_jac_sparsity.")
self._total_jac_sparsity = sparsity
def show_options_table(reference):
"""
Return the options table of the given reference path.
Parameters
----------
reference : str
Dot path of desired class or function.
Returns
-------
IPython.display
Options table of the given class or function.
"""
obj = _get_object_from_reference(reference)()
if ipy:
return display(HTML(obj.options.to_table(fmt='html')))
else:
simple_warning(
"IPython is not installed. Run `pip install openmdao[notebooks]` or "
"`pip install openmdao[docs]` to upgrade.")
def dynamic_sparsity(driver):
"""
Compute deriv sparsity during runtime.
Parameters
----------
driver : <Driver>
The driver performing the optimization.
"""
problem = driver._problem
if not problem.model._use_derivatives:
simple_warning("Derivatives have been turned off. Skipping dynamic sparsity computation.")
return
driver._total_jac = None
repeats = driver.options['dynamic_derivs_repeats']
# save the sparsity.json file for later inspection
with open("sparsity.json", "w") as f:
sparsity = get_sparsity(problem, mode=problem._mode, repeats=repeats, stream=f)
driver.set_total_jac_sparsity(sparsity)
driver._setup_tot_jac_sparsity()
def add_approximation(self, abs_key, system, kwargs, vector=None):
"""
Use this approximation scheme to approximate the derivative d(of)/d(wrt).
Parameters
----------
abs_key : tuple(str,str)
Absolute name pairing of (of, wrt) for the derivative.
system : System
Containing System.
kwargs : dict
Additional keyword arguments, to be interpreted by sub-classes.
vector : ndarray or None
Direction for difference when using directional derivatives.
"""
options = self.DEFAULT_OPTIONS.copy()
options.update(kwargs)
if options['order'] is None:
form = options['form']
if form in DEFAULT_ORDER:
options['order'] = DEFAULT_ORDER[options['form']]
else:
raise ValueError(
"{}: '{}' is not a valid form of finite difference; must be "
"one of {}".format(system.msginfo, form,
list(DEFAULT_ORDER.keys())))
options['vector'] = vector
wrt = abs_key[1]
if wrt in self._wrt_meta:
simple_warning(
f"{system.msginfo}: overriding previous approximation defined for "
f"'{wrt}.")
self._wrt_meta[wrt] = options
self._reset() # force later regen of approx_groups
def _setup_partials(self, recurse=True):
"""
Process all partials and approximations that the user declared.
Metamodel needs to declare its partials after inputs and outputs are known.
Parameters
----------
recurse : bool
Whether to call this method in subsystems.
"""
super(MetaModelUnStructuredComp, self)._setup_partials()
vec_size = self.options['vec_size']
if vec_size > 1:
# Sparse specification of partials for vectorized models.
for wrt, n_wrt in self._surrogate_input_names:
for of, shape_of in self._surrogate_output_names:
n_of = np.prod(shape_of)
rows = np.repeat(np.arange(n_of), n_wrt)
cols = np.tile(np.arange(n_wrt), n_of)
nnz = len(rows)
rows = np.tile(rows, vec_size) + np.repeat(
np.arange(vec_size), nnz) * n_of
cols = np.tile(cols, vec_size) + np.repeat(
np.arange(vec_size), nnz) * n_wrt
self._declare_partials(of=of,
wrt=wrt,
rows=rows,
cols=cols)
else:
# Dense specification of partials for non-vectorized models.
self._declare_partials(
of=[name[0] for name in self._surrogate_output_names],
wrt=[name[0] for name in self._surrogate_input_names])
# warn the user that if they don't explicitly set options for fd,
# the defaults will be used
# get a list of approximated partials
declared_partials = set()
for of, wrt, method, fd_options in self._approximated_partials:
pattern_matches = self._find_partial_matches(of, wrt)
for of_bundle, wrt_bundle in product(*pattern_matches):
of_pattern, of_matches = of_bundle
wrt_pattern, wrt_matches = wrt_bundle
for rel_key in product(of_matches, wrt_matches):
abs_key = rel_key2abs_key(self, rel_key)
declared_partials.add(abs_key)
non_declared_partials = []
for of, n_of in self._surrogate_output_names:
has_derivs = False
surrogate = self._metadata(of).get('surrogate')
if surrogate:
has_derivs = overrides_method('linearize', surrogate,
SurrogateModel)
if not has_derivs:
for wrt, n_wrt in self._surrogate_input_names:
abs_key = rel_key2abs_key(self, (of, wrt))
if abs_key not in declared_partials:
non_declared_partials.append(abs_key)
if non_declared_partials:
msg = "Because the MetaModelUnStructuredComp '{}' uses a surrogate " \
"which does not define a linearize method,\nOpenMDAO will use " \
"finite differences to compute derivatives. Some of the derivatives " \
"will be computed\nusing default finite difference " \
"options because they were not explicitly declared.\n".format(self.name)
msg += "The derivatives computed using the defaults are:\n"
for abs_key in non_declared_partials:
msg += " {}, {}\n".format(*abs_key)
simple_warning(msg, RuntimeWarning)
for out_name, out_shape in self._surrogate_output_names:
surrogate = self._metadata(out_name).get('surrogate')
if surrogate and not overrides_method('linearize', surrogate,
SurrogateModel):
self._approx_partials(
of=out_name,
wrt=[name[0] for name in self._surrogate_input_names],
method='fd')
if "fd" not in self._approx_schemes:
self._approx_schemes['fd'] = FiniteDifference()
def _get_viewer_data(data_source):
"""
Get the data needed by the N2 viewer as a dictionary.
Parameters
----------
data_source : <Problem> or <Group> or str
A Problem or Group or case recorder file name containing the model or model data.
Returns
-------
dict
A dictionary containing information about the model for use by the viewer.
"""
if isinstance(data_source, Problem):
root_group = data_source.model
if not isinstance(root_group, Group):
simple_warning(
"The model is not a Group, viewer data is unavailable.")
return {}
driver = data_source.driver
driver_name = driver.__class__.__name__
driver_type = 'doe' if isinstance(driver,
DOEDriver) else 'optimization'
driver_options = {k: driver.options[k] for k in driver.options}
driver_opt_settings = None
if driver_type is 'optimization' and 'opt_settings' in dir(driver):
driver_opt_settings = driver.opt_settings
elif isinstance(data_source, Group):
if not data_source.pathname: # root group
root_group = data_source
driver_name = None
driver_type = None
driver_options = None
driver_opt_settings = None
else:
# this function only makes sense when it is at the root
return {}
elif isinstance(data_source, str):
return CaseReader(data_source, pre_load=False).problem_metadata
else:
raise TypeError(
'_get_viewer_data only accepts Problems, Groups or filenames')
data_dict = {}
comp_exec_idx = [0] # list so pass by ref
orders = {}
data_dict['tree'] = _get_tree_dict(root_group, orders, comp_exec_idx)
connections_list = []
sys_pathnames_list = [] # list of pathnames of systems found in cycles
sys_pathnames_dict = {} # map of pathnames to index of pathname in list
G = root_group.compute_sys_graph(comps_only=True)
scc = nx.strongly_connected_components(G)
for strong_comp in scc:
if len(strong_comp) > 1:
# these IDs are only used when back edges are present
sys_pathnames_list.extend(strong_comp)
for name in strong_comp:
sys_pathnames_dict[name] = len(sys_pathnames_dict)
for src, tgt in G.edges(strong_comp):
if src in strong_comp and tgt in strong_comp:
exe_src = orders[src]
exe_tgt = orders[tgt]
if exe_tgt < exe_src:
exe_low = exe_tgt
exe_high = exe_src
else:
exe_low = exe_src
exe_high = exe_tgt
edges_list = [
(sys_pathnames_dict[s], sys_pathnames_dict[t])
for s, t in G.edges(strong_comp)
if exe_low <= orders[s] <= exe_high and exe_low <=
orders[t] <= exe_high and not (s == src and t == tgt)
]
for vsrc, vtgtlist in iteritems(
G.get_edge_data(src, tgt)['conns']):
for vtgt in vtgtlist:
connections_list.append({
'src': vsrc,
'tgt': vtgt,
'cycle_arrows': edges_list
})
else: # edge is out of the SCC
for vsrc, vtgtlist in iteritems(
G.get_edge_data(src, tgt)['conns']):
for vtgt in vtgtlist:
connections_list.append({'src': vsrc, 'tgt': vtgt})
data_dict['sys_pathnames_list'] = sys_pathnames_list
data_dict['connections_list'] = connections_list
data_dict['abs2prom'] = root_group._var_abs2prom
data_dict['driver'] = {
'name': driver_name,
'type': driver_type,
'options': driver_options,
'opt_settings': driver_opt_settings
}
data_dict['design_vars'] = root_group.get_design_vars()
data_dict['responses'] = root_group.get_responses()
data_dict['declare_partials_list'] = _get_declare_partials(root_group)
return data_dict
def _setup_solvers(self, system, depth):
"""
Assign system instance, set depth, and optionally perform setup.
Parameters
----------
system : <System>
Pointer to the owning system.
depth : int
Depth of the current system (already incremented).
"""
super(BroydenSolver, self)._setup_solvers(system, depth)
self._recompute_jacobian = True
self._computed_jacobians = 0
self._disallow_discrete_outputs()
if self.linear_solver is not None:
self.linear_solver._setup_solvers(self._system, self._depth + 1)
else:
self.linear_solver = system.linear_solver
if self.linesearch is not None:
self.linesearch._setup_solvers(self._system, self._depth + 1)
self.linesearch._do_subsolve = True
states = self.options['state_vars']
prom = system._var_allprocs_prom2abs_list['output']
# Check names of states.
bad_names = [name for name in states if name not in prom]
if len(bad_names) > 0:
msg = "The following variable names were not found: {}"
raise ValueError(msg.format(', '.join(bad_names)))
# Size linear system
outputs = system._outputs
if len(states) > 0:
n = 0
for name in states:
size = len(outputs[name])
self._idx[name] = (n, n + size)
n += size
else:
self._full_inverse = True
n = len(outputs._data)
self.n = n
self.Gm = np.empty((n, n))
self.xm = np.empty((n, ))
self.fxm = np.empty((n, ))
self.delta_xm = None
self.delta_fxm = None
if self._full_inverse:
# Can only use DirectSolver here.
from openmdao.solvers.linear.direct import DirectSolver
if not isinstance(self.linear_solver, DirectSolver):
msg = "Linear solver must be DirectSolver when solving the full model."
raise ValueError(msg.format(', '.join(bad_names)))
return
# Always look for states that aren't being solved so we can warn the user.
def sys_recurse(system, all_states):
subs = system._subsystems_myproc
if len(subs) == 0:
# Skip implicit components that appear to solve themselves.
from openmdao.core.implicitcomponent import ImplicitComponent
if overrides_method('solve_nonlinear', system, ImplicitComponent):
return
all_states.extend(system._list_states())
else:
for subsys in subs:
sub_nl = subsys.nonlinear_solver
if sub_nl and sub_nl.supports['implicit_components']:
continue
sys_recurse(subsys, all_states)
all_states = []
sys_recurse(system, all_states)
all_states = [system._var_abs2prom['output'][name] for name in all_states]
missing = set(all_states).difference(states)
if len(missing) > 0:
msg = "The following states are not covered by a solver, and may have been " + \
"omitted from the BroydenSolver 'state_vars': "
msg += ', '.join(sorted(missing))
simple_warning(msg)
def _get_viewer_data(data_source):
"""
Get the data needed by the N2 viewer as a dictionary.
Parameters
----------
data_source : <Problem> or <Group> or str
A Problem or Group or case recorder file name containing the model or model data.
Returns
-------
dict
A dictionary containing information about the model for use by the viewer.
"""
if isinstance(data_source, Problem):
root_group = data_source.model
if not isinstance(root_group, Group):
simple_warning(
"The model is not a Group, viewer data is unavailable.")
return {}
driver = data_source.driver
driver_name = driver.__class__.__name__
driver_type = 'doe' if isinstance(driver,
DOEDriver) else 'optimization'
driver_options = {k: driver.options[k] for k in driver.options}
driver_opt_settings = None
if driver_type is 'optimization' and 'opt_settings' in dir(driver):
driver_opt_settings = driver.opt_settings
elif isinstance(data_source, Group):
if not data_source.pathname: # root group
root_group = data_source
driver_name = None
driver_type = None
driver_options = None
driver_opt_settings = None
else:
# this function only makes sense when it is at the root
return {}
elif isinstance(data_source, str):
return CaseReader(data_source, pre_load=False).problem_metadata
else:
raise TypeError(
'_get_viewer_data only accepts Problems, Groups or filenames')
data_dict = {}
comp_exec_idx = [0] # list so pass by ref
comp_exec_orders = {}
data_dict['tree'] = _get_tree_dict(root_group, comp_exec_orders,
comp_exec_idx)
connections_list = []
sys_pathnames_list = [] # list of pathnames of systems found in cycles
sys_pathnames_dict = {} # map of pathnames to index of pathname in list
# sort to make deterministic for testing
sorted_abs_input2src = OrderedDict(
sorted(root_group._conn_global_abs_in2out.items()))
root_group._conn_global_abs_in2out = sorted_abs_input2src
G = root_group.compute_sys_graph(comps_only=True)
scc = nx.strongly_connected_components(G)
scc_list = [s for s in scc if len(s) > 1]
for in_abs, out_abs in iteritems(sorted_abs_input2src):
if out_abs is None:
continue
src_subsystem = out_abs.rsplit('.', 1)[0]
tgt_subsystem = in_abs.rsplit('.', 1)[0]
src_to_tgt_str = src_subsystem + ' ' + tgt_subsystem
count = 0
edges_list = []
for li in scc_list:
if src_subsystem in li and tgt_subsystem in li:
count += 1
if count > 1:
raise ValueError('Count greater than 1')
exe_tgt = comp_exec_orders[tgt_subsystem]
exe_src = comp_exec_orders[src_subsystem]
exe_low = min(exe_tgt, exe_src)
exe_high = max(exe_tgt, exe_src)
subg = G.subgraph(
n for n in li
if exe_low <= comp_exec_orders[n] <= exe_high)
for edge in subg.edges():
edge_str = ' '.join(edge)
if edge_str != src_to_tgt_str:
src, tgt = edge
# add src & tgt to pathnames list & dict if not already there
for pathname in edge:
if pathname not in sys_pathnames_dict:
sys_pathnames_list.append(pathname)
sys_pathnames_dict[pathname] = len(
sys_pathnames_list) - 1
# replace src & tgt pathnames with indices into pathname list
src = sys_pathnames_dict[src]
tgt = sys_pathnames_dict[tgt]
edges_list.append([src, tgt])
if edges_list:
edges_list.sort(
) # make deterministic so same .html file will be produced each run
connections_list.append(
dict([('src', out_abs), ('tgt', in_abs),
('cycle_arrows', edges_list)]))
else:
connections_list.append(dict([('src', out_abs), ('tgt', in_abs)]))
data_dict['sys_pathnames_list'] = sys_pathnames_list
data_dict['connections_list'] = connections_list
data_dict['abs2prom'] = root_group._var_abs2prom
data_dict['driver'] = {
'name': driver_name,
'type': driver_type,
'options': driver_options,
'opt_settings': driver_opt_settings
}
data_dict['design_vars'] = root_group.get_design_vars()
data_dict['responses'] = root_group.get_responses()
data_dict['declare_partials_list'] = _get_declare_partials(root_group)
return data_dict
def _setup_partials(self, recurse=True):
"""
Process all partials and approximations that the user declared.
Metamodel needs to declare its partials after inputs and outputs are known.
Parameters
----------
recurse : bool
Whether to call this method in subsystems.
"""
super(MetaModelUnStructuredComp, self)._setup_partials()
vec_size = self.options['vec_size']
if vec_size > 1:
# Sparse specification of partials for vectorized models.
for wrt, n_wrt in self._surrogate_input_names:
for of, shape_of in self._surrogate_output_names:
n_of = np.prod(shape_of)
rows = np.repeat(np.arange(n_of), n_wrt)
cols = np.tile(np.arange(n_wrt), n_of)
nnz = len(rows)
rows = np.tile(rows, vec_size) + np.repeat(np.arange(vec_size), nnz) * n_of
cols = np.tile(cols, vec_size) + np.repeat(np.arange(vec_size), nnz) * n_wrt
self._declare_partials(of=of, wrt=wrt, rows=rows, cols=cols)
else:
# Dense specification of partials for non-vectorized models.
self._declare_partials(of=[name[0] for name in self._surrogate_output_names],
wrt=[name[0] for name in self._surrogate_input_names])
# warn the user that if they don't explicitly set options for fd,
# the defaults will be used
# get a list of approximated partials
declared_partials = set()
for of, wrt, method, fd_options in self._approximated_partials:
pattern_matches = self._find_partial_matches(of, wrt)
for of_bundle, wrt_bundle in product(*pattern_matches):
of_pattern, of_matches = of_bundle
wrt_pattern, wrt_matches = wrt_bundle
for rel_key in product(of_matches, wrt_matches):
abs_key = rel_key2abs_key(self, rel_key)
declared_partials.add(abs_key)
non_declared_partials = []
for of, n_of in self._surrogate_output_names:
has_derivs = False
surrogate = self._metadata(of).get('surrogate')
if surrogate:
has_derivs = overrides_method('linearize', surrogate, SurrogateModel)
if not has_derivs:
for wrt, n_wrt in self._surrogate_input_names:
abs_key = rel_key2abs_key(self, (of, wrt))
if abs_key not in declared_partials:
non_declared_partials.append(abs_key)
if non_declared_partials:
msg = "Because the MetaModelUnStructuredComp '{}' uses a surrogate " \
"which does not define a linearize method,\nOpenMDAO will use " \
"finite differences to compute derivatives. Some of the derivatives " \
"will be computed\nusing default finite difference " \
"options because they were not explicitly declared.\n".format(self.name)
msg += "The derivatives computed using the defaults are:\n"
for abs_key in non_declared_partials:
msg += " {}, {}\n".format(*abs_key)
simple_warning(msg, RuntimeWarning)
for out_name, out_shape in self._surrogate_output_names:
surrogate = self._metadata(out_name).get('surrogate')
if surrogate and not overrides_method('linearize', surrogate, SurrogateModel):
self._approx_partials(of=out_name,
wrt=[name[0] for name in self._surrogate_input_names],
method='fd')
if "fd" not in self._approx_schemes:
self._approx_schemes['fd'] = FiniteDifference()
def run(self):
"""
Excute pyOptsparse.
Note that pyOpt controls the execution, and the individual optimizers
(e.g., SNOPT) control the iteration.
Returns
-------
boolean
Failure flag; True if failed to converge, False is successful.
"""
problem = self._problem()
model = problem.model
relevant = model._relevant
self.pyopt_solution = None
self._total_jac = None
self.iter_count = 0
fwd = problem._mode == 'fwd'
optimizer = self.options['optimizer']
self._quantities = []
self._check_for_missing_objective()
# Only need initial run if we have linear constraints or if we are using an optimizer that
# doesn't perform one initially.
con_meta = self._cons
model_ran = False
if optimizer in run_required or np.any(
[con['linear'] for con in self._cons.values()]):
with RecordingDebugging(self._get_name(), self.iter_count,
self) as rec:
# Initial Run
model.run_solve_nonlinear()
rec.abs = 0.0
rec.rel = 0.0
model_ran = True
self.iter_count += 1
# compute dynamic simul deriv coloring or just sparsity if option is set
if c_mod._use_total_sparsity:
coloring = None
if self._coloring_info['coloring'] is None and self._coloring_info[
'dynamic']:
coloring = c_mod.dynamic_total_coloring(
self,
run_model=not model_ran,
fname=self._get_total_coloring_fname())
if coloring is not None:
# if the improvement wasn't large enough, don't use coloring
pct = coloring._solves_info()[-1]
info = self._coloring_info
if info['min_improve_pct'] > pct:
info['coloring'] = info['static'] = None
simple_warning(
"%s: Coloring was deactivated. Improvement of %.1f%% was less "
"than min allowed (%.1f%%)." %
(self.msginfo, pct, info['min_improve_pct']))
comm = None if isinstance(problem.comm, FakeComm) else problem.comm
opt_prob = Optimization(self.options['title'],
weak_method_wrapper(self, '_objfunc'),
comm=comm)
# Add all design variables
param_meta = self._designvars
self._indep_list = indep_list = list(param_meta)
param_vals = self.get_design_var_values()
for name, meta in param_meta.items():
opt_prob.addVarGroup(name,
meta['size'],
type='c',
value=param_vals[name],
lower=meta['lower'],
upper=meta['upper'])
opt_prob.finalizeDesignVariables()
# Add all objectives
objs = self.get_objective_values()
for name in objs:
opt_prob.addObj(name)
self._quantities.append(name)
# Calculate and save derivatives for any linear constraints.
lcons = [key for (key, con) in con_meta.items() if con['linear']]
if len(lcons) > 0:
_lin_jacs = self._compute_totals(of=lcons,
wrt=indep_list,
return_format='dict')
# convert all of our linear constraint jacs to COO format. Otherwise pyoptsparse will
# do it for us and we'll end up with a fully dense COO matrix and very slow evaluation
# of linear constraints!
to_remove = []
for jacdct in _lin_jacs.values():
for n, subjac in jacdct.items():
if isinstance(subjac, np.ndarray):
# we can safely use coo_matrix to automatically convert the ndarray
# since our linear constraint jacs are constant, so zeros won't become
# nonzero during the optimization.
mat = coo_matrix(subjac)
if mat.row.size > 0:
# convert to 'coo' format here to avoid an emphatic warning
# by pyoptsparse.
jacdct[n] = {
'coo': [mat.row, mat.col, mat.data],
'shape': mat.shape
}
# Add all equality constraints
for name, meta in con_meta.items():
if meta['equals'] is None:
continue
size = meta['size']
lower = upper = meta['equals']
if fwd:
wrt = [v for v in indep_list if name in relevant[v]]
else:
rels = relevant[name]
wrt = [v for v in indep_list if v in rels]
if meta['linear']:
jac = {w: _lin_jacs[name][w] for w in wrt}
opt_prob.addConGroup(name,
size,
lower=lower,
upper=upper,
linear=True,
wrt=wrt,
jac=jac)
else:
if name in self._res_jacs:
resjac = self._res_jacs[name]
jac = {n: resjac[n] for n in wrt}
else:
jac = None
opt_prob.addConGroup(name,
size,
lower=lower,
upper=upper,
wrt=wrt,
jac=jac)
self._quantities.append(name)
# Add all inequality constraints
for name, meta in con_meta.items():
if meta['equals'] is not None:
continue
size = meta['size']
# Bounds - double sided is supported
lower = meta['lower']
upper = meta['upper']
if fwd:
wrt = [v for v in indep_list if name in relevant[v]]
else:
rels = relevant[name]
wrt = [v for v in indep_list if v in rels]
if meta['linear']:
jac = {w: _lin_jacs[name][w] for w in wrt}
opt_prob.addConGroup(name,
size,
upper=upper,
lower=lower,
linear=True,
wrt=wrt,
jac=jac)
else:
if name in self._res_jacs:
resjac = self._res_jacs[name]
jac = {n: resjac[n] for n in wrt}
else:
jac = None
opt_prob.addConGroup(name,
size,
upper=upper,
lower=lower,
wrt=wrt,
jac=jac)
self._quantities.append(name)
# Instantiate the requested optimizer
try:
_tmp = __import__('pyoptsparse', globals(), locals(), [optimizer],
0)
opt = getattr(_tmp, optimizer)()
except Exception as err:
# Change whatever pyopt gives us to an ImportError, give it a readable message,
# but raise with the original traceback.
msg = "Optimizer %s is not available in this installation." % optimizer
raise ImportError(msg)
# Process any default optimizer-specific settings.
if optimizer in DEFAULT_OPT_SETTINGS:
for name, value in DEFAULT_OPT_SETTINGS[optimizer].items():
if name not in self.opt_settings:
self.opt_settings[name] = value
# Set optimization options
for option, value in self.opt_settings.items():
opt.setOption(option, value)
# Execute the optimization problem
if self.options['gradient method'] == 'pyopt_fd':
# Use pyOpt's internal finite difference
# TODO: Need to get this from OpenMDAO
# fd_step = problem.model.deriv_options['step_size']
fd_step = 1e-6
sol = opt(opt_prob,
sens='FD',
sensStep=fd_step,
storeHistory=self.hist_file,
hotStart=self.hotstart_file)
elif self.options['gradient method'] == 'snopt_fd':
if self.options['optimizer'] == 'SNOPT':
# Use SNOPT's internal finite difference
# TODO: Need to get this from OpenMDAO
# fd_step = problem.model.deriv_options['step_size']
fd_step = 1e-6
sol = opt(opt_prob,
sens=None,
sensStep=fd_step,
storeHistory=self.hist_file,
hotStart=self.hotstart_file)
else:
raise Exception(
"SNOPT's internal finite difference can only be used with SNOPT"
)
else:
# Use OpenMDAO's differentiator for the gradient
sol = opt(opt_prob,
sens=weak_method_wrapper(self, '_gradfunc'),
storeHistory=self.hist_file,
hotStart=self.hotstart_file)
# Print results
if self.options['print_results']:
print(sol)
# Pull optimal parameters back into framework and re-run, so that
# framework is left in the right final state
dv_dict = sol.getDVs()
for name in indep_list:
self.set_design_var(name, dv_dict[name])
with RecordingDebugging(self._get_name(), self.iter_count,
self) as rec:
model.run_solve_nonlinear()
rec.abs = 0.0
rec.rel = 0.0
self.iter_count += 1
# Save the most recent solution.
self.pyopt_solution = sol
try:
exit_status = sol.optInform['value']
self.fail = False
# These are various failed statuses.
if optimizer == 'IPOPT':
if exit_status not in {0, 1}:
self.fail = True
elif exit_status > 2:
self.fail = True
except KeyError:
# optimizers other than pySNOPT may not populate this dict
pass
# revert signal handler to cached version
sigusr = self.options['user_teriminate_signal']
if sigusr is not None:
signal.signal(sigusr, self._signal_cache)
self._signal_cache = None # to prevent memory leak test from failing
return self.fail
def _setup_solvers(self, system, depth):
"""
Assign system instance, set depth, and optionally perform setup.
Parameters
----------
system : <System>
Pointer to the owning system.
depth : int
Depth of the current system (already incremented).
"""
super(BroydenSolver, self)._setup_solvers(system, depth)
self._recompute_jacobian = True
self._computed_jacobians = 0
self._disallow_discrete_outputs()
if self.linear_solver is not None:
self.linear_solver._setup_solvers(self._system, self._depth + 1)
else:
self.linear_solver = system.linear_solver
if self.linesearch is not None:
self.linesearch._setup_solvers(self._system, self._depth + 1)
self.linesearch._do_subsolve = True
states = self.options['state_vars']
prom = system._var_allprocs_prom2abs_list['output']
# Check names of states.
bad_names = [name for name in states if name not in prom]
if len(bad_names) > 0:
msg = "The following variable names were not found: {}"
raise ValueError(msg.format(', '.join(bad_names)))
# Size linear system
outputs = system._outputs
if len(states) > 0:
n = 0
for name in states:
size = len(outputs[name])
self._idx[name] = (n, n + size)
n += size
else:
self._full_inverse = True
n = len(outputs._data)
self.n = n
self.Gm = np.empty((n, n))
self.xm = np.empty((n, ))
self.fxm = np.empty((n, ))
self.delta_xm = None
self.delta_fxm = None
if self._full_inverse:
# Can only use DirectSolver here.
from openmdao.solvers.linear.direct import DirectSolver
if not isinstance(self.linear_solver, DirectSolver):
msg = "Linear solver must be DirectSolver when solving the full model."
raise ValueError(msg.format(', '.join(bad_names)))
return
# Always look for states that aren't being solved so we can warn the user.
def sys_recurse(system, all_states):
subs = system._subsystems_myproc
if len(subs) == 0:
# Skip implicit components that appear to solve themselves.
from openmdao.core.implicitcomponent import ImplicitComponent
if overrides_method('solve_nonlinear', system, ImplicitComponent):
return
all_states.extend(system._list_states())
else:
for subsys in subs:
sub_nl = subsys.nonlinear_solver
if sub_nl and sub_nl.supports['implicit_components']:
continue
sys_recurse(subsys, all_states)
all_states = []
sys_recurse(system, all_states)
all_states = [system._var_abs2prom['output'][name] for name in all_states]
missing = set(all_states).difference(states)
if len(missing) > 0:
msg = "The following states are not covered by a solver, and may have been " + \
"omitted from the BroydenSolver 'state_vars': "
msg += ', '.join(sorted(missing))
simple_warning(msg)
def _get_viewer_data(data_source):
"""
Get the data needed by the N2 viewer as a dictionary.
Parameters
----------
data_source : <Problem> or <Group> or str
A Problem or Group or case recorder file name containing the model or model data.
Returns
-------
dict
A dictionary containing information about the model for use by the viewer.
"""
if isinstance(data_source, Problem):
root_group = data_source.model
if not isinstance(root_group, Group):
simple_warning("The model is not a Group, viewer data is unavailable.")
return {}
elif isinstance(data_source, Group):
if not data_source.pathname: # root group
root_group = data_source
else:
# this function only makes sense when it is at the root
return {}
elif isinstance(data_source, str):
check_valid_sqlite3_db(data_source)
import sqlite3
con = sqlite3.connect(data_source, detect_types=sqlite3.PARSE_DECLTYPES)
cur = con.cursor()
cur.execute("SELECT format_version FROM metadata")
row = cur.fetchone()
format_version = row[0]
cur.execute("SELECT model_viewer_data FROM driver_metadata;")
model_text = cur.fetchone()
from six import PY2, PY3
if row is not None:
if format_version >= 3:
return json.loads(model_text[0])
elif format_version in (1, 2):
if PY2:
import cPickle
return cPickle.loads(str(model_text[0]))
if PY3:
import pickle
return pickle.loads(model_text[0])
else:
raise TypeError('_get_viewer_data only accepts Problems, Groups or filenames')
data_dict = {}
comp_exec_idx = [0] # list so pass by ref
comp_exec_orders = {}
data_dict['tree'] = _get_tree_dict(root_group, comp_exec_orders, comp_exec_idx)
connections_list = []
# sort to make deterministic for testing
sorted_abs_input2src = OrderedDict(sorted(root_group._conn_global_abs_in2out.items()))
root_group._conn_global_abs_in2out = sorted_abs_input2src
G = root_group.compute_sys_graph(comps_only=True)
scc = nx.strongly_connected_components(G)
scc_list = [s for s in scc if len(s) > 1]
for in_abs, out_abs in iteritems(sorted_abs_input2src):
if out_abs is None:
continue
src_subsystem = out_abs.rsplit('.', 1)[0]
tgt_subsystem = in_abs.rsplit('.', 1)[0]
src_to_tgt_str = src_subsystem + ' ' + tgt_subsystem
count = 0
edges_list = []
for li in scc_list:
if src_subsystem in li and tgt_subsystem in li:
count += 1
if count > 1:
raise ValueError('Count greater than 1')
exe_tgt = comp_exec_orders[tgt_subsystem]
exe_src = comp_exec_orders[src_subsystem]
exe_low = min(exe_tgt, exe_src)
exe_high = max(exe_tgt, exe_src)
subg = G.subgraph(n for n in li if exe_low <= comp_exec_orders[n] <= exe_high)
for edge in subg.edges():
edge_str = ' '.join(edge)
if edge_str != src_to_tgt_str:
edges_list.append(edge_str)
if edges_list:
edges_list.sort() # make deterministic so same .html file will be produced each run
connections_list.append(OrderedDict([('src', out_abs), ('tgt', in_abs),
('cycle_arrows', edges_list)]))
else:
connections_list.append(OrderedDict([('src', out_abs), ('tgt', in_abs)]))
data_dict['connections_list'] = connections_list
data_dict['abs2prom'] = root_group._var_abs2prom
return data_dict
def _setup_procs(self, pathname, comm, mode):
"""
Execute first phase of the setup process.
Distribute processors, assign pathnames, and call setup on the component.
Parameters
----------
pathname : str
Global name of the system, including the path.
comm : MPI.Comm or <FakeComm>
MPI communicator object.
mode : string
Derivatives calculation mode, 'fwd' for forward, and 'rev' for
reverse (adjoint). Default is 'rev'.
"""
self.pathname = pathname
orig_comm = comm
if self._num_par_fd > 1:
if comm.size > 1:
comm = self._setup_par_fd_procs(comm)
elif not MPI:
msg = ("'%s': MPI is not active but num_par_fd = %d. No parallel finite difference "
"will be performed." % (self.pathname, self._num_par_fd))
simple_warning(msg)
self.comm = comm
self._mode = mode
self._subsystems_proc_range = []
# Clear out old variable information so that we can call setup on the component.
self._var_rel_names = {'input': [], 'output': []}
self._var_rel2meta = {}
self._design_vars = OrderedDict()
self._responses = OrderedDict()
self._static_mode = False
self._var_rel2meta.update(self._static_var_rel2meta)
for type_ in ['input', 'output']:
self._var_rel_names[type_].extend(self._static_var_rel_names[type_])
self._design_vars.update(self._static_design_vars)
self._responses.update(self._static_responses)
self.setup()
# check to make sure that if num_par_fd > 1 that this system is actually doing FD.
# Unfortunately we have to do this check after system setup has been called because that's
# when declare_partials generally happens, so we raise an exception here instead of just
# resetting the value of num_par_fd (because the comm has already been split and possibly
# used by the system setup).
if self._num_par_fd > 1 and orig_comm.size > 1 and not (self._owns_approx_jac or
self._approximated_partials):
raise RuntimeError("'%s': num_par_fd is > 1 but no FD is active." % self.pathname)
self._static_mode = True
if self.options['distributed']:
if self._distributed_vector_class is not None:
self._vector_class = self._distributed_vector_class
else:
simple_warning("The 'distributed' option is set to True for Component %s, "
"but there is no distributed vector implementation (MPI/PETSc) "
"available. The default non-distributed vectors will be used."
% pathname)
self._vector_class = self._local_vector_class
else:
self._vector_class = self._local_vector_class
def compute_approximations(self, system, jac, total=False):
"""
Execute the system to compute the approximate sub-Jacobians.
Parameters
----------
system : System
System on which the execution is run.
jac : dict-like
Approximations are stored in the given dict-like object.
total : bool
If True total derivatives are being approximated, else partials.
"""
if len(self._exec_list) == 0:
return
if system.under_complex_step:
# If we are nested under another complex step, then warn and swap to FD.
if not self._fd:
from openmdao.approximation_schemes.finite_difference import FiniteDifference
msg = "Nested complex step detected. Finite difference will be used for '%s'."
simple_warning(msg % system.pathname)
fd = self._fd = FiniteDifference()
for item in self._exec_list:
fd.add_approximation(item[0:2], {})
self._fd.compute_approximations(system, jac, total=total)
return
if total:
current_vec = system._outputs
else:
current_vec = system._residuals
# Clean vector for results
results_clone = current_vec._clone(True)
# Turn on complex step.
system._set_complex_step_mode(True)
results_clone.set_complex_step_mode(True)
# To support driver src_indices, we need to override some checks in Jacobian, but do it
# selectively.
uses_src_indices = (system._owns_approx_of_idx or system._owns_approx_wrt_idx) and \
not isinstance(jac, dict)
use_parallel_fd = system._num_par_fd > 1 and (system._full_comm is not None and
system._full_comm.size > 1)
num_par_fd = system._num_par_fd if use_parallel_fd else 1
is_parallel = use_parallel_fd or system.comm.size > 1
results = defaultdict(list)
iproc = system.comm.rank
owns = system._owning_rank
mycomm = system._full_comm if use_parallel_fd else system.comm
fd_count = 0
approx_groups = self._get_approx_groups(system)
for tup in approx_groups:
wrt, delta, fact, in_idx, in_size, outputs = tup
for i_count, idx in enumerate(in_idx):
if fd_count % num_par_fd == system._par_fd_id:
# Run the Finite Difference
result = self._run_point_complex(system, wrt, idx, delta, results_clone, total)
if is_parallel:
for of, _, out_idx in outputs:
if owns[of] == iproc:
results[(of, wrt)].append(
(i_count, result._views_flat[of][out_idx].imag.copy()))
else:
for of, subjac, out_idx in outputs:
subjac[:, i_count] = result._views_flat[of][out_idx].imag
fd_count += 1
if is_parallel:
results = _gather_jac_results(mycomm, results)
for wrt, _, fact, _, _, outputs in approx_groups:
for of, subjac, _ in outputs:
key = (of, wrt)
if is_parallel:
for i, result in results[key]:
subjac[:, i] = result
subjac *= fact
if uses_src_indices:
jac._override_checks = True
jac[key] = subjac
jac._override_checks = False
else:
jac[key] = subjac
# Turn off complex step.
system._set_complex_step_mode(False)
def _get_viewer_data(data_source):
"""
Get the data needed by the N2 viewer as a dictionary.
Parameters
----------
data_source : <Problem> or <Group> or str
A Problem or Group or case recorder file name containing the model or model data.
Returns
-------
dict
A dictionary containing information about the model for use by the viewer.
"""
if isinstance(data_source, Problem):
root_group = data_source.model
if not isinstance(root_group, Group):
simple_warning("The model is not a Group, viewer data is unavailable.")
return {}
driver = data_source.driver
driver_name = driver.__class__.__name__
driver_type = 'doe' if isinstance(driver, DOEDriver) else 'optimization'
driver_options = {k: driver.options[k] for k in driver.options}
driver_opt_settings = None
if driver_type is 'optimization' and 'opt_settings' in dir(driver):
driver_opt_settings = driver.opt_settings
elif isinstance(data_source, Group):
if not data_source.pathname: # root group
root_group = data_source
driver_name = None
driver_type = None
driver_options = None
driver_opt_settings = None
else:
# this function only makes sense when it is at the root
return {}
elif isinstance(data_source, str):
return CaseReader(data_source, pre_load=False).problem_metadata
else:
raise TypeError('_get_viewer_data only accepts Problems, Groups or filenames')
data_dict = {}
comp_exec_idx = [0] # list so pass by ref
comp_exec_orders = {}
data_dict['tree'] = _get_tree_dict(root_group, comp_exec_orders, comp_exec_idx)
connections_list = []
sys_pathnames_list = [] # list of pathnames of systems found in cycles
sys_pathnames_dict = {} # map of pathnames to index of pathname in list
# sort to make deterministic for testing
sorted_abs_input2src = OrderedDict(sorted(root_group._conn_global_abs_in2out.items()))
root_group._conn_global_abs_in2out = sorted_abs_input2src
G = root_group.compute_sys_graph(comps_only=True)
scc = nx.strongly_connected_components(G)
scc_list = [s for s in scc if len(s) > 1]
for in_abs, out_abs in iteritems(sorted_abs_input2src):
if out_abs is None:
continue
src_subsystem = out_abs.rsplit('.', 1)[0]
tgt_subsystem = in_abs.rsplit('.', 1)[0]
src_to_tgt_str = src_subsystem + ' ' + tgt_subsystem
count = 0
edges_list = []
for li in scc_list:
if src_subsystem in li and tgt_subsystem in li:
count += 1
if count > 1:
raise ValueError('Count greater than 1')
exe_tgt = comp_exec_orders[tgt_subsystem]
exe_src = comp_exec_orders[src_subsystem]
exe_low = min(exe_tgt, exe_src)
exe_high = max(exe_tgt, exe_src)
subg = G.subgraph(n for n in li if exe_low <= comp_exec_orders[n] <= exe_high)
for edge in subg.edges():
edge_str = ' '.join(edge)
if edge_str != src_to_tgt_str:
src, tgt = edge
# add src & tgt to pathnames list & dict if not already there
for pathname in edge:
if pathname not in sys_pathnames_dict:
sys_pathnames_list.append(pathname)
sys_pathnames_dict[pathname] = len(sys_pathnames_list) - 1
# replace src & tgt pathnames with indices into pathname list
src = sys_pathnames_dict[src]
tgt = sys_pathnames_dict[tgt]
edges_list.append([src, tgt])
if edges_list:
edges_list.sort() # make deterministic so same .html file will be produced each run
connections_list.append(dict([('src', out_abs), ('tgt', in_abs),
('cycle_arrows', edges_list)]))
else:
connections_list.append(dict([('src', out_abs), ('tgt', in_abs)]))
data_dict['sys_pathnames_list'] = sys_pathnames_list
data_dict['connections_list'] = connections_list
data_dict['abs2prom'] = root_group._var_abs2prom
data_dict['driver'] = {'name': driver_name, 'type': driver_type,
'options': driver_options, 'opt_settings': driver_opt_settings}
data_dict['design_vars'] = root_group.get_design_vars()
data_dict['responses'] = root_group.get_responses()
return data_dict
