def test_n2_from_sqlite(self):
"""
Test that an n2 html file is generated from a sqlite file.
"""
p = Problem()
p.model = SellarStateConnection()
r = SqliteRecorder(self.sqlite_db_filename2)
p.driver.add_recorder(r)
p.setup()
p.final_setup()
r.shutdown()
n2(p, outfile=self.compare_html_filename, show_browser=DEBUG_BROWSER)
n2(self.sqlite_db_filename2,
outfile=self.sqlite_html_filename,
show_browser=DEBUG_BROWSER)
# Check that the html file has been created and has something in it.
self.assertTrue(os.path.isfile(self.sqlite_html_filename),
(self.sqlite_html_filename + " is not a valid file."))
self.assertGreater(os.path.getsize(self.sqlite_html_filename), 100)
# Check that there are no errors when running from the command line with a recording.
check_call('openmdao n2 --no_browser %s' % self.sqlite_db_filename2)
# Compare the sizes of the files generated from the Problem and the recording
size1 = os.path.getsize(self.sqlite_html_filename)
size2 = os.path.getsize(self.compare_html_filename)
self.assertTrue(
size1 == size2, 'File size of ' + self.sqlite_html_filename +
' is ' + str(size1) + ', but size of ' +
self.compare_html_filename + ' is ' + str(size2))
def test_n2_from_sqlite(self):
"""
Test that an n2 html file is generated from a sqlite file.
"""
p = om.Problem()
p.model = SellarStateConnection()
r = SqliteRecorder(self.sqlite_db_filename2)
p.driver.add_recorder(r)
p.setup()
p.final_setup()
r.shutdown()
n2(p, outfile=self.compare_html_filename, show_browser=DEBUG_BROWSER)
n2(self.sqlite_db_filename2,
outfile=self.sqlite_html_filename,
show_browser=DEBUG_BROWSER)
# Check that the html file has been created and has something in it.
self.assertTrue(os.path.isfile(self.sqlite_html_filename),
(self.sqlite_html_filename + " is not a valid file."))
self.assertGreater(os.path.getsize(self.sqlite_html_filename), 100)
# Check that there are no errors when running from the command line with a recording.
check_call('openmdao n2 --no_browser %s' % self.sqlite_db_filename2)
# Compare models from the files generated from the Problem and the recording
sqlite_model_data = self._extract_compressed_model(
self.sqlite_html_filename)
compare_model_data = self._extract_compressed_model(
self.compare_html_filename)
self.assertTrue(
sqlite_model_data == compare_model_data,
'Model data from sqlite does not match data from Problem.')
def test_pyxdsm_case_reading(self):
"""
Writes a recorder file, and the XDSM writer makes the diagram based on the SQL file
and not the Problem instance.
"""
from openmdao.recorders.sqlite_recorder import SqliteRecorder
filename = 'xdsm_from_sql'
case_recording_filename = filename + '.sql'
prob = Problem()
prob.model = model = SellarNoDerivatives()
model.add_design_var('z', lower=np.array([-10.0, 0.0]),
upper=np.array([10.0, 10.0]), indices=np.arange(2, dtype=int))
model.add_design_var('x', lower=0.0, upper=10.0)
model.add_objective('obj')
model.add_constraint('con1', equals=np.zeros(1))
model.add_constraint('con2', upper=0.0)
recorder = SqliteRecorder(case_recording_filename)
prob.driver.add_recorder(recorder)
prob.setup(check=False)
prob.final_setup()
# Write output
write_xdsm(case_recording_filename, filename=filename, out_format='tex',
show_browser=False, quiet=QUIET)
# Check if file was created
self.assertTrue(os.path.isfile(case_recording_filename))
self.assertTrue(os.path.isfile('.'.join([filename, 'tex'])))
# Check that there are no errors when running from the command line with a recording.
check_call('openmdao xdsm --no_browser %s' % case_recording_filename)
def test_model_viewer_has_correct_data_from_sqlite(self):
"""
Verify that the correct data exists when a model structure is recorded
and then pulled out of a sqlite db file and compared to the expected
structure. Uses the SellarStateConnection model.
"""
p = om.Problem(model=SellarStateConnection())
r = SqliteRecorder(self.sqlite_db_filename)
p.driver.add_recorder(r)
p.setup()
p.final_setup()
r.shutdown()
model_viewer_data = _get_viewer_data(self.sqlite_db_filename)
with open(os.path.join(self.parent_dir,
'sellar_tree.json')) as json_file:
expected_tree = json.load(json_file)
# check expected model tree
self.check_model_viewer_data(
model_viewer_data, expected_tree, self.expected_pathnames,
self.expected_conns, self.expected_abs2prom,
self.expected_declare_partials, self.expected_driver_name,
self.expected_design_vars_names, self.expected_responses_names)
def setUp(self):
self.dir = mkdtemp()
self.filename = os.path.join(self.dir, "sqlite_test")
self.tablename = 'openmdao'
self.recorder = SqliteRecorder(self.filename)
self.recorder.options['record_metadata'] = False
self.eps = 1e-5
def setUp(self):
recording_iteration.stack = [] # reset to avoid problems from earlier tests
self.dir = mkdtemp()
self.filename = os.path.join(self.dir, "sqlite_test")
self.recorder = SqliteRecorder(self.filename)
self.original_path = os.getcwd()
os.chdir(self.dir)
def test_view_model_from_sqlite(self):
"""
Test that an n2 html file is generated from a sqlite file.
"""
p = Problem()
p.model = SellarStateConnection()
r = SqliteRecorder(self.sqlite_db_filename2)
p.driver.add_recorder(r)
p.setup(check=False)
p.final_setup()
r.shutdown()
view_model(self.sqlite_db_filename2, outfile=self.sqlite_filename, show_browser=False)
# Check that the html file has been created and has something in it.
self.assertTrue(os.path.isfile(self.sqlite_html_filename), (self.problem_html_filename + " is not a valid file."))
self.assertGreater(os.path.getsize(self.sqlite_html_filename), 100)
def test_model_viewer_has_correct_data_from_sqlite(self):
"""
Verify that the correct data exists when a model structure is recorded
and then pulled out of a sqlite db file and compared to the expected
structure. Uses the SellarStateConnection model.
"""
p = Problem(model=SellarStateConnection())
r = SqliteRecorder(self.sqlite_db_filename)
p.driver.add_recorder(r)
p.setup()
p.final_setup()
r.shutdown()
model_viewer_data = _get_viewer_data(self.sqlite_db_filename)
print(model_viewer_data['tree'])
# check expected model tree
self.assertDictEqual(model_viewer_data['tree'], self.expected_tree)
# check expected system pathnames
pathnames = model_viewer_data['sys_pathnames_list']
self.assertListEqual(sorted(pathnames), self.expected_pathnames)
# check expected connections, after mapping cycle_arrows indices back to pathnames
connections = sorted(model_viewer_data['connections_list'],
key=lambda x: (x['tgt'], x['src']))
for conn in connections:
if 'cycle_arrows' in conn:
cycle_arrows = []
for src, tgt in conn['cycle_arrows']:
cycle_arrows.append(' '.join(
[pathnames[src], pathnames[tgt]]))
conn['cycle_arrows'] = sorted(cycle_arrows)
self.assertEqual(len(connections), len(self.expected_conns))
for c, ex in zip(connections, self.expected_conns):
self.assertEqual(c['src'], ex['src'])
self.assertEqual(c['tgt'], ex['tgt'])
self.assertEqual(c.get('cycle_arrows', []),
ex.get('cycle_arrows', []))
# check expected abs2prom map
self.assertDictEqual(model_viewer_data['abs2prom'],
self.expected_abs2prom)
def test_pyxdsm_case_reading(self):
"""
Writes a recorder file, and the XDSM writer makes the diagram based on the SQL file
and not the Problem instance.
"""
from openmdao.recorders.sqlite_recorder import SqliteRecorder
filename = 'xdsm_from_sql'
case_recording_filename = filename + '.sql'
prob = Problem()
prob.model = model = SellarNoDerivatives()
model.add_design_var('z',
lower=np.array([-10.0, 0.0]),
upper=np.array([10.0, 10.0]),
indices=np.arange(2, dtype=int))
model.add_design_var('x', lower=0.0, upper=10.0)
model.add_objective('obj')
model.add_constraint('con1', equals=np.zeros(1))
model.add_constraint('con2', upper=0.0)
recorder = SqliteRecorder(case_recording_filename)
prob.driver.add_recorder(recorder)
prob.setup(check=False)
prob.final_setup()
# Write output
msg = (
'For SQL input the XDSM writer shows only the model hierarchy, '
'and the driver, design variables and responses are not part of the '
'diagram.')
with assert_warning(Warning, msg):
write_xdsm(case_recording_filename,
filename=filename,
out_format='tex',
show_browser=False,
quiet=QUIET)
# Check if file was created
self.assertTrue(os.path.isfile(case_recording_filename))
self.assertTrue(os.path.isfile('.'.join([filename, 'tex'])))
def test_model_viewer_has_correct_data_from_sqlite(self):
"""
Verify that the correct data exists when a model structure is recorded
and then pulled out of a sqlite db file and compared to the expected
structure. Uses the SellarStateConnection model.
"""
p = Problem()
p.model = SellarStateConnection()
r = SqliteRecorder(self.sqlite_db_filename)
p.driver.add_recorder(r)
p.setup(check=False)
p.final_setup()
r.close()
model_viewer_data = _get_viewer_data(self.sqlite_db_filename)
tree_json = json.dumps(model_viewer_data['tree'])
conns_json = json.dumps(model_viewer_data['connections_list'])
self.assertEqual(self.expected_tree_json, tree_json)
self.assertEqual(self.expected_conns_json, conns_json)
def test_view_model_from_sqlite(self):
"""
Test that an n2 html file is generated from a sqlite file.
"""
p = Problem()
p.model = SellarStateConnection()
r = SqliteRecorder(self.sqlite_db_filename2)
p.driver.add_recorder(r)
p.setup()
p.final_setup()
r.shutdown()
view_model(self.sqlite_db_filename2,
outfile=self.sqlite_html_filename,
show_browser=DEBUG)
# Check that the html file has been created and has something in it.
self.assertTrue(os.path.isfile(self.sqlite_html_filename),
(self.problem_html_filename + " is not a valid file."))
self.assertGreater(os.path.getsize(self.sqlite_html_filename), 100)
# Check that there are no errors when running from the command line with a recording.
check_call('openmdao view_model --no_browser %s' %
self.sqlite_db_filename2)
# Some constraints only exit on one process so cannot record everything
recording_includes_options = ['obj.val']
for j in range(npts):
recording_includes_options.append('pt%s.ConCh' % str(j))
recording_includes_options.append('pt%s.ConDs' % str(j))
recording_includes_options.append('pt%s.ConS0' % str(j))
recording_includes_options.append('pt%s.ConS1' % str(j))
recording_includes_options.append('pt%s_con5.val' % str(j))
#from openmdao.recorders import DumpRecorder
#rec = DumpRecorder(out='data.dmp')
#model.driver.add_recorder(rec)
#rec.options['includes'] = recording_includes_options
from openmdao.recorders.sqlite_recorder import SqliteRecorder
rec = SqliteRecorder(out='data.sql')
model.driver.add_recorder(rec)
rec.options['includes'] = recording_includes_options
model.setup()
model.run()
import resource
print("Memory Usage:", resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1000.0, "MB (on unix)")
#----------------------------------------------------------------
# Below this line, code I was using for verifying and profiling.
#----------------------------------------------------------------
#profile = False
#params = model.driver.get_desvars().keys()
#unks = model.driver.get_objectives().keys() + model.driver.get_constraints().keys()
def compute(self, inputs, outputs):
# type: (Vector, Vector) -> None
"""Computation performed by the component.
Parameters
----------
inputs : all inputs coming from outside the component in the group
outputs : all outputs provided outside the component in the group"""
# Define problem of subdriver
p = self.prob
m = p.model
# Push global inputs down
for input_name in m.model_constants:
p[input_name] = inputs[input_name]
failed_experiments = {}
# sort to have outputs first
sorted_model_super_inputs = sorted(m.model_super_inputs.keys(), reverse=True)
for input_name in sorted_model_super_inputs:
if input_name in m.sm_of_training_params.keys(): # Add these inputs as training data
sm_uid = m.sm_of_training_params[input_name]
pred_param = m.find_mapped_parameter(input_name,
m.sm_prediction_inputs[sm_uid] |
m.sm_prediction_outputs[sm_uid])
sm_comp = getattr(m, str_to_valid_sys_name(sm_uid))
if sm_uid not in failed_experiments.keys():
failed_experiments[sm_uid] = (None, None)
sm_comp.options['train:'+pred_param], failed_experiments[sm_uid]\
= p.postprocess_experiments(inputs[input_name], input_name,
failed_experiments[sm_uid])
else:
p[input_name] = inputs[input_name]
# Provide message on failed experiments
warn_about_failed_experiments(failed_experiments)
# Set initial values of design variables back to original ones (to avoid using values of
# last run)
for des_var, attrbs in m.design_vars.items():
p[des_var] = attrbs['initial']
# Run the driver
print('Running subdriver {}'.format(self.options['driver_uid']))
if 'Sub-Optimizer' not in p.case_reader_path:
p.driver.cleanup()
basename, extension = os.path.splitext(p.case_reader_path)
case_reader_filename = basename + '_loop' + str(self._run_count) + extension
p.driver.add_recorder(SqliteRecorder(case_reader_filename))
p.driver.recording_options['includes'] = ['*']
p.driver.recording_options['record_model_metadata'] = True
p.driver._setup_recording()
p.run_driver()
self._add_run_count()
# Pull the value back up to the output array
doe_out_vecs = {}
for output_name in m.model_super_outputs:
# Add these outputs as vectors based on DOE driver
if output_name in m.doe_parameters.keys():
doe_out_vecs[output_name] = []
else:
if not p.driver.fail:
outputs[output_name] = p[output_name]
else:
outputs[output_name] = float('nan')
# If the driver failed (hence, optimization failed), then send message and clean
if p.driver.fail:
print('Driver run failed!')
p.clean_driver_after_failure()
# Provide DOE output vectors as output of the component, if this is expected
if doe_out_vecs:
# First read out the case reader
cr = CaseReader(case_reader_filename)
cases = cr.list_cases('driver')
for n in range(len(cases)):
cr_outputs = cr.get_case(n).outputs
doe_param_matches = {}
for output_name in doe_out_vecs.keys():
doe_param_matches[output_name] = doe_param_match \
= m.find_mapped_parameter(output_name, cr_outputs.keys())
doe_out_vecs[output_name].append(cr_outputs[doe_param_match][0])
# Then write the final vectors to the global output array
for output_name in doe_out_vecs.keys():
if output_name in p.doe_samples[p.driver_uid]['inputs']:
des_var_match = m.find_mapped_parameter(output_name, m._design_vars.keys())
doe_out_vecs[output_name] = unscale_value(doe_out_vecs[output_name],
m._design_vars[des_var_match]['ref0'],
m._design_vars[des_var_match]['ref'])
outputs[output_name] = np.array(doe_out_vecs[output_name])
