def record_iteration_problem(self, recording_requester, data, metadata):
"""
Record data and metadata from a Problem.
Parameters
----------
recording_requester : object
Problem in need of recording.
data : dict
Dictionary containing desvars, objectives, and constraints.
metadata : dict
Dictionary containing execution metadata.
"""
if self.connection:
outputs = data['out']
# convert to list so this can be dumped as JSON
if outputs is not None:
for var in outputs:
outputs[var] = make_serializable(outputs[var])
outputs_text = json.dumps(outputs)
with self.connection as c:
c = c.cursor() # need a real cursor for lastrowid
c.execute(
"INSERT INTO problem_cases(counter, case_name, "
"timestamp, success, msg, outputs) VALUES(?,?,?,?,?,?)",
(self._counter, metadata['name'], metadata['timestamp'],
metadata['success'], metadata['msg'], outputs_text))
def record_iteration_problem(self, recording_requester, data, metadata):
"""
Record data and metadata from a Problem.
Parameters
----------
recording_requester : object
Problem in need of recording.
data : dict
Dictionary containing desvars, objectives, and constraints.
metadata : dict
Dictionary containing execution metadata.
"""
if self.connection:
outputs = data['out']
# convert to list so this can be dumped as JSON
if outputs is not None:
for var in outputs:
outputs[var] = make_serializable(outputs[var])
outputs_text = json.dumps(outputs)
with self.connection as c:
c = c.cursor() # need a real cursor for lastrowid
c.execute("INSERT INTO problem_cases(counter, case_name, "
"timestamp, success, msg, outputs) VALUES(?,?,?,?,?,?)",
(self._counter, metadata['name'],
metadata['timestamp'], metadata['success'], metadata['msg'],
outputs_text))
def _cleanup_abs2meta(self):
"""
Convert all abs2meta variable properties to a form that can be dumped as JSON.
"""
for name in self._abs2meta:
for prop in self._abs2meta[name]:
self._abs2meta[name][prop] = make_serializable(self._abs2meta[name][prop])
def _cleanup_abs2meta(self):
"""
Convert all abs2meta variable properties to a form that can be dumped as JSON.
"""
for name in self._abs2meta:
for prop in self._abs2meta[name]:
self._abs2meta[name][prop] = make_serializable(self._abs2meta[name][prop])
def record_iteration_solver(self, recording_requester, data, metadata):
"""
Record data and metadata from a Solver.
Parameters
----------
recording_requester : Solver
Solver in need of recording.
data : dict
Dictionary containing outputs, residuals, and errors.
metadata : dict
Dictionary containing execution metadata.
"""
if self.connection:
abs = data['abs']
rel = data['rel']
inputs = data['input']
outputs = data['output']
residuals = data['residual']
# convert to list so this can be dumped as JSON
for i_o_r in (inputs, outputs, residuals):
if i_o_r is None:
continue
for var in i_o_r:
i_o_r[var] = make_serializable(i_o_r[var])
outputs_text = json.dumps(outputs)
inputs_text = json.dumps(inputs)
residuals_text = json.dumps(residuals)
with self.connection as c:
c = c.cursor() # need a real cursor for lastrowid
c.execute("INSERT INTO solver_iterations(counter, iteration_coordinate, "
"timestamp, success, msg, abs_err, rel_err, "
"solver_inputs, solver_output, solver_residuals) "
"VALUES(?,?,?,?,?,?,?,?,?,?)",
(self._counter, self._iteration_coordinate,
metadata['timestamp'], metadata['success'], metadata['msg'],
abs, rel, inputs_text, outputs_text, residuals_text))
# get the pathname of the source system
source_system = recording_requester._system().pathname
if source_system == '':
source_system = 'root'
# get solver type from SOLVER class attribute to determine the solver pathname
solver_type = recording_requester.SOLVER[0:2]
if solver_type == 'NL':
source_solver = source_system + '.nonlinear_solver'
elif solver_type == 'LS':
source_solver = source_system + '.nonlinear_solver.linesearch'
else:
raise RuntimeError("Solver type '%s' not recognized during recording. "
"Expecting NL or LS" % recording_requester.SOLVER)
c.execute("INSERT INTO global_iterations(record_type, rowid, source) VALUES(?,?,?)",
('solver', c.lastrowid, source_solver))
def record_iteration_solver(self, recording_requester, data, metadata):
"""
Record data and metadata from a Solver.
Parameters
----------
recording_requester : Solver
Solver in need of recording.
data : dict
Dictionary containing outputs, residuals, and errors.
metadata : dict
Dictionary containing execution metadata.
"""
if self.connection:
abs = data['abs']
rel = data['rel']
inputs = data['i']
outputs = data['o']
residuals = data['r']
# convert to list so this can be dumped as JSON
for i_o_r in (inputs, outputs, residuals):
if i_o_r is None:
continue
for var in i_o_r:
i_o_r[var] = make_serializable(i_o_r[var])
outputs_text = json.dumps(outputs)
inputs_text = json.dumps(inputs)
residuals_text = json.dumps(residuals)
with self.connection as c:
c = c.cursor() # need a real cursor for lastrowid
c.execute("INSERT INTO solver_iterations(counter, iteration_coordinate, "
"timestamp, success, msg, abs_err, rel_err, "
"solver_inputs, solver_output, solver_residuals) "
"VALUES(?,?,?,?,?,?,?,?,?,?)",
(self._counter, self._iteration_coordinate,
metadata['timestamp'], metadata['success'], metadata['msg'],
abs, rel, inputs_text, outputs_text, residuals_text))
# get the pathname of the source system
source_system = recording_requester._system.pathname
if source_system == '':
source_system = 'root'
# get solver type from SOLVER class attribute to determine the solver pathname
solver_type = recording_requester.SOLVER[0:2]
if solver_type == 'NL':
source_solver = source_system + '.nonlinear_solver'
elif solver_type == 'LS':
source_solver = source_system + '.nonlinear_solver.linesearch'
else:
raise RuntimeError("Solver type '%s' not recognized during recording. "
"Expecting NL or LS" % recording_requester.SOLVER)
c.execute("INSERT INTO global_iterations(record_type, rowid, source) VALUES(?,?,?)",
('solver', c.lastrowid, source_solver))
def record_iteration_problem(self, problem, data, metadata):
"""
Record data and metadata from a Problem.
Parameters
----------
problem : Problem
Problem in need of recording.
data : dict
Dictionary containing desvars, objectives, and constraints.
metadata : dict
Dictionary containing execution metadata.
"""
if self.connection:
outputs = data['output']
inputs = data['input']
residuals = data['residual']
driver = problem.driver
if problem.recording_options['record_derivatives'] and \
driver._designvars and driver._responses:
totals = data['totals']
else:
totals = OrderedDict([])
totals_array = dict_to_structured_array(totals)
totals_blob = array_to_blob(totals_array)
# convert to list so this can be dumped as JSON
for in_out_resid in (inputs, outputs, residuals):
if in_out_resid is None:
continue
for var in in_out_resid:
in_out_resid[var] = make_serializable(in_out_resid[var])
outputs_text = json.dumps(outputs)
inputs_text = json.dumps(inputs)
residuals_text = json.dumps(residuals)
abs_err = data['abs']
rel_err = data['rel']
with self.connection as c:
c = c.cursor() # need a real cursor for lastrowid
c.execute(
"INSERT INTO problem_cases(counter, case_name, "
"timestamp, success, msg, inputs, outputs, residuals, jacobian, "
"abs_err, rel_err ) "
"VALUES(?,?,?,?,?,?,?,?,?,?,?)",
(self._counter, metadata['name'], metadata['timestamp'],
metadata['success'], metadata['msg'], inputs_text,
outputs_text, residuals_text, totals_blob, abs_err,
rel_err))
c.execute(
"INSERT INTO global_iterations(record_type, rowid, source) VALUES(?,?,?)",
('problem', c.lastrowid, metadata['name']))
def record_iteration_system(self, system, data, metadata):
"""
Record data and metadata from a System.
Parameters
----------
system : System
System in need of recording.
data : dict
Dictionary containing inputs, outputs, and residuals.
metadata : dict
Dictionary containing execution metadata.
"""
if not self._database_initialized:
raise RuntimeError(
f"{system.msginfo} attempted to record iteration to "
f"'{self._filepath}', but database is not initialized;"
" `run_model()`, `run_driver()`, or `final_setup()` "
"must be called after adding a recorder.")
if self.connection:
inputs = data['input']
outputs = data['output']
residuals = data['residual']
# convert to list so this can be dumped as JSON
for i_o_r in (inputs, outputs, residuals):
for var, dat in i_o_r.items():
i_o_r[var] = make_serializable(dat)
outputs_text = json.dumps(outputs)
inputs_text = json.dumps(inputs)
residuals_text = json.dumps(residuals)
with self.connection as c:
c = c.cursor() # need a real cursor for lastrowid
c.execute(
"INSERT INTO system_iterations(counter, iteration_coordinate, "
"timestamp, success, msg, inputs , outputs , residuals ) "
"VALUES(?,?,?,?,?,?,?,?)",
(self._counter, self._iteration_coordinate,
metadata['timestamp'], metadata['success'],
metadata['msg'], inputs_text, outputs_text,
residuals_text))
# get the pathname of the source system
source_system = system.pathname
if source_system == '':
source_system = 'root'
c.execute(
"INSERT INTO global_iterations(record_type, rowid, source) VALUES(?,?,?)",
('system', c.lastrowid, source_system))
def record_iteration_driver(self, driver, data, metadata):
"""
Record data and metadata from a Driver.
Parameters
----------
driver : Driver
Driver in need of recording.
data : dict
Dictionary containing desvars, objectives, constraints, responses, and System vars.
metadata : dict
Dictionary containing execution metadata.
"""
if not self._database_initialized:
raise RuntimeError(
f"{driver.msginfo} attempted to record iteration to "
f"'{self._filepath}', but database is not initialized;"
" `run_model()`, `run_driver()`, or `final_setup()` "
"must be called after adding a recorder.")
if self.connection:
outputs = data['output']
inputs = data['input']
residuals = data['residual']
# convert to list so this can be dumped as JSON
for in_out_resid in (inputs, outputs, residuals):
if in_out_resid is None:
continue
for var in in_out_resid:
in_out_resid[var] = make_serializable(in_out_resid[var])
outputs_text = json.dumps(outputs)
inputs_text = json.dumps(inputs)
residuals_text = json.dumps(residuals)
with self.connection as c:
c = c.cursor() # need a real cursor for lastrowid
c.execute(
"INSERT INTO driver_iterations(counter, iteration_coordinate, "
"timestamp, success, msg, inputs, outputs, residuals) "
"VALUES(?,?,?,?,?,?,?,?)",
(self._counter, self._iteration_coordinate,
metadata['timestamp'], metadata['success'],
metadata['msg'], inputs_text, outputs_text,
residuals_text))
c.execute(
"INSERT INTO global_iterations(record_type, rowid, source) VALUES(?,?,?)",
('driver', c.lastrowid, driver._get_name()))
def record_iteration_system(self, recording_requester, data, metadata):
"""
Record data and metadata from a System.
Parameters
----------
recording_requester : System
System in need of recording.
data : dict
Dictionary containing inputs, outputs, and residuals.
metadata : dict
Dictionary containing execution metadata.
"""
if self.connection:
inputs = data['i']
outputs = data['o']
residuals = data['r']
# convert to list so this can be dumped as JSON
for i_o_r in (inputs, outputs, residuals):
if i_o_r is None:
continue
for var in i_o_r:
i_o_r[var] = make_serializable(i_o_r[var])
outputs_text = json.dumps(outputs)
inputs_text = json.dumps(inputs)
residuals_text = json.dumps(residuals)
with self.connection as c:
c = c.cursor() # need a real cursor for lastrowid
c.execute(
"INSERT INTO system_iterations(counter, iteration_coordinate, "
"timestamp, success, msg, inputs , outputs , residuals ) "
"VALUES(?,?,?,?,?,?,?,?)",
(self._counter, self._iteration_coordinate,
metadata['timestamp'], metadata['success'],
metadata['msg'], inputs_text, outputs_text,
residuals_text))
# get the pathname of the source system
source_system = recording_requester.pathname
if source_system == '':
source_system = 'root'
c.execute(
"INSERT INTO global_iterations(record_type, rowid, source) VALUES(?,?,?)",
('system', c.lastrowid, source_system))
def record_iteration_system(self, recording_requester, data, metadata):
"""
Record data and metadata from a System.
Parameters
----------
recording_requester : System
System in need of recording.
data : dict
Dictionary containing inputs, outputs, and residuals.
metadata : dict
Dictionary containing execution metadata.
"""
if self.connection:
inputs = data['i']
outputs = data['o']
residuals = data['r']
# convert to list so this can be dumped as JSON
for i_o_r in (inputs, outputs, residuals):
if i_o_r is None:
continue
for var in i_o_r:
i_o_r[var] = make_serializable(i_o_r[var])
outputs_text = json.dumps(outputs)
inputs_text = json.dumps(inputs)
residuals_text = json.dumps(residuals)
with self.connection as c:
c = c.cursor() # need a real cursor for lastrowid
c.execute("INSERT INTO system_iterations(counter, iteration_coordinate, "
"timestamp, success, msg, inputs , outputs , residuals ) "
"VALUES(?,?,?,?,?,?,?,?)",
(self._counter, self._iteration_coordinate,
metadata['timestamp'], metadata['success'], metadata['msg'],
inputs_text, outputs_text, residuals_text))
# get the pathname of the source system
source_system = recording_requester.pathname
if source_system == '':
source_system = 'root'
c.execute("INSERT INTO global_iterations(record_type, rowid, source) VALUES(?,?,?)",
('system', c.lastrowid, source_system))
def record_iteration_driver(self, driver, data, metadata):
"""
Record data and metadata from a Driver.
Parameters
----------
driver : Driver
Driver in need of recording.
data : dict
Dictionary containing desvars, objectives, constraints, responses, and System vars.
metadata : dict
Dictionary containing execution metadata.
"""
if self.connection:
outputs = data['output']
inputs = data['input']
residuals = data['residual']
# convert to list so this can be dumped as JSON
for in_out_resid in (inputs, outputs, residuals):
if in_out_resid is None:
continue
for var in in_out_resid:
in_out_resid[var] = make_serializable(in_out_resid[var])
outputs_text = json.dumps(outputs)
inputs_text = json.dumps(inputs)
residuals_text = json.dumps(residuals)
with self.connection as c:
c = c.cursor() # need a real cursor for lastrowid
c.execute(
"INSERT INTO driver_iterations(counter, iteration_coordinate, "
"timestamp, success, msg, inputs, outputs, residuals) "
"VALUES(?,?,?,?,?,?,?,?)",
(self._counter, self._iteration_coordinate,
metadata['timestamp'], metadata['success'],
metadata['msg'], inputs_text, outputs_text,
residuals_text))
c.execute(
"INSERT INTO global_iterations(record_type, rowid, source) VALUES(?,?,?)",
('driver', c.lastrowid, driver._get_name()))
def _cleanup_var_settings(self, var_settings):
"""
Convert all var_settings variable properties to a form that can be dumped as JSON.
Parameters
----------
var_settings : dict
Dictionary mapping absolute variable names to variable settings.
Returns
-------
var_settings : dict
Dictionary mapping absolute variable names to var settings that are JSON compatible.
"""
# otherwise we trample on values that are used elsewhere
var_settings = deepcopy(var_settings)
for name in var_settings:
for prop in var_settings[name]:
var_settings[name][prop] = make_serializable(var_settings[name][prop])
return var_settings
def _cleanup_var_settings(self, var_settings):
"""
Convert all var_settings variable properties to a form that can be dumped as JSON.
Parameters
----------
var_settings : dict
Dictionary mapping absolute variable names to variable settings.
Returns
-------
var_settings : dict
Dictionary mapping absolute variable names to var settings that are JSON compatible.
"""
# otherwise we trample on values that are used elsewhere
var_settings = deepcopy(var_settings)
for name in var_settings:
for prop in var_settings[name]:
var_settings[name][prop] = make_serializable(var_settings[name][prop])
return var_settings
def record_iteration_driver(self, recording_requester, data, metadata):
"""
Record data and metadata from a Driver.
Parameters
----------
recording_requester : object
Driver in need of recording.
data : dict
Dictionary containing desvars, objectives, constraints, responses, and System vars.
metadata : dict
Dictionary containing execution metadata.
"""
if self.connection:
outputs = data['out']
inputs = data['in']
# convert to list so this can be dumped as JSON
for in_out in (inputs, outputs):
if in_out is None:
continue
for var in in_out:
in_out[var] = make_serializable(in_out[var])
outputs_text = json.dumps(outputs)
inputs_text = json.dumps(inputs)
with self.connection as c:
c = c.cursor() # need a real cursor for lastrowid
c.execute("INSERT INTO driver_iterations(counter, iteration_coordinate, "
"timestamp, success, msg, inputs, outputs) VALUES(?,?,?,?,?,?,?)",
(self._counter, self._iteration_coordinate,
metadata['timestamp'], metadata['success'], metadata['msg'],
inputs_text, outputs_text))
c.execute("INSERT INTO global_iterations(record_type, rowid, source) VALUES(?,?,?)",
('driver', c.lastrowid, recording_requester._get_name()))