def run_system(component: System,
input_vars: om.IndepVarComp,
setup_mode="auto",
add_solvers=False):
"""Runs and returns an OpenMDAO problem with provided component and data"""
problem = om.Problem()
model = problem.model
model.add_subsystem("inputs", input_vars, promotes=["*"])
model.add_subsystem("component", component, promotes=["*"])
if add_solvers:
model.nonlinear_solver = om.NewtonSolver(solve_subsystems=False)
model.linear_solver = om.DirectSolver()
problem.setup(mode=setup_mode)
variable_names = [
var.name
for var in VariableList.from_problem(problem, io_status="inputs")
if np.any(np.isnan(var.val))
]
assert not variable_names, "These inputs are not provided: %s" % variable_names
problem.run_model()
return problem
def write_outputs(self):
"""
Writes all outputs in the configured output file.
"""
if self.output_file_path:
writer = VariableIO(self.output_file_path)
variables = VariableList.from_problem(self)
writer.write(variables)
def check_problem_variables(cls, problem: om.Problem):
"""
Checks variable values in provided problem and logs warnings for each variable
that is out of registered limits.
:param problem:
:return:
"""
variables = VariableList.from_problem(problem)
records = cls.check_variables(variables)
cls.log_records(records)
def write_outputs(self):
"""
Writes all outputs in the configured output file.
"""
if self.output_file_path:
writer = VariableIO(self.output_file_path)
if self.additional_variables is None:
self.additional_variables = []
variables = VariableList(self.additional_variables)
for var in variables:
var.is_input = None
variables.update(
VariableList.from_problem(self, promoted_only=True), add_variables=True
)
writer.write(variables)
def check_problem_variables(cls, problem: om.Problem) -> List[CheckRecord]:
"""
Checks variable values in provided problem.
Logs warnings for each variable that is out of registered limits.
problem.setup() must have been run.
:param problem:
:return: the list of checks
"""
cls._update_problem_limit_definitions(problem)
variables = VariableList.from_problem(problem)
records = cls.check_variables(variables)
cls.log_records(records)
return records
def _update_problem_limit_definitions(cls, problem: om.Problem):
"""
Updates limit definitions using variable declarations of provided OpenMDAO system.
problem.setup() must have been run.
:param problem:
"""
variables = VariableList.from_problem(problem,
get_promoted_names=False,
promoted_only=False)
for var in variables:
system_path = var.name.split(".")
system = problem.model
for system_name in system_path[:-1]:
system = getattr(system, system_name)
var_name = system_path[-1]
if hasattr(system, "_fastoad_limit_definitions"):
limit_definitions = (
system._fastoad_limit_definitions # pylint: disable=protected-access # We added it
)
if var_name in limit_definitions:
# Get units for already defined limits
limit_def = limit_definitions[var_name]
if limit_def.units is None and var.units is not None:
limit_def.units = var.units
elif "lower" in var.metadata or "upper" in var.metadata:
# Get bounds if defined in add_output.
lower = var.metadata.get("lower")
# lower can be None if it is not found OR if it defined and set to None
if lower is None:
lower = -np.inf
upper = var.metadata.get("upper")
# upper can be None if it is not found OR if it defined and set to None
if upper is None:
upper = np.inf
units = var.metadata.get("units")
if lower > -np.inf or upper < np.inf:
limit_definitions[var_name] = _LimitDefinition(
lower, upper, units)
def write_needed_inputs(
self, source_file_path: str = None, source_formatter: IVariableIOFormatter = None
):
"""
Writes the input file of the problem with unconnected inputs of the
configured problem.
Written value of each variable will be taken:
1. from input_data if it contains the variable
2. from defined default values in component definitions
:param source_file_path: if provided, variable values will be read from it
:param source_formatter: the class that defines format of input file. if
not provided, expected format will be the default one.
"""
problem = self.get_problem(read_inputs=False)
problem.setup()
variables = DataFile(self.input_file_path, load_data=False)
unconnected_inputs = VariableList.from_problem(
problem,
use_initial_values=True,
get_promoted_names=True,
promoted_only=True,
io_status="inputs",
)
variables.update(
unconnected_inputs,
add_variables=True,
)
if source_file_path:
ref_vars = DataFile(source_file_path, formatter=source_formatter)
variables.update(ref_vars, add_variables=False)
for var in variables:
var.is_input = True
variables.save()
def list_variables(
configuration_file_path: str,
out: Union[IO, str] = None,
overwrite: bool = False,
force_text_output: bool = False,
tablefmt: str = "grid",
):
"""
Writes list of variables for the :class:`FASTOADProblem` specified in configuration_file_path.
List is generally written as text. It can be displayed as a scrollable table view if:
- function is used in an interactive IPython shell
- out == sys.stdout
- force_text_output == False
:param configuration_file_path:
:param out: the output stream or a path for the output file (None means sys.stdout)
:param overwrite: if True and out parameter is a file path, the file will be written even if one
already exists
:param force_text_output: if True, list will be written as text, even if command is used in an
interactive IPython shell (Jupyter notebook). Has no effect in other
shells or if out parameter is not sys.stdout
:param tablefmt: The formatting of the requested table. Options are the same as those available
to the tabulate package. See tabulate.tabulate_formats for a complete list.
:raise FastFileExistsError: if overwrite==False and out parameter is a file path and the file
exists
"""
if out is None:
out = sys.stdout
conf = FASTOADProblemConfigurator(configuration_file_path)
conf._set_configuration_modifier(_PROBLEM_CONFIGURATOR)
problem = conf.get_problem()
problem.setup()
# Extracting inputs and outputs
variables = VariableList.from_problem(problem)
variables.sort(key=lambda var: var.name)
input_variables = VariableList([var for var in variables if var.is_input])
output_variables = VariableList([var for var in variables if not var.is_input])
for var in input_variables:
var.metadata["I/O"] = "IN"
for var in output_variables:
var.metadata["I/O"] = "OUT"
variables_df = (
(input_variables + output_variables)
.to_dataframe()[["name", "I/O", "desc"]]
.rename(columns={"name": "NAME", "desc": "DESCRIPTION"})
)
if isinstance(out, str):
if not overwrite and pth.exists(out):
raise FastFileExistsError(
"File %s not written because it already exists. "
"Use overwrite=True to bypass." % out,
out,
)
make_parent_dir(out)
out_file = open(out, "w")
else:
if out == sys.stdout and InteractiveShell.initialized() and not force_text_output:
display(HTML(variables_df.to_html(index=False)))
return
# Here we continue with text output
out_file = out
# For a terminal output, we limit width of NAME column
variables_df["NAME"] = variables_df["NAME"].apply(lambda s: "\n".join(tw.wrap(s, 50)))
# In any case, let's break descriptions that are too long
variables_df["DESCRIPTION"] = variables_df["DESCRIPTION"].apply(
lambda s: "\n".join(tw.wrap(s, 100,))
)
out_file.write(
tabulate(variables_df, headers=variables_df.columns, showindex=False, tablefmt=tablefmt)
)
out_file.write("\n")
if isinstance(out, str):
out_file.close()
_LOGGER.info("Output list written in %s", out_file)
def load(
self,
problem_configuration: FASTOADProblemConfigurator,
):
"""
Loads the FAST-OAD problem and stores its data.
:param problem_configuration: the FASTOADProblem instance.
"""
self.problem_configuration = problem_configuration
if pth.isfile(self.problem_configuration.input_file_path):
input_variables = DataFile(
self.problem_configuration.input_file_path)
else:
# TODO: generate the input file by default ?
raise FastMissingFile(
"Please generate input file before using the optimization viewer"
)
if pth.isfile(self.problem_configuration.output_file_path):
output_variables = DataFile(
self.problem_configuration.output_file_path)
else:
problem = self.problem_configuration.get_problem()
problem.setup()
output_variables = VariableList.from_problem(problem)
optimization_variables = VariableList()
opt_def = problem_configuration.get_optimization_definition()
# Design Variables
if KEY_DESIGN_VARIABLES in opt_def:
for name, design_var in opt_def[KEY_DESIGN_VARIABLES].items():
metadata = {
"type": "design_var",
"initial_value": input_variables[name].value,
"lower": design_var.get("lower"),
"value": output_variables[name].value,
"upper": design_var.get("upper"),
"units": input_variables[name].units,
"desc": input_variables[name].description,
}
optimization_variables[name] = metadata
# Constraints
if KEY_CONSTRAINTS in opt_def:
for name, constr in opt_def[KEY_CONSTRAINTS].items():
metadata = {
"type": "constraint",
"initial_value": None,
"lower": constr.get("lower"),
"value": output_variables[name].value,
"upper": constr.get("upper"),
"units": output_variables[name].units,
"desc": output_variables[name].description,
}
optimization_variables[name] = metadata
# Objectives
if KEY_OBJECTIVE in opt_def:
for name in opt_def[KEY_OBJECTIVE]:
metadata = {
"type": "objective",
"initial_value": None,
"lower": None,
"value": output_variables[name].value,
"upper": None,
"units": output_variables[name].units,
"desc": output_variables[name].description,
}
optimization_variables[name] = metadata
self.load_variables(optimization_variables)
def load(
self,
problem_configuration: FASTOADProblemConfigurator,
):
"""
Loads the FAST-OAD problem and stores its data.
:param problem_configuration: the FASTOADProblem instance.
:param file_formatter: the formatter that defines file format. If not provided,
default format will be assumed.
"""
self.problem_configuration = problem_configuration
problem = self.problem_configuration.get_problem()
if pth.isfile(problem.input_file_path):
input_variables = VariableIO(problem.input_file_path).read()
else:
# TODO: generate the input file by default ?
raise FastMissingFile(
"Please generate input file before using the optimization viewer"
)
if pth.isfile(problem.output_file_path):
output_variables = VariableIO(problem.output_file_path).read()
else:
output_variables = VariableList.from_problem(problem)
optimization_variables = VariableList()
opt_def = problem_configuration.get_optimization_definition()
# Design Variables
if "design_var" in opt_def:
for name, design_var in opt_def["design_var"].items():
initial_value = input_variables[name].value
if "lower" in design_var:
lower = design_var["lower"]
else:
lower = None
value = output_variables[name].value
if "upper" in design_var:
upper = design_var["upper"]
else:
upper = None
units = input_variables[name].units
desc = input_variables[name].description
metadata = {
"type": "design_var",
"initial_value": initial_value,
"lower": lower,
"value": value,
"upper": upper,
"units": units,
"desc": desc,
}
optimization_variables[name] = metadata
# Constraints
if "constraint" in opt_def:
for name, constr in opt_def["constraint"].items():
if "lower" in constr:
lower = constr["lower"]
else:
lower = None
value = output_variables[name].value
if "upper" in constr:
upper = constr["upper"]
else:
upper = None
units = output_variables[name].units
desc = output_variables[name].description
metadata = {
"type": "constraint",
"initial_value": None,
"lower": lower,
"value": value,
"upper": upper,
"units": units,
"desc": desc,
}
optimization_variables[name] = metadata
# Objectives
if "objective" in opt_def:
for name, obj in opt_def["objective"].items():
value = output_variables[name].value
units = output_variables[name].units
desc = output_variables[name].description
metadata = {
"type": "objective",
"initial_value": None,
"lower": None,
"value": value,
"upper": None,
"units": units,
"desc": desc,
}
optimization_variables[name] = metadata
self.load_variables(optimization_variables)
def list_variables(
configuration_file_path: str,
out: Union[IO, str] = sys.stdout,
overwrite: bool = False,
force_text_output: bool = False,
):
"""
Writes list of variables for the :class:`FASTOADProblem` specified in configuration_file_path.
List is generally written as text. It can be displayed as a scrollable table view if:
- function is used in an interactive IPython shell
- out == sys.stdout
- force_text_output == False
:param configuration_file_path:
:param out: the output stream or a path for the output file
:param overwrite: if True and out parameter is a file path, the file will be written even if one
already exists
:param force_text_output: if True, list will be written as text, even if command is used in an
interactive IPython shell (Jupyter notebook). Has no effect in other
shells or if out parameter is not sys.stdout
:raise FastFileExistsError: if overwrite==False and out parameter is a file path and the file
exists
"""
conf = FASTOADProblemConfigurator(configuration_file_path)
problem = conf.get_problem()
problem.setup()
# Extracting inputs and outputs
variables = VariableList.from_problem(problem, get_promoted_names=False)
variables.sort(key=lambda var: var.name)
input_variables = VariableList([var for var in variables if var.is_input])
output_variables = VariableList(
[var for var in variables if not var.is_input])
if isinstance(out, str):
if not overwrite and pth.exists(out):
raise FastFileExistsError(
"File %s not written because it already exists. "
"Use overwrite=True to bypass." % out,
out,
)
make_parent_dir(out)
out_file = open(out, "w")
table_width = MAX_TABLE_WIDTH
else:
if out == sys.stdout and InteractiveShell.initialized(
) and not force_text_output:
# Here we display the variable list as VariableViewer in a notebook
for var in input_variables:
var.metadata["I/O"] = "IN"
for var in output_variables:
var.metadata["I/O"] = "OUT"
df = ((input_variables + output_variables).to_dataframe()[[
"I/O", "name", "desc"
]].rename(columns={
"name": "Name",
"desc": "Description"
}))
display(HTML(df.to_html()))
return
# Here we continue with text output
out_file = out
table_width = min(get_terminal_size().columns, MAX_TABLE_WIDTH) - 1
pd.set_option("display.max_colwidth", 1000)
max_name_length = np.max([
len(name)
for name in input_variables.names() + output_variables.names()
])
description_text_width = table_width - max_name_length - 2
def _write_variables(out_f, variables):
"""Writes variables and their description as a pandas DataFrame"""
df = variables.to_dataframe()
# Create a new Series where description are wrapped on several lines if needed.
# Each line becomes an element of the Series
df["desc"] = [
"\n".join(tw.wrap(s, description_text_width)) for s in df["desc"]
]
new_desc = df.desc.str.split("\n", expand=True).stack()
# Create a Series for name that will match new_desc Series. Variable name will be in front of
# first line of description. An empty string will be in front of other lines.
new_name = [
df.name.loc[i] if j == 0 else "" for i, j in new_desc.index
]
# Create the DataFrame that will be displayed
new_df = pd.DataFrame({"NAME": new_name, "DESCRIPTION": new_desc})
out_f.write(
new_df.to_string(
index=False,
columns=["NAME", "DESCRIPTION"],
justify="center",
formatters={ # Formatters are needed for enforcing left justification
"NAME": ("{:%s}" % max_name_length).format,
"DESCRIPTION": ("{:%s}" % description_text_width).format,
},
)
)
out_file.write("\n")
def _write_text_with_line(txt: str, line_length: int):
""" Writes a line of given length with provided text inside """
out_file.write("-" + txt + "-" * (line_length - 1 - len(txt)) + "\n")
# Inputs
_write_text_with_line(" INPUTS OF THE PROBLEM ", table_width)
_write_variables(out_file, input_variables)
# Outputs
out_file.write("\n")
_write_text_with_line(" OUTPUTS OF THE PROBLEM ", table_width)
_write_variables(out_file, output_variables)
_write_text_with_line("", table_width)
if isinstance(out, str):
out_file.close()
_LOGGER.info("Output list written in %s", out_file)
def test_register_checks_as_decorator(cleanup):
log_file_path = pth.join(RESULTS_FOLDER_PATH, "log4.txt")
set_logger_file(log_file_path)
@ValidityDomainChecker({
"input1": (1.0, 5.0),
"output2": (300.0, 500.0)
}, "main.dec")
class Comp1(om.ExplicitComponent):
def setup(self):
self.add_input("input1", 0.5, units="km")
self.add_output("output1", 150.0, units="km/h", upper=130.0)
self.add_output("output2", 200.0, units="K", lower=0.0)
self.add_output("output3",
1000.0,
units="kg",
lower=0.0,
upper=5000.0)
comp = Comp1()
problem = om.Problem()
problem.model.add_subsystem("comp", comp, promotes=["*"])
problem.setup()
# Just checking there is no side effect. VariableList.from_system() uses
# setup(), even if it is made to have no effect, and ValidityDomainChecker
# modifies setup(), so is is worth checking.
variables = VariableList.from_problem(problem)
assert len(variables) == 4
# Now for the real test
# We test that upper and lower bounds are retrieved from OpenMDAO component,
# overwritten when required and that units are correctly taken into account.
ValidityDomainChecker._update_problem_limit_definitions(problem)
variables = VariableList([
Variable("input1", value=3.0, units="m"),
Variable("output1", value=40.0, units="m/s"),
Variable("output2", value=310.0, units="degC"),
Variable("output3", value=6.0, units="t"),
])
records = ValidityDomainChecker.check_variables(variables)
assert [(
rec.variable_name,
rec.status,
rec.limit_value,
rec.value,
rec.source_file,
rec.logger_name,
) for rec in records] == [
("input1", ValidityStatus.TOO_LOW, 1.0, 3.0, __file__, "main.dec"),
("output1", ValidityStatus.TOO_HIGH, 130.0, 40.0, __file__,
"main.dec"),
("output2", ValidityStatus.TOO_HIGH, 500.0, 310.0, __file__,
"main.dec"),
("output3", ValidityStatus.TOO_HIGH, 5000.0, 6.0, __file__,
"main.dec"),
]
ValidityDomainChecker.log_records(records)
with open(log_file_path) as log_file:
assert len(log_file.readlines()) == 4
def test_problem_with_dynamically_shaped_inputs(cleanup):
class MyComp1(om.ExplicitComponent):
def setup(self):
self.add_input("x", shape_by_conn=True, copy_shape="y")
self.add_output("y", shape_by_conn=True, copy_shape="x")
def compute(self,
inputs,
outputs,
discrete_inputs=None,
discrete_outputs=None):
outputs["y"] = 10 * inputs["x"]
class MyComp2(om.ExplicitComponent):
def setup(self):
self.add_input("y", shape_by_conn=True, copy_shape="z")
self.add_output("z", shape_by_conn=True, copy_shape="y")
def compute(self,
inputs,
outputs,
discrete_inputs=None,
discrete_outputs=None):
outputs["z"] = 0.1 * inputs["y"]
# --------------------------------------------------------------------------
# With these 2 components, an OpenMDAO problem won't pass the setup due to
# the non-determined shapes
vanilla_problem = om.Problem()
vanilla_problem.model.add_subsystem("comp1", MyComp1(), promotes=["*"])
vanilla_problem.model.add_subsystem("comp2", MyComp2(), promotes=["*"])
with pytest.raises(RuntimeError):
vanilla_problem.setup()
# --------------------------------------------------------------------------
# ... But fastoad problem will do the setup and provide dummy shapes
# when needed
fastoad_problem = FASTOADProblem()
fastoad_problem.model.add_subsystem("comp1", MyComp1(), promotes=["*"])
fastoad_problem.model.add_subsystem("comp2", MyComp2(), promotes=["*"])
fastoad_problem.setup()
assert (fastoad_problem["x"].shape == fastoad_problem["y"].shape ==
fastoad_problem["z"].shape == (2, ))
# In such case, reading inputs after the setup will make run_model fail, because dummy shapes
# have already been provided, and will probably not match the ones in input file.
fastoad_problem.input_file_path = pth.join(DATA_FOLDER_PATH,
"dynamic_shape_inputs_1.xml")
fastoad_problem.read_inputs()
with pytest.raises(ValueError):
fastoad_problem.run_model()
# --------------------------------------------------------------------------
# If input reading is done before setup, all is fine.
fastoad_problem = FASTOADProblem()
fastoad_problem.model.add_subsystem("comp1", MyComp1(), promotes=["*"])
fastoad_problem.model.add_subsystem("comp2", MyComp2(), promotes=["*"])
fastoad_problem.input_file_path = pth.join(DATA_FOLDER_PATH,
"dynamic_shape_inputs_1.xml")
fastoad_problem.read_inputs()
fastoad_problem.setup()
inputs = VariableList.from_problem(fastoad_problem, io_status="inputs")
assert inputs.names() == ["x"]
outputs = VariableList.from_problem(fastoad_problem, io_status="outputs")
assert outputs.names() == ["y", "z"]
variables = VariableList.from_problem(fastoad_problem)
assert variables.names() == ["x", "y", "z"]
fastoad_problem.run_model()
assert_allclose(fastoad_problem["x"], [1.0, 2.0, 5.0])
assert_allclose(fastoad_problem["y"], [10.0, 20.0, 50.0])
assert_allclose(fastoad_problem["z"], [1.0, 2.0, 5.0])
# --------------------------------------------------------------------------
# In the case variables are shaped from "downstream", OpenMDAO works OK.
class MyComp3(om.ExplicitComponent):
def setup(self):
self.add_input("z", shape=(3, ))
self.add_output("a")
def compute(self,
inputs,
outputs,
discrete_inputs=None,
discrete_outputs=None):
outputs["a"] = np.sum(inputs["z"])
fastoad_problem = FASTOADProblem()
fastoad_problem.model.add_subsystem("comp1", MyComp1(), promotes=["*"])
fastoad_problem.model.add_subsystem("comp2", MyComp2(), promotes=["*"])
fastoad_problem.model.add_subsystem("comp3", MyComp3(), promotes=["*"])
inputs = VariableList.from_problem(fastoad_problem, io_status="inputs")
assert inputs.names() == ["x"]
outputs = VariableList.from_problem(fastoad_problem, io_status="outputs")
assert outputs.names() == ["y", "z", "a"]
variables = VariableList.from_problem(fastoad_problem)
assert variables.names() == ["x", "y", "z", "a"]
fastoad_problem.setup()
fastoad_problem.run_model()
