def _get_problem_inputs(self) -> Tuple[VariableList, VariableList]:
"""
Reads input file for the configured problem.
Needed variables and unused variables are
returned as a VariableList instance.
:return: VariableList of needed input variables, VariableList with unused variables.
"""
problem_variables = VariableList().from_problem(self)
problem_inputs_names = [var.name for var in problem_variables if var.is_input]
input_variables = DataFile(self.input_file_path)
unused_variables = VariableList(
[var for var in input_variables if var.name not in problem_inputs_names]
)
for name in unused_variables.names():
del input_variables[name]
nan_variable_names = [var.name for var in input_variables if np.all(np.isnan(var.value))]
if nan_variable_names:
raise FASTOpenMDAONanInInputFile(self.input_file_path, nan_variable_names)
return input_variables, unused_variables
def _get_problem_inputs(
self,
problem: FASTOADProblem) -> Tuple[om.IndepVarComp, VariableList]:
"""
Reads input file for the configure problem.
Needed variables are returned as an IndepVarComp instance while unused variables are
returned as a VariableList instance.
:param problem: problem with missing inputs. setup() must have been run.
:return: IVC of needed input variables, VariableList with unused variables.
"""
mandatory, optional = get_unconnected_input_names(problem,
promoted_names=True)
needed_variable_names = mandatory + optional
input_variables = DataFile(self.input_file_path)
unused_variables = VariableList([
var for var in input_variables
if var.name not in needed_variable_names
])
for name in unused_variables.names():
del input_variables[name]
nan_variable_names = [
var.name for var in input_variables if np.all(np.isnan(var.value))
]
if nan_variable_names:
raise FASTConfigurationNanInInputFile(self.input_file_path,
nan_variable_names)
input_ivc = input_variables.to_ivc()
return input_ivc, unused_variables
def test_register_checks_as_decorator(cleanup):
log_file_path = pth.join(RESULTS_FOLDER_PATH, "log4.txt")
set_logger_file(log_file_path)
@ValidityDomainChecker({"input1": (1.0, 5.0), "output2": (300.0, 500.0)}, "main.dec")
class Comp1(om.ExplicitComponent):
def setup(self):
self.add_input("input1", 0.5, units="km")
self.add_output("output1", 150.0, units="km/h", upper=130.0)
self.add_output("output2", 200.0, units="K", lower=0.0)
self.add_output("output3", 1000.0, units="kg", lower=0.0, upper=5000.0)
comp = Comp1()
# Just checking there is no side effect. VariableList.from_system() uses
# setup(), even if it is made to have no effect, and ValidityDomainChecker
# modifies setup(), so is is worth checking.
variables = VariableList.from_system(comp)
assert len(variables) == 4
# Now for the real test
# We test that upper and lower bounds are retrieved from OpenMDAO component,
# overwritten when required and that units are correctly taken into account.
comp.setup()
variables = VariableList(
[
Variable("input1", value=3.0, units="m"),
Variable("output1", value=40.0, units="m/s"),
Variable("output2", value=310.0, units="degC"),
Variable("output3", value=6.0, units="t"),
]
)
records = ValidityDomainChecker.check_variables(variables)
assert [
(
rec.variable_name,
rec.status,
rec.limit_value,
rec.value,
rec.source_file,
rec.logger_name,
)
for rec in records
] == [
("input1", ValidityStatus.TOO_LOW, 1.0, 3.0, __file__, "main.dec"),
("output1", ValidityStatus.TOO_HIGH, 130.0, 40.0, __file__, "main.dec"),
("output2", ValidityStatus.TOO_HIGH, 500.0, 310.0, __file__, "main.dec"),
("output3", ValidityStatus.TOO_HIGH, 5000.0, 6.0, __file__, "main.dec"),
]
ValidityDomainChecker.log_records(records)
with open(log_file_path) as log_file:
assert len(log_file.readlines()) == 4
def read_variables(self, data_source: Union[str, IO]) -> VariableList:
variables = VariableList()
parser = etree.XMLParser(remove_blank_text=True, remove_comments=True)
tree = etree.parse(data_source, parser)
root = tree.getroot()
for elem in root.iter():
units = elem.attrib.get(self.xml_unit_attribute, None)
is_input = elem.attrib.get(self.xml_io_attribute, None)
if units:
# Ensures compatibility with OpenMDAO units
for legacy_chars, om_chars in self.unit_translation.items():
units = re.sub(legacy_chars, om_chars, units)
units = units.replace(legacy_chars, om_chars)
value = None
if elem.text:
value = get_float_list_from_string(elem.text)
if value is not None:
try:
path_tags = [
ancestor.tag for ancestor in elem.iterancestors()
]
path_tags.reverse()
path_tags.append(elem.tag)
xpath = "/".join(path_tags[1:]) # Do not use root tag
name = self._translator.get_variable_name(xpath)
except FastXpathTranslatorXPathError as err:
_LOGGER.warning(
"The xpath %s does not have any variable "
"affected in the translator.",
err.xpath,
)
continue
if name not in variables.names():
# Add Variable
if is_input is not None:
is_input = is_input == "True"
variables[name] = {
"value": value,
"units": units,
"is_input": is_input
}
else:
raise FastXmlFormatterDuplicateVariableError(
"Variable %s is defined in more than one place in file %s"
% (name, data_source))
return variables
def run_system(component: System,
input_vars: om.IndepVarComp,
setup_mode="auto",
add_solvers=False):
"""Runs and returns an OpenMDAO problem with provided component and data"""
problem = om.Problem()
model = problem.model
model.add_subsystem("inputs", input_vars, promotes=["*"])
model.add_subsystem("component", component, promotes=["*"])
if add_solvers:
model.nonlinear_solver = om.NewtonSolver(solve_subsystems=False)
model.linear_solver = om.DirectSolver()
problem.setup(mode=setup_mode)
variable_names = [
var.name
for var in VariableList.from_problem(problem, io_status="inputs")
if np.any(np.isnan(var.val))
]
assert not variable_names, "These inputs are not provided: %s" % variable_names
problem.run_model()
return problem
def write_needed_inputs(
self,
source_file_path: str = None,
source_formatter: IVariableIOFormatter = None,
):
"""
Writes the input file of the problem with unconnected inputs of the problem.
.. warning::
:meth:`setup` must have been run on this Problem instance.
Written value of each variable will be taken:
1. from input_data if it contains the variable
2. from defined default values in component definitions
WARNING: if inputs have already been read, they won't be needed any more
and won't be in written file.
:param source_file_path: if provided, variable values will be read from it
:param source_formatter: the class that defines format of input file. if not provided,
expected format will be the default one.
"""
variables = VariableList.from_unconnected_inputs(
self, with_optional_inputs=True)
if source_file_path:
ref_vars = VariableIO(source_file_path, source_formatter).read()
variables.update(ref_vars)
for var in variables:
var.is_input = True
writer = VariableIO(self.input_file_path)
writer.write(variables)
def write_outputs(self):
"""
Writes all outputs in the configured output file.
"""
if self.output_file_path:
writer = VariableIO(self.output_file_path)
if self.additional_variables is None:
self.additional_variables = []
variables = VariableList(self.additional_variables)
for var in variables:
var.is_input = None
variables.update(
VariableList.from_problem(self, promoted_only=True), add_variables=True
)
writer.write(variables)
def get_variables(self,
column_to_attribute: Dict[str,
str] = None) -> VariableList:
"""
:param column_to_attribute: dictionary keys tell what columns are kept and the values tell what
variable attribute it corresponds to. If not provided, default translation
will apply.
:return: a variable list from current data set
"""
if not column_to_attribute:
column_to_attribute = {
value: key
for key, value in self._DEFAULT_COLUMN_RENAMING.items()
}
df = self.dataframe.copy()
df["is_input"] = None
df.loc[df["I/O"] == INPUT, "is_input"] = True
df.loc[df["I/O"] == OUTPUT, "is_input"] = False
df.drop(columns=["I/O"], inplace=True)
variables = VariableList.from_dataframe(
df[column_to_attribute.keys()].rename(columns=column_to_attribute))
return variables
def write_needed_inputs(self,
source_file_path: str = None,
source_formatter: IVariableIOFormatter = None):
"""
Writes the input file of the problem with unconnected inputs of the
configured problem.
Written value of each variable will be taken:
1. from input_data if it contains the variable
2. from defined default values in component definitions
:param source_file_path: if provided, variable values will be read from it
:param source_formatter: the class that defines format of input file. if
not provided, expected format will be the default one.
"""
problem = self.get_problem(read_inputs=False)
problem.setup()
variables = DataFile(self.input_file_path, load_data=False)
variables.update(
VariableList.from_unconnected_inputs(problem,
with_optional_inputs=True),
add_variables=True,
)
if source_file_path:
ref_vars = DataFile(source_file_path, formatter=source_formatter)
variables.update(ref_vars, add_variables=False)
for var in variables:
var.is_input = True
variables.save()
def _data_weight_decomposition(variables: VariableList, owe=None):
"""
Returns the two level weight decomposition of MTOW and optionally the decomposition of owe
subcategories.
:param variables: instance containing variables information
:param owe: value of OWE, if provided names of owe subcategories will be provided
:return: variable values, names and optionally owe subcategories names
"""
category_values = []
category_names = []
owe_subcategory_names = []
for variable in variables.names():
name_split = variable.split(":")
if isinstance(name_split, list) and len(name_split) == 4:
if name_split[0] + name_split[1] + name_split[
3] == "dataweightmass" and not ("aircraft"
in name_split[2]):
category_values.append(
convert_units(variables[variable].value[0],
variables[variable].units, "kg"))
category_names.append(name_split[2])
if owe:
owe_subcategory_names.append(
name_split[2] + "<br>" +
str(int(variables[variable].value[0])) + " [kg] (" +
str(round(variables[variable].value[0] / owe *
100, 1)) + "%)")
if owe:
result = category_values, category_names, owe_subcategory_names
else:
result = category_values, category_names, None
return result
def list_inputs(component: Union[om.ExplicitComponent, om.Group]) -> list:
""" Reads input variables from a component/problem and return as a list """
register_wrappers()
variables = VariableList.from_system(component)
input_names = [var.name for var in variables if var.is_input]
return input_names
def _update_df(self, change=None):
"""
Updates the stored DataFrame with respect to the actual values of the Sheet.
Then updates the file with respect to the stored DataFrame.
"""
frames = [
self._sheet_to_df(self._design_var_sheet),
self._sheet_to_df(self._constraint_sheet),
self._sheet_to_df(self._objective_sheet),
]
df = pd.concat(frames, sort=True)
columns = {}
columns.update(self._DEFAULT_COLUMN_RENAMING)
columns.pop("type")
column_to_attribute = {value: key for key, value in columns.items()}
variables = VariableList.from_dataframe(
df[column_to_attribute.keys()].rename(columns=column_to_attribute))
attribute_to_column = columns
df = (variables.to_dataframe().rename(columns=attribute_to_column)[
attribute_to_column.values()].reset_index(drop=True))
rows = df.index.tolist()
columns = df.columns.tolist()
for r in rows:
for c in columns:
self.dataframe.loc[int(r), c] = df.loc[r, c]
def _filter_variables(variables: VariableList,
only: Sequence[str] = None,
ignore: Sequence[str] = None) -> VariableList:
"""
filters the variables such that the ones in arg only are kept and the ones in
arg ignore are removed.
Elements of `only` and `ignore` can be variable names or Unix-shell-style patterns.
In any case, filter is case-sensitive.
:param variables:
:param only: List of OpenMDAO variable names that should be written. Other names will be
ignored. If None, all variables will be written.
:param ignore: List of OpenMDAO variable names that should be ignored when writing
:return: filtered variables
"""
# Dev note: We use sets, but sets of Variable instances do
# not work. Do we work with variable names instead.
# FIXME: Variable instances are now hashable, so set of Variable instances should now work
var_names = variables.names()
if only is None:
used_var_names = set(var_names)
else:
used_var_names = set()
for pattern in only:
used_var_names.update([
variable.name for variable in variables
if fnmatchcase(variable.name, pattern)
])
if ignore is not None:
for pattern in ignore:
used_var_names.difference_update([
variable.name for variable in variables
if fnmatchcase(variable.name, pattern)
])
# It could be simpler, but I want to keep the order
used_variables = VariableList()
for var in variables:
if var.name in used_var_names:
used_variables.append(var)
return used_variables
def write_outputs(self):
"""
Writes all outputs in the configured output file.
"""
if self.output_file_path:
writer = VariableIO(self.output_file_path)
variables = VariableList.from_problem(self)
writer.write(variables)
def test_basic_xml_read_and_write_from_vars(cleanup):
"""
Tests the creation of an XML file from a VariableList instance
"""
result_folder = pth.join(RESULTS_FOLDER_PATH, "basic_xml")
# Check write hand-made component
vars = VariableList()
vars["geometry/total_surface"] = {"value": [780.3], "units": "m**2"}
vars["geometry/wing/span"] = {"value": 42.0, "units": "m", "description": "span of the wing"}
vars["geometry/wing/aspect_ratio"] = {"value": [9.8]}
vars["geometry/fuselage/length"] = {"value": 40.0, "units": "m"}
vars["constants"] = {"value": [-42.0], "description": "the answer"}
vars["constants/k1"] = {"value": [1.0, 2.0, 3.0], "units": "kg"}
vars["constants/k2"] = {"value": [10.0, 20.0], "description": "Geronimo!"}
vars["constants/k3"] = {"value": np.array([100.0, 200.0, 300.0, 400.0]), "units": "m/s"}
vars["constants/k4"] = {"value": [-1.0, -2.0, -3.0]}
vars["constants/k5"] = {"value": [100.0, 200.0, 400.0, 500.0, 600.0]}
vars["constants/k8"] = {"value": [[1e2, 3.4e5], [5.4e3, 2.1]]}
# Try writing with non-existing folder
assert not pth.exists(result_folder)
filename = pth.join(result_folder, "handmade.xml")
xml_write = VariableIO(filename, formatter=VariableXmlStandardFormatter())
xml_write.path_separator = "/"
xml_write.write(vars)
# check (read another IndepVarComp instance from xml)
xml_check = VariableIO(filename, formatter=VariableXmlStandardFormatter())
xml_check.path_separator = ":"
new_vars = xml_check.read()
_check_basic_vars(new_vars)
# Check reading hand-made XML (with some format twists)
filename = pth.join(DATA_FOLDER_PATH, "basic.xml")
xml_read = VariableIO(filename, formatter=VariableXmlStandardFormatter())
xml_read.path_separator = ":"
vars = xml_read.read()
_check_basic_vars(vars)
# write it (with existing destination folder)
new_filename = pth.join(result_folder, "basic.xml")
xml_write = VariableIO(new_filename, formatter=VariableXmlStandardFormatter())
xml_write.path_separator = ":"
xml_write.write(vars)
# check (read another IndepVarComp instance from new xml)
xml_check = VariableIO(new_filename, formatter=VariableXmlStandardFormatter())
xml_check.path_separator = ":"
new_vars = xml_check.read()
_check_basic_vars(new_vars)
# try to write with bad separator
xml_write.formatter.path_separator = "/"
with pytest.raises(FastXPathEvalError):
xml_write.write(vars)
def list_variables(component: Union[om.ExplicitComponent, om.Group]) -> list:
""" Reads all variables from a component/problem and return as a list """
register_wrappers()
if isinstance(component, om.Group):
new_component = AutoUnitsDefaultGroup()
new_component.add_subsystem("system", component, promotes=['*'])
component = new_component
variables = VariableList.from_system(component)
return variables
def check_problem_variables(cls, problem: om.Problem):
"""
Checks variable values in provided problem and logs warnings for each variable
that is out of registered limits.
:param problem:
:return:
"""
variables = VariableList.from_problem(problem)
records = cls.check_variables(variables)
cls.log_records(records)
def get_variables(self, column_to_attribute: Dict[str, str] = None) -> VariableList:
"""
:param column_to_attribute: dictionary keys tell what columns are kept and the values
tell whatvariable attribute it corresponds to. If not
provided, default translation will apply.
:return: a variable list from current data set
"""
if not column_to_attribute:
column_to_attribute = {
value: key for key, value in self._DEFAULT_COLUMN_RENAMING.items()
}
return VariableList.from_dataframe(
self.dataframe[column_to_attribute.keys()].rename(columns=column_to_attribute)
)
def load_variables(self,
variables: VariableList,
attribute_to_column: Dict[str, str] = None):
"""
Loads provided variable list and replace current data set.
:param variables: the variables to load
:param attribute_to_column: dictionary keys tell what variable attributes are
kept and the values tell what name will be displayed. If not provided,
default translation will apply.
"""
if not attribute_to_column:
attribute_to_column = self._DEFAULT_COLUMN_RENAMING
self.dataframe = (variables.to_dataframe().rename(
columns=attribute_to_column)[
attribute_to_column.values()].reset_index(drop=True))
def check_problem_variables(cls, problem: om.Problem) -> List[CheckRecord]:
"""
Checks variable values in provided problem.
Logs warnings for each variable that is out of registered limits.
problem.setup() must have been run.
:param problem:
:return: the list of checks
"""
cls._update_problem_limit_definitions(problem)
variables = VariableList.from_problem(problem)
records = cls.check_variables(variables)
cls.log_records(records)
return records
def _update_problem_limit_definitions(cls, problem: om.Problem):
"""
Updates limit definitions using variable declarations of provided OpenMDAO system.
problem.setup() must have been run.
:param problem:
"""
variables = VariableList.from_problem(problem,
get_promoted_names=False,
promoted_only=False)
for var in variables:
system_path = var.name.split(".")
system = problem.model
for system_name in system_path[:-1]:
system = getattr(system, system_name)
var_name = system_path[-1]
if hasattr(system, "_fastoad_limit_definitions"):
limit_definitions = (
system._fastoad_limit_definitions # pylint: disable=protected-access # We added it
)
if var_name in limit_definitions:
# Get units for already defined limits
limit_def = limit_definitions[var_name]
if limit_def.units is None and var.units is not None:
limit_def.units = var.units
elif "lower" in var.metadata or "upper" in var.metadata:
# Get bounds if defined in add_output.
lower = var.metadata.get("lower")
# lower can be None if it is not found OR if it defined and set to None
if lower is None:
lower = -np.inf
upper = var.metadata.get("upper")
# upper can be None if it is not found OR if it defined and set to None
if upper is None:
upper = np.inf
units = var.metadata.get("units")
if lower > -np.inf or upper < np.inf:
limit_definitions[var_name] = _LimitDefinition(
lower, upper, units)
def run_system(
component: System, input_vars: om.IndepVarComp, setup_mode="auto", add_solvers=False
):
"""Runs and returns an OpenMDAO problem with provided component and data"""
problem = om.Problem()
model = problem.model
model.add_subsystem("inputs", input_vars, promotes=["*"])
model.add_subsystem("component", component, promotes=["*"])
if add_solvers:
model.nonlinear_solver = om.NewtonSolver(solve_subsystems=False)
model.linear_solver = om.DirectSolver()
problem.setup(mode=setup_mode)
vars = VariableList.from_unconnected_inputs(problem)
assert not vars, "These inputs are not provided: %s" % vars.names()
problem.run_model()
return problem
def setup(self):
"""Will replace the original setup method."""
# Ok kid, this is where it gets maybe too complicated...
# We need to get variables of the current OpenMDAO instance.
# This is done with VariableList.from_system() which we know does
# setup() on a copy of given instance (not much choice to be sure
# from_system() will work also for Group instances)
# But by doing this, it will call this setup() and be stuck in an
# infinite recursion.
# So we create a copy of current instance where we put back the
# original setup so we can safely use VariableList.from_system()
original_self = deepcopy(self)
setattr(original_self, "setup", getattr(original_self, setup_new_name))
variables = VariableList.from_system(original_self)
# Now update limit definition
limit_definitions = ValidityDomainChecker._limit_definitions[checker_id]
for var in variables:
if var.name in limit_definitions:
# Get units for already defined limits
limit_def = limit_definitions[var.name]
if limit_def.units is None and var.units is not None:
limit_def.units = var.units
elif "lower" in var.metadata or "upper" in var.metadata:
# Get bounds if defined in add_output.
lower = var.metadata.get("lower")
# lower can be None if it is not found OR if it defined and set to None
if lower is None:
lower = -np.inf
upper = var.metadata.get("upper")
# upper can be None if it is not found OR if it defined and set to None
if upper is None:
upper = np.inf
units = var.metadata.get("units")
if lower > -np.inf or upper < np.inf:
limit_definitions[var.name] = _LimitDefinition(lower, upper, units)
# Now run original setup that has been moved to setup_new_name
getattr(self, setup_new_name)()
def load(
self,
problem_configuration: FASTOADProblemConfigurator,
):
"""
Loads the FAST-OAD problem and stores its data.
:param problem_configuration: the FASTOADProblem instance.
"""
self.problem_configuration = problem_configuration
if pth.isfile(self.problem_configuration.input_file_path):
input_variables = DataFile(
self.problem_configuration.input_file_path)
else:
# TODO: generate the input file by default ?
raise FastMissingFile(
"Please generate input file before using the optimization viewer"
)
if pth.isfile(self.problem_configuration.output_file_path):
output_variables = DataFile(
self.problem_configuration.output_file_path)
else:
problem = self.problem_configuration.get_problem()
problem.setup()
output_variables = VariableList.from_problem(problem)
optimization_variables = VariableList()
opt_def = problem_configuration.get_optimization_definition()
# Design Variables
if KEY_DESIGN_VARIABLES in opt_def:
for name, design_var in opt_def[KEY_DESIGN_VARIABLES].items():
metadata = {
"type": "design_var",
"initial_value": input_variables[name].value,
"lower": design_var.get("lower"),
"value": output_variables[name].value,
"upper": design_var.get("upper"),
"units": input_variables[name].units,
"desc": input_variables[name].description,
}
optimization_variables[name] = metadata
# Constraints
if KEY_CONSTRAINTS in opt_def:
for name, constr in opt_def[KEY_CONSTRAINTS].items():
metadata = {
"type": "constraint",
"initial_value": None,
"lower": constr.get("lower"),
"value": output_variables[name].value,
"upper": constr.get("upper"),
"units": output_variables[name].units,
"desc": output_variables[name].description,
}
optimization_variables[name] = metadata
# Objectives
if KEY_OBJECTIVE in opt_def:
for name in opt_def[KEY_OBJECTIVE]:
metadata = {
"type": "objective",
"initial_value": None,
"lower": None,
"value": output_variables[name].value,
"upper": None,
"units": output_variables[name].units,
"desc": output_variables[name].description,
}
optimization_variables[name] = metadata
self.load_variables(optimization_variables)
def list_variables(
configuration_file_path: str,
out: Union[IO, str] = sys.stdout,
overwrite: bool = False,
force_text_output: bool = False,
):
"""
Writes list of variables for the :class:`FASTOADProblem` specified in configuration_file_path.
List is generally written as text. It can be displayed as a scrollable table view if:
- function is used in an interactive IPython shell
- out == sys.stdout
- force_text_output == False
:param configuration_file_path:
:param out: the output stream or a path for the output file
:param overwrite: if True and out parameter is a file path, the file will be written even if one
already exists
:param force_text_output: if True, list will be written as text, even if command is used in an
interactive IPython shell (Jupyter notebook). Has no effect in other
shells or if out parameter is not sys.stdout
:raise FastFileExistsError: if overwrite==False and out parameter is a file path and the file
exists
"""
conf = FASTOADProblemConfigurator(configuration_file_path)
problem = conf.get_problem()
problem.setup()
# Extracting inputs and outputs
variables = VariableList.from_problem(problem, get_promoted_names=False)
variables.sort(key=lambda var: var.name)
input_variables = VariableList([var for var in variables if var.is_input])
output_variables = VariableList(
[var for var in variables if not var.is_input])
if isinstance(out, str):
if not overwrite and pth.exists(out):
raise FastFileExistsError(
"File %s not written because it already exists. "
"Use overwrite=True to bypass." % out,
out,
)
make_parent_dir(out)
out_file = open(out, "w")
table_width = MAX_TABLE_WIDTH
else:
if out == sys.stdout and InteractiveShell.initialized(
) and not force_text_output:
# Here we display the variable list as VariableViewer in a notebook
for var in input_variables:
var.metadata["I/O"] = "IN"
for var in output_variables:
var.metadata["I/O"] = "OUT"
df = ((input_variables + output_variables).to_dataframe()[[
"I/O", "name", "desc"
]].rename(columns={
"name": "Name",
"desc": "Description"
}))
display(HTML(df.to_html()))
return
# Here we continue with text output
out_file = out
table_width = min(get_terminal_size().columns, MAX_TABLE_WIDTH) - 1
pd.set_option("display.max_colwidth", 1000)
max_name_length = np.max([
len(name)
for name in input_variables.names() + output_variables.names()
])
description_text_width = table_width - max_name_length - 2
def _write_variables(out_f, variables):
"""Writes variables and their description as a pandas DataFrame"""
df = variables.to_dataframe()
# Create a new Series where description are wrapped on several lines if needed.
# Each line becomes an element of the Series
df["desc"] = [
"\n".join(tw.wrap(s, description_text_width)) for s in df["desc"]
]
new_desc = df.desc.str.split("\n", expand=True).stack()
# Create a Series for name that will match new_desc Series. Variable name will be in front of
# first line of description. An empty string will be in front of other lines.
new_name = [
df.name.loc[i] if j == 0 else "" for i, j in new_desc.index
]
# Create the DataFrame that will be displayed
new_df = pd.DataFrame({"NAME": new_name, "DESCRIPTION": new_desc})
out_f.write(
new_df.to_string(
index=False,
columns=["NAME", "DESCRIPTION"],
justify="center",
formatters={ # Formatters are needed for enforcing left justification
"NAME": ("{:%s}" % max_name_length).format,
"DESCRIPTION": ("{:%s}" % description_text_width).format,
},
)
)
out_file.write("\n")
def _write_text_with_line(txt: str, line_length: int):
""" Writes a line of given length with provided text inside """
out_file.write("-" + txt + "-" * (line_length - 1 - len(txt)) + "\n")
# Inputs
_write_text_with_line(" INPUTS OF THE PROBLEM ", table_width)
_write_variables(out_file, input_variables)
# Outputs
out_file.write("\n")
_write_text_with_line(" OUTPUTS OF THE PROBLEM ", table_width)
_write_variables(out_file, output_variables)
_write_text_with_line("", table_width)
if isinstance(out, str):
out_file.close()
_LOGGER.info("Output list written in %s", out_file)
def test_problem_with_dynamically_shaped_inputs(cleanup):
class MyComp1(om.ExplicitComponent):
def setup(self):
self.add_input("x", shape_by_conn=True, copy_shape="y")
self.add_output("y", shape_by_conn=True, copy_shape="x")
def compute(self,
inputs,
outputs,
discrete_inputs=None,
discrete_outputs=None):
outputs["y"] = 10 * inputs["x"]
class MyComp2(om.ExplicitComponent):
def setup(self):
self.add_input("y", shape_by_conn=True, copy_shape="z")
self.add_output("z", shape_by_conn=True, copy_shape="y")
def compute(self,
inputs,
outputs,
discrete_inputs=None,
discrete_outputs=None):
outputs["z"] = 0.1 * inputs["y"]
# --------------------------------------------------------------------------
# With these 2 components, an OpenMDAO problem won't pass the setup due to
# the non-determined shapes
vanilla_problem = om.Problem()
vanilla_problem.model.add_subsystem("comp1", MyComp1(), promotes=["*"])
vanilla_problem.model.add_subsystem("comp2", MyComp2(), promotes=["*"])
with pytest.raises(RuntimeError):
vanilla_problem.setup()
# --------------------------------------------------------------------------
# ... But fastoad problem will do the setup and provide dummy shapes
# when needed
fastoad_problem = FASTOADProblem()
fastoad_problem.model.add_subsystem("comp1", MyComp1(), promotes=["*"])
fastoad_problem.model.add_subsystem("comp2", MyComp2(), promotes=["*"])
fastoad_problem.setup()
assert (fastoad_problem["x"].shape == fastoad_problem["y"].shape ==
fastoad_problem["z"].shape == (2, ))
# In such case, reading inputs after the setup will make run_model fail, because dummy shapes
# have already been provided, and will probably not match the ones in input file.
fastoad_problem.input_file_path = pth.join(DATA_FOLDER_PATH,
"dynamic_shape_inputs_1.xml")
fastoad_problem.read_inputs()
with pytest.raises(ValueError):
fastoad_problem.run_model()
# --------------------------------------------------------------------------
# If input reading is done before setup, all is fine.
fastoad_problem = FASTOADProblem()
fastoad_problem.model.add_subsystem("comp1", MyComp1(), promotes=["*"])
fastoad_problem.model.add_subsystem("comp2", MyComp2(), promotes=["*"])
fastoad_problem.input_file_path = pth.join(DATA_FOLDER_PATH,
"dynamic_shape_inputs_1.xml")
fastoad_problem.read_inputs()
fastoad_problem.setup()
inputs = VariableList.from_problem(fastoad_problem, io_status="inputs")
assert inputs.names() == ["x"]
outputs = VariableList.from_problem(fastoad_problem, io_status="outputs")
assert outputs.names() == ["y", "z"]
variables = VariableList.from_problem(fastoad_problem)
assert variables.names() == ["x", "y", "z"]
fastoad_problem.run_model()
assert_allclose(fastoad_problem["x"], [1.0, 2.0, 5.0])
assert_allclose(fastoad_problem["y"], [10.0, 20.0, 50.0])
assert_allclose(fastoad_problem["z"], [1.0, 2.0, 5.0])
# --------------------------------------------------------------------------
# In the case variables are shaped from "downstream", OpenMDAO works OK.
class MyComp3(om.ExplicitComponent):
def setup(self):
self.add_input("z", shape=(3, ))
self.add_output("a")
def compute(self,
inputs,
outputs,
discrete_inputs=None,
discrete_outputs=None):
outputs["a"] = np.sum(inputs["z"])
fastoad_problem = FASTOADProblem()
fastoad_problem.model.add_subsystem("comp1", MyComp1(), promotes=["*"])
fastoad_problem.model.add_subsystem("comp2", MyComp2(), promotes=["*"])
fastoad_problem.model.add_subsystem("comp3", MyComp3(), promotes=["*"])
inputs = VariableList.from_problem(fastoad_problem, io_status="inputs")
assert inputs.names() == ["x"]
outputs = VariableList.from_problem(fastoad_problem, io_status="outputs")
assert outputs.names() == ["y", "z", "a"]
variables = VariableList.from_problem(fastoad_problem)
assert variables.names() == ["x", "y", "z", "a"]
fastoad_problem.setup()
fastoad_problem.run_model()
def load(
self,
problem_configuration: FASTOADProblemConfigurator,
):
"""
Loads the FAST-OAD problem and stores its data.
:param problem_configuration: the FASTOADProblem instance.
:param file_formatter: the formatter that defines file format. If not provided,
default format will be assumed.
"""
self.problem_configuration = problem_configuration
problem = self.problem_configuration.get_problem()
if pth.isfile(problem.input_file_path):
input_variables = VariableIO(problem.input_file_path).read()
else:
# TODO: generate the input file by default ?
raise FastMissingFile(
"Please generate input file before using the optimization viewer"
)
if pth.isfile(problem.output_file_path):
output_variables = VariableIO(problem.output_file_path).read()
else:
output_variables = VariableList.from_problem(problem)
optimization_variables = VariableList()
opt_def = problem_configuration.get_optimization_definition()
# Design Variables
if "design_var" in opt_def:
for name, design_var in opt_def["design_var"].items():
initial_value = input_variables[name].value
if "lower" in design_var:
lower = design_var["lower"]
else:
lower = None
value = output_variables[name].value
if "upper" in design_var:
upper = design_var["upper"]
else:
upper = None
units = input_variables[name].units
desc = input_variables[name].description
metadata = {
"type": "design_var",
"initial_value": initial_value,
"lower": lower,
"value": value,
"upper": upper,
"units": units,
"desc": desc,
}
optimization_variables[name] = metadata
# Constraints
if "constraint" in opt_def:
for name, constr in opt_def["constraint"].items():
if "lower" in constr:
lower = constr["lower"]
else:
lower = None
value = output_variables[name].value
if "upper" in constr:
upper = constr["upper"]
else:
upper = None
units = output_variables[name].units
desc = output_variables[name].description
metadata = {
"type": "constraint",
"initial_value": None,
"lower": lower,
"value": value,
"upper": upper,
"units": units,
"desc": desc,
}
optimization_variables[name] = metadata
# Objectives
if "objective" in opt_def:
for name, obj in opt_def["objective"].items():
value = output_variables[name].value
units = output_variables[name].units
desc = output_variables[name].description
metadata = {
"type": "objective",
"initial_value": None,
"lower": None,
"value": value,
"upper": None,
"units": units,
"desc": desc,
}
optimization_variables[name] = metadata
self.load_variables(optimization_variables)
def list_variables(
configuration_file_path: str,
out: Union[IO, str] = None,
overwrite: bool = False,
force_text_output: bool = False,
tablefmt: str = "grid",
):
"""
Writes list of variables for the :class:`FASTOADProblem` specified in configuration_file_path.
List is generally written as text. It can be displayed as a scrollable table view if:
- function is used in an interactive IPython shell
- out == sys.stdout
- force_text_output == False
:param configuration_file_path:
:param out: the output stream or a path for the output file (None means sys.stdout)
:param overwrite: if True and out parameter is a file path, the file will be written even if one
already exists
:param force_text_output: if True, list will be written as text, even if command is used in an
interactive IPython shell (Jupyter notebook). Has no effect in other
shells or if out parameter is not sys.stdout
:param tablefmt: The formatting of the requested table. Options are the same as those available
to the tabulate package. See tabulate.tabulate_formats for a complete list.
:raise FastFileExistsError: if overwrite==False and out parameter is a file path and the file
exists
"""
if out is None:
out = sys.stdout
conf = FASTOADProblemConfigurator(configuration_file_path)
conf._set_configuration_modifier(_PROBLEM_CONFIGURATOR)
problem = conf.get_problem()
problem.setup()
# Extracting inputs and outputs
variables = VariableList.from_problem(problem)
variables.sort(key=lambda var: var.name)
input_variables = VariableList([var for var in variables if var.is_input])
output_variables = VariableList([var for var in variables if not var.is_input])
for var in input_variables:
var.metadata["I/O"] = "IN"
for var in output_variables:
var.metadata["I/O"] = "OUT"
variables_df = (
(input_variables + output_variables)
.to_dataframe()[["name", "I/O", "desc"]]
.rename(columns={"name": "NAME", "desc": "DESCRIPTION"})
)
if isinstance(out, str):
if not overwrite and pth.exists(out):
raise FastFileExistsError(
"File %s not written because it already exists. "
"Use overwrite=True to bypass." % out,
out,
)
make_parent_dir(out)
out_file = open(out, "w")
else:
if out == sys.stdout and InteractiveShell.initialized() and not force_text_output:
display(HTML(variables_df.to_html(index=False)))
return
# Here we continue with text output
out_file = out
# For a terminal output, we limit width of NAME column
variables_df["NAME"] = variables_df["NAME"].apply(lambda s: "\n".join(tw.wrap(s, 50)))
# In any case, let's break descriptions that are too long
variables_df["DESCRIPTION"] = variables_df["DESCRIPTION"].apply(
lambda s: "\n".join(tw.wrap(s, 100,))
)
out_file.write(
tabulate(variables_df, headers=variables_df.columns, showindex=False, tablefmt=tablefmt)
)
out_file.write("\n")
if isinstance(out, str):
out_file.close()
_LOGGER.info("Output list written in %s", out_file)
def test_register_checks_instantiation(cleanup):
ValidityDomainChecker(
{
"var_with_upper_and_lower": (-1.0, 10.0),
"var_with_lower": (-10.0, None),
"var_with_upper": (None, 5.0),
},
"main.logger1",
)
# Now registering is done through instantiation, and with another logger
ValidityDomainChecker(
{"var_with_upper_and_lower": (-3.0, 5.0), "var_with_lower": (0.0, None)}, "main.logger2"
)
# Third call with same logger as the first one
ValidityDomainChecker({"other_var": (-10.0, 1.0)}, "main.logger1")
# Check 1 --------------------------------------------------------------------------------------
log_file_path = pth.join(RESULTS_FOLDER_PATH, "log1.txt")
set_logger_file(log_file_path)
variables = VariableList(
[
Variable("var_with_upper_and_lower", value=-2.0, units="m"), # too low for logger1
Variable("var_with_lower", value=-1.0), # too low for logger2
Variable("var_with_upper", value=10.0), # too high, only for logger1
Variable("other_var", value=1.1), # too high, only for logger1 (second registering)
Variable("unbound_var", value=42.0),
]
)
records = ValidityDomainChecker.check_variables(variables)
assert [
(
rec.variable_name,
rec.status,
rec.limit_value,
rec.value,
rec.source_file,
rec.logger_name,
)
for rec in records
] == [
("var_with_upper_and_lower", ValidityStatus.TOO_LOW, -1.0, -2.0, __file__, "main.logger1"),
("var_with_upper_and_lower", ValidityStatus.OK, None, -2.0, __file__, "main.logger2"),
("var_with_lower", ValidityStatus.OK, None, -1.0, __file__, "main.logger1"),
("var_with_lower", ValidityStatus.TOO_LOW, 0.0, -1.0, __file__, "main.logger2"),
("var_with_upper", ValidityStatus.TOO_HIGH, 5.0, 10.0, __file__, "main.logger1"),
("other_var", ValidityStatus.TOO_HIGH, 1.0, 1.1, __file__, "main.logger1"),
]
ValidityDomainChecker.log_records(records)
with open(log_file_path) as log_file:
assert len(log_file.readlines()) == 4
# Check 2 --------------------------------------------------------------------------------------
log_file_path = pth.join(RESULTS_FOLDER_PATH, "log2.txt")
set_logger_file(log_file_path)
variables = VariableList(
[
Variable("other_var", value=-11.0), # too low, only for logger1 (second registering)
Variable("var_with_lower", value=-15.0), # too low for logger1 and logger2
Variable("var_with_upper", value=0.0), # Ok
Variable("unbound_var", value=1e42),
Variable("var_with_upper_and_lower", value=7.0), # too high for logger2
]
)
records = ValidityDomainChecker.check_variables(variables)
assert [
(
rec.variable_name,
rec.status,
rec.limit_value,
rec.value,
rec.source_file,
rec.logger_name,
)
for rec in records
] == [
("other_var", ValidityStatus.TOO_LOW, -10.0, -11.0, __file__, "main.logger1"),
("var_with_lower", ValidityStatus.TOO_LOW, -10.0, -15.0, __file__, "main.logger1"),
("var_with_lower", ValidityStatus.TOO_LOW, 0.0, -15.0, __file__, "main.logger2"),
("var_with_upper", ValidityStatus.OK, None, 0.0, __file__, "main.logger1"),
("var_with_upper_and_lower", ValidityStatus.OK, None, 7.0, __file__, "main.logger1"),
("var_with_upper_and_lower", ValidityStatus.TOO_HIGH, 5.0, 7.0, __file__, "main.logger2"),
]
ValidityDomainChecker.log_records(records)
with open(log_file_path) as log_file:
assert len(log_file.readlines()) == 4
# Check 3 --------------------------------------------------------------------------------------
log_file_path = pth.join(RESULTS_FOLDER_PATH, "log3.txt")
set_logger_file(log_file_path)
variables = VariableList(
[
Variable("var_with_upper_and_lower", value=1.0), # Ok
Variable("other_var", value=-5.0), # Ok
Variable("unbound_var", value=-1e42),
Variable("var_with_lower", value=1.0), # Ok
]
)
records = ValidityDomainChecker.check_variables(variables)
assert [
(
rec.variable_name,
rec.status,
rec.limit_value,
rec.value,
rec.source_file,
rec.logger_name,
)
for rec in records
] == [
("var_with_upper_and_lower", ValidityStatus.OK, None, 1.0, __file__, "main.logger1"),
("var_with_upper_and_lower", ValidityStatus.OK, None, 1.0, __file__, "main.logger2"),
("other_var", ValidityStatus.OK, None, -5.0, __file__, "main.logger1"),
("var_with_lower", ValidityStatus.OK, None, 1.0, __file__, "main.logger1"),
("var_with_lower", ValidityStatus.OK, None, 1.0, __file__, "main.logger2"),
]
ValidityDomainChecker.log_records(records)
with open(log_file_path) as log_file:
assert len(log_file.readlines()) == 0
