def _get_problem_inputs(self) -> Tuple[VariableList, VariableList]: """ Reads input file for the configured problem. Needed variables and unused variables are returned as a VariableList instance. :return: VariableList of needed input variables, VariableList with unused variables. """ problem_variables = VariableList().from_problem(self) problem_inputs_names = [var.name for var in problem_variables if var.is_input] input_variables = DataFile(self.input_file_path) unused_variables = VariableList( [var for var in input_variables if var.name not in problem_inputs_names] ) for name in unused_variables.names(): del input_variables[name] nan_variable_names = [var.name for var in input_variables if np.all(np.isnan(var.value))] if nan_variable_names: raise FASTOpenMDAONanInInputFile(self.input_file_path, nan_variable_names) return input_variables, unused_variables
def _get_problem_inputs( self, problem: FASTOADProblem) -> Tuple[om.IndepVarComp, VariableList]: """ Reads input file for the configure problem. Needed variables are returned as an IndepVarComp instance while unused variables are returned as a VariableList instance. :param problem: problem with missing inputs. setup() must have been run. :return: IVC of needed input variables, VariableList with unused variables. """ mandatory, optional = get_unconnected_input_names(problem, promoted_names=True) needed_variable_names = mandatory + optional input_variables = DataFile(self.input_file_path) unused_variables = VariableList([ var for var in input_variables if var.name not in needed_variable_names ]) for name in unused_variables.names(): del input_variables[name] nan_variable_names = [ var.name for var in input_variables if np.all(np.isnan(var.value)) ] if nan_variable_names: raise FASTConfigurationNanInInputFile(self.input_file_path, nan_variable_names) input_ivc = input_variables.to_ivc() return input_ivc, unused_variables
def test_basic_xml_read_and_write_from_vars(cleanup): """ Tests the creation of an XML file from a VariableList instance """ result_folder = pth.join(RESULTS_FOLDER_PATH, "basic_xml") # Check write hand-made component vars = VariableList() vars["geometry/total_surface"] = {"value": [780.3], "units": "m**2"} vars["geometry/wing/span"] = {"value": 42.0, "units": "m", "description": "span of the wing"} vars["geometry/wing/aspect_ratio"] = {"value": [9.8]} vars["geometry/fuselage/length"] = {"value": 40.0, "units": "m"} vars["constants"] = {"value": [-42.0], "description": "the answer"} vars["constants/k1"] = {"value": [1.0, 2.0, 3.0], "units": "kg"} vars["constants/k2"] = {"value": [10.0, 20.0], "description": "Geronimo!"} vars["constants/k3"] = {"value": np.array([100.0, 200.0, 300.0, 400.0]), "units": "m/s"} vars["constants/k4"] = {"value": [-1.0, -2.0, -3.0]} vars["constants/k5"] = {"value": [100.0, 200.0, 400.0, 500.0, 600.0]} vars["constants/k8"] = {"value": [[1e2, 3.4e5], [5.4e3, 2.1]]} # Try writing with non-existing folder assert not pth.exists(result_folder) filename = pth.join(result_folder, "handmade.xml") xml_write = VariableIO(filename, formatter=VariableXmlStandardFormatter()) xml_write.path_separator = "/" xml_write.write(vars) # check (read another IndepVarComp instance from xml) xml_check = VariableIO(filename, formatter=VariableXmlStandardFormatter()) xml_check.path_separator = ":" new_vars = xml_check.read() _check_basic_vars(new_vars) # Check reading hand-made XML (with some format twists) filename = pth.join(DATA_FOLDER_PATH, "basic.xml") xml_read = VariableIO(filename, formatter=VariableXmlStandardFormatter()) xml_read.path_separator = ":" vars = xml_read.read() _check_basic_vars(vars) # write it (with existing destination folder) new_filename = pth.join(result_folder, "basic.xml") xml_write = VariableIO(new_filename, formatter=VariableXmlStandardFormatter()) xml_write.path_separator = ":" xml_write.write(vars) # check (read another IndepVarComp instance from new xml) xml_check = VariableIO(new_filename, formatter=VariableXmlStandardFormatter()) xml_check.path_separator = ":" new_vars = xml_check.read() _check_basic_vars(new_vars) # try to write with bad separator xml_write.formatter.path_separator = "/" with pytest.raises(FastXPathEvalError): xml_write.write(vars)
def test_register_checks_as_decorator(cleanup): log_file_path = pth.join(RESULTS_FOLDER_PATH, "log4.txt") set_logger_file(log_file_path) @ValidityDomainChecker({"input1": (1.0, 5.0), "output2": (300.0, 500.0)}, "main.dec") class Comp1(om.ExplicitComponent): def setup(self): self.add_input("input1", 0.5, units="km") self.add_output("output1", 150.0, units="km/h", upper=130.0) self.add_output("output2", 200.0, units="K", lower=0.0) self.add_output("output3", 1000.0, units="kg", lower=0.0, upper=5000.0) comp = Comp1() # Just checking there is no side effect. VariableList.from_system() uses # setup(), even if it is made to have no effect, and ValidityDomainChecker # modifies setup(), so is is worth checking. variables = VariableList.from_system(comp) assert len(variables) == 4 # Now for the real test # We test that upper and lower bounds are retrieved from OpenMDAO component, # overwritten when required and that units are correctly taken into account. comp.setup() variables = VariableList( [ Variable("input1", value=3.0, units="m"), Variable("output1", value=40.0, units="m/s"), Variable("output2", value=310.0, units="degC"), Variable("output3", value=6.0, units="t"), ] ) records = ValidityDomainChecker.check_variables(variables) assert [ ( rec.variable_name, rec.status, rec.limit_value, rec.value, rec.source_file, rec.logger_name, ) for rec in records ] == [ ("input1", ValidityStatus.TOO_LOW, 1.0, 3.0, __file__, "main.dec"), ("output1", ValidityStatus.TOO_HIGH, 130.0, 40.0, __file__, "main.dec"), ("output2", ValidityStatus.TOO_HIGH, 500.0, 310.0, __file__, "main.dec"), ("output3", ValidityStatus.TOO_HIGH, 5000.0, 6.0, __file__, "main.dec"), ] ValidityDomainChecker.log_records(records) with open(log_file_path) as log_file: assert len(log_file.readlines()) == 4
def read_variables(self, data_source: Union[str, IO]) -> VariableList: variables = VariableList() parser = etree.XMLParser(remove_blank_text=True, remove_comments=True) tree = etree.parse(data_source, parser) root = tree.getroot() for elem in root.iter(): units = elem.attrib.get(self.xml_unit_attribute, None) is_input = elem.attrib.get(self.xml_io_attribute, None) if units: # Ensures compatibility with OpenMDAO units for legacy_chars, om_chars in self.unit_translation.items(): units = re.sub(legacy_chars, om_chars, units) units = units.replace(legacy_chars, om_chars) value = None if elem.text: value = get_float_list_from_string(elem.text) if value is not None: try: path_tags = [ ancestor.tag for ancestor in elem.iterancestors() ] path_tags.reverse() path_tags.append(elem.tag) xpath = "/".join(path_tags[1:]) # Do not use root tag name = self._translator.get_variable_name(xpath) except FastXpathTranslatorXPathError as err: _LOGGER.warning( "The xpath %s does not have any variable " "affected in the translator.", err.xpath, ) continue if name not in variables.names(): # Add Variable if is_input is not None: is_input = is_input == "True" variables[name] = { "value": value, "units": units, "is_input": is_input } else: raise FastXmlFormatterDuplicateVariableError( "Variable %s is defined in more than one place in file %s" % (name, data_source)) return variables
def write_outputs(self): """ Writes all outputs in the configured output file. """ if self.output_file_path: writer = VariableIO(self.output_file_path) if self.additional_variables is None: self.additional_variables = [] variables = VariableList(self.additional_variables) for var in variables: var.is_input = None variables.update( VariableList.from_problem(self, promoted_only=True), add_variables=True ) writer.write(variables)
def _filter_variables(variables: VariableList, only: Sequence[str] = None, ignore: Sequence[str] = None) -> VariableList: """ filters the variables such that the ones in arg only are kept and the ones in arg ignore are removed. Elements of `only` and `ignore` can be variable names or Unix-shell-style patterns. In any case, filter is case-sensitive. :param variables: :param only: List of OpenMDAO variable names that should be written. Other names will be ignored. If None, all variables will be written. :param ignore: List of OpenMDAO variable names that should be ignored when writing :return: filtered variables """ # Dev note: We use sets, but sets of Variable instances do # not work. Do we work with variable names instead. # FIXME: Variable instances are now hashable, so set of Variable instances should now work var_names = variables.names() if only is None: used_var_names = set(var_names) else: used_var_names = set() for pattern in only: used_var_names.update([ variable.name for variable in variables if fnmatchcase(variable.name, pattern) ]) if ignore is not None: for pattern in ignore: used_var_names.difference_update([ variable.name for variable in variables if fnmatchcase(variable.name, pattern) ]) # It could be simpler, but I want to keep the order used_variables = VariableList() for var in variables: if var.name in used_var_names: used_variables.append(var) return used_variables
def list_variables( configuration_file_path: str, out: Union[IO, str] = None, overwrite: bool = False, force_text_output: bool = False, tablefmt: str = "grid", ): """ Writes list of variables for the :class:`FASTOADProblem` specified in configuration_file_path. List is generally written as text. It can be displayed as a scrollable table view if: - function is used in an interactive IPython shell - out == sys.stdout - force_text_output == False :param configuration_file_path: :param out: the output stream or a path for the output file (None means sys.stdout) :param overwrite: if True and out parameter is a file path, the file will be written even if one already exists :param force_text_output: if True, list will be written as text, even if command is used in an interactive IPython shell (Jupyter notebook). Has no effect in other shells or if out parameter is not sys.stdout :param tablefmt: The formatting of the requested table. Options are the same as those available to the tabulate package. See tabulate.tabulate_formats for a complete list. :raise FastFileExistsError: if overwrite==False and out parameter is a file path and the file exists """ if out is None: out = sys.stdout conf = FASTOADProblemConfigurator(configuration_file_path) conf._set_configuration_modifier(_PROBLEM_CONFIGURATOR) problem = conf.get_problem() problem.setup() # Extracting inputs and outputs variables = VariableList.from_problem(problem) variables.sort(key=lambda var: var.name) input_variables = VariableList([var for var in variables if var.is_input]) output_variables = VariableList([var for var in variables if not var.is_input]) for var in input_variables: var.metadata["I/O"] = "IN" for var in output_variables: var.metadata["I/O"] = "OUT" variables_df = ( (input_variables + output_variables) .to_dataframe()[["name", "I/O", "desc"]] .rename(columns={"name": "NAME", "desc": "DESCRIPTION"}) ) if isinstance(out, str): if not overwrite and pth.exists(out): raise FastFileExistsError( "File %s not written because it already exists. " "Use overwrite=True to bypass." % out, out, ) make_parent_dir(out) out_file = open(out, "w") else: if out == sys.stdout and InteractiveShell.initialized() and not force_text_output: display(HTML(variables_df.to_html(index=False))) return # Here we continue with text output out_file = out # For a terminal output, we limit width of NAME column variables_df["NAME"] = variables_df["NAME"].apply(lambda s: "\n".join(tw.wrap(s, 50))) # In any case, let's break descriptions that are too long variables_df["DESCRIPTION"] = variables_df["DESCRIPTION"].apply( lambda s: "\n".join(tw.wrap(s, 100,)) ) out_file.write( tabulate(variables_df, headers=variables_df.columns, showindex=False, tablefmt=tablefmt) ) out_file.write("\n") if isinstance(out, str): out_file.close() _LOGGER.info("Output list written in %s", out_file)
def load( self, problem_configuration: FASTOADProblemConfigurator, ): """ Loads the FAST-OAD problem and stores its data. :param problem_configuration: the FASTOADProblem instance. """ self.problem_configuration = problem_configuration if pth.isfile(self.problem_configuration.input_file_path): input_variables = DataFile( self.problem_configuration.input_file_path) else: # TODO: generate the input file by default ? raise FastMissingFile( "Please generate input file before using the optimization viewer" ) if pth.isfile(self.problem_configuration.output_file_path): output_variables = DataFile( self.problem_configuration.output_file_path) else: problem = self.problem_configuration.get_problem() problem.setup() output_variables = VariableList.from_problem(problem) optimization_variables = VariableList() opt_def = problem_configuration.get_optimization_definition() # Design Variables if KEY_DESIGN_VARIABLES in opt_def: for name, design_var in opt_def[KEY_DESIGN_VARIABLES].items(): metadata = { "type": "design_var", "initial_value": input_variables[name].value, "lower": design_var.get("lower"), "value": output_variables[name].value, "upper": design_var.get("upper"), "units": input_variables[name].units, "desc": input_variables[name].description, } optimization_variables[name] = metadata # Constraints if KEY_CONSTRAINTS in opt_def: for name, constr in opt_def[KEY_CONSTRAINTS].items(): metadata = { "type": "constraint", "initial_value": None, "lower": constr.get("lower"), "value": output_variables[name].value, "upper": constr.get("upper"), "units": output_variables[name].units, "desc": output_variables[name].description, } optimization_variables[name] = metadata # Objectives if KEY_OBJECTIVE in opt_def: for name in opt_def[KEY_OBJECTIVE]: metadata = { "type": "objective", "initial_value": None, "lower": None, "value": output_variables[name].value, "upper": None, "units": output_variables[name].units, "desc": output_variables[name].description, } optimization_variables[name] = metadata self.load_variables(optimization_variables)
def load( self, problem_configuration: FASTOADProblemConfigurator, ): """ Loads the FAST-OAD problem and stores its data. :param problem_configuration: the FASTOADProblem instance. :param file_formatter: the formatter that defines file format. If not provided, default format will be assumed. """ self.problem_configuration = problem_configuration problem = self.problem_configuration.get_problem() if pth.isfile(problem.input_file_path): input_variables = VariableIO(problem.input_file_path).read() else: # TODO: generate the input file by default ? raise FastMissingFile( "Please generate input file before using the optimization viewer" ) if pth.isfile(problem.output_file_path): output_variables = VariableIO(problem.output_file_path).read() else: output_variables = VariableList.from_problem(problem) optimization_variables = VariableList() opt_def = problem_configuration.get_optimization_definition() # Design Variables if "design_var" in opt_def: for name, design_var in opt_def["design_var"].items(): initial_value = input_variables[name].value if "lower" in design_var: lower = design_var["lower"] else: lower = None value = output_variables[name].value if "upper" in design_var: upper = design_var["upper"] else: upper = None units = input_variables[name].units desc = input_variables[name].description metadata = { "type": "design_var", "initial_value": initial_value, "lower": lower, "value": value, "upper": upper, "units": units, "desc": desc, } optimization_variables[name] = metadata # Constraints if "constraint" in opt_def: for name, constr in opt_def["constraint"].items(): if "lower" in constr: lower = constr["lower"] else: lower = None value = output_variables[name].value if "upper" in constr: upper = constr["upper"] else: upper = None units = output_variables[name].units desc = output_variables[name].description metadata = { "type": "constraint", "initial_value": None, "lower": lower, "value": value, "upper": upper, "units": units, "desc": desc, } optimization_variables[name] = metadata # Objectives if "objective" in opt_def: for name, obj in opt_def["objective"].items(): value = output_variables[name].value units = output_variables[name].units desc = output_variables[name].description metadata = { "type": "objective", "initial_value": None, "lower": None, "value": value, "upper": None, "units": units, "desc": desc, } optimization_variables[name] = metadata self.load_variables(optimization_variables)
def list_variables( configuration_file_path: str, out: Union[IO, str] = sys.stdout, overwrite: bool = False, force_text_output: bool = False, ): """ Writes list of variables for the :class:`FASTOADProblem` specified in configuration_file_path. List is generally written as text. It can be displayed as a scrollable table view if: - function is used in an interactive IPython shell - out == sys.stdout - force_text_output == False :param configuration_file_path: :param out: the output stream or a path for the output file :param overwrite: if True and out parameter is a file path, the file will be written even if one already exists :param force_text_output: if True, list will be written as text, even if command is used in an interactive IPython shell (Jupyter notebook). Has no effect in other shells or if out parameter is not sys.stdout :raise FastFileExistsError: if overwrite==False and out parameter is a file path and the file exists """ conf = FASTOADProblemConfigurator(configuration_file_path) problem = conf.get_problem() problem.setup() # Extracting inputs and outputs variables = VariableList.from_problem(problem, get_promoted_names=False) variables.sort(key=lambda var: var.name) input_variables = VariableList([var for var in variables if var.is_input]) output_variables = VariableList( [var for var in variables if not var.is_input]) if isinstance(out, str): if not overwrite and pth.exists(out): raise FastFileExistsError( "File %s not written because it already exists. " "Use overwrite=True to bypass." % out, out, ) make_parent_dir(out) out_file = open(out, "w") table_width = MAX_TABLE_WIDTH else: if out == sys.stdout and InteractiveShell.initialized( ) and not force_text_output: # Here we display the variable list as VariableViewer in a notebook for var in input_variables: var.metadata["I/O"] = "IN" for var in output_variables: var.metadata["I/O"] = "OUT" df = ((input_variables + output_variables).to_dataframe()[[ "I/O", "name", "desc" ]].rename(columns={ "name": "Name", "desc": "Description" })) display(HTML(df.to_html())) return # Here we continue with text output out_file = out table_width = min(get_terminal_size().columns, MAX_TABLE_WIDTH) - 1 pd.set_option("display.max_colwidth", 1000) max_name_length = np.max([ len(name) for name in input_variables.names() + output_variables.names() ]) description_text_width = table_width - max_name_length - 2 def _write_variables(out_f, variables): """Writes variables and their description as a pandas DataFrame""" df = variables.to_dataframe() # Create a new Series where description are wrapped on several lines if needed. # Each line becomes an element of the Series df["desc"] = [ "\n".join(tw.wrap(s, description_text_width)) for s in df["desc"] ] new_desc = df.desc.str.split("\n", expand=True).stack() # Create a Series for name that will match new_desc Series. Variable name will be in front of # first line of description. An empty string will be in front of other lines. new_name = [ df.name.loc[i] if j == 0 else "" for i, j in new_desc.index ] # Create the DataFrame that will be displayed new_df = pd.DataFrame({"NAME": new_name, "DESCRIPTION": new_desc}) out_f.write( new_df.to_string( index=False, columns=["NAME", "DESCRIPTION"], justify="center", formatters={ # Formatters are needed for enforcing left justification "NAME": ("{:%s}" % max_name_length).format, "DESCRIPTION": ("{:%s}" % description_text_width).format, }, ) ) out_file.write("\n") def _write_text_with_line(txt: str, line_length: int): """ Writes a line of given length with provided text inside """ out_file.write("-" + txt + "-" * (line_length - 1 - len(txt)) + "\n") # Inputs _write_text_with_line(" INPUTS OF THE PROBLEM ", table_width) _write_variables(out_file, input_variables) # Outputs out_file.write("\n") _write_text_with_line(" OUTPUTS OF THE PROBLEM ", table_width) _write_variables(out_file, output_variables) _write_text_with_line("", table_width) if isinstance(out, str): out_file.close() _LOGGER.info("Output list written in %s", out_file)
def test_register_checks_instantiation(cleanup): ValidityDomainChecker( { "var_with_upper_and_lower": (-1.0, 10.0), "var_with_lower": (-10.0, None), "var_with_upper": (None, 5.0), }, "main.logger1", ) # Now registering is done through instantiation, and with another logger ValidityDomainChecker( {"var_with_upper_and_lower": (-3.0, 5.0), "var_with_lower": (0.0, None)}, "main.logger2" ) # Third call with same logger as the first one ValidityDomainChecker({"other_var": (-10.0, 1.0)}, "main.logger1") # Check 1 -------------------------------------------------------------------------------------- log_file_path = pth.join(RESULTS_FOLDER_PATH, "log1.txt") set_logger_file(log_file_path) variables = VariableList( [ Variable("var_with_upper_and_lower", value=-2.0, units="m"), # too low for logger1 Variable("var_with_lower", value=-1.0), # too low for logger2 Variable("var_with_upper", value=10.0), # too high, only for logger1 Variable("other_var", value=1.1), # too high, only for logger1 (second registering) Variable("unbound_var", value=42.0), ] ) records = ValidityDomainChecker.check_variables(variables) assert [ ( rec.variable_name, rec.status, rec.limit_value, rec.value, rec.source_file, rec.logger_name, ) for rec in records ] == [ ("var_with_upper_and_lower", ValidityStatus.TOO_LOW, -1.0, -2.0, __file__, "main.logger1"), ("var_with_upper_and_lower", ValidityStatus.OK, None, -2.0, __file__, "main.logger2"), ("var_with_lower", ValidityStatus.OK, None, -1.0, __file__, "main.logger1"), ("var_with_lower", ValidityStatus.TOO_LOW, 0.0, -1.0, __file__, "main.logger2"), ("var_with_upper", ValidityStatus.TOO_HIGH, 5.0, 10.0, __file__, "main.logger1"), ("other_var", ValidityStatus.TOO_HIGH, 1.0, 1.1, __file__, "main.logger1"), ] ValidityDomainChecker.log_records(records) with open(log_file_path) as log_file: assert len(log_file.readlines()) == 4 # Check 2 -------------------------------------------------------------------------------------- log_file_path = pth.join(RESULTS_FOLDER_PATH, "log2.txt") set_logger_file(log_file_path) variables = VariableList( [ Variable("other_var", value=-11.0), # too low, only for logger1 (second registering) Variable("var_with_lower", value=-15.0), # too low for logger1 and logger2 Variable("var_with_upper", value=0.0), # Ok Variable("unbound_var", value=1e42), Variable("var_with_upper_and_lower", value=7.0), # too high for logger2 ] ) records = ValidityDomainChecker.check_variables(variables) assert [ ( rec.variable_name, rec.status, rec.limit_value, rec.value, rec.source_file, rec.logger_name, ) for rec in records ] == [ ("other_var", ValidityStatus.TOO_LOW, -10.0, -11.0, __file__, "main.logger1"), ("var_with_lower", ValidityStatus.TOO_LOW, -10.0, -15.0, __file__, "main.logger1"), ("var_with_lower", ValidityStatus.TOO_LOW, 0.0, -15.0, __file__, "main.logger2"), ("var_with_upper", ValidityStatus.OK, None, 0.0, __file__, "main.logger1"), ("var_with_upper_and_lower", ValidityStatus.OK, None, 7.0, __file__, "main.logger1"), ("var_with_upper_and_lower", ValidityStatus.TOO_HIGH, 5.0, 7.0, __file__, "main.logger2"), ] ValidityDomainChecker.log_records(records) with open(log_file_path) as log_file: assert len(log_file.readlines()) == 4 # Check 3 -------------------------------------------------------------------------------------- log_file_path = pth.join(RESULTS_FOLDER_PATH, "log3.txt") set_logger_file(log_file_path) variables = VariableList( [ Variable("var_with_upper_and_lower", value=1.0), # Ok Variable("other_var", value=-5.0), # Ok Variable("unbound_var", value=-1e42), Variable("var_with_lower", value=1.0), # Ok ] ) records = ValidityDomainChecker.check_variables(variables) assert [ ( rec.variable_name, rec.status, rec.limit_value, rec.value, rec.source_file, rec.logger_name, ) for rec in records ] == [ ("var_with_upper_and_lower", ValidityStatus.OK, None, 1.0, __file__, "main.logger1"), ("var_with_upper_and_lower", ValidityStatus.OK, None, 1.0, __file__, "main.logger2"), ("other_var", ValidityStatus.OK, None, -5.0, __file__, "main.logger1"), ("var_with_lower", ValidityStatus.OK, None, 1.0, __file__, "main.logger1"), ("var_with_lower", ValidityStatus.OK, None, 1.0, __file__, "main.logger2"), ] ValidityDomainChecker.log_records(records) with open(log_file_path) as log_file: assert len(log_file.readlines()) == 0