def setup_module(m_args): """Run the simulation up to module E0 and save dict_values before and after evaluation""" if os.path.exists(TEST_OUTPUT_PATH): shutil.rmtree(TEST_OUTPUT_PATH, ignore_errors=True) user_input = A0.process_user_arguments() logging.debug("Accessing script: B0_data_input_json") dict_values = B0.load_json( user_input[PATH_INPUT_FILE], path_input_folder=user_input[PATH_INPUT_FOLDER], path_output_folder=user_input[PATH_OUTPUT_FOLDER], move_copy=False, ) logging.debug("Accessing script: C0_data_processing") C0.all(dict_values) logging.debug("Accessing script: D0_modelling_and_optimization") results_meta, results_main = D0.run_oemof(dict_values) with open(DICT_BEFORE, "wb") as handle: pickle.dump(dict_values, handle, protocol=pickle.HIGHEST_PROTOCOL) logging.debug("Accessing script: E0_evaluation") E0.evaluate_dict(dict_values, results_main, results_meta) with open(DICT_AFTER, "wb") as handle: pickle.dump(dict_values, handle, protocol=pickle.HIGHEST_PROTOCOL)
def setup_class(m_args): """Run the simulation up to module E0 and prepare bus_data for E1""" if os.path.exists(TEST_OUTPUT_PATH): shutil.rmtree(TEST_OUTPUT_PATH, ignore_errors=True) logging.debug("Accessing script: A0_initialization") user_input = A0.process_user_arguments() A1.create_input_json(input_directory=os.path.join( user_input[PATH_INPUT_FOLDER], CSV_ELEMENTS)) logging.debug("Accessing script: B0_data_input_json") dict_values = B0.load_json( user_input[PATH_INPUT_FILE], path_input_folder=user_input[PATH_INPUT_FOLDER], path_output_folder=user_input[PATH_OUTPUT_FOLDER], move_copy=True, set_default_values=True, ) logging.debug("Accessing script: C0_data_processing") C0.all(dict_values) logging.debug("Accessing script: D0_modelling_and_optimization") results_meta, results_main = D0.run_oemof(dict_values) bus_data = {} # Store all information related to busses in bus_data for bus in dict_values[ENERGY_BUSSES]: # Read all energy flows from busses bus_data.update({bus: solph.views.node(results_main, bus)}) # Pickle dump bus data with open(BUS_DATA_DUMP, "wb") as handle: pickle.dump(bus_data, handle, protocol=pickle.HIGHEST_PROTOCOL)
def test_processing_dict_for_json_export_parse_pandas_series(self): """ """ expr = B0.convert_from_special_types_to_json(JSON_TEST_DICTIONARY[TYPE_SERIES]) assert expr == { DATA_TYPE_JSON_KEY: TYPE_SERIES, "value": [0, 1, 2], }
def test_processing_dict_for_json_export_parse_pandas_Timestamp(self): """ """ expr = B0.convert_from_special_types_to_json( JSON_TEST_DICTIONARY[TYPE_TIMESTAMP]) assert expr == { DATA_TYPE_JSON_KEY: TYPE_TIMESTAMP, "value": "2020-01-01 00:00:00", }
def test_processing_dict_for_json_export_parse_pandas_DatetimeIndex(self): """ """ expr = B0.convert_from_special_types_to_json( JSON_TEST_DICTIONARY[TYPE_DATETIMEINDEX] ) assert expr == { DATA_TYPE_JSON_KEY: TYPE_DATETIMEINDEX, "value": [1577836800000000000, 1577840400000000000, 1577844000000000000], }
def setup_class(self): """ """ if os.path.exists(OUTPUT_PATH): shutil.rmtree(OUTPUT_PATH, ignore_errors=True) os.mkdir(OUTPUT_PATH) self.file_name = "test_json_converter" F0.store_as_json(JSON_TEST_DICTIONARY, OUTPUT_PATH, self.file_name) self.value_dict = B0.load_json( os.path.join(OUTPUT_PATH, self.file_name + ".json"))
def run_simulation(json_dict, **kwargs): r""" Starts MVS tool simulation from an input json file Parameters ----------- json_dict: dict json from http request Other Parameters ---------------- pdf_report: bool, optional Can generate an automatic pdf report of the simulation's results (True) or not (False) Default: False. display_output : str, optional Sets the level of displayed logging messages. Options: "debug", "info", "warning", "error". Default: "info". lp_file_output : bool, optional Specifies whether linear equation system generated is saved as lp file. Default: False. """ welcome_text = ( "\n \n Multi-Vector Simulation Tool (MVS) V" + version_num + " " + "\n Version: " + version_date + " " + '\n Part of the toolbox of H2020 project "E-LAND", ' + "Integrated multi-vector management system for Energy isLANDs" + "\n Coded at: Reiner Lemoine Institute (Berlin) " + "\n Contributors: Martha M. Hoffmann \n \n ") logging.info(welcome_text) logging.debug("Accessing script: B0_data_input_json") dict_values = data_input.convert_from_json_to_special_types(json_dict) print("") logging.debug("Accessing script: C0_data_processing") data_processing.all(dict_values) print("") logging.debug("Accessing script: D0_modelling_and_optimization") results_meta, results_main = modelling.run_oemof(dict_values) print("") logging.debug("Accessing script: E0_evaluation") evaluation.evaluate_dict(dict_values, results_main, results_meta) logging.debug("Convert results to json") epa_dict_values = data_parser.convert_mvs_params_to_epa(dict_values) output_processing.select_essential_results(epa_dict_values) json_values = output_processing.store_as_json(epa_dict_values) return json.loads(json_values)
def test_processing_dict_for_json_export_parse_pandas_Dataframe(self): """ """ expr = B0.convert_from_special_types_to_json( JSON_TEST_DICTIONARY[TYPE_DATAFRAME]) assert expr == { DATA_TYPE_JSON_KEY: TYPE_DATAFRAME, "columns": ["a", "b"], "index": [0, 1, 2], "data": [[0, 0], [1, 1], [2, 2]], }
def test_retrieve_datetimeindex_for_simulation(): simulation_settings = { START_DATE: "2020-01-01", EVALUATED_PERIOD: { VALUE: 1 }, TIMESTEP: { VALUE: 60 }, } B0.retrieve_date_time_info(simulation_settings) for k in (START_DATE, END_DATE, TIME_INDEX): assert (k in simulation_settings.keys() ), f"Function does not add {k} to the simulation settings." assert simulation_settings[START_DATE] == pd.Timestamp( "2020-01-01 00:00:00"), f"Function incorrectly parses the timestamp." assert simulation_settings[END_DATE] == pd.Timestamp( "2020-01-01 23:00:00"), f"Function incorrectly parses the timestamp." assert ( simulation_settings[PERIODS] == 24 ), f"Function incorrectly identifies the number of evaluated periods."
def test_load_json_copies_json_file_to_output_folder(self, m_args): A0.process_user_arguments() A1.create_input_json(input_directory=CSV_PATH, pass_back=True) dict_values = B0.load_json(JSON_CSV_PATH, path_output_folder=self.test_out_path, move_copy=True) assert (os.path.exists( os.path.join( dict_values[SIMULATION_SETTINGS][PATH_OUTPUT_FOLDER_INPUTS], CSV_FNAME, )) is True)
def test_parse_pandas_series_provided_longer_time_index(self, caplog): pd_series = B0.convert_from_json_to_special_types( self.test_dict_series, time_index=self.ti_long) # collect warning message log_msg = caplog.record_tuples[0] # check it is a warning assert log_msg[1] == 30 assert ( "The time index inferred from simulation_settings is shorter as the timeserie under the field series" in log_msg[2]) assert (pd_series["series"].values == self.test_result_series.values ).all()
def test_load_json_removes_json_file_from_inputs_folder(self, m_args): A0.process_user_arguments() A1.create_input_json(input_directory=CSV_PATH, pass_back=True) dict_values = B0.load_json(JSON_CSV_PATH, path_output_folder=self.test_out_path, move_copy=True) assert os.path.exists(os.path.join( CSV_PATH, CSV_ELEMENTS, CSV_FNAME, )) is False assert os.path.exists(os.path.join( CSV_PATH, CSV_FNAME, )) is False
def run_parts(margs): if os.path.exists(TEST_OUTPUT_PATH): shutil.rmtree(TEST_OUTPUT_PATH, ignore_errors=True) user_input = A0.process_user_arguments() logging.debug("Accessing script: B0_data_input_json") dict_values = B0.load_json( user_input[PATH_INPUT_FILE], path_input_folder=user_input[PATH_INPUT_FOLDER], path_output_folder=user_input[PATH_OUTPUT_FOLDER], move_copy=False, ) logging.debug("Accessing script: C0_data_processing") C0.all(dict_values) logging.debug("Run parts of D0_modelling_and_optimization") model, dict_model = D0.model_building.initialize(dict_values) model = D0.model_building.adding_assets_to_energysystem_model( dict_values, dict_model, model) return dict_values, model, dict_model
def test_load_json_overwrite_input_folder_from_json(): dict_values = B0.load_json(JSON_PATH, path_input_folder="test") assert dict_values[SIMULATION_SETTINGS][PATH_INPUT_FOLDER] == "test"
def test_load_json_overwrite_output_folder_from_json(): dict_values = B0.load_json(JSON_PATH, path_output_folder="test") assert dict_values[SIMULATION_SETTINGS][PATH_OUTPUT_FOLDER] == "test" assert dict_values[SIMULATION_SETTINGS][ PATH_OUTPUT_FOLDER_INPUTS] == os.path.join("test", "inputs")
def test_parse_pandas_series_provided_time_index(self): pd_series = B0.convert_from_json_to_special_types( self.test_dict_series, time_index=self.ti) assert (pd_series["series"] == self.test_result_series).all()
def test_create_input_json_required_fields_are_filled(): js_file = A1.create_input_json(input_directory=CSV_PATH, pass_back=True) js = data_input.load_json(js_file) for k in js.keys(): assert k in REQUIRED_CSV_FILES + (PATHS_TO_PLOTS, )
def main(**kwargs): r""" Starts MVS tool simulations. Other Parameters ---------------- overwrite : bool, optional Determines whether to replace existing results in `path_output_folder` with the results of the current simulation (True) or not (False). Default: False. pdf_report: bool, optional Can generate an automatic pdf report of the simulation's results (True) or not (False) Default: False. input_type : str, optional Defines whether the input is taken from the `mvs_config.json` file ("json") or from csv files ('csv') located within <path_input_folder>/csv_elements/. Default: 'json'. path_input_folder : str, optional The path to the directory where the input CSVs/JSON files are located. Default: 'inputs/'. path_output_folder : str, optional The path to the directory where the results of the simulation such as the plots, time series, results JSON files are saved by MVS E-Lands. Default: 'MVS_outputs/' display_output : str, optional Sets the level of displayed logging messages. Options: "debug", "info", "warning", "error". Default: "info". lp_file_output : bool, optional Specifies whether linear equation system generated is saved as lp file. Default: False. """ welcome_text = ( "\n \n Multi-Vector Simulation Tool (MVS) V" + version_num + " " + "\n Version: " + version_date + " " + '\n Part of the toolbox of H2020 project "E-LAND", ' + "Integrated multi-vector management system for Energy isLANDs" + "\n Coded at: Reiner Lemoine Institute (Berlin) " + "\n Contributors: Martha M. Hoffmann \n \n ") logging.debug("Accessing script: A0_initialization") user_input = A0.process_user_arguments(welcome_text=welcome_text, **kwargs) # Read all inputs # print("") # # todo: is user input completely used? # dict_values = data_input.load_json(user_input[PATH_INPUT_FILE ]) move_copy_config_file = False if user_input[INPUT_TYPE] == CSV_EXT: logging.debug("Accessing script: A1_csv_to_json") move_copy_config_file = True A1.create_input_json(input_directory=os.path.join( user_input[PATH_INPUT_FOLDER], CSV_ELEMENTS)) logging.debug("Accessing script: B0_data_input_json") dict_values = B0.load_json( user_input[PATH_INPUT_FILE], path_input_folder=user_input[PATH_INPUT_FOLDER], path_output_folder=user_input[PATH_OUTPUT_FOLDER], move_copy=move_copy_config_file, set_default_values=True, ) F0.store_as_json( dict_values, dict_values[SIMULATION_SETTINGS][PATH_OUTPUT_FOLDER_INPUTS], MVS_CONFIG, ) print("") logging.debug("Accessing script: C0_data_processing") C0.all(dict_values) F0.store_as_json( dict_values, dict_values[SIMULATION_SETTINGS][PATH_OUTPUT_FOLDER], JSON_PROCESSED, ) if "path_pdf_report" in user_input or "path_png_figs" in user_input: save_energy_system_graph = True else: save_energy_system_graph = False print("") logging.debug("Accessing script: D0_modelling_and_optimization") results_meta, results_main = D0.run_oemof( dict_values, save_energy_system_graph=save_energy_system_graph, ) print("") logging.debug("Accessing script: E0_evaluation") E0.evaluate_dict(dict_values, results_main, results_meta) logging.debug("Accessing script: F0_outputs") F0.evaluate_dict( dict_values, path_pdf_report=user_input.get("path_pdf_report", None), path_png_figs=user_input.get("path_png_figs", None), ) return 1
def report(pdf=None, path_simulation_output_json=None, path_pdf_report=None): """Display the report of a MVS simulation Command line use: .. code-block:: bash mvs_report [-h] [-i [PATH_SIM_OUTPUT]] [-o [REPORT_PATH]] [-pdf] optional command line arguments: -h, --help show this help message and exit -pdf [PRINT_REPORT] print the report as pdf (default: False) -i [OUTPUT_FOLDER] path to the simulation result json file 'json_with_results.json' -o [REPORT_PATH] path to save the pdf report -d [DEBUG_REPORT] run the dash app in debug mode with hot-reload Parameters ---------- pdf: bool if True a pdf report should be generated Default: False path_simulation_output_json: str path to the simulation result json file 'json_with_results.json' path_pdf_report: str path to save the pdf report Returns ------- Save a pdf report if option -pdf is provided, otherwise display the report as an app """ # Parse the arguments from the command line parser = A0.report_arg_parser() args = vars(parser.parse_args()) # Give priority from user input kwargs over command line arguments # However the command line arguments have priority over default kwargs if pdf is None: pdf = args.get(ARG_PDF, False) if path_simulation_output_json is None: path_simulation_output_json = args.get(ARG_PATH_SIM_OUTPUT) if path_pdf_report is None: path_pdf_report = args.get(ARG_REPORT_PATH) # if the user only provided the path to the folder, we complete with default json file if os.path.isdir(path_simulation_output_json) is True: path_simulation_output_json = os.path.join( path_simulation_output_json, JSON_WITH_RESULTS + JSON_FILE_EXTENSION) logging.warning( f"Only path to a folder provided ({args.get(ARG_PATH_SIM_OUTPUT)}), looking now for default {JSON_WITH_RESULTS + JSON_FILE_EXTENSION} file within this folder" ) if os.path.exists(path_simulation_output_json) is False: raise FileNotFoundError( "Simulation results file {} not found. You need to run a simulation to generate " "the data before you can generate a report\n\n\tsee `mvs_tool -h` for help on how " "to run a simulation\n".format(path_simulation_output_json)) else: # path to the mvs simulation output files path_sim_output = os.path.dirname(path_simulation_output_json) # if report path is not specified it will be included in the mvs simulation outputs folder if path_pdf_report == "": path_pdf_report = os.path.join(path_sim_output, REPORT_FOLDER, PDF_REPORT) # load the results of a simulation dict_values = B0.load_json(path_simulation_output_json, flag_missing_values=False) test_app = create_app(dict_values, path_sim_output=path_sim_output) banner = "*" * 40 print(banner + "\nPress ctrl+c to stop the report server\n" + banner) if pdf is True: print_pdf(test_app, path_pdf_report=path_pdf_report) else: if args.get(ARG_DEBUG_REPORT) is True: test_app.run_server(debug=True) else: # run the dash server for 600s before shutting it down open_in_browser(test_app, timeout=600) print( banner + "\nThe report server has timed out.\nTo start it again run " "`mvs_report`.\nTo let it run for a longer time, change timeout setting in " "the cli.py file\n" + banner)
def test_processing_dict_for_json_export_parse_numpy_int64(self): """ """ expr = B0.convert_from_special_types_to_json( JSON_TEST_DICTIONARY["numpy_int64"] ) assert expr == SCALAR
def test_processing_dict_for_json_export_parse_numpy_array(self): """ """ expr = B0.convert_from_special_types_to_json(JSON_TEST_DICTIONARY[TYPE_NDARRAY]) assert expr == {DATA_TYPE_JSON_KEY: TYPE_NDARRAY, "value": [0, 1, 2]}
def run_simulation(json_dict, epa_format=True, **kwargs): r""" Starts MVS tool simulation from an input json file Parameters ----------- json_dict: dict json from http request epa_format: bool, optional Specifies whether the output is formatted for EPA standards Default: True Other Parameters ---------------- pdf_report: bool, optional Can generate an automatic pdf report of the simulation's results (True) or not (False) Default: False. display_output : str, optional Sets the level of displayed logging messages. Options: "debug", "info", "warning", "error". Default: "info". lp_file_output : bool, optional Specifies whether linear equation system generated is saved as lp file. Default: False. """ display_output = kwargs.get("display_output", None) if display_output == "debug": screen_level = logging.DEBUG elif display_output == "info": screen_level = logging.INFO elif display_output == "warning": screen_level = logging.WARNING elif display_output == "error": screen_level = logging.ERROR else: screen_level = logging.INFO # Define logging settings and path for saving log logger.define_logging(screen_level=screen_level) welcome_text = ( "\n \n Multi-Vector Simulation Tool (MVS) V" + version_num + " " + "\n Version: " + version_date + " " + '\n Part of the toolbox of H2020 project "E-LAND", ' + "Integrated multi-vector management system for Energy isLANDs" + "\n Coded at: Reiner Lemoine Institute (Berlin) " + "\n Reference: https://zenodo.org/record/4610237 \n \n ") logging.info(welcome_text) logging.debug("Accessing script: B0_data_input_json") dict_values = B0.convert_from_json_to_special_types(json_dict) print("") logging.debug("Accessing script: C0_data_processing") C0.all(dict_values) print("") logging.debug("Accessing script: D0_modelling_and_optimization") results_meta, results_main = D0.run_oemof(dict_values) print("") logging.debug("Accessing script: E0_evaluation") E0.evaluate_dict(dict_values, results_main, results_meta) logging.debug("Convert results to json") if epa_format is True: epa_dict_values = data_parser.convert_mvs_params_to_epa(dict_values) json_values = F0.store_as_json(epa_dict_values) answer = json.loads(json_values) else: answer = dict_values return answer
def report(pdf=None, path_simulation_output_json=None, path_pdf_report=None): """Display the report of a MVS simulation Command line use: .. code-block:: bash mvs_report [-h] [-i [PATH_SIM_OUTPUT]] [-o [REPORT_PATH]] [-pdf] optional command line arguments: -h, --help show this help message and exit -pdf [PRINT_REPORT] print the report as pdf (default: False) -i [OUTPUT_FOLDER] path to the simulation result json file 'json_with_results.json' -o [REPORT_PATH] path to save the pdf report Parameters ---------- pdf: bool if True a pdf report should be generated Default: False path_simulation_output_json: str path to the simulation result json file 'json_with_results.json' path_pdf_report: str path to save the pdf report Returns ------- Save a pdf report if option -pdf is provided, otherwise display the report as an app """ # Parse the arguments from the command line parser = initializing.report_arg_parser() args = vars(parser.parse_args()) print(args) # Give priority from user input kwargs over command line arguments # However the command line arguments have priority over default kwargs if pdf is None: pdf = args.get(ARG_PDF, False) if path_simulation_output_json is None: path_simulation_output_json = args.get(ARG_PATH_SIM_OUTPUT) if path_pdf_report is None: path_pdf_report = args.get(ARG_REPORT_PATH) # if the user only provided the path to the folder, we complete with default json file if os.path.isdir(path_simulation_output_json) is True: path_simulation_output_json = os.path.join(path_simulation_output_json, JSON_WITH_RESULTS) if os.path.exists(path_simulation_output_json) is False: raise FileNotFoundError( "{} not found. You need to run a simulation to generate the data to report" "see `python mvs_tool.py -h` for help".format( path_simulation_output_json)) else: # path to the mvs simulation output files path_sim_output = os.path.dirname(path_simulation_output_json) # if report path is not specified it will be included in the mvs simulation outputs folder if path_pdf_report == "": path_pdf_report = os.path.join(path_sim_output, REPORT_FOLDER, PDF_REPORT) # load the results of a simulation dict_values = data_input.load_json(path_simulation_output_json) test_app = create_app(dict_values, path_sim_output=path_sim_output) banner = "*" * 40 print(banner + "\nPress ctrl+c to stop the report server\n" + banner) if pdf is True: print_pdf(test_app, path_pdf_report=path_pdf_report) else: # run the dash server for 600s before shutting it down open_in_browser(test_app, timeout=600) print( banner + "\nThe report server has timed out.\nTo start it again run `python " "mvs_report.py`.\nTo let it run for a longer time, change timeout setting in " "the mvs_report.py file\n" + banner)
def test_processing_dict_for_json_export_parse_unknown(self): """ """ with pytest.raises(TypeError): B0.convert_from_special_types_to_json(UNKNOWN_TYPE)