def _make_schema( db_name: Optional[str] = None, ) -> Tuple[DatabaseConnector, Dict[str, Dict[str, str]]]: """ Create the schema (i.e. make all the tables) of the database. Parameters ---------- db_name : None or str Name of the database Returns ------- db_connector : DatabaseConnector The database connection object final_parameters_as_sql_types : dict Final parameters as sql types """ db_connector = make_test_database(db_name) default_parameters = get_default_parameters final_parameters = FinalParameters(default_parameters) final_parameters_dict = final_parameters.get_final_parameters() final_parameters_as_sql_types = final_parameters.cast_to_sql_type( final_parameters_dict ) db_creator = DatabaseCreator(db_connector) db_creator.create_all_schema_tables(final_parameters_as_sql_types) return db_connector, final_parameters_as_sql_types
def test_final_parameters(get_default_parameters: DefaultParameters) -> None: """ Test that RunParameters overwrites DefaultParameters. Parameters ---------- get_default_parameters : DefaultParameters The DefaultParameters object """ default_parameters = get_default_parameters run_parameters = RunParameters({"global": {"timestep": False}}) final_parameters = FinalParameters(default_parameters, run_parameters) final_parameters_dict = final_parameters.get_final_parameters() assert final_parameters_dict["global"]["timestep"] == 0
def make_run_group( name: str, make_project: Path, run_graph: Optional[RunGraph] = None, restart_from: Optional[Path] = None, waiting_for: Optional[Union[str, Iterable[str]]] = None, ) -> RunGroup: """ Return a basic RunGroup. Parameters ---------- run_graph name : str Name of RunGroup and DatabaseConnector make_project : Path The path to the conduction example run_graph : RunGraph The RunGraph object restart_from : Path or None The path to copy the restart files from waiting_for : None or str or iterable Name of nodes this node will wait for to finish before executing Returns ------- run_group : RunGroup A basic run group """ # Make project to save time project_path = make_project # Create the `bout_paths` object bout_paths = BoutPaths( project_path=project_path, bout_inp_src_dir=project_path.joinpath("data"), bout_inp_dst_dir=project_path.joinpath(name), ) # Create the input objects run_parameters = RunParameters({"global": {"nout": 0}}) default_parameters = DefaultParameters(bout_paths) final_parameters = FinalParameters(default_parameters, run_parameters) executor = Executor( bout_paths=bout_paths, submitter=LocalSubmitter(bout_paths.project_path), run_parameters=run_parameters, restart_from=restart_from, ) db_connector = DatabaseConnector(name) bout_run_setup = BoutRunSetup(executor, db_connector, final_parameters) # Create the `run_group` run_graph = run_graph if run_graph is not None else RunGraph() run_group = RunGroup(run_graph, bout_run_setup, name=name, waiting_for=waiting_for) return run_group
def test_cast_parameters_to_sql_type( get_default_parameters: DefaultParameters) -> None: """ Test that obtain_project_parameters returns expected output. Parameters ---------- get_default_parameters : DefaultParameters The DefaultParameters object """ default_parameters = get_default_parameters final_parameters = FinalParameters(default_parameters) final_parameters_dict = final_parameters.get_final_parameters() parameter_as_sql = final_parameters.cast_to_sql_type(final_parameters_dict) assert isinstance(parameter_as_sql, dict) assert "global" in parameter_as_sql.keys() assert isinstance(parameter_as_sql["global"], dict) assert parameter_as_sql["global"]["append"] == "INTEGER" # bool assert parameter_as_sql["conduction"]["chi"] == "REAL" # float assert parameter_as_sql["global"]["mxg"] == "INTEGER" # int assert parameter_as_sql["global"]["datadir"] == "TEXT" # str
def __init__( self, executor: Optional[BoutRunExecutor] = None, db_connector: Optional[DatabaseConnector] = None, final_parameters: Optional[FinalParameters] = None, ) -> None: """ Set the member data. This constructor will also create the schema if it does not exist. Parameters ---------- executor : BoutRunExecutor or None Object executing the run If None, default parameters will be used db_connector : DatabaseConnector or None The connection to the database If None: Default database connector will be used final_parameters : FinalParameters or None The object containing the parameters which are going to be used in the run If None, default parameters will be used """ # Set member data # NOTE: We are not setting the default as a keyword argument # as this would mess up the paths logging.info("Start: Making a BoutRunSetup object") self.__executor = executor if executor is not None else BoutRunExecutor( ) self.__final_parameters = (final_parameters if final_parameters is not None else FinalParameters()) self.__db_connector = ( db_connector if db_connector is not None else DatabaseConnector( name=self.__executor.exec_name, db_root_path=self.__executor.bout_paths.project_path, )) self.__db_creator = DatabaseCreator(self.db_connector) self.__metadata_recorder = MetadataRecorder(self.__db_connector, self.executor.bout_paths, self.final_parameters) if not self.__metadata_recorder.db_reader.check_tables_created(): logging.info( "Creating schema as no tables were found in " "%s", self.__metadata_recorder.db_reader.db_connector.db_path, ) self.__create_schema() logging.info("Done: Making a BoutRunSetup object")
def _get_bout_run_setup(tmp_path_name: str) -> BoutRunSetup: """ Create BoutRunSetup based on the conduction directory. Parameters ---------- tmp_path_name : str Name of the temporary directory Returns ------- bout_run_setup : BoutRunSetup The BoutRunSetup object """ executor = get_executor(tmp_path_name) db_connector = make_test_database(tmp_path_name) final_parameters = FinalParameters(get_default_parameters) bout_run_setup = BoutRunSetup(executor, db_connector, final_parameters) return bout_run_setup
def test_metadata_recorder( yield_bout_path_conduction: Callable[[str], BoutPaths], get_default_parameters: DefaultParameters, make_project: Path, make_test_schema: Callable[[str], Tuple[DatabaseConnector, MetadataReader]], yield_number_of_rows_for_all_tables: Callable[[DatabaseReader], Dict[str, int]], ) -> None: """ Test the metadata recorder. Specifically this test will test that: 1. It is possible to make a new entry 2. The newly created schema only have 1 row from the created entry 3. No new entries are made when trying to make the entry again 4. All the tables still have one row 5. Create a new entry after changing the run parameters 6. Check that a new entry is created 7. See that the `split` table and the `run` table have two rows, whereas the rest have one 8. Check that it is possible to forcefully make an entry to the run table Parameters ---------- yield_bout_path_conduction : function Function which makes the BoutPaths object for the conduction example get_default_parameters : DefaultParameters The DefaultParameters object make_project : Path The path to the conduction example make_test_schema : function The function making the schema (i.e. making all the tables) yield_number_of_rows_for_all_tables : function Function which returns the number of rows for all tables in a schema """ # NOTE: If the project is not made, the metadata recorder will # fail when the get_file_modification is trying to get the last # edited time of the executable _ = make_project db_connector, _ = make_test_schema("test_metadata_recorder") bout_paths = yield_bout_path_conduction("test_metadata_recorder") default_parameters = get_default_parameters final_parameters = FinalParameters(default_parameters) metadata_recorder = MetadataRecorder(db_connector, bout_paths, final_parameters) # Assert that this is a new entry run_id = metadata_recorder.capture_new_data_from_run(ProcessorSplit()) assert run_id is None # Assert that all the values are 1 number_of_rows_dict = yield_number_of_rows_for_all_tables( metadata_recorder.db_reader ) assert sum(number_of_rows_dict.values()) == len(number_of_rows_dict.keys()) # Assert that this is not a new entry run_id = metadata_recorder.capture_new_data_from_run(ProcessorSplit()) assert run_id == 1 # Assert that all the values are 1 number_of_rows_dict = yield_number_of_rows_for_all_tables( metadata_recorder.db_reader ) assert sum(number_of_rows_dict.values()) == len(number_of_rows_dict.keys()) # Create a new entry in the split table run_id = metadata_recorder.capture_new_data_from_run( ProcessorSplit(number_of_nodes=2) ) # Assert that a new entry has been made (the number of rows in # the tables will be counted when checking the forceful entry) assert run_id is None # Forcefully make an entry run_id = metadata_recorder.capture_new_data_from_run( ProcessorSplit(number_of_nodes=2), force=True ) # Assert that a new entry has been made assert run_id == 2 number_of_rows_dict = yield_number_of_rows_for_all_tables( metadata_recorder.db_reader ) tables_with_2 = dict() tables_with_2["split"] = number_of_rows_dict.pop("split") tables_with_3 = dict() tables_with_3["run"] = number_of_rows_dict.pop("run") # Assert that all the values are 1 assert sum(number_of_rows_dict.values()) == len(number_of_rows_dict.keys()) # Assert that all the values are 2 assert sum(tables_with_2.values()) == 2 * len(tables_with_2.keys()) # Assert that all the values are 3 assert sum(tables_with_3.values()) == 3 * len(tables_with_3.keys())
def make_run_group( run_group_parameters: Dict[str, Union[str, Optional[RunGraph], Optional[Union[str, Iterable[str]]]]], make_project: Path, file_state_restorer: FileStateRestorer, restart_from: Optional[Path] = None, ) -> RunGroup: """ Return a basic RunGroup. Parameters ---------- run_group_parameters : dict Parameters to the run_group containing the keys - name : str Name of the run_group Note that the name will also be used for the destination dir and the name of the database - run_graph: None or RunGraph The run_graph to use - waiting_for : None or str or iterable of str Name of nodes this node will wait for to finish before executing make_project : Path The path to the conduction example file_state_restorer : FileStateRestorer Object for restoring files to original state restart_from : Path or None The path to copy the restart files from Returns ------- run_group : RunGroup A basic run group Raises ------ ValueError If the shape or types of the run_group_parameters are wrong """ # NOTE: The following is a mypy guard which could be solved with TypedDict # However, TypedDict is new from 3.8 if "name" not in run_group_parameters.keys() or not isinstance( run_group_parameters["name"], str): raise ValueError( "'name' must be of string type in run_group_parameters") if "run_graph" not in run_group_parameters.keys() or not ( isinstance(run_group_parameters["run_graph"], RunGraph) or run_group_parameters["run_graph"] is None): raise ValueError( "'run_graph' must be of RunGroup type or None in run_group_parameters" ) if ("waiting_for" not in run_group_parameters.keys() or not (hasattr(run_group_parameters["waiting_for"], "__iter__") or run_group_parameters["waiting_for"] is None) or isinstance(run_group_parameters["waiting_for"], RunGraph)): raise ValueError( "'waiting_for' must be of RunGroup type or None in run_group_parameters" ) # Make project to save time project_path = make_project # Create the `bout_paths` object bout_paths = BoutPaths( project_path=project_path, bout_inp_src_dir=project_path.joinpath("data"), bout_inp_dst_dir=project_path.joinpath(run_group_parameters["name"]), ) # Create the input objects run_parameters = RunParameters({"global": {"nout": 0}}) default_parameters = DefaultParameters(bout_paths) final_parameters = FinalParameters(default_parameters, run_parameters) submitter = get_submitter() if isinstance(submitter, LocalSubmitter): submitter.run_path = bout_paths.project_path executor = BoutRunExecutor( bout_paths=bout_paths, submitter=submitter, run_parameters=run_parameters, restart_from=restart_from, ) db_connector = DatabaseConnector(name=run_group_parameters["name"], db_root_path=project_path) bout_run_setup = BoutRunSetup(executor, db_connector, final_parameters) # Create the `run_group` run_group = RunGroup( run_group_parameters["run_graph"] if run_group_parameters["run_graph"] is not None else RunGraph(), bout_run_setup, name=run_group_parameters["name"], waiting_for=run_group_parameters["waiting_for"], ) file_state_restorer.add(executor.bout_paths.bout_inp_dst_dir, force_mark_removal=True) file_state_restorer.add(db_connector.db_path, force_mark_removal=True) file_state_restorer.add( executor.bout_paths.project_path.joinpath("settings_run"), force_mark_removal=True, ) return run_group
def test_restart_documentation( clean_up_bout_inp_src_and_dst: Callable[[str, str], Tuple[Path, Path, Path]] ) -> None: """ Test that the restart documentation runs without error. Parameters ---------- clean_up_bout_inp_src_and_dst : function Function which adds temporary BOUT.inp directories to removal. """ # NOTE: We are aware of the number of locals, and are here only testing the docs # pylint: disable=too-many-locals project_path, bout_inp_src_dir, bout_inp_dst_dir = clean_up_bout_inp_src_and_dst( "test_restart_documentation_src", "test_restart_documentation_dst") bout_paths = BoutPaths( project_path=project_path, bout_inp_src_dir=bout_inp_src_dir, bout_inp_dst_dir=bout_inp_dst_dir, ) default_parameters = DefaultParameters(bout_paths) run_parameters = RunParameters({"global": {"nout": 0}}) final_parameters = FinalParameters(default_parameters, run_parameters) basic_executor = Executor( bout_paths=bout_paths, submitter=LocalSubmitter(bout_paths.project_path), run_parameters=run_parameters, ) # NOTE: We set the database to bout_inp_dst_dir as this will be removed later db_connector = DatabaseConnector("name_of_database", db_root_path=bout_inp_dst_dir) basic_bout_run_setup = BoutRunSetup(basic_executor, db_connector, final_parameters) run_graph = RunGraph() name = "my_restart_runs" basic_run_group = RunGroup(run_graph, basic_bout_run_setup, name=name) # New section in the documentation restart_executor = Executor( bout_paths=bout_paths, submitter=LocalSubmitter(bout_paths.project_path), run_parameters=run_parameters, restart_from=bout_paths.bout_inp_dst_dir, ) restart_bout_run_setup = BoutRunSetup(restart_executor, db_connector, final_parameters) RunGroup( run_graph, restart_bout_run_setup, name=name, waiting_for=basic_run_group.bout_run_node_name, ) # New section in the documentation new_run_parameters = RunParameters({"solver": {"adams_moulton": True}}) new_final_parameters = FinalParameters(default_parameters, run_parameters) restart_with_changing_parameters_executor = Executor( bout_paths=bout_paths, submitter=LocalSubmitter(bout_paths.project_path), run_parameters=new_run_parameters, restart_from=bout_paths.bout_inp_dst_dir, ) BoutRunSetup(restart_with_changing_parameters_executor, db_connector, new_final_parameters) RunGroup( run_graph, restart_bout_run_setup, name=name, waiting_for=basic_run_group.bout_run_node_name, ) # New section in the documentation run_graph.get_dot_string() # New section in the documentation runner = BoutRunner(run_graph) runner.run()
def test_pre_and_post_documentation( clean_up_bout_inp_src_and_dst: Callable[[str, str], Tuple[Path, Path, Path]] ) -> None: """ Test that the pre and post documentation runs without error. Parameters ---------- clean_up_bout_inp_src_and_dst : function Function which adds temporary BOUT.inp directories to removal. """ # NOTE: We are aware of the number of locals, and are here only testing the docs # pylint: disable=too-many-locals project_path, bout_inp_src_dir, bout_inp_dst_dir = clean_up_bout_inp_src_and_dst( "test_pre_post_documentation_src", "test_pre_post_documentation_dst") bout_paths = BoutPaths( project_path=project_path, bout_inp_src_dir=bout_inp_src_dir, bout_inp_dst_dir=bout_inp_dst_dir, ) default_parameters = DefaultParameters(bout_paths) run_parameters = RunParameters({"global": {"nout": 0}}) final_parameters = FinalParameters(default_parameters, run_parameters) basic_executor = Executor( bout_paths=bout_paths, submitter=LocalSubmitter(bout_paths.project_path), run_parameters=run_parameters, ) # NOTE: We set the database to bout_inp_dst_dir as this will be removed later db_connector = DatabaseConnector("name_of_database", db_root_path=bout_inp_dst_dir) basic_bout_run_setup = BoutRunSetup(basic_executor, db_connector, final_parameters) run_graph = RunGraph() name = "my_restart_runs" basic_run_group = RunGroup(run_graph, basic_bout_run_setup, name=name) # New section in the documentation basic_run_group.add_post_processor({ "function": return_none, "args": None, "kwargs": None }) expanded_noise_restarts_dir = bout_paths.bout_inp_dst_dir.parent.joinpath( "expanded_noise_restarts") kwargs = { "newNz": 16, "path": bout_paths.bout_inp_dst_dir, "output": expanded_noise_restarts_dir, } expand_node_name = basic_run_group.add_post_processor({ "function": mock_expand, "args": None, "kwargs": kwargs }) # New section in the documentation # NOTE: Add these for removal clean_up_bout_inp_src_and_dst("expanded_noise_restarts", "expanded_noise_restarts") # Create the RunGroup restart_executor = Executor( bout_paths=bout_paths, submitter=LocalSubmitter(bout_paths.project_path), run_parameters=run_parameters, restart_from=expanded_noise_restarts_dir, ) restart_bout_run_setup = BoutRunSetup(restart_executor, db_connector, final_parameters) restart_run_group = RunGroup(run_graph, restart_bout_run_setup, name=name) kwargs = {"path": expanded_noise_restarts_dir, "scale": 1e-5} restart_run_group.add_pre_processor( { "function": return_none, "args": None, "kwargs": kwargs }, waiting_for=expand_node_name, ) # New section in the documentation run_graph.get_dot_string() # New section in the documentation runner = BoutRunner(run_graph) runner.run()
def test_restart_documentation( make_project: Path, copy_bout_inp: Callable[[Path, str], Path], file_state_restorer: FileStateRestorer, ) -> None: """ Test that the restart documentation runs without error. Parameters ---------- make_project : Path The path to the conduction example copy_bout_inp : function Function which copies BOUT.inp and returns the path to the temporary directory file_state_restorer : FileStateRestorer Object for restoring files to original state """ # NOTE: We are aware of the number of locals, and are here only testing the docs # pylint: disable=too-many-locals project_path = make_project bout_inp_src_dir = copy_bout_inp(project_path, "test_restart_documentation_src") bout_inp_dst_dir = project_path.joinpath("test_restart_documentation_dst") # NOTE: bout_inp_src_dir removed by copy_bout_inp teardown file_state_restorer.add(bout_inp_dst_dir, force_mark_removal=True) bout_paths = BoutPaths( project_path=project_path, bout_inp_src_dir=bout_inp_src_dir, bout_inp_dst_dir=bout_inp_dst_dir, ) default_parameters = DefaultParameters(bout_paths) run_parameters = RunParameters({"global": {"nout": 0}}) final_parameters = FinalParameters(default_parameters, run_parameters) basic_executor = BoutRunExecutor( bout_paths=bout_paths, submitter=LocalSubmitter(bout_paths.project_path), run_parameters=run_parameters, ) # NOTE: We set the database to bout_inp_dst_dir as this will be removed later db_connector = DatabaseConnector("name_of_database", db_root_path=bout_inp_dst_dir) file_state_restorer.add(db_connector.db_path, force_mark_removal=True) basic_bout_run_setup = BoutRunSetup(basic_executor, db_connector, final_parameters) run_graph = RunGraph() name = "my_restart_runs" basic_run_group = RunGroup(run_graph, basic_bout_run_setup, name=name) # New section in the documentation restart_executor = BoutRunExecutor( bout_paths=bout_paths, submitter=LocalSubmitter(bout_paths.project_path), run_parameters=run_parameters, restart_from=bout_paths.bout_inp_dst_dir, ) file_state_restorer.add(restart_executor.bout_paths.bout_inp_dst_dir, force_mark_removal=True) restart_bout_run_setup = BoutRunSetup(restart_executor, db_connector, final_parameters) RunGroup( run_graph, restart_bout_run_setup, name=name, waiting_for=basic_run_group.bout_run_node_name, ) # New section in the documentation new_run_parameters = RunParameters({"solver": {"adams_moulton": True}}) new_final_parameters = FinalParameters(default_parameters, run_parameters) restart_with_changing_parameters_executor = BoutRunExecutor( bout_paths=bout_paths, submitter=LocalSubmitter(bout_paths.project_path), run_parameters=new_run_parameters, restart_from=bout_paths.bout_inp_dst_dir, ) file_state_restorer.add( restart_with_changing_parameters_executor.bout_paths.bout_inp_dst_dir, force_mark_removal=True, ) BoutRunSetup(restart_with_changing_parameters_executor, db_connector, new_final_parameters) RunGroup( run_graph, restart_bout_run_setup, name=name, waiting_for=basic_run_group.bout_run_node_name, ) # New section in the documentation run_graph.get_dot_string() # New section in the documentation runner = BoutRunner(run_graph) runner.run()
def test_pre_and_post_documentation( make_project: Path, copy_bout_inp: Callable[[Path, str], Path], file_state_restorer: FileStateRestorer, ) -> None: """ Test that the pre and post documentation runs without error. Parameters ---------- make_project : Path The path to the conduction example copy_bout_inp : function Function which copies BOUT.inp and returns the path to the temporary directory file_state_restorer : FileStateRestorer Object for restoring files to original state """ # NOTE: We are aware of the number of locals, and are here only testing the docs # pylint: disable=too-many-locals project_path = make_project bout_inp_src_dir = copy_bout_inp(project_path, "test_pre_post_documentation_src") bout_inp_dst_dir = project_path.joinpath("test_pre_post_documentation_dst") # NOTE: bout_inp_src_dir removed by copy_bout_inp teardown file_state_restorer.add(bout_inp_dst_dir, force_mark_removal=True) bout_paths = BoutPaths( project_path=project_path, bout_inp_src_dir=bout_inp_src_dir, bout_inp_dst_dir=bout_inp_dst_dir, ) default_parameters = DefaultParameters(bout_paths) run_parameters = RunParameters({"global": {"nout": 0}}) final_parameters = FinalParameters(default_parameters, run_parameters) basic_executor = BoutRunExecutor( bout_paths=bout_paths, submitter=LocalSubmitter(bout_paths.project_path), run_parameters=run_parameters, ) # NOTE: We set the database to bout_inp_dst_dir as this will be removed later db_connector = DatabaseConnector("name_of_database", db_root_path=bout_inp_dst_dir) file_state_restorer.add(db_connector.db_path, force_mark_removal=True) basic_bout_run_setup = BoutRunSetup(basic_executor, db_connector, final_parameters) run_graph = RunGraph() name = "my_restart_runs" basic_run_group = RunGroup(run_graph, basic_bout_run_setup, name=name) # New section in the documentation basic_run_group.add_post_processor({ "function": return_none, "args": None, "kwargs": None }) expanded_noise_restarts_dir = bout_paths.bout_inp_dst_dir.parent.joinpath( "expanded_noise_restarts") file_state_restorer.add(expanded_noise_restarts_dir, force_mark_removal=True) kwargs = { "newNz": 16, "path": bout_paths.bout_inp_dst_dir, "output": expanded_noise_restarts_dir, } expand_node_name = basic_run_group.add_post_processor( { "function": mock_expand, "args": None, "kwargs": kwargs, }, ) # New section in the documentation # Create the RunGroup restart_executor = BoutRunExecutor( bout_paths=bout_paths, submitter=LocalSubmitter(bout_paths.project_path), run_parameters=run_parameters, restart_from=expanded_noise_restarts_dir, ) file_state_restorer.add(restart_executor.bout_paths.bout_inp_dst_dir, force_mark_removal=True) restart_bout_run_setup = BoutRunSetup(restart_executor, db_connector, final_parameters) restart_run_group = RunGroup(run_graph, restart_bout_run_setup, name=name) kwargs = {"path": expanded_noise_restarts_dir, "scale": 1e-5} restart_run_group.add_pre_processor( { "function": return_none, "args": None, "kwargs": kwargs, }, waiting_for=expand_node_name, ) # New section in the documentation run_graph.get_dot_string() # New section in the documentation runner = BoutRunner(run_graph) runner.run()