def __check_submitted(self, metadata_updater: MetadataUpdater,
submitted_to_check: DataFrame) -> None:
"""
Check the status of all runs which has status `submitted`.
Parameters
----------
metadata_updater : MetadataUpdater
Object which updates the database
submitted_to_check : DataFrame
DataFrame containing the `id` and `name` of the runs with status `submitted`
Raises
------
RuntimeError
In case log_reader.started() is True and log_reader.start_time is None
"""
for name, run_id in submitted_to_check.itertuples(index=False):
metadata_updater.run_id = run_id
log_path = self.__project_path.joinpath(name, "BOUT.log.0")
if log_path.is_file():
log_reader = LogReader(log_path)
if log_reader.started():
start_time = log_reader.start_time
# Assert to prevent "Incompatible types in assignment" with Optional
if start_time is None:
msg = ("log_reader.start_time is None although "
"log_reader.started is True")
logging.critical(msg)
raise RuntimeError(msg)
metadata_updater.update_start_time(start_time)
latest_status = self.__check_if_stopped(
log_reader, metadata_updater)
else:
# No started time is found in the log
latest_status = self.check_if_running_or_errored(
log_reader)
else:
# No log file exists
# NOTE: This means that the execution is either in a
# queue or has failed the submission.
# For now, we still consider this as submitted
# This can maybe be decided by checking either the
# pid or the status from the submitter
latest_status = "submitted"
metadata_updater.update_latest_status(latest_status)
def __check_if_stopped(self, log_reader: LogReader,
metadata_updater: MetadataUpdater) -> str:
"""
Check if a run has stopped.
Parameters
----------
log_reader : LogReader
The object which reads log files
metadata_updater : MetadataUpdater
Object which updates the database
Returns
-------
latest_status : str
The latest status
Raises
------
RuntimeError
In case log_reader.ended() is True and log_reader.end_time is None
"""
if log_reader.ended():
end_time = log_reader.end_time
# Assert to prevent "Incompatible types in assignment" with Optional
if end_time is None:
raise RuntimeError("log_reader.end_time is None although "
"log_reader.ended() is True")
metadata_updater.update_stop_time(end_time)
latest_status = "complete"
else:
latest_status = self.check_if_running_or_errored(log_reader)
return latest_status
def test_pid(yield_logs: Dict[str, Path]) -> None:
"""
Test self.pid.
Parameters
----------
yield_logs : dict of Path
A dictionary containing the log paths used for testing
"""
log_paths = yield_logs
success_log_reader = LogReader(log_paths["success_log"])
assert success_log_reader.pid == 1191
failed_log_reader = LogReader(log_paths["fail_log"])
assert failed_log_reader.pid == 1190
unfinished_no_pid_log_reader = LogReader(
log_paths["unfinished_no_pid_log"])
assert unfinished_no_pid_log_reader.pid is None
def test_end_time(yield_logs: Dict[str, Path]) -> None:
"""
Test self.end_time.
Parameters
----------
yield_logs : dict of Path
A dictionary containing the log paths used for testing
"""
log_paths = yield_logs
end_time = datetime(2020, 5, 1, 17, 7, 14)
success_log_reader = LogReader(log_paths["success_log"])
assert success_log_reader.end_time == end_time
failed_log_reader = LogReader(log_paths["fail_log"])
assert failed_log_reader.end_time is None
unfinished_no_pid_log_reader = LogReader(
log_paths["unfinished_no_pid_log"])
assert unfinished_no_pid_log_reader.end_time is None
def test_get_simulation_steps(yield_logs: Dict[str, Path]) -> None:
"""
Test self.get_simulation_steps.
Parameters
----------
yield_logs : dict of Path
A dictionary containing the log paths used for testing
"""
log_paths = yield_logs
success_log_reader = LogReader(log_paths["success_log"])
assert success_log_reader.get_simulation_steps().shape == (101, 8)
failed_log_reader = LogReader(log_paths["fail_log"])
assert failed_log_reader.get_simulation_steps().shape == (1, 8)
unfinished_no_pid_log_reader = LogReader(
log_paths["unfinished_no_pid_log"])
assert unfinished_no_pid_log_reader.get_simulation_steps().empty
def __check_running(self, metadata_updater: MetadataUpdater,
running_to_check: DataFrame) -> None:
"""
Check the status of all runs which has status `running`.
Parameters
----------
metadata_updater : MetadataUpdater
Object which updates the database
running_to_check : DataFrame
DataFrame containing the `id` and `name` of the runs with status `running`
"""
for name, run_id in running_to_check.itertuples(index=False):
metadata_updater.run_id = run_id
log_path = self.__project_path.joinpath(name, "BOUT.log.0")
log_reader = LogReader(log_path)
latest_status = self.check_if_running_or_errored(log_reader)
metadata_updater.update_latest_status(latest_status)
def large_graph_tester(
submitter_type: Type[AbstractSubmitter],
make_project: Path,
yield_number_of_rows_for_all_tables: Callable[[DatabaseReader], Dict[str, int]],
file_state_restorer: FileStateRestorer,
) -> None:
"""
Test that the graph with 10 nodes work as expected.
The node setup can be found in node_functions.py
Parameters
----------
submitter_type : type
Used to assert that the correct submitter is used
make_project : Path
The path to the conduction example
yield_number_of_rows_for_all_tables : function
Function which returns the number of rows for all tables in a schema
file_state_restorer : FileStateRestorer
Object for restoring files to original state
"""
name = f"test_large_graph_{submitter_type.__name__}"
node_adder = LargeGraphNodeAdder(
name, make_project, submitter_type, file_state_restorer
)
# RunGroup belonging to node 2
node_adder.add_and_assert_node_group_2()
# RunGroup belonging to node 3 and 4
node_adder.add_and_assert_node_group_3_and_4()
# RunGroup belonging to node 6
node_8 = node_adder.add_and_assert_node_group_6()
# RunGroup belonging to node 9
node_adder.add_and_assert_node_node_9(node_8)
# Run the project
runner = BoutRunner(node_adder.run_graph)
runner.run()
runner.wait_until_completed()
# Check that all the nodes have changed status
with pytest.raises(RuntimeError):
runner.run()
runner.wait_until_completed()
# Check that all files are present
# Check that the pre and post files are present
for node in (0, 1, 5, 7, 8, 10):
assert (
node_adder.paths["pre_and_post_directory"].joinpath(f"{node}.txt").is_file()
)
# Check that all the dump files are present
for restart_str in ("", "_restart_0", "_restart_1", "_restart_2"):
assert (
node_adder.paths["project_path"]
.joinpath(f"{name}{restart_str}", "BOUT.dmp.0.nc")
.is_file()
or node_adder.paths["project_path"]
.joinpath(f"{name}{restart_str}", "BOUT.dmp.0.h5")
.is_file()
)
# NOTE: We will only have 4 runs as node 4 is a duplicate of node 2 and will
# therefore be skipped
number_of_runs = 4
assert_tables_have_expected_len(
DatabaseReader(node_adder.run_groups["run_group_2"].db_connector),
yield_number_of_rows_for_all_tables,
expected_run_number=number_of_runs,
restarted=True,
)
simulation_steps = LogReader(
node_adder.paths["project_path"].joinpath(f"{name}_restart_2", "BOUT.log.0")
).get_simulation_steps()
# NOTE: nout=0 set in the function tests.utils.run.make_run_group
assert np.isclose(simulation_steps.loc[simulation_steps.index[-1], "Sim_time"], 0.0)
def bout_runner_from_path_tester(
submitter_type: Type[AbstractSubmitter],
project_path: Path,
yield_number_of_rows_for_all_tables: Callable[[DatabaseReader], Dict[str, int]],
file_state_restorer: FileStateRestorer,
) -> None:
"""
Test that the minimal BoutRunners setup works.
This test will test that:
1. We can execute a run from the (mocked) current work directory
2. The correct submitter has been used
3. The metadata is properly stored
4. We cannot execute the run again...
5. ...unless we set force=True
6. Check the restart functionality twice
Parameters
----------
submitter_type : type
Submitter type to check for
project_path : Path
The path to the conduction example
yield_number_of_rows_for_all_tables : function
Function which returns the number of rows for all tables in a schema
file_state_restorer : FileStateRestorer
Object for restoring files to original state
"""
# NOTE: This triggers too-many-statements (51/50)
# pylint: disable=too-many-statements
logging.info("Start: First run")
# Make project to save time
_ = project_path
file_state_restorer.add(project_path.joinpath("conduction.db"))
file_state_restorer.add(project_path.joinpath("settings_run"))
with change_directory(project_path):
runner = BoutRunner()
bout_run_setup = runner.run_graph["bout_run_0"]["bout_run_setup"]
file_state_restorer.add(
bout_run_setup.bout_paths.bout_inp_dst_dir, force_mark_removal=True
)
runner.run()
runner.wait_until_completed()
assert isinstance(bout_run_setup.executor.submitter, submitter_type)
bout_paths = bout_run_setup.bout_paths
db_connector = bout_run_setup.db_connector
# Assert that the run went well
db_reader = assert_first_run(bout_paths, db_connector)
# Assert that the number of runs is 1
assert_tables_have_expected_len(
db_reader, yield_number_of_rows_for_all_tables, expected_run_number=1
)
logging.info("Done: First run")
logging.info("Start: Check RuntimeError")
# Check that all the nodes have changed status
with pytest.raises(RuntimeError):
runner.run()
runner.wait_until_completed()
logging.info("Done: Check RuntimeError")
logging.info("Start: Assert that run will not be run again")
# Check that the run will not be executed again
runner.reset()
runner.run()
runner.wait_until_completed()
# Assert that the number of runs is 1
assert_tables_have_expected_len(
db_reader, yield_number_of_rows_for_all_tables, expected_run_number=1
)
logging.info("Done: Assert that run will not be run again")
logging.info("Start: Run with force=True")
# Check that force overrides the behaviour
runner.run(force=True)
runner.wait_until_completed()
assert_tables_have_expected_len(
db_reader, yield_number_of_rows_for_all_tables, expected_run_number=2
)
logging.info("Done: Run with force=True")
logging.info("Start: Run with restart_all=True the first time")
dump_dir_parent = bout_paths.bout_inp_dst_dir.parent
dump_dir_name = bout_paths.bout_inp_dst_dir.name
file_state_restorer.add(dump_dir_parent.joinpath(f"{dump_dir_name}_restart_0"))
# Check that the restart functionality works
runner.run(restart_all=True)
runner.wait_until_completed()
assert_tables_have_expected_len(
db_reader,
yield_number_of_rows_for_all_tables,
expected_run_number=3,
restarted=True,
)
# NOTE: The test in tests.unit.bout_runners.runners.test_bout_runner is testing
# restart_from_bout_inp_dst=True, whether this is testing restart_all=True
assert_dump_files_exist(dump_dir_parent.joinpath(f"{dump_dir_name}_restart_0"))
logging.info("Done: Run with restart_all=True the first time")
logging.info("Start: Run with restart_all=True the second time")
# ...twice
file_state_restorer.add(dump_dir_parent.joinpath(f"{dump_dir_name}_restart_1"))
runner.run(restart_all=True)
runner.wait_until_completed()
assert_tables_have_expected_len(
db_reader,
yield_number_of_rows_for_all_tables,
expected_run_number=4,
restarted=True,
)
# NOTE: The test in tests.unit.bout_runners.runners.test_bout_runner is testing
# restart_from_bout_inp_dst=True, whether this is testing restart_all=True
assert_dump_files_exist(dump_dir_parent.joinpath(f"{dump_dir_name}_restart_1"))
simulation_steps = LogReader(
dump_dir_parent.joinpath(f"{dump_dir_name}_restart_1", "BOUT.log.0")
).get_simulation_steps()
assert np.isclose(
simulation_steps.loc[simulation_steps.index[-1], "Sim_time"], 30.0
)
logging.info("Done: Run with restart_all=True the second time")