def test_submitter_factory() -> None:
"""Test that the SubmitterFactory returns Submitter objects."""
submitter = get_submitter(name="local", argument_dict=dict())
assert isinstance(submitter, LocalSubmitter)
with pytest.raises(NotImplementedError):
get_submitter(name="not a class", argument_dict=dict())
def __make_restart_files_node(self, to_node_name: str,
copy_restart_from: Path,
copy_restart_to: Path) -> str:
"""
Make nodes which copies restart files.
Parameters
----------
to_node_name : str
Name of the node which will wait for a restart node
copy_restart_from : Path
Path to copy restart files from
copy_restart_to : Path
Path to copy restart files to
Returns
-------
current_node_name : str
Name of the node which copies files
"""
current_node_name = (
f"copy_restart_files_from_{copy_restart_from.name}_to_"
f"{copy_restart_to.name}_for_{to_node_name}")
function_dict: Dict[str, Optional[Union[Callable, Tuple[Any, ...],
Dict[str, Any]]]] = {
"function":
copy_restart_files,
"args": (copy_restart_from,
copy_restart_to),
"kwargs": None,
}
path = copy_restart_to.joinpath(f"{current_node_name}.py")
submitter = get_submitter()
if isinstance(submitter, AbstractClusterSubmitter):
submitter.store_dir = copy_restart_to
submitter.job_name = current_node_name
self.__run_graph.add_function_node(
name=current_node_name,
function_dict=function_dict,
path=path,
submitter=submitter,
)
return current_node_name
def __init__(
self,
bout_paths: Optional[BoutPaths] = None,
submitter: Optional[AbstractSubmitter] = None,
run_parameters: Optional[RunParameters] = None,
restart_from: Optional[Path] = None,
) -> None:
"""
Set the input parameters.
Parameters
----------
bout_paths : BoutPaths or None
Object containing the paths
If None, default BoutPaths values will be used
submitter : AbstractSubmitter
Object containing the submitter
run_parameters : RunParameters or None
Object containing the run parameters
If None, default parameters will be used
restart_from : Path or None
The path to copy the restart files from
"""
# NOTE: We are not setting the default as a keyword argument
# as this would mess up the paths
# NOTE: We are deepcopying bout_paths as it may be altered by for
# example the self.restart_from setter
logging.info("Start: Making an BoutRunExecutor object")
self.__bout_paths = (
deepcopy(bout_paths) if bout_paths is not None else BoutPaths()
)
self.__run_parameters = (
run_parameters if run_parameters is not None else RunParameters()
)
self.__make = Make(self.__bout_paths.project_path)
self.submitter = submitter if submitter is not None else get_submitter()
if isinstance(self.submitter, AbstractClusterSubmitter):
self.submitter.store_dir = self.__bout_paths.bout_inp_dst_dir
self.__restart_from = None
self.restart_from = restart_from
logging.info("Done: Making an BoutRunExecutor object")
def add_post_processor(
self,
function_dict: Dict[str, Optional[Union[Callable, Tuple[Any, ...],
Dict[str, Any]]]],
directory: Optional[Path] = None,
submitter: Optional[AbstractSubmitter] = None,
waiting_for: Optional[Union[str, Iterable[str]]] = None,
) -> str:
"""
Add a post-processor to the BOUT++ run.
The function and the parameters will be saved to a python script which will
be submitted
Parameters
----------
function_dict : dict
Dict with the function to call
On the form
>>> {'function': callable,
... 'args': None or tuple,
... 'kwargs': None or dict}
directory : None or Path
Absolute path to directory to store the python script
If None, the destination directory of BoutRun will be used
waiting_for : None or str or iterable
Name of nodes this node will wait for to finish before executing
submitter : None or AbstractSubmitter
Submitter to submit the function with
If None, the default LocalSubmitter will be used
Returns
-------
post_processor_node_name : str
The node name of the pre-processor
Raises
------
ValueError
If the function in the function_dict is not callable
"""
if directory is None:
directory = self.__dst_dir
if "function" not in function_dict.keys() or not callable(
function_dict["function"]):
msg = 'function_dict["function"] must be callable'
logging.error(msg)
raise ValueError(msg)
post_processor_node_name = (
f"post_processor_{self.__name}_{len(self.__post_processors)}")
path = directory.joinpath(
f"{function_dict['function'].__name__}_{post_processor_node_name}.py"
)
if submitter is None:
submitter = get_submitter()
if isinstance(submitter, AbstractClusterSubmitter):
submitter.job_name = post_processor_node_name
submitter.store_dir = self.__bout_run_setup.bout_paths.bout_inp_dst_dir
self.__run_graph.add_function_node(
post_processor_node_name,
function_dict=function_dict,
path=path,
submitter=submitter,
)
self.__run_graph.add_edge(self.bout_run_node_name,
post_processor_node_name)
self.__run_graph.add_waiting_for(post_processor_node_name, waiting_for)
self.__post_processors.append(post_processor_node_name)
return post_processor_node_name
def make_run_group(
run_group_parameters: Dict[str, Union[str, Optional[RunGraph],
Optional[Union[str,
Iterable[str]]]]],
make_project: Path,
file_state_restorer: FileStateRestorer,
restart_from: Optional[Path] = None,
) -> RunGroup:
"""
Return a basic RunGroup.
Parameters
----------
run_group_parameters : dict
Parameters to the run_group containing the keys
- name : str
Name of the run_group
Note that the name will also be used for the destination dir and
the name of the database
- run_graph: None or RunGraph
The run_graph to use
- waiting_for : None or str or iterable of str
Name of nodes this node will wait for to finish before executing
make_project : Path
The path to the conduction example
file_state_restorer : FileStateRestorer
Object for restoring files to original state
restart_from : Path or None
The path to copy the restart files from
Returns
-------
run_group : RunGroup
A basic run group
Raises
------
ValueError
If the shape or types of the run_group_parameters are wrong
"""
# NOTE: The following is a mypy guard which could be solved with TypedDict
# However, TypedDict is new from 3.8
if "name" not in run_group_parameters.keys() or not isinstance(
run_group_parameters["name"], str):
raise ValueError(
"'name' must be of string type in run_group_parameters")
if "run_graph" not in run_group_parameters.keys() or not (
isinstance(run_group_parameters["run_graph"], RunGraph)
or run_group_parameters["run_graph"] is None):
raise ValueError(
"'run_graph' must be of RunGroup type or None in run_group_parameters"
)
if ("waiting_for" not in run_group_parameters.keys()
or not (hasattr(run_group_parameters["waiting_for"], "__iter__")
or run_group_parameters["waiting_for"] is None)
or isinstance(run_group_parameters["waiting_for"], RunGraph)):
raise ValueError(
"'waiting_for' must be of RunGroup type or None in run_group_parameters"
)
# Make project to save time
project_path = make_project
# Create the `bout_paths` object
bout_paths = BoutPaths(
project_path=project_path,
bout_inp_src_dir=project_path.joinpath("data"),
bout_inp_dst_dir=project_path.joinpath(run_group_parameters["name"]),
)
# Create the input objects
run_parameters = RunParameters({"global": {"nout": 0}})
default_parameters = DefaultParameters(bout_paths)
final_parameters = FinalParameters(default_parameters, run_parameters)
submitter = get_submitter()
if isinstance(submitter, LocalSubmitter):
submitter.run_path = bout_paths.project_path
executor = BoutRunExecutor(
bout_paths=bout_paths,
submitter=submitter,
run_parameters=run_parameters,
restart_from=restart_from,
)
db_connector = DatabaseConnector(name=run_group_parameters["name"],
db_root_path=project_path)
bout_run_setup = BoutRunSetup(executor, db_connector, final_parameters)
# Create the `run_group`
run_group = RunGroup(
run_group_parameters["run_graph"]
if run_group_parameters["run_graph"] is not None else RunGraph(),
bout_run_setup,
name=run_group_parameters["name"],
waiting_for=run_group_parameters["waiting_for"],
)
file_state_restorer.add(executor.bout_paths.bout_inp_dst_dir,
force_mark_removal=True)
file_state_restorer.add(db_connector.db_path, force_mark_removal=True)
file_state_restorer.add(
executor.bout_paths.project_path.joinpath("settings_run"),
force_mark_removal=True,
)
return run_group