def test_given_2_studies_when_launch_controller_launch_all_studies_is_called_then_connection_upload_file_is_called_twice(
self,
):
# given
connection = mock.Mock()
slurm_script_features = SlurmScriptFeatures()
environment = RemoteEnvironmentWithSlurm(connection, slurm_script_features)
study1 = mock.Mock()
study1.zipfile_path = "filepath"
study1.zip_is_sent = False
study1.path = "path"
study2 = mock.Mock()
study2.zipfile_path = "filepath"
study2.zip_is_sent = False
study2.path = "path"
data_repo = mock.Mock()
data_repo.get_list_of_studies = mock.Mock(return_value=[study1, study2])
file_manager = mock.Mock()
display = DisplayTerminal()
launch_controller = LaunchController(
repo=data_repo,
env=environment,
file_manager=file_manager,
display=display,
)
# when
launch_controller.launch_all_studies()
# then
assert connection.upload_file.call_count == 2
def launch_controller(self):
connection = mock.Mock()
slurm_script_features = SlurmScriptFeatures("slurm_script_path")
environment = RemoteEnvironmentWithSlurm(connection,
slurm_script_features)
study1 = mock.Mock()
study1.zipfile_path = "filepath"
study1.zip_is_sent = False
study1.path = "path"
study2 = mock.Mock()
study2.zipfile_path = "filepath"
study2.zip_is_sent = False
study2.path = "path"
data_repo = mock.Mock()
data_repo.get_list_of_studies = mock.Mock(
return_value=[study1, study2])
file_manager = mock.Mock()
display = DisplayTerminal()
launch_controller = LaunchController(
repo=data_repo,
env=environment,
file_manager=file_manager,
display=display,
)
return launch_controller
def my_remote_env_with_slurm_mock(self):
remote_home_dir = "remote_home_dir"
connection = mock.Mock()
connection.make_dir = mock.Mock(return_value=True)
connection.check_file_not_empty = mock.Mock(return_value=True)
connection.home_dir = remote_home_dir
slurm_script_features = SlurmScriptFeatures()
return RemoteEnvironmentWithSlurm(connection, slurm_script_features)
def test_given_time_limit_lower_than_min_duration_when_convert_time_is_called_return_min_duration(
self, ):
# given
time_lim_sec = 42
# when
output = RemoteEnvironmentWithSlurm.convert_time_limit_from_seconds_to_minutes(
time_lim_sec)
# then
assert output == 1
def test_when_constructor_is_called_and_remote_base_path_cannot_be_created_then_exception_is_raised(
self, ):
# given
connection = mock.Mock()
slurm_script_features = SlurmScriptFeatures("slurm_script_path")
# when
connection.make_dir = mock.Mock(return_value=False)
# then
with pytest.raises(NoRemoteBaseDirException):
RemoteEnvironmentWithSlurm(connection, slurm_script_features)
def test_when_constructor_is_called_and_connection_check_file_not_empty_is_false_then_exception_is_raised(
self, ):
# given
remote_home_dir = "/applis/antares/"
connection = mock.Mock()
connection.home_dir = remote_home_dir
connection.make_dir = mock.Mock(return_value=True)
slurm_script_features = SlurmScriptFeatures()
# when
connection.check_file_not_empty = mock.Mock(return_value=False)
# then
with pytest.raises(NoLaunchScriptFoundException):
RemoteEnvironmentWithSlurm(connection, slurm_script_features)
def test_when_constructor_is_called_then_connection_check_file_not_empty_is_called_with_correct_arguments(
self, ):
# given
connection = mock.Mock()
connection.make_dir = mock.Mock(return_value=True)
connection.check_file_not_empty = mock.Mock(return_value=True)
slurm_script_features = SlurmScriptFeatures("slurm_script_path")
# when
RemoteEnvironmentWithSlurm(connection, slurm_script_features)
# then
remote_script_antares = "slurm_script_path"
connection.check_file_not_empty.assert_called_with(
remote_script_antares)
def setup_method(self):
connection = mock.Mock()
connection.home_dir = "Submitted"
slurm_script_features = SlurmScriptFeatures()
self.env = RemoteEnvironmentWithSlurm(
_connection=connection,
slurm_script_features=slurm_script_features)
state_updater = StateUpdater(env=self.env, display=mock.Mock())
self.retrieve_controller = RetrieveController(
repo=mock.Mock(),
env=self.env,
file_manager=mock.Mock(),
display=mock.Mock(),
state_updater=state_updater,
)
def test_when_constructor_is_called_then_connection_check_file_not_empty_is_called_with_correct_arguments(
self, ):
# given
remote_home_dir = "/applis/antares"
connection = mock.Mock()
connection.home_dir = remote_home_dir
connection.make_dir = mock.Mock(return_value=True)
connection.check_file_not_empty = mock.Mock(return_value=True)
slurm_script_features = SlurmScriptFeatures()
# when
RemoteEnvironmentWithSlurm(connection, slurm_script_features)
# then
remote_script_antares = definitions.SLURM_SCRIPT_PATH
connection.check_file_not_empty.assert_called_with(
remote_script_antares)
def test_initialise_remote_path_calls_connection_make_dir_with_correct_arguments(
self, ):
# given
remote_home_dir = "remote_home_dir"
remote_base_dir = (str(remote_home_dir) + "/REMOTE_" +
getpass.getuser() + "_" + socket.gethostname())
connection = mock.Mock()
connection.home_dir = remote_home_dir
connection.make_dir = mock.Mock(return_value=True)
connection.check_file_not_empty = mock.Mock(return_value=True)
slurm_script_features = SlurmScriptFeatures()
# when
RemoteEnvironmentWithSlurm(connection, slurm_script_features)
# then
connection.make_dir.assert_called_with(remote_base_dir)
def setup_method(self):
self.connection_mock = mock.Mock()
self.connection_mock.username = "******"
self.connection_mock.execute_command = mock.Mock(return_value=("", ""))
slurm_script_features = SlurmScriptFeatures("slurm_script_path")
env_mock = RemoteEnvironmentWithSlurm(
_connection=self.connection_mock,
slurm_script_features=slurm_script_features,
)
display_mock = mock.Mock()
slurm_queue_show = SlurmQueueShow(env_mock, display_mock)
state_updater = StateUpdater(env_mock, display_mock)
repo = mock.MagicMock(spec=IDataRepo)
self.check_queue_controller = CheckQueueController(
slurm_queue_show, state_updater, repo)
def test_given_a_study_when_launch_controller_submit_job_is_called_then_connection_execute_command_is_called_once_with_correct_command(
self,
):
# given
connection = mock.Mock()
connection.execute_command = mock.Mock(return_value=["Submitted 42", ""])
connection.home_dir = "Submitted"
slurm_script_features = SlurmScriptFeatures()
environment = RemoteEnvironmentWithSlurm(connection, slurm_script_features)
study1 = StudyDTO(
path="dummy_path",
zipfile_path=str(Path("base_path") / "zip_name"),
zip_is_sent=False,
n_cpu=12,
antares_version=700,
time_limit=120,
)
home_dir = "Submitted"
remote_base_path = (
str(home_dir) + "/REMOTE_" + getpass.getuser() + "_" + socket.gethostname()
)
study_type = "ANTARES"
post_processing = False
command = (
f"cd {remote_base_path} && "
f'sbatch --job-name="{Path(study1.path).name}" --time={study1.time_limit // 60} --cpus-per-task={study1.n_cpu}'
f" {environment.slurm_script_features.solver_script_path}"
f' "{Path(study1.zipfile_path).name}" {study1.antares_version} {study_type} {post_processing}'
)
data_repo = mock.Mock()
data_repo.get_list_of_studies = mock.Mock(return_value=[study1])
file_manager = mock.Mock()
display = DisplayTerminal()
launch_controller = LaunchController(
repo=data_repo,
env=environment,
file_manager=file_manager,
display=display,
)
# when
launch_controller.launch_all_studies() # _submit_job(study1)
# then
connection.execute_command.assert_called_once_with(command)
def run_with(
arguments: argparse.Namespace, parameters: MainParameters, show_banner=False
):
"""Instantiates all the objects necessary to antares-launcher, and runs the program"""
if arguments.version:
print(f"Antares_Launcher v{VERSION}")
return
if show_banner:
print(ANTARES_LAUNCHER_BANNER)
display = DisplayTerminal()
file_manager = FileManager(display)
db_json_file_path = parameters.json_dir / parameters.default_json_db_name
tree_structure_initializer = TreeStructureInitializer(
display,
file_manager,
arguments.studies_in,
arguments.log_dir,
arguments.output_dir,
)
tree_structure_initializer.init_tree_structure()
logger_initializer = LoggerInitializer(
str(Path(arguments.log_dir) / "antares_launcher.log")
)
logger_initializer.init_logger()
# connection
ssh_dict = get_ssh_config_dict(
file_manager,
arguments.json_ssh_config,
parameters.default_ssh_dict,
)
connection = ssh_connection.SshConnection(config=ssh_dict)
verify_connection(connection, display)
slurm_script_features = SlurmScriptFeatures(parameters.slurm_script_path)
environment = RemoteEnvironmentWithSlurm(connection, slurm_script_features)
data_repo = DataRepoTinydb(
database_file_path=db_json_file_path, db_primary_key=parameters.db_primary_key
)
study_list_composer = StudyListComposer(
repo=data_repo,
file_manager=file_manager,
display=display,
parameters=StudyListComposerParameters(
studies_in_dir=arguments.studies_in,
time_limit=arguments.time_limit,
log_dir=arguments.log_dir,
n_cpu=arguments.n_cpu,
xpansion_mode=arguments.xpansion_mode,
output_dir=arguments.output_dir,
post_processing=arguments.post_processing,
antares_versions_on_remote_server=parameters.antares_versions_on_remote_server,
),
)
launch_controller = LaunchController(
repo=data_repo,
env=environment,
file_manager=file_manager,
display=display,
)
state_updater = StateUpdater(env=environment, display=display)
retrieve_controller = RetrieveController(
repo=data_repo,
env=environment,
file_manager=file_manager,
display=display,
state_updater=state_updater,
)
slurm_queue_show = SlurmQueueShow(env=environment, display=display)
check_queue_controller = CheckQueueController(
slurm_queue_show=slurm_queue_show,
state_updater=state_updater,
repo=data_repo,
)
job_kill_controller = JobKillController(
env=environment,
display=display,
repo=data_repo,
)
wait_controller = WaitController(display=display)
launcher = AntaresLauncher(
study_list_composer=study_list_composer,
launch_controller=launch_controller,
retrieve_controller=retrieve_controller,
job_kill_controller=job_kill_controller,
check_queue_controller=check_queue_controller,
wait_controller=wait_controller,
wait_mode=arguments.wait_mode,
wait_time=arguments.wait_time,
job_id_to_kill=arguments.job_id_to_kill,
xpansion_mode=arguments.xpansion_mode,
check_queue_bool=arguments.check_queue,
)
launcher.run()
def my_remote_env_with_slurm_mock(self):
remote_home_dir = "remote_home_dir"
connection = mock.Mock()
connection.home_dir = remote_home_dir
slurm_script_features = SlurmScriptFeatures("slurm_script_path")
return RemoteEnvironmentWithSlurm(connection, slurm_script_features)
def setup_method(self):
slurm_script_features = SlurmScriptFeatures("slurm_script_path")
env = RemoteEnvironmentWithSlurm(mock.Mock(), slurm_script_features)
self.job_kill_controller = JobKillController(env,
mock.Mock(),
repo=mock.Mock())
def run_with(arguments):
"""Instantiates all the objects necessary to antares-launcher, and runs the program"""
if arguments.version:
print(f"Antares_Launcher v{VERSION}")
return
print(ANTARES_LAUNCHER_BANNER)
studies_in = Path(arguments.studies_in).resolve()
display = DisplayTerminal()
file_manager = FileManager(display)
json_file_name = JSON_DIR / DEFAULT_JSON_DB_NAME
tree_structure_initializer = TreeStructureInitializer(
display,
file_manager,
arguments.studies_in,
definitions.LOG_DIR,
arguments.output_dir,
)
tree_structure_initializer.init_tree_structure()
logger_initializer = LoggerInitializer(
Path(arguments.log_dir) / "antares_launcher.log")
logger_initializer.init_logger()
# connection
ssh_dict = get_ssh_config_dict(file_manager, arguments.json_ssh_config)
connection = ssh_connection.SshConnection(config=ssh_dict)
verify_connection(connection, display)
slurm_script_features = SlurmScriptFeatures()
environment = RemoteEnvironmentWithSlurm(connection, slurm_script_features)
data_repo = DataRepoTinydb(database_name=json_file_name)
study_list_composer = StudyListComposer(
repo=data_repo,
file_manager=file_manager,
display=display,
studies_in_dir=studies_in,
time_limit=arguments.time_limit,
n_cpu=arguments.n_cpu,
log_dir=arguments.log_dir,
output_dir=arguments.output_dir,
xpansion_mode=arguments.xpansion_mode,
post_processing=arguments.post_processing,
)
launch_controller = LaunchController(
repo=data_repo,
env=environment,
file_manager=file_manager,
display=display,
)
state_updater = StateUpdater(env=environment, display=display)
retrieve_controller = RetrieveController(
repo=data_repo,
env=environment,
file_manager=file_manager,
display=display,
state_updater=state_updater,
)
slurm_queue_show = SlurmQueueShow(env=environment, display=display)
check_queue_controller = CheckQueueController(
slurm_queue_show=slurm_queue_show,
state_updater=state_updater,
repo=data_repo)
job_kill_controller = JobKillController(
env=environment,
display=display,
repo=data_repo,
)
wait_controller = WaitController(display=display)
launcher = AntaresLauncher(
study_list_composer=study_list_composer,
launch_controller=launch_controller,
retrieve_controller=retrieve_controller,
job_kill_controller=job_kill_controller,
check_queue_controller=check_queue_controller,
wait_controller=wait_controller,
wait_mode=arguments.wait_mode,
wait_time=arguments.wait_time,
job_id_to_kill=arguments.job_id_to_kill,
xpansion_mode=arguments.xpansion_mode,
check_queue_bool=arguments.check_queue,
)
launcher.run()
