def test_run_on_list_calls_run_start__processing_studies_that_are_done():
# given
my_study1 = StudyDTO(path="study_path1", job_id=1, done=False)
my_study2 = StudyDTO(path="study_path2", job_id=None)
my_study3 = StudyDTO(path="study_path3", job_id=2, done=True)
study_list = [my_study1, my_study2, my_study3]
remote_env_mock = mock.Mock()
remote_env_mock.get_job_state_flags = mock.Mock(return_value=(True, False,
False))
display = mock.Mock()
display.show_message = mock.Mock()
state_updater = StateUpdater(remote_env_mock, display)
# when
state_updater.run_on_list(study_list)
# then
welcome_message = "Checking status of the studies:"
message1 = f'"{Path(my_study1.path).name}" (JOBID={my_study1.job_id}): Running'
message3 = (
f'"{Path(my_study3.path).name}" (JOBID={my_study3.job_id}): everything is done'
)
calls = [
call(welcome_message, mock.ANY),
call(message3, mock.ANY),
call(message1, mock.ANY),
]
display.show_message.assert_has_calls(calls)
def test_state_updater_run_on_empty_list_of_studies_write_one_message():
# given
study_list = []
remote_env_mock = mock.Mock()
display = mock.Mock()
state_updater = StateUpdater(remote_env_mock, display)
message = "Checking status of the studies:"
# when
state_updater.run_on_list(study_list)
# then
display.show_message.assert_called_once_with(message, mock.ANY)
def test_with_a_list_of_one_submitted_study_run_on_list_calls_run_once_on_study(
):
# given
my_study1 = StudyDTO(path="study_path1", job_id=1)
study_list = [my_study1]
remote_env_mock = mock.Mock()
remote_env_mock.get_job_state_flags = mock.Mock(return_value=(1, 2, 3))
display = mock.Mock()
state_updater = StateUpdater(remote_env_mock, display)
# when
state_updater.run_on_list(study_list)
# then
remote_env_mock.get_job_state_flags.assert_called_once_with(my_study1)
def test_given_a_non_submitted_study_then_get_job_state_flags_is_not_called():
# given
my_study = StudyDTO(path="study_path", job_id=None)
remote_env_mock = mock.Mock()
remote_env_mock.get_job_state_flags = mock.Mock()
display = mock.Mock()
display.show_error = mock.Mock()
state_updater = StateUpdater(remote_env_mock, display)
message = f'"{Path(my_study.path).name}": Job was not submitted'
# when
state_updater.run(my_study)
# then
remote_env_mock.get_job_state_flags.assert_not_called()
display.show_error.assert_called_once_with(message, mock.ANY)
def test_run_on_list_calls_run_on_all_submitted_studies_of_the_list():
# given
my_study1 = StudyDTO(path="study_path1", job_id=1)
my_study2 = StudyDTO(path="study_path2", job_id=None)
my_study3 = StudyDTO(path="study_path3", job_id=2)
study_list = [my_study1, my_study2, my_study3]
remote_env_mock = mock.Mock()
remote_env_mock.get_job_state_flags = mock.Mock(return_value=(1, 2, 3))
display = mock.Mock()
state_updater = StateUpdater(remote_env_mock, display)
# when
state_updater.run_on_list(study_list)
# then
calls = [call(my_study1), call(my_study3)]
remote_env_mock.get_job_state_flags.assert_has_calls(calls)
def setup_method(self):
self.remote_env_mock = mock.Mock(
spec=iremote_environment.IRemoteEnvironment)
self.file_manager = mock.Mock()
self.data_repo = mock.Mock()
self.display = mock.Mock()
self.state_updater_mock = StateUpdater(self.remote_env_mock,
self.display)
def test_given_a_done_study_then_get_job_state_flags_is_not_called():
# given
my_study = StudyDTO(path="study_path", job_id=42, done=True)
remote_env_mock = mock.Mock()
remote_env_mock.get_job_state_flags = mock.Mock(return_value=(True, False,
False))
display = mock.Mock()
display.show_message = mock.Mock()
state_updater = StateUpdater(remote_env_mock, display)
message = (
f'"{Path(my_study.path).name}" (JOBID={my_study.job_id}): everything is done'
)
# when
state_updater.run(my_study)
# then
remote_env_mock.get_job_state_flags.assert_not_called()
display.show_message.assert_called_once_with(message, mock.ANY)
def test_given_a_submitted_study_then_study_flags_are_updated(
started_flag, finished_flag, with_error_flag, status):
# given
my_study = StudyDTO(path="study_path", job_id=42)
remote_env_mock = mock.Mock()
display = mock.Mock()
remote_env_mock.get_job_state_flags = mock.Mock(
return_value=(started_flag, finished_flag, with_error_flag))
display.show_message = mock.Mock()
state_updater = StateUpdater(remote_env_mock, display)
message = f'"{Path(my_study.path).name}" (JOBID={my_study.job_id}): {status}'
# when
study_test = state_updater.run(my_study)
# then
remote_env_mock.get_job_state_flags.assert_called_once_with(my_study)
display.show_message.assert_called_once_with(message, mock.ANY)
assert study_test.started == started_flag
assert study_test.finished == finished_flag
assert study_test.with_error == with_error_flag
assert study_test.job_state == status
def setup_method(self):
self.repo_mock = mock.Mock(spec=IDataRepo)
self.env = mock.Mock()
self.display = mock.Mock()
self.slurm_queue_show = SlurmQueueShow(env=self.env,
display=self.display)
self.state_updater = StateUpdater(env=mock.Mock(),
display=self.display)
self.check_queue_controller = CheckQueueController(
slurm_queue_show=self.slurm_queue_show,
state_updater=self.state_updater,
repo=self.repo_mock,
)
def setup_method(self):
connection = mock.Mock()
connection.home_dir = "Submitted"
slurm_script_features = SlurmScriptFeatures()
self.env = RemoteEnvironmentWithSlurm(
_connection=connection,
slurm_script_features=slurm_script_features)
state_updater = StateUpdater(env=self.env, display=mock.Mock())
self.retrieve_controller = RetrieveController(
repo=mock.Mock(),
env=self.env,
file_manager=mock.Mock(),
display=mock.Mock(),
state_updater=state_updater,
)
def setup_method(self):
self.connection_mock = mock.Mock()
self.connection_mock.username = "******"
self.connection_mock.execute_command = mock.Mock(return_value=("", ""))
slurm_script_features = SlurmScriptFeatures("slurm_script_path")
env_mock = RemoteEnvironmentWithSlurm(
_connection=self.connection_mock,
slurm_script_features=slurm_script_features,
)
display_mock = mock.Mock()
slurm_queue_show = SlurmQueueShow(env_mock, display_mock)
state_updater = StateUpdater(env_mock, display_mock)
repo = mock.MagicMock(spec=IDataRepo)
self.check_queue_controller = CheckQueueController(
slurm_queue_show, state_updater, repo)
def setup_method(self):
env = mock.Mock(spec_set=IRemoteEnvironment)
display = mock.Mock(spec_set=IDisplay)
file_manager = mock.Mock(spec_set=FileManager)
repo = mock.Mock(spec_set=IDataRepo)
self.reporter = DataReporter(repo)
self.state_updater = StateUpdater(env, display)
self.logs_downloader = LogDownloader(env, file_manager, display)
self.final_zip_downloader = FinalZipDownloader(env, display)
self.remote_server_cleaner = RemoteServerCleaner(env, display)
self.zip_extractor = FinalZipExtractor(file_manager, display)
self.study_retriever = StudyRetriever(
self.state_updater,
self.logs_downloader,
self.final_zip_downloader,
self.remote_server_cleaner,
self.zip_extractor,
self.reporter,
)
def run_with(
arguments: argparse.Namespace, parameters: MainParameters, show_banner=False
):
"""Instantiates all the objects necessary to antares-launcher, and runs the program"""
if arguments.version:
print(f"Antares_Launcher v{VERSION}")
return
if show_banner:
print(ANTARES_LAUNCHER_BANNER)
display = DisplayTerminal()
file_manager = FileManager(display)
db_json_file_path = parameters.json_dir / parameters.default_json_db_name
tree_structure_initializer = TreeStructureInitializer(
display,
file_manager,
arguments.studies_in,
arguments.log_dir,
arguments.output_dir,
)
tree_structure_initializer.init_tree_structure()
logger_initializer = LoggerInitializer(
str(Path(arguments.log_dir) / "antares_launcher.log")
)
logger_initializer.init_logger()
# connection
ssh_dict = get_ssh_config_dict(
file_manager,
arguments.json_ssh_config,
parameters.default_ssh_dict,
)
connection = ssh_connection.SshConnection(config=ssh_dict)
verify_connection(connection, display)
slurm_script_features = SlurmScriptFeatures(parameters.slurm_script_path)
environment = RemoteEnvironmentWithSlurm(connection, slurm_script_features)
data_repo = DataRepoTinydb(
database_file_path=db_json_file_path, db_primary_key=parameters.db_primary_key
)
study_list_composer = StudyListComposer(
repo=data_repo,
file_manager=file_manager,
display=display,
parameters=StudyListComposerParameters(
studies_in_dir=arguments.studies_in,
time_limit=arguments.time_limit,
log_dir=arguments.log_dir,
n_cpu=arguments.n_cpu,
xpansion_mode=arguments.xpansion_mode,
output_dir=arguments.output_dir,
post_processing=arguments.post_processing,
antares_versions_on_remote_server=parameters.antares_versions_on_remote_server,
),
)
launch_controller = LaunchController(
repo=data_repo,
env=environment,
file_manager=file_manager,
display=display,
)
state_updater = StateUpdater(env=environment, display=display)
retrieve_controller = RetrieveController(
repo=data_repo,
env=environment,
file_manager=file_manager,
display=display,
state_updater=state_updater,
)
slurm_queue_show = SlurmQueueShow(env=environment, display=display)
check_queue_controller = CheckQueueController(
slurm_queue_show=slurm_queue_show,
state_updater=state_updater,
repo=data_repo,
)
job_kill_controller = JobKillController(
env=environment,
display=display,
repo=data_repo,
)
wait_controller = WaitController(display=display)
launcher = AntaresLauncher(
study_list_composer=study_list_composer,
launch_controller=launch_controller,
retrieve_controller=retrieve_controller,
job_kill_controller=job_kill_controller,
check_queue_controller=check_queue_controller,
wait_controller=wait_controller,
wait_mode=arguments.wait_mode,
wait_time=arguments.wait_time,
job_id_to_kill=arguments.job_id_to_kill,
xpansion_mode=arguments.xpansion_mode,
check_queue_bool=arguments.check_queue,
)
launcher.run()
def run_with(arguments):
"""Instantiates all the objects necessary to antares-launcher, and runs the program"""
if arguments.version:
print(f"Antares_Launcher v{VERSION}")
return
print(ANTARES_LAUNCHER_BANNER)
studies_in = Path(arguments.studies_in).resolve()
display = DisplayTerminal()
file_manager = FileManager(display)
json_file_name = JSON_DIR / DEFAULT_JSON_DB_NAME
tree_structure_initializer = TreeStructureInitializer(
display,
file_manager,
arguments.studies_in,
definitions.LOG_DIR,
arguments.output_dir,
)
tree_structure_initializer.init_tree_structure()
logger_initializer = LoggerInitializer(
Path(arguments.log_dir) / "antares_launcher.log")
logger_initializer.init_logger()
# connection
ssh_dict = get_ssh_config_dict(file_manager, arguments.json_ssh_config)
connection = ssh_connection.SshConnection(config=ssh_dict)
verify_connection(connection, display)
slurm_script_features = SlurmScriptFeatures()
environment = RemoteEnvironmentWithSlurm(connection, slurm_script_features)
data_repo = DataRepoTinydb(database_name=json_file_name)
study_list_composer = StudyListComposer(
repo=data_repo,
file_manager=file_manager,
display=display,
studies_in_dir=studies_in,
time_limit=arguments.time_limit,
n_cpu=arguments.n_cpu,
log_dir=arguments.log_dir,
output_dir=arguments.output_dir,
xpansion_mode=arguments.xpansion_mode,
post_processing=arguments.post_processing,
)
launch_controller = LaunchController(
repo=data_repo,
env=environment,
file_manager=file_manager,
display=display,
)
state_updater = StateUpdater(env=environment, display=display)
retrieve_controller = RetrieveController(
repo=data_repo,
env=environment,
file_manager=file_manager,
display=display,
state_updater=state_updater,
)
slurm_queue_show = SlurmQueueShow(env=environment, display=display)
check_queue_controller = CheckQueueController(
slurm_queue_show=slurm_queue_show,
state_updater=state_updater,
repo=data_repo)
job_kill_controller = JobKillController(
env=environment,
display=display,
repo=data_repo,
)
wait_controller = WaitController(display=display)
launcher = AntaresLauncher(
study_list_composer=study_list_composer,
launch_controller=launch_controller,
retrieve_controller=retrieve_controller,
job_kill_controller=job_kill_controller,
check_queue_controller=check_queue_controller,
wait_controller=wait_controller,
wait_mode=arguments.wait_mode,
wait_time=arguments.wait_time,
job_id_to_kill=arguments.job_id_to_kill,
xpansion_mode=arguments.xpansion_mode,
check_queue_bool=arguments.check_queue,
)
launcher.run()
