def test_run_on_list_calls_run_start__processing_studies_that_are_done():
# given
my_study1 = StudyDTO(path="study_path1", job_id=1, done=False)
my_study2 = StudyDTO(path="study_path2", job_id=None)
my_study3 = StudyDTO(path="study_path3", job_id=2, done=True)
study_list = [my_study1, my_study2, my_study3]
remote_env_mock = mock.Mock()
remote_env_mock.get_job_state_flags = mock.Mock(return_value=(True, False,
False))
display = mock.Mock()
display.show_message = mock.Mock()
state_updater = StateUpdater(remote_env_mock, display)
# when
state_updater.run_on_list(study_list)
# then
welcome_message = "Checking status of the studies:"
message1 = f'"{Path(my_study1.path).name}" (JOBID={my_study1.job_id}): Running'
message3 = (
f'"{Path(my_study3.path).name}" (JOBID={my_study3.job_id}): everything is done'
)
calls = [
call(welcome_message, mock.ANY),
call(message3, mock.ANY),
call(message1, mock.ANY),
]
display.show_message.assert_has_calls(calls)
def test_launch_study_calls_all_four_steps(self):
study = StudyDTO(path="hello")
study1 = StudyDTO(path="hello1")
self.zipper.zip = mock.Mock(return_value=study1)
study2 = StudyDTO(path="hello2", zip_is_sent=True)
self.study_uploader.upload = mock.Mock(return_value=study2)
study3 = StudyDTO(path="hello3")
self.zipfile_cleaner.remove_input_zipfile = mock.Mock(
return_value=study3)
study4 = StudyDTO(path="hello4")
self.study_submitter.submit_job = mock.Mock(return_value=study4)
self.reporter.save_study = mock.Mock()
self.study_launcher.launch_study(study)
self.zipper.zip.assert_called_once_with(study)
self.study_uploader.upload.assert_called_once_with(study1)
self.zipfile_cleaner.remove_input_zipfile.assert_called_once_with(
study2)
self.study_submitter.submit_job.assert_called_once_with(study3)
assert self.reporter.save_study.call_count == 4
calls = self.reporter.save_study.call_args_list
assert calls[0] == call(study1)
assert calls[1] == call(study2)
assert calls[2] == call(study3)
assert calls[3] == call(study4)
def test_given_a_list_of_done_studies_when_retrieve_all_studies_called_then_message_is_shown(
self, ):
# given
study = StudyDTO("path")
study.done = True
study_list = [deepcopy(study), deepcopy(study)]
display_mock = mock.Mock(spec=IDisplay)
my_retriever = RetrieveController(
self.data_repo,
self.remote_env_mock,
self.file_manager,
display_mock,
self.state_updater_mock,
)
my_retriever.repo.get_list_of_studies = mock.Mock(
return_value=study_list)
display_mock.show_message = mock.Mock()
# when
output = my_retriever.retrieve_all_studies()
# then
expected_message1 = "Retrieving all studies"
expected_message2 = "Everything is done"
calls = [
call(expected_message1, mock.ANY),
call(expected_message2, mock.ANY),
] # , call(my_study3)]
display_mock.show_message.assert_has_calls(calls)
assert output is True
def test_remote_env_not_called_if_study_has_already_a_jobid(self):
self.remote_env.submit_job = mock.Mock()
study = StudyDTO(path="hello")
study.job_id = 42
self.study_submitter.submit_job(study)
self.remote_env.submit_job.assert_not_called()
def test_file_manager_and_remote_env_not_called_if_study_not_started(self):
self.remote_env_mock.download_logs = mock.Mock()
self.file_manager.make_dir = mock.Mock()
study = StudyDTO(path="hello")
study.started = False
self.log_downloader.run(study)
self.remote_env_mock.download_logs.assert_not_called()
self.file_manager.make_dir.assert_not_called()
def test_remote_env_not_called_if_upload_was_done(self):
self.remote_env.upload_file = mock.Mock()
study = StudyDTO(path="hello")
study.zip_is_sent = True
new_study = self.study_uploader.upload(study)
self.remote_env.upload_file.assert_not_called()
assert new_study == study
def test_remote_environment_not_called_if_remote_server_is_already_clean(self):
self.remote_env_mock.clean_remote_server = mock.Mock()
study = StudyDTO(path="hello")
study.local_final_zipfile_path = "hello.zip"
study.remote_server_is_clean = True
new_study = self.remote_server_cleaner.clean(study)
self.remote_env_mock.clean_remote_server.assert_not_called()
assert new_study == study
def test_remote_env_not_called_if_final_zip_already_downloaded(self):
self.remote_env.download_final_zip = mock.Mock()
downloaded_study = StudyDTO("hello")
downloaded_study.local_final_zipfile_path = "results.zip"
new_study = self.final_zip_downloader.download(downloaded_study)
self.remote_env.download_final_zip.assert_not_called()
assert new_study == downloaded_study
def test_file_manager_not_called_if_zip_exists(self):
self.file_manager.zip_dir_excluding_subdir = mock.Mock()
study = StudyDTO(path="hello")
study.zipfile_path = "ciao.zip"
new_zip = self.study_zipper.zip(study)
self.file_manager.zip_dir_excluding_subdir.assert_not_called()
self.display_mock.show_error.assert_not_called()
self.display_mock.show_message.assert_not_called()
assert new_zip == study
def test_given_tinydb_document_when_doc_to_study_called_then_return_corresponding_study(
self, ):
# given
expected_study = StudyDTO(path="path")
expected_study.job_id = 42
expected_study.n_cpu = 999
doc = tinydb.database.Document(expected_study.__dict__, 42)
# when
output_study = DataRepoTinydb.doc_to_study(doc)
# then
assert expected_study.__dict__ == output_study.__dict__
def test_remote_env_is_called_if_upload_was_not_done(self):
self.remote_env.upload_file = mock.Mock()
study = StudyDTO(path="hello")
study.zip_is_sent = False
expected_study = copy(study)
expected_study.zip_is_sent = True
new_study = self.study_uploader.upload(study)
self.remote_env.upload_file.assert_called_once_with(study.zipfile_path)
assert new_study == expected_study
def test_given_two_studies_in_repo_check_queue_controller_calls_updater_twice(
self):
# given
study_list = [StudyDTO(""), StudyDTO("")]
self.slurm_queue_show.run = mock.Mock()
self.state_updater.run = mock.Mock()
self.repo_mock.get_list_of_studies = mock.Mock(return_value=study_list)
self.display.show_message = mock.Mock()
# when
self.check_queue_controller.check_queue()
# then
assert self.state_updater.run.call_count == 2
def doc_to_study(doc: tinydb.database.Document):
"""Create a studyDTO from a tinydb.database.Document
Args:
doc: Document representing a study
Returns:
studyDTO object
"""
study = StudyDTO(path="empty/path")
study.__dict__ = doc
return study
def test_check_queue_controller_writes_message_before_returning_state_of_studies(
self, ):
# given
study_list = [StudyDTO(""), StudyDTO("")]
self.slurm_queue_show.run = mock.Mock()
self.repo_mock.get_list_of_studies = mock.Mock(return_value=study_list)
self.state_updater.run_on_list = mock.Mock()
self.display.show_message = mock.Mock()
# when
self.check_queue_controller.check_queue()
# then
self.state_updater.run_on_list.assert_called_once_with(study_list)
def test_remote_env_is_called_if_study_has_no_jobid(self):
self.remote_env.submit_job = mock.Mock(return_value=42)
study = StudyDTO(path="hello")
study.zipfile_path = "ciao.zip"
study.job_id = None
new_study = self.study_submitter.submit_job(study)
self.remote_env.submit_job.assert_called_once()
first_call = self.remote_env.submit_job.call_args_list[0]
first_argument = first_call[0][0]
assert asdict(first_argument) == asdict(study)
assert new_study.job_id is 42
def test_is_job_id_inside_database_returns_true_only_if_one_job_id_is_found(
self):
# given
repo = DataRepoTinydb("")
type(repo).db = mock.PropertyMock()
study_dto = StudyDTO(path="path")
study_dto.job_id = 6381
repo.get_list_of_studies = mock.Mock(return_value=[study_dto])
repo.save_study(study_dto)
# when
output = repo.is_job_id_inside_database(6381)
# then
assert output is True
def test_file_manager_is_called_if_zip_doesnt_exist(self):
self.file_manager.zip_dir_excluding_subdir = mock.Mock(
return_value=True)
study_path = "hello"
study = StudyDTO(path=study_path)
study.zipfile_path = ""
new_study = self.study_zipper.zip(study)
expected_zipfile_path = study.path + "-" + getpass.getuser() + ".zip"
self.file_manager.zip_dir_excluding_subdir.assert_called_once_with(
study_path, expected_zipfile_path, "output")
assert new_study.zipfile_path == expected_zipfile_path
def test_remote_environment_not_called_if_final_zip_not_downloaded(self):
self.remote_env_mock.clean_remote_server = mock.Mock()
study = StudyDTO(path="hello")
study.local_final_zipfile_path = ""
new_study = self.remote_server_cleaner.clean(study)
self.remote_env_mock.clean_remote_server.assert_not_called()
assert new_study == study
study.local_final_zipfile_path = None
new_study = self.remote_server_cleaner.clean(study)
self.remote_env_mock.clean_remote_server.assert_not_called()
assert new_study == study
def test_run_on_list_calls_run_on_all_submitted_studies_of_the_list():
# given
my_study1 = StudyDTO(path="study_path1", job_id=1)
my_study2 = StudyDTO(path="study_path2", job_id=None)
my_study3 = StudyDTO(path="study_path3", job_id=2)
study_list = [my_study1, my_study2, my_study3]
remote_env_mock = mock.Mock()
remote_env_mock.get_job_state_flags = mock.Mock(return_value=(1, 2, 3))
display = mock.Mock()
state_updater = StateUpdater(remote_env_mock, display)
# when
state_updater.run_on_list(study_list)
# then
calls = [call(my_study1), call(my_study3)]
remote_env_mock.get_job_state_flags.assert_has_calls(calls)
def _create_study(self, path, antares_version, xpansion_study):
"""Generate a study dto from study directory path, antares version and directory hash
Args:
path: path of the study directory
antares_version: version number of antares
Returns:
study dto filled with the specified data
"""
if self.xpansion_mode:
run_mode = Modes.xpansion
else:
run_mode = Modes.antares
new_study = StudyDTO(
path=str(path),
n_cpu=self.n_cpu,
time_limit=self.time_limit,
antares_version=antares_version,
job_log_dir=self.DEFAULT_JOB_LOG_DIR_PATH,
output_dir=str(self.output_dir),
xpansion_study=xpansion_study,
run_mode=run_mode,
post_processing=self.post_processing,
)
return new_study
def study_to_extract(self):
local_zip = "results.zip"
study = StudyDTO(
path="hello",
local_final_zipfile_path=local_zip,
final_zip_extracted=False,
)
return study
def test_file_manager_not_called_if_study_should_not_be_extracted(self):
self.file_manager.unzip = mock.Mock()
empty_study = StudyDTO("hello")
new_study = self.zip_extractor.extract_final_zip(empty_study)
self.file_manager.unzip.assert_not_called()
self.display_mock.show_error.assert_not_called()
self.display_mock.show_message.assert_not_called()
assert new_study == empty_study
def downloaded_zip_study(self):
study = StudyDTO(
path=Path("path") / "hello",
started=True,
finished=True,
job_id=42,
local_final_zipfile_path=str(Path("final") / "zip" / "path.zip"),
)
return study
def successfully_finished_zip_study(self):
study = StudyDTO(
path=Path("path") / "hello",
started=True,
finished=True,
with_error=False,
job_id=42,
)
return study
def test_if_zipfile_does_not_exist_then_unzip_returns_false(self, ):
# given
study = StudyDTO("path")
display_terminal = DisplayTerminal()
my_file_manager = file_manager.FileManager(display_terminal)
# when
output = my_file_manager.unzip(study.local_final_zipfile_path)
# then
assert output is False
def test_data_provider_return_list_of_studies_obtained_from_repo():
# given
data_repo = Mock(spec_set=IDataRepo)
study = StudyDTO(path="empty_path")
data_repo.get_list_of_studies = Mock(return_value=[study])
data_provider = DataProvider(data_repo)
# when
list_of_studies = data_provider.get_list_of_studies()
# then
assert list_of_studies == [study]
def test_given_a_list_of_done_studies_when_all_studies_done_called_then_return_true(
self, ):
# given
study = StudyDTO("path")
study.done = True
study_list = [deepcopy(study), deepcopy(study)]
my_retriever = RetrieveController(
self.data_repo,
self.remote_env_mock,
self.file_manager,
self.display,
self.state_updater_mock,
)
my_retriever.repo.get_list_of_studies = mock.Mock(
return_value=study_list)
# when
output = my_retriever.all_studies_done
# then
assert output is True
def test_data_reporter_calls_repo_to_save_study():
# given
data_repo = Mock(spec_set=IDataRepo)
data_repo.save_study = Mock()
data_reporter = DataReporter(data_repo)
study = StudyDTO(path="empty_path")
# when
data_reporter.save_study(study)
# then
data_repo.save_study.assert_called_once_with(study)
def test_zip_study_show_message_if_zip_succeeds(self):
self.file_manager.zip_dir_excluding_subdir = mock.Mock(
return_value=True)
study = StudyDTO(path="hello")
self.study_zipper.zip(study)
expected_message = f'"hello": was zipped'
self.display_mock.show_message.assert_called_once_with(
expected_message, mock.ANY)
def test_submit_study_shows_message_if_submit_succeeds(self):
self.remote_env.submit_job = mock.Mock(return_value=42)
study = StudyDTO(path="hello")
new_study = self.study_submitter.submit_job(study)
expected_message = f'"hello": was submitted'
self.display_mock.show_message.assert_called_once_with(
expected_message, mock.ANY)
assert new_study.job_id == 42
