def test_regression_test(test_output_dirs: OutputFolderForTests) -> None:
"""
Test that the file comparison for regression tests is actually called in the workflow.
"""
container = DummyContainerWithModel()
container.local_dataset = test_output_dirs.root_dir
container.regression_test_folder = Path(str(uuid.uuid4().hex))
runner = MLRunner(container=container)
runner.setup()
with pytest.raises(ValueError) as ex:
runner.run()
assert "Folder with expected files does not exist" in str(ex)
def test_container_hooks(test_output_dirs: OutputFolderForTests) -> None:
"""
Test if the hooks before training are called at the right place and in the right order.
"""
container = DummyContainerWithHooks()
container.local_dataset = test_output_dirs.root_dir
runner = MLRunner(model_config=None, container=container)
runner.setup()
runner.run()
# The hooks in DummyContainerWithHooks itself check that the hooks are called in the right order. Here,
# only check that they have all been called.
for file in [
"global_rank_zero.txt", "local_rank_zero.txt", "all_ranks.txt"
]:
assert (runner.container.outputs_folder /
file).is_file(), f"Missing file: {file}"
def test_model_inference_on_single_run(test_output_dirs: OutputFolderForTests) -> None:
falllback_run_id = FALLBACK_HELLO_CONTAINER_RUN
files_to_check = ["test_mse.txt", "test_mae.txt"]
training_run = get_most_recent_run(fallback_run_id_for_local_execution=falllback_run_id)
all_training_files = training_run.get_file_names()
for file in files_to_check:
assert f"outputs/{file}" in all_training_files, f"{file} is missing"
training_folder = test_output_dirs.root_dir / "training"
training_folder.mkdir()
training_files = [training_folder / file for file in files_to_check]
for file, download_path in zip(files_to_check, training_files):
training_run.download_file(f"outputs/{file}", output_file_path=str(download_path))
container = HelloContainer()
container.set_output_to(test_output_dirs.root_dir)
container.model_id = get_most_recent_model_id(fallback_run_id_for_local_execution=falllback_run_id)
azure_config = get_default_azure_config()
azure_config.train = False
ml_runner = MLRunner(container=container, azure_config=azure_config, project_root=test_output_dirs.root_dir)
ml_runner.setup()
ml_runner.run()
inference_files = [container.outputs_folder / file for file in files_to_check]
for inference_file in inference_files:
assert inference_file.exists(), f"{inference_file} is missing"
for training_file, inference_file in zip(training_files, inference_files):
training_lines = training_file.read_text().splitlines()
inference_lines = inference_file.read_text().splitlines()
# We expect all the files we are reading to have a single float value
assert len(training_lines) == 1
train_value = float(training_lines[0].strip())
assert len(inference_lines) == 1
inference_value = float(inference_lines[0].strip())
assert inference_value == pytest.approx(train_value, 1e-6)