def test_regression_test(test_output_dirs: OutputFolderForTests) -> None: """ Test that the file comparison for regression tests is actually called in the workflow. """ container = DummyContainerWithModel() container.local_dataset = test_output_dirs.root_dir container.regression_test_folder = Path(str(uuid.uuid4().hex)) runner = MLRunner(container=container) runner.setup() with pytest.raises(ValueError) as ex: runner.run() assert "Folder with expected files does not exist" in str(ex)
def test_container_hooks(test_output_dirs: OutputFolderForTests) -> None: """ Test if the hooks before training are called at the right place and in the right order. """ container = DummyContainerWithHooks() container.local_dataset = test_output_dirs.root_dir runner = MLRunner(model_config=None, container=container) runner.setup() runner.run() # The hooks in DummyContainerWithHooks itself check that the hooks are called in the right order. Here, # only check that they have all been called. for file in [ "global_rank_zero.txt", "local_rank_zero.txt", "all_ranks.txt" ]: assert (runner.container.outputs_folder / file).is_file(), f"Missing file: {file}"
def test_model_inference_on_single_run(test_output_dirs: OutputFolderForTests) -> None: falllback_run_id = FALLBACK_HELLO_CONTAINER_RUN files_to_check = ["test_mse.txt", "test_mae.txt"] training_run = get_most_recent_run(fallback_run_id_for_local_execution=falllback_run_id) all_training_files = training_run.get_file_names() for file in files_to_check: assert f"outputs/{file}" in all_training_files, f"{file} is missing" training_folder = test_output_dirs.root_dir / "training" training_folder.mkdir() training_files = [training_folder / file for file in files_to_check] for file, download_path in zip(files_to_check, training_files): training_run.download_file(f"outputs/{file}", output_file_path=str(download_path)) container = HelloContainer() container.set_output_to(test_output_dirs.root_dir) container.model_id = get_most_recent_model_id(fallback_run_id_for_local_execution=falllback_run_id) azure_config = get_default_azure_config() azure_config.train = False ml_runner = MLRunner(container=container, azure_config=azure_config, project_root=test_output_dirs.root_dir) ml_runner.setup() ml_runner.run() inference_files = [container.outputs_folder / file for file in files_to_check] for inference_file in inference_files: assert inference_file.exists(), f"{inference_file} is missing" for training_file, inference_file in zip(training_files, inference_files): training_lines = training_file.read_text().splitlines() inference_lines = inference_file.read_text().splitlines() # We expect all the files we are reading to have a single float value assert len(training_lines) == 1 train_value = float(training_lines[0].strip()) assert len(inference_lines) == 1 inference_value = float(inference_lines[0].strip()) assert inference_value == pytest.approx(train_value, 1e-6)