def test_download_checkpoints(test_output_dirs: OutputFolderForTests, is_ensemble: bool, runner_config: AzureConfig) -> None: output_dir = test_output_dirs.root_dir assert get_results_blob_path("some_run_id") == "azureml/ExperimentRun/dcid.some_run_id" # Any recent run ID from a PR build will do. Use a PR build because the checkpoint files are small there. config = ModelConfigBase(should_validate=False) config.set_output_to(output_dir) runner_config.run_recovery_id = DEFAULT_ENSEMBLE_RUN_RECOVERY_ID if is_ensemble else DEFAULT_RUN_RECOVERY_ID run_recovery = RunRecovery.download_checkpoints_from_recovery_run(runner_config, config) run_to_recover = fetch_run(workspace=runner_config.get_workspace(), run_recovery_id=runner_config.run_recovery_id) expected_checkpoint_file = "1" + CHECKPOINT_FILE_SUFFIX if is_ensemble: child_runs = fetch_child_runs(run_to_recover) expected_files = [config.checkpoint_folder / OTHER_RUNS_SUBDIR_NAME / str(x.get_tags()['cross_validation_split_index']) / expected_checkpoint_file for x in child_runs] else: expected_files = [config.checkpoint_folder / run_to_recover.id / expected_checkpoint_file] checkpoint_paths = run_recovery.get_checkpoint_paths(1) if is_ensemble: assert len(run_recovery.checkpoints_roots) == len(expected_files) assert all([(x in [y.parent for y in expected_files]) for x in run_recovery.checkpoints_roots]) assert len(checkpoint_paths) == len(expected_files) assert all([x in expected_files for x in checkpoint_paths]) else: assert len(checkpoint_paths) == 1 assert checkpoint_paths[0] == expected_files[0] assert all([expected_file.exists() for expected_file in expected_files])
def save_build_info_for_dotnet_consumers(self) -> None: results_container = get_results_blob_path(RUN_CONTEXT.id) result_location = ExperimentResultLocation( azure_job_name=RUN_CONTEXT.id, dataset_folder=self.model_config.azure_dataset_id, results_container_name=results_container, commandline_overrides=str(self.model_config.overrides), dataset_uri=self.model_config.azure_dataset_id, results_uri="", ) # Fill in the missing information in the build config (everything that is not available at the time # of evoking the runner), and then save in the format needed for the .NET consumers build_information_to_dot_net_json_file( self.azure_config, result_location, folder=self.model_config.outputs_folder)
def test_get_results_blob_path() -> None: assert get_results_blob_path("some_run_id") == "azureml/ExperimentRun/dcid.some_run_id"