def test_download_checkpoints(test_output_dirs: OutputFolderForTests, is_ensemble: bool,
runner_config: AzureConfig) -> None:
output_dir = test_output_dirs.root_dir
assert get_results_blob_path("some_run_id") == "azureml/ExperimentRun/dcid.some_run_id"
# Any recent run ID from a PR build will do. Use a PR build because the checkpoint files are small there.
config = ModelConfigBase(should_validate=False)
config.set_output_to(output_dir)
runner_config.run_recovery_id = DEFAULT_ENSEMBLE_RUN_RECOVERY_ID if is_ensemble else DEFAULT_RUN_RECOVERY_ID
run_recovery = RunRecovery.download_checkpoints_from_recovery_run(runner_config, config)
run_to_recover = fetch_run(workspace=runner_config.get_workspace(), run_recovery_id=runner_config.run_recovery_id)
expected_checkpoint_file = "1" + CHECKPOINT_FILE_SUFFIX
if is_ensemble:
child_runs = fetch_child_runs(run_to_recover)
expected_files = [config.checkpoint_folder
/ OTHER_RUNS_SUBDIR_NAME
/ str(x.get_tags()['cross_validation_split_index']) / expected_checkpoint_file
for x in child_runs]
else:
expected_files = [config.checkpoint_folder / run_to_recover.id / expected_checkpoint_file]
checkpoint_paths = run_recovery.get_checkpoint_paths(1)
if is_ensemble:
assert len(run_recovery.checkpoints_roots) == len(expected_files)
assert all([(x in [y.parent for y in expected_files]) for x in run_recovery.checkpoints_roots])
assert len(checkpoint_paths) == len(expected_files)
assert all([x in expected_files for x in checkpoint_paths])
else:
assert len(checkpoint_paths) == 1
assert checkpoint_paths[0] == expected_files[0]
assert all([expected_file.exists() for expected_file in expected_files])
def save_build_info_for_dotnet_consumers(self) -> None:
results_container = get_results_blob_path(RUN_CONTEXT.id)
result_location = ExperimentResultLocation(
azure_job_name=RUN_CONTEXT.id,
dataset_folder=self.model_config.azure_dataset_id,
results_container_name=results_container,
commandline_overrides=str(self.model_config.overrides),
dataset_uri=self.model_config.azure_dataset_id,
results_uri="",
)
# Fill in the missing information in the build config (everything that is not available at the time
# of evoking the runner), and then save in the format needed for the .NET consumers
build_information_to_dot_net_json_file(
self.azure_config, result_location, folder=self.model_config.outputs_folder)
def test_get_results_blob_path() -> None:
assert get_results_blob_path("some_run_id") == "azureml/ExperimentRun/dcid.some_run_id"
