def download_checkpoints_from_recovery_run(azure_config: AzureConfig,
config: DeepLearningConfig,
run_context: Optional[Run] = None) -> RunRecovery:
"""
Downloads checkpoints of run corresponding to the run_recovery_id in azure_config, and any
checkpoints of the child runs if they exist.
:param azure_config: Azure related configs.
:param config: Model related configs.
:param run_context: Context of the current run (will be used to find the target AML workspace)
:return:RunRecovery
"""
run_context = run_context or RUN_CONTEXT
workspace = azure_config.get_workspace()
# Find the run to recover in AML workspace
if not azure_config.run_recovery_id:
raise ValueError("A valid run_recovery_id is required to download recovery checkpoints, found None")
run_to_recover = fetch_run(workspace, azure_config.run_recovery_id.strip())
# Handle recovery of a HyperDrive cross validation run (from within a successor HyperDrive run,
# not in ensemble creation). In this case, run_recovery_id refers to the parent prior run, so we
# need to set run_to_recover to the child of that run whose split index is the same as that of
# the current (child) run.
if is_cross_validation_child_run(run_context):
run_to_recover = next(x for x in fetch_child_runs(run_to_recover) if
get_cross_validation_split_index(x) == get_cross_validation_split_index(run_context))
return RunRecovery.download_checkpoints_from_run(config, run_to_recover)
def test_get_cross_validation_split_index_single_run() -> None:
"""
Test that retrieved cross validation split index is as expected, for single runs.
"""
run = get_most_recent_run()
# check for offline run
assert get_cross_validation_split_index(
Run.get_context()) == DEFAULT_CROSS_VALIDATION_SPLIT_INDEX
# check for online runs
assert get_cross_validation_split_index(
run) == DEFAULT_CROSS_VALIDATION_SPLIT_INDEX
def test_get_cross_validation_split_index_ensemble_run() -> None:
"""
Test that retrieved cross validation split index is as expected, for ensembles.
"""
# check for offline run
assert get_cross_validation_split_index(
Run.get_context()) == DEFAULT_CROSS_VALIDATION_SPLIT_INDEX
# check for online runs
run = get_most_recent_run(
fallback_run_id_for_local_execution=FALLBACK_ENSEMBLE_RUN)
assert get_cross_validation_split_index(
run) == DEFAULT_CROSS_VALIDATION_SPLIT_INDEX
assert all([
get_cross_validation_split_index(x) >
DEFAULT_CROSS_VALIDATION_SPLIT_INDEX for x in fetch_child_runs(run)
])
def test_get_cross_validation_split_index(is_ensemble: bool) -> None:
"""
Test that retrieved cross validation split index is as expected, for single runs and ensembles.
"""
run = fetch_run(workspace=get_default_workspace(),
run_recovery_id=DEFAULT_ENSEMBLE_RUN_RECOVERY_ID
if is_ensemble else DEFAULT_RUN_RECOVERY_ID)
# check for offline run
assert get_cross_validation_split_index(
Run.get_context()) == DEFAULT_CROSS_VALIDATION_SPLIT_INDEX
# check for online runs
assert get_cross_validation_split_index(
run) == DEFAULT_CROSS_VALIDATION_SPLIT_INDEX
if is_ensemble:
assert all([
get_cross_validation_split_index(x) >
DEFAULT_CROSS_VALIDATION_SPLIT_INDEX for x in fetch_child_runs(run)
])