def test_submit_for_inference(test_output_dirs: OutputFolderForTests) -> None:
"""
Execute the submit_for_inference script on the model that was recently trained. This starts an AzureML job,
and downloads the segmentation. Then check if the segmentation was actually produced.
:return:
"""
model = get_most_recent_model(
fallback_run_id_for_local_execution=FALLBACK_SINGLE_RUN)
image_file = fixed_paths_for_tests.full_ml_test_data_path(
) / "train_and_test_data" / "id1_channel1.nii.gz"
assert image_file.exists(), f"Image file not found: {image_file}"
settings_file = fixed_paths.SETTINGS_YAML_FILE
assert settings_file.exists(), f"Settings file not found: {settings_file}"
# Read the name of the branch from environment, so that the inference experiment is also listed alongside
# all other AzureML runs that belong to the current PR.
build_branch = os.environ.get("BUILD_BRANCH", None)
experiment_name = to_azure_friendly_string(
build_branch) if build_branch else "model_inference"
args = [
"--image_file",
str(image_file), "--model_id", model.id, "--settings",
str(settings_file), "--download_folder",
str(test_output_dirs.root_dir), "--cluster", "training-nc12",
"--experiment", experiment_name
]
seg_path = test_output_dirs.root_dir / DEFAULT_RESULT_IMAGE_NAME
assert not seg_path.exists(
), f"Result file {seg_path} should not yet exist"
submit_for_inference.main(
args, project_root=fixed_paths.repository_root_directory())
assert seg_path.exists(), f"Result file {seg_path} was not created"
def get_run_and_download_pytest(branch: str,
number: int) -> Optional[Path]:
experiment = Experiment(workspace,
name=to_azure_friendly_string(branch))
runs = [run for run in experiment.get_runs() if run.number == number]
if len(runs) != 1:
raise ValueError(
f"Expected to get exactly 1 run in experiment {experiment.name}"
)
return download_pytest_result(runs[0], output_dir)
def get_experiment_name_from_environment() -> str:
"""
Reads the name of the present branch from environment variable "BUILD_BRANCH". This must be set in the YML file.
If the variable is not found, return an empty string.
With this setup, all AML runs that belong to a given pull request are listed at the same place.
"""
env_branch = "BUILD_BRANCH"
build_branch = os.environ.get(env_branch, None)
if not build_branch:
if is_running_on_azure_agent():
raise ValueError(f"Environment variable {env_branch} should be set when running on Azure agents.")
return ""
return to_azure_friendly_string(build_branch) or ""
def create_and_submit_experiment(
azure_config: AzureConfig,
source_config: SourceConfig,
model_config_overrides: str,
azure_dataset_id: str) -> Run:
"""
Creates an AzureML experiment in the workspace and submits it for execution.
:param azure_config: azure related configurations to setup valid workspace
:param source_config: The information about which code should be submitted, and which arguments should be used.
:param model_config_overrides: A string that describes which model parameters were overwritten by commandline
arguments in the present run. This is only used for diagnostic purposes (it is set as a Tag on the run).
:param azure_dataset_id: The name of the dataset in blob storage to be used for this run.
:returns: Run object for the submitted AzureML run
"""
workspace = azure_config.get_workspace()
experiment_name = create_experiment_name(azure_config)
exp = Experiment(workspace=workspace, name=azure_util.to_azure_friendly_string(experiment_name))
script_run_config = create_run_config(azure_config, source_config, azure_dataset_id)
# submit a training/testing run associated with the experiment
run: Run = exp.submit(script_run_config)
# set metadata for the run
set_run_tags(run, azure_config, model_config_overrides)
print("\n==============================================================================")
print(f"Successfully queued new run {run.id} in experiment: {exp.name}")
if azure_config.run_recovery_id:
print(f"\nRecovered from: {azure_config.run_recovery_id}")
recovery_id = azure_util.create_run_recovery_id(run)
recovery_file = Path(RUN_RECOVERY_FILE)
if recovery_file.exists():
recovery_file.unlink()
recovery_file.write_text(recovery_id)
print("Experiment URL: {}".format(exp.get_portal_url()))
print("Run URL: {}".format(run.get_portal_url()))
print("If this run fails, re-start runner.py and supply these additional arguments: "
f"--run_recovery_id={recovery_id}")
print(f"The run recovery ID has been written to this file: {recovery_file}")
print("==============================================================================")
if azure_config.tensorboard and azure_config.azureml:
print("Starting TensorBoard now because you specified --tensorboard")
monitor(monitor_config=AMLTensorBoardMonitorConfig(run_ids=[run.id]), azure_config=azure_config)
else:
print(f"To monitor this run locally using TensorBoard, run the script: "
f"InnerEye/Azure/tensorboard_monitor.py --run_ids={run.id}")
print("==============================================================================")
return run
def test_submit_for_inference(use_dicom: bool,
test_output_dirs: OutputFolderForTests) -> None:
"""
Execute the submit_for_inference script on the model that was recently trained. This starts an AzureML job,
and downloads the segmentation. Then check if the segmentation was actually produced.
:param use_dicom: True to test DICOM in/out, False otherwise.
:param test_output_dirs: Test output directories.
"""
model = get_most_recent_model(
fallback_run_id_for_local_execution=FALLBACK_SINGLE_RUN)
assert PYTHON_ENVIRONMENT_NAME in model.tags, "Environment name not present in model properties"
if use_dicom:
size = (64, 64, 64)
spacing = (1., 1., 2.5)
image_file = test_output_dirs.root_dir / "temp_pack_dicom_series" / "dicom_series.zip"
scratch_folder = test_output_dirs.root_dir / "temp_dicom_series"
zip_random_dicom_series(size, spacing, image_file, scratch_folder)
else:
image_file = fixed_paths_for_tests.full_ml_test_data_path(
) / "train_and_test_data" / "id1_channel1.nii.gz"
assert image_file.exists(), f"Image file not found: {image_file}"
settings_file = fixed_paths.SETTINGS_YAML_FILE
assert settings_file.exists(), f"Settings file not found: {settings_file}"
# Read the name of the branch from environment, so that the inference experiment is also listed alongside
# all other AzureML runs that belong to the current PR.
build_branch = os.environ.get("BUILD_BRANCH", None)
experiment_name = to_azure_friendly_string(
build_branch) if build_branch else "model_inference"
args = [
"--image_file",
str(image_file), "--model_id", model.id, "--settings",
str(settings_file), "--download_folder",
str(test_output_dirs.root_dir), "--cluster", "training-nc12",
"--experiment", experiment_name, "--use_dicom",
str(use_dicom)
]
download_file = DEFAULT_RESULT_ZIP_DICOM_NAME if use_dicom else DEFAULT_RESULT_IMAGE_NAME
seg_path = test_output_dirs.root_dir / download_file
assert not seg_path.exists(
), f"Result file {seg_path} should not yet exist"
submit_for_inference.main(
args, project_root=fixed_paths.repository_root_directory())
assert seg_path.exists(), f"Result file {seg_path} was not created"
def create_and_submit_experiment(azure_config: AzureConfig,
script_run_config: ScriptRunConfig,
commandline_args: str) -> Run:
"""
Creates an AzureML experiment in the workspace and submits it for execution.
:param azure_config: azure related configurations to setup a valid workspace.
:param script_run_config: The configuration for the script that should be run inside of AzureML.
:param commandline_args: A string with all commandline arguments that were provided to the runner. These are only
used to set a tag on the submitted AzureML run.
:returns: Run object for the submitted AzureML run
"""
workspace = azure_config.get_workspace()
experiment_name = create_experiment_name(azure_config)
exp = Experiment(workspace=workspace,
name=azure_util.to_azure_friendly_string(experiment_name))
# submit a training/testing run associated with the experiment
run: Run = exp.submit(script_run_config)
if is_offline_run_context(run):
# This codepath will only be executed in unit tests, when exp.submit is mocked.
return run
# Set metadata for the run.
set_run_tags(run, azure_config, commandline_args=commandline_args)
print(
"\n=============================================================================="
)
print(f"Successfully queued new run {run.id} in experiment: {exp.name}")
if azure_config.run_recovery_id:
print(f"\nRecovered from: {azure_config.run_recovery_id}")
recovery_id = azure_util.create_run_recovery_id(run)
recovery_file = Path(RUN_RECOVERY_FILE)
if recovery_file.exists():
recovery_file.unlink()
recovery_file.write_text(recovery_id)
print("Experiment URL: {}".format(exp.get_portal_url()))
print("Run URL: {}".format(run.get_portal_url()))
print(
"If this run fails, re-start runner.py and supply these additional arguments: "
f"--run_recovery_id={recovery_id}")
print(
f"The run recovery ID has been written to this file: {recovery_file}")
print(
"=============================================================================="
)
if azure_config.tensorboard and azure_config.azureml:
print("Starting TensorBoard now because you specified --tensorboard")
monitor(monitor_config=AMLTensorBoardMonitorConfig(run_ids=[run.id]),
azure_config=azure_config)
else:
print(
f"To monitor this run locally using TensorBoard, run the script: "
f"InnerEye/Azure/tensorboard_monitor.py --run_ids={run.id}")
print(
"=============================================================================="
)
return run
