def test_source_config_set_params() -> None:
"""
Check that commandline arguments are set correctly when submitting the script to AzureML.
In particular, the azureml flag should be omitted, irrespective of how the argument is written.
"""
s = SourceConfig(root_folder=Path(""),
entry_script=Path("something.py"),
conda_dependencies_files=[])
def assert_has_params(expected_args: str) -> None:
assert s.script_params is not None
# Arguments are in the keys of the dictionary only, and should have been added in the right order
assert " ".join(s.script_params.keys()) == expected_args
with mock.patch("sys.argv", [
"", "some", "--param", "1",
f"--{AZURECONFIG_SUBMIT_TO_AZUREML}=True", "more"
]):
s.set_script_params_except_submit_flag()
assert_has_params("some --param 1 more")
with mock.patch("sys.argv", [
"", "some", "--param", "1", f"--{AZURECONFIG_SUBMIT_TO_AZUREML}",
"False", "more"
]):
s.set_script_params_except_submit_flag()
assert_has_params("some --param 1 more")
# Arguments where azureml is just the prefix should not be removed.
with mock.patch(
"sys.argv",
["", "some", f"--{AZURECONFIG_SUBMIT_TO_AZUREML}foo", "False", "more"
]):
s.set_script_params_except_submit_flag()
assert_has_params(f"some --{AZURECONFIG_SUBMIT_TO_AZUREML}foo False more")
def test_create_python_env() -> None:
"""
Checks if environment variables in the SourceConfig are correctly passed through to the Python environment.
Environment variables in SourceConfig are only used in the internal InnerEye repo.
:return:
"""
foo = "foo"
bar = "bar"
entry_script = Path("something.py")
conda_file = get_environment_yaml_file()
s = SourceConfig(root_folder=Path(""),
entry_script=entry_script,
conda_dependencies_files=[conda_file],
environment_variables={foo: bar})
env = get_or_create_python_environment(
source_config=s,
azure_config=get_default_azure_config(),
register_environment=False)
assert foo in env.environment_variables
assert env.environment_variables[foo] == bar
# Check that some of the basic packages that we expect to always exist are picked up correctly in the Conda env
def remove_version_number(items: Iterator[str]) -> Set[str]:
return set(c.split("=")[0] for c in items)
assert "pytorch" in remove_version_number(
env.python.conda_dependencies.conda_packages)
assert "pytorch-lightning" in remove_version_number(
env.python.conda_dependencies.pip_packages)
def create_run_config(azure_config: AzureConfig,
source_config: SourceConfig,
all_azure_dataset_ids: List[str],
all_dataset_mountpoints: List[str],
environment_name: str = "") -> ScriptRunConfig:
"""
Creates a configuration to run the InnerEye training script in AzureML.
:param azure_config: azure related configurations to use for model scale-out behaviour
:param source_config: configurations for model execution, such as name and execution mode
:param all_azure_dataset_ids: The name of all datasets on blob storage that will be used for this run.
:param all_dataset_mountpoints: When using the datasets in AzureML, these are the per-dataset mount points.
:param environment_name: If specified, try to retrieve the existing Python environment with this name. If that
is not found, create one from the Conda files provided in `source_config`. This parameter is meant to be used
when running inference for an existing model.
:return: The configured script run.
"""
dataset_consumptions = create_dataset_consumptions(
azure_config, all_azure_dataset_ids, all_dataset_mountpoints)
# AzureML seems to sometimes expect the entry script path in Linux format, hence convert to posix path
entry_script_relative_path = source_config.entry_script.relative_to(
source_config.root_folder).as_posix()
logging.info(
f"Entry script {entry_script_relative_path} ({source_config.entry_script} relative to "
f"source directory {source_config.root_folder})")
max_run_duration = None
if azure_config.max_run_duration:
max_run_duration = run_duration_string_to_seconds(
azure_config.max_run_duration)
workspace = azure_config.get_workspace()
run_config = RunConfiguration(
script=entry_script_relative_path,
arguments=source_config.script_params,
)
run_config.environment = get_or_create_python_environment(
azure_config, source_config, environment_name=environment_name)
run_config.target = azure_config.cluster
run_config.max_run_duration_seconds = max_run_duration
if azure_config.num_nodes > 1:
distributed_job_config = MpiConfiguration(
node_count=azure_config.num_nodes)
run_config.mpi = distributed_job_config
run_config.framework = "Python"
run_config.communicator = "IntelMpi"
run_config.node_count = distributed_job_config.node_count
if len(dataset_consumptions) > 0:
run_config.data = {
dataset.name: dataset
for dataset in dataset_consumptions
}
# Use blob storage for storing the source, rather than the FileShares section of the storage account.
run_config.source_directory_data_store = workspace.datastores.get(
WORKSPACE_DEFAULT_BLOB_STORE_NAME).name
script_run_config = ScriptRunConfig(
source_directory=str(source_config.root_folder),
run_config=run_config,
)
if azure_config.hyperdrive:
script_run_config = source_config.hyperdrive_config_func(
script_run_config) # type: ignore
return script_run_config
def submit_to_azureml(self) -> Run:
"""
Submit a job to AzureML, returning the resulting Run object, or exiting if we were asked to wait for
completion and the Run did not succeed.
"""
# The adal package creates a logging.info line each time it gets an authentication token, avoid that.
logging.getLogger('adal-python').setLevel(logging.WARNING)
if not self.model_config.azure_dataset_id:
raise ValueError(
"When running on AzureML, the 'azure_dataset_id' property must be set."
)
model_config_overrides = str(self.model_config.overrides)
source_config = SourceConfig(
root_folder=self.project_root,
entry_script=Path(sys.argv[0]).resolve(),
conda_dependencies_files=[
get_environment_yaml_file(),
self.project_root / fixed_paths.ENVIRONMENT_YAML_FILE_NAME
],
hyperdrive_config_func=lambda estimator: self.model_config.
get_hyperdrive_config(estimator),
# For large jobs, upload of results times out frequently because of large checkpoint files. Default is 600
upload_timeout_seconds=86400,
)
source_config.set_script_params_except_submit_flag()
assert self.model_config.azure_dataset_id is not None # to stop mypy complaining about next line
azure_run = submit_to_azureml(self.azure_config, source_config,
model_config_overrides,
self.model_config.azure_dataset_id)
logging.info("Job submission to AzureML done.")
if self.azure_config.pytest_mark:
# The AzureML job can optionally run pytest. Attempt to download it to the current directory.
# A build step will pick up that file and publish it to Azure DevOps.
# If pytest_mark is set, this file must exist.
logging.info("Downloading pytest result file.")
download_pytest_result(azure_run)
else:
logging.info(
"No pytest_mark present, hence not downloading the pytest result file."
)
status = azure_run.get_status()
# For PR builds where we wait for job completion, the job must have ended in a COMPLETED state.
# If a pytest failed, the runner has exited with code -1 (see below)
if self.azure_config.wait_for_completion and status != RunStatus.COMPLETED:
logging.error(f"Job completed with status {status}. Exiting.")
exit(-1)
return azure_run
def submit_for_inference(args: SubmitForInferenceConfig,
azure_config: AzureConfig) -> Optional[Path]:
"""
Create and submit an inference to AzureML, and optionally download the resulting segmentation.
:param azure_config: An object with all necessary information for accessing Azure.
:param args: configuration, see SubmitForInferenceConfig
:return: path to downloaded segmentation on local disc, or None if none.
"""
logging.info(f"Building Azure configuration from {args.settings}")
logging.info("Getting workspace")
workspace = azure_config.get_workspace()
logging.info("Identifying model")
model = Model(workspace=workspace, id=args.model_id)
model_id = model.id
logging.info(f"Identified model {model_id}")
source_directory = tempfile.TemporaryDirectory()
source_directory_name = source_directory.name
logging.info(
f"Building inference run submission in {source_directory_name}")
source_directory_path = Path(source_directory_name)
copy_image_file(args.image_file,
source_directory_path / DEFAULT_DATA_FOLDER)
# We copy over run_scoring.py, and score.py as well in case the model we're using
# does not have sufficiently recent versions of those files.
for base in ["run_scoring.py", "score.py"]:
shutil.copyfile(base, str(source_directory_path / base))
source_config = SourceConfig(
root_folder=source_directory_name,
entry_script=str(source_directory_path / "run_scoring.py"),
script_params={
"--data-folder": ".",
"--spawnprocess": "python",
"--model-id": model_id,
"score.py": ""
},
conda_dependencies_files=download_conda_dependency_files(
model, source_directory_path))
estimator = create_estimator_from_configs(workspace, azure_config,
source_config, [])
exp = Experiment(workspace=workspace, name=args.experiment_name)
run = exp.submit(estimator)
logging.info(f"Submitted run {run.id} in experiment {run.experiment.name}")
logging.info(f"Run URL: {run.get_portal_url()}")
if not args.keep_upload_folder:
source_directory.cleanup()
logging.info(f"Deleted submission directory {source_directory_name}")
if args.download_folder is None:
return None
logging.info("Awaiting run completion")
run.wait_for_completion()
logging.info(f"Run has completed with status {run.get_status()}")
download_path = choose_download_path(args.download_folder)
logging.info(f"Attempting to download segmentation to {download_path}")
run.download_file(DEFAULT_RESULT_IMAGE_NAME, str(download_path))
if download_path.exists():
logging.info(f"Downloaded segmentation to {download_path}")
else:
logging.warning("Segmentation NOT downloaded")
return download_path
def create_run_config(azure_config: AzureConfig,
source_config: SourceConfig,
azure_dataset_id: str = "",
environment_name: str = "") -> ScriptRunConfig:
"""
Creates a configuration to run the InnerEye training script in AzureML.
:param azure_config: azure related configurations to use for model scale-out behaviour
:param source_config: configurations for model execution, such as name and execution mode
:param azure_dataset_id: The name of the dataset in blob storage to be used for this run. This can be an empty
string to not use any datasets.
:param environment_name: If specified, try to retrieve the existing Python environment with this name. If that
is not found, create one from the Conda files provided in `source_config`. This parameter is meant to be used
when running inference for an existing model.
:return: The configured script run.
"""
if azure_dataset_id:
azureml_dataset = get_or_create_dataset(azure_config, azure_dataset_id=azure_dataset_id)
if not azureml_dataset:
raise ValueError(f"AzureML dataset {azure_dataset_id} could not be found or created.")
named_input = azureml_dataset.as_named_input(INPUT_DATA_KEY)
dataset_consumption = named_input.as_mount() if azure_config.use_dataset_mount else named_input.as_download()
else:
dataset_consumption = None
# AzureML seems to sometimes expect the entry script path in Linux format, hence convert to posix path
entry_script_relative_path = source_config.entry_script.relative_to(source_config.root_folder).as_posix()
logging.info(f"Entry script {entry_script_relative_path} ({source_config.entry_script} relative to "
f"source directory {source_config.root_folder})")
max_run_duration = None
if azure_config.max_run_duration:
max_run_duration = run_duration_string_to_seconds(azure_config.max_run_duration)
workspace = azure_config.get_workspace()
run_config = RunConfiguration(
script=entry_script_relative_path,
arguments=source_config.script_params,
)
run_config.environment = get_or_create_python_environment(azure_config, source_config,
environment_name=environment_name)
run_config.target = azure_config.cluster
run_config.max_run_duration_seconds = max_run_duration
if azure_config.num_nodes > 1:
distributed_job_config = MpiConfiguration(node_count=azure_config.num_nodes)
run_config.mpi = distributed_job_config
run_config.framework = "Python"
run_config.communicator = "IntelMpi"
run_config.node_count = distributed_job_config.node_count
if dataset_consumption:
run_config.data = {dataset_consumption.name: dataset_consumption}
# Use blob storage for storing the source, rather than the FileShares section of the storage account.
run_config.source_directory_data_store = workspace.datastores.get(WORKSPACE_DEFAULT_BLOB_STORE_NAME).name
script_run_config = ScriptRunConfig(
source_directory=str(source_config.root_folder),
run_config=run_config,
)
if azure_config.hyperdrive:
script_run_config = source_config.hyperdrive_config_func(script_run_config) # type: ignore
return script_run_config
def test_get_hyperdrive_config(
number_of_cross_validation_splits: int,
number_of_cross_validation_splits_per_fold: int,
test_output_dirs: TestOutputDirectories) -> None:
"""
Test to make sure the number of dataset reader workers are set correctly
"""
if number_of_cross_validation_splits_per_fold > 0:
config = HyperDriveTestModelScalar()
config.number_of_cross_validation_splits_per_fold = number_of_cross_validation_splits_per_fold
else:
config = HyperDriveTestModelSegmentation()
config.number_of_cross_validation_splits = number_of_cross_validation_splits
# create HyperDrive config with dummy estimator for testing
source_config = SourceConfig(root_folder=test_output_dirs.root_dir,
entry_script="something.py",
conda_dependencies_files=[])
estimator = Estimator(source_directory=source_config.root_folder,
entry_script=source_config.entry_script,
compute_target="Local")
hd_config = config.get_hyperdrive_config(estimator=estimator)
assert hd_config.estimator.source_directory == source_config.root_folder
assert hd_config.estimator.run_config.script == source_config.entry_script
assert hd_config.estimator._script_params == source_config.script_params
if number_of_cross_validation_splits > 0 and number_of_cross_validation_splits_per_fold > 0:
assert hd_config._max_total_runs == number_of_cross_validation_splits * \
number_of_cross_validation_splits_per_fold
elif number_of_cross_validation_splits > 0:
assert hd_config._max_total_runs == number_of_cross_validation_splits
else:
assert hd_config._max_total_runs == HYPERDRIVE_TOTAL_RUNS
if config.perform_cross_validation:
# check sampler is as expected
sampler = config.get_cross_validation_hyperdrive_sampler()
expected_sampler_dict = {
CROSS_VALIDATION_SPLIT_INDEX_TAG_KEY:
choice(list(range(number_of_cross_validation_splits)))
}
if number_of_cross_validation_splits_per_fold > 0:
expected_sampler_dict[
CROSS_VALIDATION_SUB_FOLD_SPLIT_INDEX_TAG_KEY] = choice(
list(range(number_of_cross_validation_splits_per_fold)))
assert sampler._parameter_space == expected_sampler_dict
else:
assert vars(config.get_hyperdrive_config(estimator)) \
== vars(_create_dummy_hyperdrive_param_search_config(estimator))
def submit_to_azureml(self) -> Run:
"""
Submit a job to AzureML, returning the resulting Run object, or exiting if we were asked to wait for
completion and the Run did not succeed.
"""
# The adal package creates a logging.info line each time it gets an authentication token, avoid that.
logging.getLogger('adal-python').setLevel(logging.WARNING)
# Azure core prints full HTTP requests even in INFO mode
logging.getLogger('azure').setLevel(logging.WARNING)
# PyJWT prints out warnings that are beyond our control
warnings.filterwarnings("ignore", category=DeprecationWarning)
if isinstance(self.model_config, DeepLearningConfig) and not self.lightning_container.azure_dataset_id:
raise ValueError("When running an InnerEye built-in model in AzureML, the 'azure_dataset_id' "
"property must be set.")
hyperdrive_func = lambda run_config: self.model_config.get_hyperdrive_config(run_config) # type: ignore
source_config = SourceConfig(
root_folder=self.project_root,
entry_script=Path(sys.argv[0]).resolve(),
conda_dependencies_files=get_all_environment_files(self.project_root),
hyperdrive_config_func=hyperdrive_func,
# For large jobs, upload of results can time out because of large checkpoint files. Default is 600
upload_timeout_seconds=86400,
)
source_config.set_script_params_except_submit_flag()
azure_run = submit_to_azureml(self.azure_config, source_config,
self.lightning_container.all_azure_dataset_ids(),
self.lightning_container.all_dataset_mountpoints())
logging.info("Job submission to AzureML done.")
if self.azure_config.pytest_mark and self.azure_config.wait_for_completion:
# The AzureML job can optionally run pytest. Attempt to download it to the current directory.
# A build step will pick up that file and publish it to Azure DevOps.
# If pytest_mark is set, this file must exist.
logging.info("Downloading pytest result file.")
download_pytest_result(azure_run)
else:
logging.info("No pytest_mark present, hence not downloading the pytest result file.")
# For PR builds where we wait for job completion, the job must have ended in a COMPLETED state.
if self.azure_config.wait_for_completion and not is_run_and_child_runs_completed(azure_run):
raise ValueError(f"Run {azure_run.id} in experiment {azure_run.experiment.name} or one of its child "
"runs failed.")
return azure_run
def create_estimator_from_configs(
azure_config: AzureConfig, source_config: SourceConfig,
estimator_inputs: List[DatasetConsumptionConfig]) -> PyTorch:
"""
Create an return a PyTorch estimator from the provided configuration information.
:param azure_config: Azure configuration, used to store various values for the job to be submitted
:param source_config: source configutation, for other needed values
:param estimator_inputs: value for the "inputs" field of the estimator.
:return:
"""
# AzureML seems to sometimes expect the entry script path in Linux format, hence convert to posix path
entry_script_relative_path = Path(source_config.entry_script).relative_to(
source_config.root_folder).as_posix()
logging.info(
f"Entry script {entry_script_relative_path} ({source_config.entry_script} relative to "
f"source directory {source_config.root_folder})")
environment_variables = {
"AZUREML_OUTPUT_UPLOAD_TIMEOUT_SEC":
str(source_config.upload_timeout_seconds),
"MKL_SERVICE_FORCE_INTEL":
"1",
**(source_config.environment_variables or {})
}
# Merge the project-specific dependencies with the packages that InnerEye itself needs. This should not be
# necessary if the innereye package is installed. It is necessary when working with an outer project and
# InnerEye as a git submodule and submitting jobs from the local machine.
# In case of version conflicts, the package version in the outer project is given priority.
conda_dependencies = merge_conda_dependencies(
source_config.conda_dependencies_files) # type: ignore
if azure_config.pip_extra_index_url:
# When an extra-index-url is supplied, swap the order in which packages are searched for.
# This is necessary if we need to consume packages from extra-index that clash with names of packages on
# pypi
conda_dependencies.set_pip_option(
f"--index-url {azure_config.pip_extra_index_url}")
conda_dependencies.set_pip_option(
"--extra-index-url https://pypi.org/simple")
# create Estimator environment
framework_version = pytorch_version_from_conda_dependencies(
conda_dependencies)
logging.info(f"PyTorch framework version: {framework_version}")
max_run_duration = None
if azure_config.max_run_duration:
max_run_duration = run_duration_string_to_seconds(
azure_config.max_run_duration)
workspace = azure_config.get_workspace()
estimator = PyTorch(
source_directory=source_config.root_folder,
entry_script=entry_script_relative_path,
script_params=source_config.script_params,
compute_target=azure_config.cluster,
# Use blob storage for storing the source, rather than the FileShares section of the storage account.
source_directory_data_store=workspace.datastores.get(
WORKSPACE_DEFAULT_BLOB_STORE_NAME),
inputs=estimator_inputs,
environment_variables=environment_variables,
shm_size=azure_config.docker_shm_size,
use_docker=True,
use_gpu=True,
framework_version=framework_version,
max_run_duration_seconds=max_run_duration)
estimator.run_config.environment.python.conda_dependencies = conda_dependencies
# We'd like to log the estimator config, but conversion to string fails when the Estimator has some inputs.
# logging.info(azure_util.estimator_to_string(estimator))
if azure_config.hyperdrive:
estimator = source_config.hyperdrive_config_func(
estimator) # type: ignore
return estimator
def submit_to_azureml_if_needed(self) -> AzureRunInfo:
"""
Submit a job to AzureML, returning the resulting Run object, or exiting if we were asked to wait for
completion and the Run did not succeed.
"""
if self.azure_config.azureml and isinstance(self.model_config, DeepLearningConfig) \
and not self.lightning_container.azure_dataset_id:
raise ValueError(
"When running an InnerEye built-in model in AzureML, the 'azure_dataset_id' "
"property must be set.")
# https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility
env_variables = {
"CUBLAS_WORKSPACE_CONFIG": ":4096:8"
} if self.lightning_container.pl_deterministic else {}
source_config = SourceConfig(
root_folder=self.project_root,
entry_script=Path(sys.argv[0]).resolve(),
script_params=sys.argv[1:],
conda_dependencies_files=get_all_environment_files(
self.project_root),
hyperdrive_config_func=(
self.model_config.get_hyperdrive_config if self.model_config
else self.lightning_container.get_hyperdrive_config),
# For large jobs, upload of results can time out because of large checkpoint files. Default is 600
upload_timeout_seconds=86400,
environment_variables=env_variables)
# Reduce the size of the snapshot by adding unused folders to amlignore. The Test* subfolders are only needed
# when running pytest.
ignored_folders = []
if not self.azure_config.pytest_mark:
ignored_folders.extend(["Tests", "TestsOutsidePackage"])
if not self.lightning_container.regression_test_folder:
ignored_folders.append("RegressionTestResults")
all_local_datasets = self.lightning_container.all_local_dataset_paths()
input_datasets = \
create_dataset_configs(self.azure_config,
all_azure_dataset_ids=self.lightning_container.all_azure_dataset_ids(),
all_dataset_mountpoints=self.lightning_container.all_dataset_mountpoints(),
all_local_datasets=all_local_datasets) # type: ignore
def after_submission_hook(azure_run: Run) -> None:
"""
A function that will be called right after job submission.
"""
# Set the default display name to what was provided as the "tag". This will affect single runs
# and Hyperdrive parent runs
if self.azure_config.tag:
azure_run.display_name = self.azure_config.tag
# Add an extra tag that depends on the run that was actually submitted. This is used for later filtering
# run in cross validation analysis
recovery_id = create_run_recovery_id(azure_run)
azure_run.tag(RUN_RECOVERY_ID_KEY_NAME, recovery_id)
print(
"If this run fails, re-start runner.py and supply these additional arguments: "
f"--run_recovery_id={recovery_id}")
if self.azure_config.tensorboard:
print(
"Starting TensorBoard now because you specified --tensorboard"
)
monitor(monitor_config=AMLTensorBoardMonitorConfig(
run_ids=[azure_run.id]),
azure_config=self.azure_config)
else:
print(
f"To monitor this run locally using TensorBoard, run the script: "
f"InnerEye/Azure/tensorboard_monitor.py --run_ids={azure_run.id}"
)
if self.azure_config.wait_for_completion:
# We want the job output to be visible on the console. Do not exit yet if the job fails, because we
# may need to download the pytest result file.
azure_run.wait_for_completion(show_output=True,
raise_on_error=False)
if self.azure_config.pytest_mark:
# The AzureML job can optionally run pytest. Attempt to download it to the current directory.
# A build step will pick up that file and publish it to Azure DevOps.
# If pytest_mark is set, this file must exist.
logging.info("Downloading pytest result file.")
download_pytest_result(azure_run)
if azure_run.status == RunStatus.FAILED:
raise ValueError(
f"The AzureML run failed. Please check this URL for details: "
f"{azure_run.get_portal_url()}")
hyperdrive_config = None
if self.azure_config.hyperdrive:
hyperdrive_config = self.lightning_container.get_hyperdrive_config(
ScriptRunConfig(source_directory=""))
# Create a temporary file for the merged conda file, that will be removed after submission of the job.
temp_conda: Optional[Path] = None
try:
if len(source_config.conda_dependencies_files) > 1:
temp_conda = source_config.root_folder / f"temp_environment-{uuid.uuid4().hex[:8]}.yml"
# Merge the project-specific dependencies with the packages that InnerEye itself needs. This should not
# be necessary if the innereye package is installed. It is necessary when working with an outer project
# and InnerEye as a git submodule and submitting jobs from the local machine.
# In case of version conflicts, the package version in the outer project is given priority.
merge_conda_files(source_config.conda_dependencies_files,
temp_conda)
# Calls like `self.azure_config.get_workspace()` will fail if we have no AzureML credentials set up, and so
# we should only attempt them if we intend to elevate this to AzureML
if self.azure_config.azureml:
if not self.azure_config.cluster:
raise ValueError(
"self.azure_config.cluster not set, but we need a compute_cluster_name to submit"
"the script to run in AzureML")
azure_run_info = submit_to_azure_if_needed(
entry_script=source_config.entry_script,
snapshot_root_directory=source_config.root_folder,
script_params=source_config.script_params,
conda_environment_file=temp_conda
or source_config.conda_dependencies_files[0],
aml_workspace=self.azure_config.get_workspace(),
compute_cluster_name=self.azure_config.cluster,
environment_variables=source_config.environment_variables,
default_datastore=self.azure_config.azureml_datastore,
experiment_name=to_azure_friendly_string(
create_experiment_name(self.azure_config)),
max_run_duration=self.azure_config.max_run_duration,
input_datasets=input_datasets,
num_nodes=self.azure_config.num_nodes,
wait_for_completion=False,
ignored_folders=ignored_folders,
pip_extra_index_url=self.azure_config.pip_extra_index_url,
submit_to_azureml=self.azure_config.azureml,
docker_base_image=DEFAULT_DOCKER_BASE_IMAGE,
docker_shm_size=self.azure_config.docker_shm_size,
tags=additional_run_tags(azure_config=self.azure_config,
commandline_args=" ".join(
source_config.script_params)),
after_submission=after_submission_hook,
hyperdrive_config=hyperdrive_config)
if self.azure_config.tag and azure_run_info.run:
if self.lightning_container.perform_cross_validation:
# This code is only reached inside Azure. Set display name again - this will now affect
# Hypdrive child runs (for other jobs, this has already been done after submission)
cv_index = self.lightning_container.cross_validation_split_index
full_display_name = f"{self.azure_config.tag} {cv_index}"
azure_run_info.run.display_name = full_display_name
else:
azure_run_info = submit_to_azure_if_needed(
input_datasets=input_datasets, submit_to_azureml=False)
finally:
if temp_conda:
temp_conda.unlink()
# submit_to_azure_if_needed calls sys.exit after submitting to AzureML. We only reach this when running
# the script locally or in AzureML.
return azure_run_info
def submit_for_inference(args: SubmitForInferenceConfig,
azure_config: AzureConfig) -> Optional[Path]:
"""
Create and submit an inference to AzureML, and optionally download the resulting segmentation.
:param azure_config: An object with all necessary information for accessing Azure.
:param args: configuration, see SubmitForInferenceConfig
:return: path to downloaded segmentation on local disc, or None if none.
"""
logging.info(f"Building Azure configuration from {args.settings}")
logging.info("Getting workspace")
workspace = azure_config.get_workspace()
logging.info("Identifying model")
model = Model(workspace=workspace, id=args.model_id)
model_id = model.id
logging.info(f"Identified model {model_id}")
source_directory = tempfile.TemporaryDirectory()
source_directory_path = Path(source_directory.name)
logging.info(
f"Building inference run submission in {source_directory_path}")
image_folder = source_directory_path / DEFAULT_DATA_FOLDER
image = copy_image_file(args.image_file, image_folder, args.use_dicom)
model_sas_urls = model.get_sas_urls()
# Identifies all the files with basename "environment.yml" in the model and downloads them.
# These downloads should go into a temp folder that will most likely not be included in the model itself,
# because the AzureML run will later download the model into the same folder structure, and the file names might
# clash.
temp_folder = source_directory_path / "temp_for_scoring"
conda_files = download_files_from_model(model_sas_urls,
ENVIRONMENT_YAML_FILE_NAME,
dir_path=temp_folder)
if not conda_files:
raise ValueError(
"At least 1 Conda environment definition must exist in the model.")
# Retrieve the name of the Python environment that the training run used. This environment should have been
# registered. If no such environment exists, it will be re-create from the Conda files provided.
python_environment_name = model.tags.get(PYTHON_ENVIRONMENT_NAME, "")
# Copy the scoring script from the repository. This will start the model download from Azure, and invoke the
# scoring script.
entry_script = source_directory_path / Path(RUN_SCORING_SCRIPT).name
shutil.copyfile(str(repository_root_directory(RUN_SCORING_SCRIPT)),
str(entry_script))
source_config = SourceConfig(
root_folder=source_directory_path,
entry_script=entry_script,
script_params=[
"--model-folder",
".",
"--model-id",
model_id,
SCORE_SCRIPT,
# The data folder must be relative to the root folder of the AzureML job. test_image_files
# is then just the file relative to the data_folder
"--data_folder",
image.parent.name,
"--image_files",
image.name,
"--use_dicom",
str(args.use_dicom)
],
conda_dependencies_files=conda_files,
)
run_config = create_run_config(azure_config,
source_config,
environment_name=python_environment_name)
exp = Experiment(workspace=workspace, name=args.experiment_name)
run = exp.submit(run_config)
logging.info(f"Submitted run {run.id} in experiment {run.experiment.name}")
logging.info(f"Run URL: {run.get_portal_url()}")
if not args.keep_upload_folder:
source_directory.cleanup()
logging.info(f"Deleted submission directory {source_directory_path}")
if args.download_folder is None:
return None
logging.info("Awaiting run completion")
run.wait_for_completion()
logging.info(f"Run has completed with status {run.get_status()}")
download_path = choose_download_path(args.download_folder)
logging.info(f"Attempting to download segmentation to {download_path}")
run.download_file(DEFAULT_RESULT_IMAGE_NAME, str(download_path))
if download_path.exists():
logging.info(f"Downloaded segmentation to {download_path}")
else:
logging.warning("Segmentation NOT downloaded")
return download_path