def test_merge_conda(test_output_dirs: OutputFolderForTests) -> None:
"""
Tests the logic for merging Conda environment files.
"""
env1 = """
channels:
- defaults
- pytorch
dependencies:
- conda1=1.0
- conda2=2.0
- conda_both=3.0
- pip:
- azureml-sdk==1.7.0
- foo==1.0
"""
env2 = """
channels:
- defaults
dependencies:
- conda1=1.1
- conda_both=3.0
- pip:
- azureml-sdk==1.6.0
- bar==2.0
"""
file1 = test_output_dirs.root_dir / "env1.yml"
file1.write_text(env1)
file2 = test_output_dirs.root_dir / "env2.yml"
file2.write_text(env2)
# Spurious test failures on Linux build agents, saying that they can't read the file. Wait a bit.
time.sleep(1)
files = [file1, file2]
merged_file = test_output_dirs.root_dir / "merged.yml"
merge_conda_files(files, merged_file)
assert merged_file.read_text().splitlines() == """channels:
- defaults
- pytorch
dependencies:
- conda1=1.0
- conda1=1.1
- conda2=2.0
- conda_both=3.0
- pip:
- azureml-sdk==1.6.0
- azureml-sdk==1.7.0
- bar==2.0
- foo==1.0
""".splitlines()
conda_dep = merge_conda_dependencies(files)
# We expect to see the union of channels.
assert list(conda_dep.conda_channels) == ["defaults", "pytorch"]
# Package version conflicts are not resolved, both versions are retained.
assert list(conda_dep.conda_packages) == [
"conda1=1.0", "conda1=1.1", "conda2=2.0", "conda_both=3.0"
]
assert list(conda_dep.pip_packages) == [
"azureml-sdk==1.6.0", "azureml-sdk==1.7.0", "bar==2.0", "foo==1.0"
]
def get_or_create_python_environment(azure_config: AzureConfig,
source_config: SourceConfig,
environment_name: str = "",
register_environment: bool = True) -> Environment:
"""
Creates a description for the Python execution environment in AzureML, based on the Conda environment
definition files that are specified in `source_config`. If such environment with this Conda environment already
exists, it is retrieved, otherwise created afresh.
:param azure_config: azure related configurations to use for model scale-out behaviour
:param source_config: configurations for model execution, such as name and execution mode
:param environment_name: If specified, try to retrieve the existing Python environment with this name. If that
is not found, create one from the Conda files provided. This parameter is meant to be used when running
inference for an existing model.
:param register_environment: If True, the Python environment will be registered in the AzureML workspace. If
False, it will only be created, but not registered. Use this for unit testing.
"""
# Merge the project-specific dependencies with the packages that InnerEye itself needs. This should not be
# necessary if the innereye package is installed. It is necessary when working with an outer project and
# InnerEye as a git submodule and submitting jobs from the local machine.
# In case of version conflicts, the package version in the outer project is given priority.
conda_dependencies, merged_yaml = merge_conda_dependencies(source_config.conda_dependencies_files) # type: ignore
if azure_config.pip_extra_index_url:
# When an extra-index-url is supplied, swap the order in which packages are searched for.
# This is necessary if we need to consume packages from extra-index that clash with names of packages on
# pypi
conda_dependencies.set_pip_option(f"--index-url {azure_config.pip_extra_index_url}")
conda_dependencies.set_pip_option("--extra-index-url https://pypi.org/simple")
env_variables = {
"AZUREML_OUTPUT_UPLOAD_TIMEOUT_SEC": str(source_config.upload_timeout_seconds),
"MKL_SERVICE_FORCE_INTEL": "1",
**(source_config.environment_variables or {})
}
base_image = "mcr.microsoft.com/azureml/openmpi3.1.2-cuda10.2-cudnn8-ubuntu18.04"
# Create a name for the environment that will likely uniquely identify it. AzureML does hashing on top of that,
# and will re-use existing environments even if they don't have the same name.
# Hashing should include everything that can reasonably change. Rely on hashlib here, because the built-in
# hash function gives different results for the same string in different python instances.
hash_string = "\n".join([merged_yaml, azure_config.docker_shm_size, base_image, str(env_variables)])
sha1 = hashlib.sha1(hash_string.encode("utf8"))
overall_hash = sha1.hexdigest()[:32]
unique_env_name = f"InnerEye-{overall_hash}"
try:
env_name_to_find = environment_name or unique_env_name
env = Environment.get(azure_config.get_workspace(), name=env_name_to_find, version=ENVIRONMENT_VERSION)
logging.info(f"Using existing Python environment '{env.name}'.")
return env
except Exception:
logging.info(f"Python environment '{unique_env_name}' does not yet exist, creating and registering it.")
env = Environment(name=unique_env_name)
env.docker.enabled = True
env.docker.shm_size = azure_config.docker_shm_size
env.python.conda_dependencies = conda_dependencies
env.docker.base_image = base_image
env.environment_variables = env_variables
if register_environment:
env.register(azure_config.get_workspace())
return env
def test_merge_conda(test_output_dirs: OutputFolderForTests) -> None:
"""
Tests the logic for merging Conda environment files.
"""
env1 = """
channels:
- defaults
- pytorch
dependencies:
- conda1=1.0
- conda2=2.0
- conda_both=3.0
- pip:
- azureml-sdk==1.7.0
- foo==1.0
"""
env2 = """
channels:
- defaults
dependencies:
- conda1=1.1
- conda_both=3.0
- pip:
- azureml-sdk==1.6.0
- bar==2.0
"""
file1 = test_output_dirs.root_dir / "env1.yml"
file1.write_text(env1)
file2 = test_output_dirs.root_dir / "env2.yml"
file2.write_text(env2)
conda_dep = merge_conda_dependencies([file1, file2])
# We expect to see the union of channels.
assert list(conda_dep.conda_channels) == ["defaults", "pytorch"]
# Conda package version conflicts are not resolved, but both versions are retained.
assert list(conda_dep.conda_packages) == [
"conda1=1.0", "conda2=2.0", "conda_both=3.0", "conda1=1.1"
]
# For pip packages, the version in the second argument takes precedence.
assert list(conda_dep.pip_packages) == [
"foo==1.0", "azureml-sdk==1.6.0", "bar==2.0"
]
def create_estimator_from_configs(
azure_config: AzureConfig, source_config: SourceConfig,
estimator_inputs: List[DatasetConsumptionConfig]) -> PyTorch:
"""
Create an return a PyTorch estimator from the provided configuration information.
:param azure_config: Azure configuration, used to store various values for the job to be submitted
:param source_config: source configutation, for other needed values
:param estimator_inputs: value for the "inputs" field of the estimator.
:return:
"""
# AzureML seems to sometimes expect the entry script path in Linux format, hence convert to posix path
entry_script_relative_path = Path(source_config.entry_script).relative_to(
source_config.root_folder).as_posix()
logging.info(
f"Entry script {entry_script_relative_path} ({source_config.entry_script} relative to "
f"source directory {source_config.root_folder})")
environment_variables = {
"AZUREML_OUTPUT_UPLOAD_TIMEOUT_SEC":
str(source_config.upload_timeout_seconds),
"MKL_SERVICE_FORCE_INTEL":
"1",
**(source_config.environment_variables or {})
}
# Merge the project-specific dependencies with the packages that InnerEye itself needs. This should not be
# necessary if the innereye package is installed. It is necessary when working with an outer project and
# InnerEye as a git submodule and submitting jobs from the local machine.
# In case of version conflicts, the package version in the outer project is given priority.
conda_dependencies = merge_conda_dependencies(
source_config.conda_dependencies_files) # type: ignore
if azure_config.pip_extra_index_url:
# When an extra-index-url is supplied, swap the order in which packages are searched for.
# This is necessary if we need to consume packages from extra-index that clash with names of packages on
# pypi
conda_dependencies.set_pip_option(
f"--index-url {azure_config.pip_extra_index_url}")
conda_dependencies.set_pip_option(
"--extra-index-url https://pypi.org/simple")
# create Estimator environment
framework_version = pytorch_version_from_conda_dependencies(
conda_dependencies)
logging.info(f"PyTorch framework version: {framework_version}")
max_run_duration = None
if azure_config.max_run_duration:
max_run_duration = run_duration_string_to_seconds(
azure_config.max_run_duration)
workspace = azure_config.get_workspace()
estimator = PyTorch(
source_directory=source_config.root_folder,
entry_script=entry_script_relative_path,
script_params=source_config.script_params,
compute_target=azure_config.cluster,
# Use blob storage for storing the source, rather than the FileShares section of the storage account.
source_directory_data_store=workspace.datastores.get(
WORKSPACE_DEFAULT_BLOB_STORE_NAME),
inputs=estimator_inputs,
environment_variables=environment_variables,
shm_size=azure_config.docker_shm_size,
use_docker=True,
use_gpu=True,
framework_version=framework_version,
max_run_duration_seconds=max_run_duration)
estimator.run_config.environment.python.conda_dependencies = conda_dependencies
# We'd like to log the estimator config, but conversion to string fails when the Estimator has some inputs.
# logging.info(azure_util.estimator_to_string(estimator))
if azure_config.hyperdrive:
estimator = source_config.hyperdrive_config_func(
estimator) # type: ignore
return estimator