def test_create_python_env() -> None:
"""
Checks if environment variables in the SourceConfig are correctly passed through to the Python environment.
Environment variables in SourceConfig are only used in the internal InnerEye repo.
:return:
"""
foo = "foo"
bar = "bar"
entry_script = Path("something.py")
conda_file = get_environment_yaml_file()
s = SourceConfig(root_folder=Path(""),
entry_script=entry_script,
conda_dependencies_files=[conda_file],
environment_variables={foo: bar})
env = get_or_create_python_environment(
source_config=s,
azure_config=get_default_azure_config(),
register_environment=False)
assert foo in env.environment_variables
assert env.environment_variables[foo] == bar
# Check that some of the basic packages that we expect to always exist are picked up correctly in the Conda env
def remove_version_number(items: Iterator[str]) -> Set[str]:
return set(c.split("=")[0] for c in items)
assert "pytorch" in remove_version_number(
env.python.conda_dependencies.conda_packages)
assert "pytorch-lightning" in remove_version_number(
env.python.conda_dependencies.pip_packages)
def get_all_environment_files(project_root: Path) -> List[Path]:
"""
Returns a list of all Conda environment files that should be used. This is firstly the InnerEye conda file,
and possibly a second environment.yml file that lives at the project root folder.
:param project_root: The root folder of the code that starts the present training run.
:return: A list with 1 or 2 entries that are conda environment files.
"""
innereye_yaml = fixed_paths.get_environment_yaml_file()
project_yaml = project_root / fixed_paths.ENVIRONMENT_YAML_FILE_NAME
files = [innereye_yaml]
if innereye_yaml != project_yaml:
files.append(project_yaml)
return files
def submit_to_azureml(self) -> Run:
"""
Submit a job to AzureML, returning the resulting Run object, or exiting if we were asked to wait for
completion and the Run did not succeed.
"""
# The adal package creates a logging.info line each time it gets an authentication token, avoid that.
logging.getLogger('adal-python').setLevel(logging.WARNING)
if not self.model_config.azure_dataset_id:
raise ValueError(
"When running on AzureML, the 'azure_dataset_id' property must be set."
)
model_config_overrides = str(self.model_config.overrides)
source_config = SourceConfig(
root_folder=self.project_root,
entry_script=Path(sys.argv[0]).resolve(),
conda_dependencies_files=[
get_environment_yaml_file(),
self.project_root / fixed_paths.ENVIRONMENT_YAML_FILE_NAME
],
hyperdrive_config_func=lambda estimator: self.model_config.
get_hyperdrive_config(estimator),
# For large jobs, upload of results times out frequently because of large checkpoint files. Default is 600
upload_timeout_seconds=86400,
)
source_config.set_script_params_except_submit_flag()
assert self.model_config.azure_dataset_id is not None # to stop mypy complaining about next line
azure_run = submit_to_azureml(self.azure_config, source_config,
model_config_overrides,
self.model_config.azure_dataset_id)
logging.info("Job submission to AzureML done.")
if self.azure_config.pytest_mark:
# The AzureML job can optionally run pytest. Attempt to download it to the current directory.
# A build step will pick up that file and publish it to Azure DevOps.
# If pytest_mark is set, this file must exist.
logging.info("Downloading pytest result file.")
download_pytest_result(azure_run)
else:
logging.info(
"No pytest_mark present, hence not downloading the pytest result file."
)
status = azure_run.get_status()
# For PR builds where we wait for job completion, the job must have ended in a COMPLETED state.
# If a pytest failed, the runner has exited with code -1 (see below)
if self.azure_config.wait_for_completion and status != RunStatus.COMPLETED:
logging.error(f"Job completed with status {status}. Exiting.")
exit(-1)
return azure_run