def test_create_python_env() -> None: """ Checks if environment variables in the SourceConfig are correctly passed through to the Python environment. Environment variables in SourceConfig are only used in the internal InnerEye repo. :return: """ foo = "foo" bar = "bar" entry_script = Path("something.py") conda_file = get_environment_yaml_file() s = SourceConfig(root_folder=Path(""), entry_script=entry_script, conda_dependencies_files=[conda_file], environment_variables={foo: bar}) env = get_or_create_python_environment( source_config=s, azure_config=get_default_azure_config(), register_environment=False) assert foo in env.environment_variables assert env.environment_variables[foo] == bar # Check that some of the basic packages that we expect to always exist are picked up correctly in the Conda env def remove_version_number(items: Iterator[str]) -> Set[str]: return set(c.split("=")[0] for c in items) assert "pytorch" in remove_version_number( env.python.conda_dependencies.conda_packages) assert "pytorch-lightning" in remove_version_number( env.python.conda_dependencies.pip_packages)
def get_all_environment_files(project_root: Path) -> List[Path]: """ Returns a list of all Conda environment files that should be used. This is firstly the InnerEye conda file, and possibly a second environment.yml file that lives at the project root folder. :param project_root: The root folder of the code that starts the present training run. :return: A list with 1 or 2 entries that are conda environment files. """ innereye_yaml = fixed_paths.get_environment_yaml_file() project_yaml = project_root / fixed_paths.ENVIRONMENT_YAML_FILE_NAME files = [innereye_yaml] if innereye_yaml != project_yaml: files.append(project_yaml) return files
def submit_to_azureml(self) -> Run: """ Submit a job to AzureML, returning the resulting Run object, or exiting if we were asked to wait for completion and the Run did not succeed. """ # The adal package creates a logging.info line each time it gets an authentication token, avoid that. logging.getLogger('adal-python').setLevel(logging.WARNING) if not self.model_config.azure_dataset_id: raise ValueError( "When running on AzureML, the 'azure_dataset_id' property must be set." ) model_config_overrides = str(self.model_config.overrides) source_config = SourceConfig( root_folder=self.project_root, entry_script=Path(sys.argv[0]).resolve(), conda_dependencies_files=[ get_environment_yaml_file(), self.project_root / fixed_paths.ENVIRONMENT_YAML_FILE_NAME ], hyperdrive_config_func=lambda estimator: self.model_config. get_hyperdrive_config(estimator), # For large jobs, upload of results times out frequently because of large checkpoint files. Default is 600 upload_timeout_seconds=86400, ) source_config.set_script_params_except_submit_flag() assert self.model_config.azure_dataset_id is not None # to stop mypy complaining about next line azure_run = submit_to_azureml(self.azure_config, source_config, model_config_overrides, self.model_config.azure_dataset_id) logging.info("Job submission to AzureML done.") if self.azure_config.pytest_mark: # The AzureML job can optionally run pytest. Attempt to download it to the current directory. # A build step will pick up that file and publish it to Azure DevOps. # If pytest_mark is set, this file must exist. logging.info("Downloading pytest result file.") download_pytest_result(azure_run) else: logging.info( "No pytest_mark present, hence not downloading the pytest result file." ) status = azure_run.get_status() # For PR builds where we wait for job completion, the job must have ended in a COMPLETED state. # If a pytest failed, the runner has exited with code -1 (see below) if self.azure_config.wait_for_completion and status != RunStatus.COMPLETED: logging.error(f"Job completed with status {status}. Exiting.") exit(-1) return azure_run