def set_run_tags_from_parent(self) -> None: """ Set metadata for the run """ assert PARENT_RUN_CONTEXT, "This function should only be called in a Hyperdrive run." run_tags_parent = PARENT_RUN_CONTEXT.get_tags() tags_to_copy = [ "tag", "model_name", "execution_mode", "recovered_from", "friendly_name", "build_number", "build_user", "source_repository", "source_branch", "source_id", "source_message", "source_author", "source_dirty", RUN_RECOVERY_FROM_ID_KEY_NAME ] new_tags = {tag: run_tags_parent.get(tag, "") for tag in tags_to_copy} new_tags[RUN_RECOVERY_ID_KEY_NAME] = create_run_recovery_id( run=RUN_CONTEXT) new_tags[CROSS_VALIDATION_SPLIT_INDEX_TAG_KEY] = str( self.model_config.cross_validation_split_index) new_tags[EFFECTIVE_RANDOM_SEED_KEY_NAME] = str( self.model_config.get_effective_random_seed()) if isinstance(self.model_config, ScalarModelBase): new_tags[NUMBER_OF_CROSS_VALIDATION_SPLITS_PER_FOLD_KEY_NAME] = str( self.model_config.number_of_cross_validation_splits_per_fold) new_tags[CROSS_VALIDATION_SUB_FOLD_SPLIT_INDEX_TAG_KEY] = str( self.model_config.cross_validation_sub_fold_split_index) RUN_CONTEXT.set_tags(new_tags)
def set_run_tags(run: Run, azure_config: AzureConfig, model_config_overrides: str) -> None: """ Set metadata for the run :param run: Run to set metadata for. :param azure_config: The configurations for the present AzureML job :param model_config_overrides: A string that describes which model parameters were overwritten by commandline arguments in the present run. """ git_information = azure_config.get_git_information() run.set_tags({ "tag": azure_config.tag, "model_name": azure_config.model, "execution_mode": ModelExecutionMode.TRAIN.value if azure_config.train else ModelExecutionMode.TEST.value, RUN_RECOVERY_ID_KEY_NAME: azure_util.create_run_recovery_id(run=run), RUN_RECOVERY_FROM_ID_KEY_NAME: azure_config.run_recovery_id, "build_number": str(azure_config.build_number), "build_user": azure_config.build_user, "build_user_email": azure_config.build_user_email, "source_repository": git_information.repository, "source_branch": git_information.branch, "source_id": git_information.commit_id, "source_message": git_information.commit_message, "source_author": git_information.commit_author, "source_dirty": str(git_information.is_dirty), "overrides": model_config_overrides, CROSS_VALIDATION_SPLIT_INDEX_TAG_KEY: -1, })
def set_run_tags(run: Run, azure_config: AzureConfig, commandline_args: str) -> None: """ Set metadata for the run :param run: Run to set metadata for. :param azure_config: The configurations for the present AzureML job :param commandline_args: A string that holds all commandline arguments that were used for the present run. """ git_information = get_git_tags(azure_config) run.set_tags({ "tag": azure_config.tag, "model_name": azure_config.model, "execution_mode": ModelExecutionMode.TRAIN.value if azure_config.train else ModelExecutionMode.TEST.value, RUN_RECOVERY_ID_KEY_NAME: azure_util.create_run_recovery_id(run=run), RUN_RECOVERY_FROM_ID_KEY_NAME: azure_config.run_recovery_id, "build_number": str(azure_config.build_number), "build_user": azure_config.build_user, "build_user_email": azure_config.build_user_email, **git_information, "commandline_args": commandline_args, CROSS_VALIDATION_SPLIT_INDEX_TAG_KEY: -1, })
def create_and_submit_experiment( azure_config: AzureConfig, source_config: SourceConfig, model_config_overrides: str, azure_dataset_id: str) -> Run: """ Creates an AzureML experiment in the workspace and submits it for execution. :param azure_config: azure related configurations to setup valid workspace :param source_config: The information about which code should be submitted, and which arguments should be used. :param model_config_overrides: A string that describes which model parameters were overwritten by commandline arguments in the present run. This is only used for diagnostic purposes (it is set as a Tag on the run). :param azure_dataset_id: The name of the dataset in blob storage to be used for this run. :returns: Run object for the submitted AzureML run """ workspace = azure_config.get_workspace() experiment_name = create_experiment_name(azure_config) exp = Experiment(workspace=workspace, name=azure_util.to_azure_friendly_string(experiment_name)) script_run_config = create_run_config(azure_config, source_config, azure_dataset_id) # submit a training/testing run associated with the experiment run: Run = exp.submit(script_run_config) # set metadata for the run set_run_tags(run, azure_config, model_config_overrides) print("\n==============================================================================") print(f"Successfully queued new run {run.id} in experiment: {exp.name}") if azure_config.run_recovery_id: print(f"\nRecovered from: {azure_config.run_recovery_id}") recovery_id = azure_util.create_run_recovery_id(run) recovery_file = Path(RUN_RECOVERY_FILE) if recovery_file.exists(): recovery_file.unlink() recovery_file.write_text(recovery_id) print("Experiment URL: {}".format(exp.get_portal_url())) print("Run URL: {}".format(run.get_portal_url())) print("If this run fails, re-start runner.py and supply these additional arguments: " f"--run_recovery_id={recovery_id}") print(f"The run recovery ID has been written to this file: {recovery_file}") print("==============================================================================") if azure_config.tensorboard and azure_config.azureml: print("Starting TensorBoard now because you specified --tensorboard") monitor(monitor_config=AMLTensorBoardMonitorConfig(run_ids=[run.id]), azure_config=azure_config) else: print(f"To monitor this run locally using TensorBoard, run the script: " f"InnerEye/Azure/tensorboard_monitor.py --run_ids={run.id}") print("==============================================================================") return run
def create_and_submit_experiment(azure_config: AzureConfig, script_run_config: ScriptRunConfig, commandline_args: str) -> Run: """ Creates an AzureML experiment in the workspace and submits it for execution. :param azure_config: azure related configurations to setup a valid workspace. :param script_run_config: The configuration for the script that should be run inside of AzureML. :param commandline_args: A string with all commandline arguments that were provided to the runner. These are only used to set a tag on the submitted AzureML run. :returns: Run object for the submitted AzureML run """ workspace = azure_config.get_workspace() experiment_name = create_experiment_name(azure_config) exp = Experiment(workspace=workspace, name=azure_util.to_azure_friendly_string(experiment_name)) # submit a training/testing run associated with the experiment run: Run = exp.submit(script_run_config) if is_offline_run_context(run): # This codepath will only be executed in unit tests, when exp.submit is mocked. return run # Set metadata for the run. set_run_tags(run, azure_config, commandline_args=commandline_args) print( "\n==============================================================================" ) print(f"Successfully queued new run {run.id} in experiment: {exp.name}") if azure_config.run_recovery_id: print(f"\nRecovered from: {azure_config.run_recovery_id}") recovery_id = azure_util.create_run_recovery_id(run) recovery_file = Path(RUN_RECOVERY_FILE) if recovery_file.exists(): recovery_file.unlink() recovery_file.write_text(recovery_id) print("Experiment URL: {}".format(exp.get_portal_url())) print("Run URL: {}".format(run.get_portal_url())) print( "If this run fails, re-start runner.py and supply these additional arguments: " f"--run_recovery_id={recovery_id}") print( f"The run recovery ID has been written to this file: {recovery_file}") print( "==============================================================================" ) if azure_config.tensorboard and azure_config.azureml: print("Starting TensorBoard now because you specified --tensorboard") monitor(monitor_config=AMLTensorBoardMonitorConfig(run_ids=[run.id]), azure_config=azure_config) else: print( f"To monitor this run locally using TensorBoard, run the script: " f"InnerEye/Azure/tensorboard_monitor.py --run_ids={run.id}") print( "==============================================================================" ) return run