def fetch_child_runs(run: Run, status: Optional[str] = None,
expected_number_cross_validation_splits: int = 0) -> List[Run]:
"""
Fetch child runs for the provided runs that have the provided AML status (or fetch all by default)
and have a run_recovery_id tag value set (this is to ignore superfluous AML infrastructure platform runs).
:param run: parent run to fetch child run from
:param status: if provided, returns only child runs with this status
:param expected_number_cross_validation_splits: when recovering child runs from AML hyperdrive
sometimes the get_children function fails to retrieve all children. If the number of child runs
retrieved by AML is lower than the expected number of splits, we try to retrieve them manually.
"""
if is_ensemble_run(run):
run_recovery_id = run.get_tags().get(RUN_RECOVERY_FROM_ID_KEY_NAME, None)
if run_recovery_id:
run = fetch_run(run.experiment.workspace, run_recovery_id)
elif PARENT_RUN_CONTEXT:
run = PARENT_RUN_CONTEXT
children_runs = list(run.get_children(tags=RUN_RECOVERY_ID_KEY_NAME))
if 0 < expected_number_cross_validation_splits != len(children_runs):
logging.warning(
f"The expected number of child runs was {expected_number_cross_validation_splits}."
f"Fetched only: {len(children_runs)} runs. Now trying to fetch them manually.")
run_ids_to_evaluate = [f"{create_run_recovery_id(run)}_{i}"
for i in range(expected_number_cross_validation_splits)]
children_runs = [fetch_run(run.experiment.workspace, id) for id in run_ids_to_evaluate]
if status is not None:
children_runs = [child_run for child_run in children_runs if child_run.get_status() == status]
return children_runs
def is_run_and_child_runs_completed(run: Run) -> bool:
"""
Checks if the given run has successfully completed. If the run has child runs, it also checks if the child runs
completed successfully.
:param run: The AzureML run to check.
:return: True if the run and all child runs completed successfully.
"""
def is_completed(run: Run) -> bool:
status = run.get_status()
if run.status == RunStatus.COMPLETED:
return True
logging.info(f"Run {run.id} in experiment {run.experiment.name} finished with status {status}.")
return False
runs = list(run.get_children())
runs.append(run)
return all(is_completed(run) for run in runs)
training_step_name = sys.argv[4]
parentrun = run.parent
print("model_name:", model_name)
print("training_step_name:", training_step_name)
# The required metrics should be present in the parent run, the below condition has been included
# to show an alternative approach by getting those metrics from the prior training step directly.
training_run_id = None
tagsdict = parentrun.get_tags()
if (tagsdict.get("best_model")) != None:
model_type = tagsdict['best_model']
model_accuracy = float(tagsdict['accuracy'])
training_run_id = parentrun.id
else:
for step in parentrun.get_children():
print("Outputs of step " + step.name)
if step.name == training_step_name:
tagsdict = step.get_tags()
model_type = tagsdict['best_model']
model_accuracy = float(tagsdict['accuracy'])
training_run_id = step.id
if (training_run_id == None):
sys.exit("Failed to retrieve model information from run.")
# A sample dataset can be included with the registered model for reference.
# To get the dataset, the data reference has to be a cloud storage path, therefore a local run won't work.
mntpath = os.environ['AZUREML_DATAREFERENCE_irisdata']
dataset = None
def get_automl_environment(workspace: Workspace, training_pipeline_run_id: str, training_experiment_name: str):
from azureml.core import Experiment, Run
experiment = Experiment(workspace, training_experiment_name)
run = Run(experiment, training_pipeline_run_id)
step_run = list(run.get_children())[0]
return step_run.get_environment()
bestModel.write().overwrite().save(model_output)
# COMMAND ----------
model_name, model_ext = model_dbfs.split(".")
# COMMAND ----------
model_zip = model_name + ".zip"
shutil.make_archive(model_name, 'zip', model_dbfs)
azRun.upload_file("outputs/" + model_nm, model_zip)
# COMMAND ----------
azRun.register_model(model_name='model_nm', model_path="outputs/" + model_nm)
# COMMAND ----------
# now delete the serialized model from local folder since it is already uploaded to run history
shutil.rmtree(model_dbfs)
os.remove(model_zip)
# COMMAND ----------
for a in azRun.get_children():
Run(exp, a.id).complete()
# COMMAND ----------
mlflow.end_run()
