def get_best_model_in_experiment(ws, experiment_name, asset_name, asset_label,
score_metric, greater_is_better):
best_score_run_id = None
best_score = None
tags = {'asset_name': asset_name}
if asset_label is not None:
tags['asset_label'] = asset_label
# for run in experiment.get_runs(tags=tags, include_children=True):
experiment = ws.experiments[experiment_name]
for run in Run.list(experiment,
tags=tags,
include_children=True,
status='Completed'):
run_metrics = run.get_metrics()
run_details = run.get_details()
# each logged metric becomes a key in this returned dict
run_score = run_metrics.get(score_metric)
if run_score is not None:
run_id = run_metrics["run_id"]
if best_score is None:
best_score = run_score
best_score_run_id = run_id
else:
if greater_is_better:
if run_score > best_score:
best_score = run_score
best_score_run_id = run_id
else:
if run_score < best_score:
best_score = run_score
best_score_run_id = run_id
if not best_score:
raise Exception(
f"Error: score metric '{score_metric}' was not found in any run!")
if not best_score_run_id:
raise Exception(
f"Error: haven't found a run with score metric '{score_metric}' score metric."
)
print("Best model run_id: " + best_score_run_id)
print("Best model score: " + str(best_score))
return best_score_run_id
def history(self):
ws = AzureProject(self.ctx)._get_ws()
experiment_name = self.ctx.config.get('experiment/name', None)
if experiment_name is None:
raise AzureException('Please specify Experiment name...')
experiment = Experiment(ws, experiment_name)
runs = Run.list(experiment)
result = []
for run in runs:
details = run.get_details()
st = dt.datetime.strptime(
details['startTimeUtc'], '%Y-%m-%dT%H:%M:%S.%fZ')
et = dt.datetime.strptime(
details['endTimeUtc'], '%Y-%m-%dT%H:%M:%S.%fZ')
duratin = str(et-st)
result.append({
'id': run.id,
'start time': details['startTimeUtc'],
'duratin': duratin,
'status': details['status']
})
print_table(self.ctx.log, result)
return {'history': result}
def deploy_model(ws, aci_service_name, experiment_name, asset_name,
asset_label, run_id, cpu_cores, memory_gb, entry_script):
env = create_env_from_requirements(endpoint=True)
inference_config = InferenceConfig(source_directory=os.getcwd(),
entry_script=entry_script,
environment=env)
deployment_config = AciWebservice.deploy_configuration(cpu_cores=cpu_cores,
memory_gb=memory_gb)
# model name
model_name = get_model_register_name(run_id)
try:
model = Model(ws, name=model_name)
except:
# creating directory for download Model files for Model register
tmp_path = create_tempdir(name='download_tmp')
register_path = create_directory(AML_MLAPP_FOLDER, path=tmp_path)
# getting RUN context
experiment = Experiment(workspace=ws, name=experiment_name)
tags = {"run_id": run_id, "asset_name": asset_name}
if asset_label is not None:
tags["asset_label"] = asset_label
selected_run_id = None
for run in Run.list(experiment,
tags=tags,
include_children=True,
status='Completed'):
run_metrics = run.get_metrics()
exp_saved_run_id = run_metrics.get("run_id")
if exp_saved_run_id == run_id:
selected_run_id = run.id
break
if selected_run_id is None:
raise Exception(
'ERROR: there is no matching Run object that associated with the run id %s in this experiment.'
% str(run_id))
current_run = Run(experiment=experiment, run_id=selected_run_id)
# download files from run object
current_run.download_files(output_directory=register_path)
# register model
model = Model.register(ws,
model_path=register_path,
model_name=model_name,
tags=tags,
description=asset_name)
# deletes tmp dir and all content
delete_directory_with_all_contents(tmp_path)
# deploy model
service = None
try:
service = Webservice(ws, name=aci_service_name)
service.update(models=[model], inference_config=inference_config)
except WebserviceException as e:
if service:
service.delete()
service = Model.deploy(ws, aci_service_name, [model], inference_config,
deployment_config)
service.wait_for_deployment(True)
