# Download files for the remote run.
download_run_files(experiment_names=[
'automl-bikeshareforecasting', 'automl-bikeshareforecasting_test'
],
download_all_runs=True)
checkExperimentResult(experiment_name='automl-bikeshareforecasting',
expected_num_iteration='1000',
expected_minimum_score=0.01,
expected_maximum_score=0.3,
metric_name='normalized_root_mean_squared_error',
absolute_minimum_score=0.0,
absolute_maximum_score=1.0)
check_experiment_model_explanation_of_best_run(
experiment_name='automl-bikeshareforecasting')
# Check the output cells of the notebook.
checkNotebookOutput(
"auto-ml-forecasting-bike-share.nbconvert.ipynb"
if len(sys.argv) < 2 else sys.argv[1],
"warning[except]retrying[except]UserWarning: Matplotlib is building the font cache"
"[except]warning: a newer version of conda exists"
"[except]UserWarning: Starting from version 2.2.1, "
"the library file in distribution wheels for macOS is built by the Apple Clang"
"[except]The following algorithms are not compatibile with lags and rolling windows"
"[except]brew install libomp"
"[except]If 'script' has been provided here"
"[except]If 'arguments' has been provided here")
remote_run = experiment.submit(automl_config)
# Canceling runs
#
# You can cancel ongoing remote runs using the *cancel()* and *cancel_iteration()* functions
print(remote_run.id)
time.sleep(180)
# Cancel the ongoing experiment and stop scheduling new iterations
remote_run.cancel()
print('run cancelled')
# Wait for the run to complete. It should complete soon because it has been canceled.
remote_run.wait_for_completion()
children = list(remote_run.get_children())
print(len(children))
if (len(children) == 100):
raise Exception(
'Run wasnt cancelled properly, child run count is 100 should have been less than 100'
)
# Check the output cells of the notebook.
checkNotebookOutput('auto-ml-remote-attach.nbconvert.ipynb', 'warning',
'nan')
# Test classification
from checknotebookoutput import checkNotebookOutput
from checkexperimentresult import checkExperimentResult
checkExperimentResult(experiment_name='automl-classification',
expected_num_iteration='1000',
expected_minimum_score=0.5,
metric_name='AUC_weighted')
# Check the output cells of the notebook.
checkNotebookOutput('auto-ml-classification.nbconvert.ipynb', 'warning',
'nan[except]\'missing\': nan')
from checknotebookoutput import checkNotebookOutput
from checkexperimentresult import checkExperimentResult
checkExperimentResult(experiment_name='automl-classification-deployment',
expected_num_iteration='10',
expected_minimum_score=0.5,
metric_name='AUC_weighted')
# Check the output cells of the notebook.
checkNotebookOutput('auto-ml-classification-with-deployment.nbconvert.ipynb',
'warning', 'nan')
absolute_maximum_score=1.0)
check_experiment_model_explanation_of_best_run(
experiment_name='automl-ojforecasting')
# Check the output cells of the notebook.
# We need to suppress this warning '[except]Warning, azureml-defaults not detected' since it's popped
# from the Aci deploy_configuration(), as the azureml-defaults is a required package by Aci.
# This won't impact the customer and it's not critical.
# [stderr] checks for any messages written to stderr, typically logger.warning() will output this.
allowed_warn_str = (
'[except]warning - retrying'
'[except]UserWarning: Matplotlib is building the font cache'
'[except]warning: a newer version of conda exists'
'[except]Warning, azureml-defaults not detected'
'[except]UserWarning: Starting from version 2.2.1, '
'the library file in distribution wheels for macOS is built by the Apple Clang'
'[except]brew install libomp'
'[except]Using different time series parameters in AutoML configs'
'[except]Forecasting parameter country_or_region will be deprecated in the future,'
'[except]Forecasting parameter max_horizon will be deprecated in the future,'
'[except]reg:linear is now deprecated'
'[except]Forecasting parameter grain_column_names will be deprecated in the future'
)
checkNotebookOutput(
'auto-ml-forecasting-orange-juice-sales.nbconvert.ipynb' if
len(sys.argv) < 2 else sys.argv[1], 'warning{}'.format(allowed_warn_str),
'[stderr]{}{}'.format(allowed_warn_str, '[except]Importing plotly failed'))
from checknotebookoutput import checkNotebookOutput
from checkexperimentresult import checkExperimentResult
checkExperimentResult(experiment_name='non_sample_weight_experiment',
expected_num_iteration='10',
expected_minimum_score=0.3,
metric_name='AUC_weighted')
checkExperimentResult(experiment_name='sample_weight_experiment',
expected_num_iteration='10',
expected_minimum_score=0.3,
metric_name='AUC_weighted')
# Check the output cells of the notebook.
checkNotebookOutput('auto-ml-sample-weight.nbconvert.ipynb', 'warning', 'nan')
from checknotebookoutput import checkNotebookOutput
from checkexperimentresult import checkExperimentResult
from checkcelloutput import checkCellOutput
checkExperimentResult(experiment_name='automl-local-regression',
expected_num_iteration='10',
expected_minimum_score=0.45,
metric_name='spearman_correlation',
absolute_minimum_score=-1.0)
# Check the output cells of the notebook.
checkNotebookOutput('auto-ml-regression.nbconvert.ipynb', 'warning', 'nan')
# Check expected cell output contents.
expected_cells = [
[], ["Found the config file in: "], [], [],
[
"Running on local machine\n", "Parent Run ID: ",
"Current status: DatasetCrossValidationSplit. Generating individually featurized CV splits.\n",
"Current status: ModelSelection. Beginning model selection.\n", "\n",
"***********************************************************************"
+ "*****************************\n",
"ITERATION: The iteration being evaluated.\n",
"PIPELINE: A summary description of the pipeline being evaluated.\n",
"DURATION: Time taken for the current iteration.\n",
"METRIC: The result of computing score on the fitted pipeline.\n",
"BEST: The best observed score thus far.\n",
"***********************************************************************"
+ "*****************************\n", "\n",
" ITERATION PIPELINE DURATION"
+ " METRIC BEST\n", " 0 ", " 1 ",
automl_runs = list(experiment.get_runs(type='automl'))
assert (len(automl_runs) == 1)
ml_run = AutoMLRun(experiment=experiment, run_id=automl_runs[0].id)
properties = ml_run.get_properties()
status = ml_run.get_details()
assert (status['status'] == 'Completed')
assert (properties['num_iterations'] == '10')
children = list(ml_run.get_children())
for iteration in children:
metrics = iteration.get_metrics()
iteration_status = iteration.get_status()
iteration_properties = iteration.get_properties()
pipeline_spec = iteration_properties['pipeline_spec']
print(iteration.id)
print(metrics['AUC_weighted'])
assert (metrics['AUC_weighted'] > 0.4)
assert (metrics['AUC_weighted'] <= 1.0)
print(iteration_status)
assert (iteration_status == 'Completed')
assert ('TFLinearClassifierWrapper' in pipeline_spec
or 'TFDNNClassifierWrapper' in pipeline_spec
or 'LightGBM' in pipeline_spec or 'Ensemble' in pipeline_spec)
# Check the output cells of the notebook.
checkNotebookOutput('auto-ml-classification-with-whitelisting.nbconvert.ipynb',
'warning', 'nan')
experiment_name = 'automl-remote-datastore-file'
# project folder
project_folder = './sample_projects/automl-remote-datastore-file'
experiment = Experiment(ws, experiment_name)
automl_runs = list(experiment.get_runs(type='automl'))
assert (len(automl_runs) == 1)
ml_run = AutoMLRun(experiment=experiment, run_id=automl_runs[0].id)
properties = ml_run.get_properties()
status = ml_run.get_details()
assert (status['status'] == 'Completed')
assert (properties['num_iterations'] == '4')
children = list(ml_run.get_children())
for iteration in children:
metrics = iteration.get_metrics()
iteration_status = iteration.get_status()
print(iteration.id)
print(metrics['AUC_weighted'])
assert (metrics['AUC_weighted'] > 0.6)
assert (metrics['AUC_weighted'] <= 1.0)
print(iteration_status)
assert (iteration_status == 'Completed')
# Check the output cells of the notebook.
checkNotebookOutput('auto-ml-remote-execution-with-datastore.nbconvert.ipynb',
'warning', 'nan')
from checknotebookoutput import checkNotebookOutput
# Check the output cells of the notebook.
checkNotebookOutput(
'auto-ml-missing-data-blacklist-early-termination.nbconvert.ipynb',
'warning', 'nan')
from checknotebookoutput import checkNotebookOutput
from checkexperimentresult import checkExperimentResult
checkExperimentResult(experiment_name='automl-dataprep-local',
expected_num_iteration='2',
expected_minimum_score=0.5,
metric_name='AUC_weighted')
# Check the output cells of the notebook.
checkNotebookOutput('auto-ml-dataprep.nbconvert.ipynb', 'warning', 'nan')