def test_wide_deep(notebooks, tmp):
notebook_path = notebooks["wide_deep"]
params = {
"MOVIELENS_DATA_SIZE": "100k",
"EPOCHS": 1,
"EVALUATE_WHILE_TRAINING": False,
"MODEL_DIR": tmp,
"EXPORT_DIR_BASE": tmp,
"RATING_METRICS": ["rmse", "mae"],
"RANKING_METRICS": ["ndcg_at_k", "precision_at_k"],
}
pm.execute_notebook(
notebook_path, OUTPUT_NOTEBOOK, kernel_name=KERNEL_NAME, parameters=params
)
results = pm.read_notebook(OUTPUT_NOTEBOOK).dataframe.set_index("name")["value"]
# Model performance is highly dependant on the initial random weights
# when epochs is small with a small dataset.
# Therefore, in the smoke-test context, rather check if the model training is working
# with minimum performance metrics as follows:
assert results["rmse"] < 2.0
assert results["mae"] < 2.0
assert results["ndcg_at_k"] > 0.0
assert results["precision_at_k"] > 0.0
def test_fastai(notebooks):
notebook_path = notebooks["fastai"]
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
parameters=dict(TOP_K=10, MOVIELENS_DATA_SIZE="100k", EPOCHS=1),
)
def test_lightgbm(notebooks):
notebook_path = notebooks["lightgbm_quickstart"]
pm.execute_notebook(notebook_path, OUTPUT_NOTEBOOK, kernel_name=KERNEL_NAME,
parameters=dict(MAX_LEAF=32,
MIN_DATA=20,
NUM_OF_TREES=10,
TREE_LEARNING_RATE=0.15,
EARLY_STOPPING_ROUNDS=20,
METRIC="auc"), )
def task_wrapper(*args, **kwargs):
''' In case that some tasks are executed repeatedly'''
start = datetime.now()
fin_flag_path = os.path.join(os.path.dirname(kwargs['output']), 'fin_flag')
if os.path.exists(fin_flag_path):
print("Task has been finished before.")
else:
pm.execute_notebook(*args, **kwargs)
open(fin_flag_path, 'a').close() # touch to indicate the task has been finished
return datetime.now() - start
def test_ncf_deep_dive(notebooks):
notebook_path = notebooks["ncf_deep_dive"]
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
parameters=dict(
TOP_K=10, MOVIELENS_DATA_SIZE="100k", EPOCHS=1, BATCH_SIZE=2048
),
)
def test_mmlspark_lightgbm_criteo_runs(notebooks):
notebook_path = notebooks["mmlspark_lightgbm_criteo"]
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
parameters=dict(
DATA_SIZE="sample",
NUM_ITERATIONS=10,
EARLY_STOPPING_ROUND=2,
)
)
def test_notebook_dkn(notebooks):
notebook_path = notebooks["dkn_quickstart"]
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
parameters=dict(epoch=1),
)
results = pm.read_notebook(OUTPUT_NOTEBOOK).dataframe.set_index("name")["value"]
assert results["res"]["auc"] == pytest.approx(0.4707, rel=TOL, abs=ABS_TOL)
assert results["res"]["acc"] == pytest.approx(0.5725, rel=TOL, abs=ABS_TOL)
def test_template_runs(notebooks):
notebook_path = notebooks["template"]
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
parameters=dict(PM_VERSION=pm.__version__),
kernel_name=KERNEL_NAME,
)
nb = pm.read_notebook(OUTPUT_NOTEBOOK)
df = nb.dataframe
assert df.shape[0] == 2
check_version = df.loc[df["name"] == "checked_version", "value"].values[0]
assert check_version is True
def test_spark_tuning(notebooks):
notebook_path = notebooks["spark_tuning"]
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
parameters=dict(
NUMBER_CORES="*",
NUMBER_ITERATIONS=3,
RANK=[5, 5],
REG=[0.1, 0.01]
)
)
def test_fastai_integration(notebooks, size, epochs, expected_values):
notebook_path = notebooks["fastai"]
pm.execute_notebook(notebook_path, OUTPUT_NOTEBOOK, kernel_name=KERNEL_NAME)
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
parameters=dict(TOP_K=10, MOVIELENS_DATA_SIZE=size, EPOCHS=epochs),
)
results = pm.read_notebook(OUTPUT_NOTEBOOK).dataframe.set_index("name")["value"]
for key, value in expected_values.items():
assert results[key] == pytest.approx(value, rel=TOL, abs=ABS_TOL)
def test_ncf_smoke(notebooks):
notebook_path = notebooks["ncf"]
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
parameters=dict(TOP_K=10, MOVIELENS_DATA_SIZE="100k", EPOCHS=1, BATCH_SIZE=256),
)
results = pm.read_notebook(OUTPUT_NOTEBOOK).dataframe.set_index("name")["value"]
# There is too much variability to do an approx equal, just adding top values
assert results["map"] < 0.05
assert results["ndcg"] < 0.20
assert results["precision"] < 0.17
assert results["recall"] < 0.10
def test_mmlspark_lightgbm_criteo_smoke(notebooks):
notebook_path = notebooks["mmlspark_lightgbm_criteo"]
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
parameters=dict(
DATA_SIZE="sample",
NUM_ITERATIONS=50,
EARLY_STOPPING_ROUND=10
)
)
nb = pm.read_notebook(OUTPUT_NOTEBOOK)
results = nb.dataframe.set_index("name")["value"]
assert results["auc"] == pytest.approx(0.68895, rel=TOL, abs=ABS_TOL)
def test_fastai(notebooks):
notebook_path = notebooks["fastai"]
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
parameters=dict(TOP_K=10, MOVIELENS_DATA_SIZE="100k", EPOCHS=1),
)
results = pm.read_notebook(OUTPUT_NOTEBOOK).dataframe.set_index("name")["value"]
assert results["rmse"] == pytest.approx(0.959352, rel=TOL, abs=ABS_TOL)
assert results["mae"] == pytest.approx(0.766504, rel=TOL, abs=ABS_TOL)
assert results["rsquared"] == pytest.approx(0.287902, rel=TOL, abs=ABS_TOL)
assert results["exp_var"] == pytest.approx(0.289008, rel=TOL, abs=ABS_TOL)
assert results["map"] == pytest.approx(0.024379, rel=TOL, abs=ABS_TOL)
assert results["ndcg"] == pytest.approx(0.148380, rel=TOL, abs=ABS_TOL)
assert results["precision"] == pytest.approx(0.138494, rel=TOL, abs=ABS_TOL)
assert results["recall"] == pytest.approx(0.058747, rel=TOL, abs=ABS_TOL)
def test_is_jupyter():
# Test on the terminal
assert is_jupyter() is False
assert is_databricks() is False
# Test on Jupyter notebook
path = os.path.join("tests", "unit", "test_notebook_utils.ipynb")
pm.execute_notebook(
path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
)
nb = pm.read_notebook(OUTPUT_NOTEBOOK)
df = nb.dataframe
result_is_jupyter = df.loc[df["name"] == "is_jupyter", "value"].values[0]
assert result_is_jupyter is True
result_is_databricks = df.loc[df["name"] == "is_databricks", "value"].values[0]
assert result_is_databricks is False
def test_wide_deep(notebooks, size, epochs, expected_values, tmp):
notebook_path = notebooks["wide_deep"]
params = {
"MOVIELENS_DATA_SIZE": size,
"EPOCHS": epochs,
"EVALUATE_WHILE_TRAINING": False,
"MODEL_DIR": tmp,
"EXPORT_DIR_BASE": tmp,
"RATING_METRICS": ["rmse", "mae", "rsquared", "exp_var"],
"RANKING_METRICS": ["ndcg_at_k", "map_at_k", "precision_at_k", "recall_at_k"],
}
pm.execute_notebook(
notebook_path, OUTPUT_NOTEBOOK, kernel_name=KERNEL_NAME, parameters=params
)
results = pm.read_notebook(OUTPUT_NOTEBOOK).dataframe.set_index("name")["value"]
for key, value in expected_values.items():
assert results[key] == pytest.approx(value, rel=TOL, abs=ABS_TOL)
def test_notebook_xdeepfm(notebooks):
notebook_path = notebooks["xdeepfm_quickstart"]
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
parameters=dict(
EPOCHS_FOR_SYNTHETIC_RUN=20,
EPOCHS_FOR_CRITEO_RUN=1,
BATCH_SIZE_SYNTHETIC=128,
BATCH_SIZE_CRITEO=512,
),
)
results = pm.read_notebook(OUTPUT_NOTEBOOK).dataframe.set_index("name")["value"]
assert results["res_syn"]["auc"] == pytest.approx(0.982, rel=TOL, abs=ABS_TOL)
assert results["res_syn"]["logloss"] == pytest.approx(0.2306, rel=TOL, abs=ABS_TOL)
assert results["res_real"]["auc"] == pytest.approx(0.628, rel=TOL, abs=ABS_TOL)
assert results["res_real"]["logloss"] == pytest.approx(0.5589, rel=TOL, abs=ABS_TOL)
def test_als_pyspark_integration(notebooks):
notebook_path = notebooks["als_pyspark"]
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
parameters=dict(TOP_K=10, MOVIELENS_DATA_SIZE="1m"),
)
nb = pm.read_notebook(OUTPUT_NOTEBOOK)
results = nb.dataframe.set_index("name")["value"]
start_or_get_spark("ALS PySpark").stop()
assert results["map"] == pytest.approx(0.00201, rel=TOL, abs=ABS_TOL)
assert results["ndcg"] == pytest.approx(0.02516, rel=TOL, abs=ABS_TOL)
assert results["precision"] == pytest.approx(0.03172, rel=TOL, abs=ABS_TOL)
assert results["recall"] == pytest.approx(0.009302, rel=TOL, abs=ABS_TOL)
assert results["rmse"] == pytest.approx(0.8621, rel=TOL, abs=ABS_TOL)
assert results["mae"] == pytest.approx(0.68023, rel=TOL, abs=ABS_TOL)
assert results["exp_var"] == pytest.approx(0.4094, rel=TOL, abs=ABS_TOL)
assert results["rsquared"] == pytest.approx(0.4038, rel=TOL, abs=ABS_TOL)
def test_als_pyspark_smoke(notebooks):
notebook_path = notebooks["als_pyspark"]
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
parameters=dict(TOP_K=10, MOVIELENS_DATA_SIZE="100k"),
)
nb = pm.read_notebook(OUTPUT_NOTEBOOK)
results = nb.dataframe.set_index("name")["value"]
start_or_get_spark("ALS PySpark").stop()
assert results["map"] == pytest.approx(0.0052, rel=TOL, abs=ABS_TOL)
assert results["ndcg"] == pytest.approx(0.0463, rel=TOL, abs=ABS_TOL)
assert results["precision"] == pytest.approx(0.0487, rel=TOL, abs=ABS_TOL)
assert results["recall"] == pytest.approx(0.0177, rel=TOL, abs=ABS_TOL)
assert results["rmse"] == pytest.approx(0.9636, rel=TOL, abs=ABS_TOL)
assert results["mae"] == pytest.approx(0.7508, rel=TOL, abs=ABS_TOL)
assert results["exp_var"] == pytest.approx(0.2672, rel=TOL, abs=ABS_TOL)
assert results["rsquared"] == pytest.approx(0.2611, rel=TOL, abs=ABS_TOL)
def test_ic_22_notebook_run(
classification_notebooks,
subscription_id,
resource_group,
workspace_name,
workspace_region,
):
notebook_path = classification_notebooks[
"22_deployment_on_azure_kubernetes_service"]
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
parameters=dict(
PM_VERSION=pm.__version__,
subscription_id=subscription_id,
resource_group=resource_group,
workspace_name=workspace_name,
workspace_region=workspace_region,
),
kernel_name=KERNEL_NAME,
)
def test_boston(self):
notebook_path = "tasks/random-forest-regressor/Experiment.ipynb"
papermill.execute_notebook(
notebook_path,
"/dev/null",
parameters=dict(
dataset="/tmp/data/boston.csv",
target="medv",
filter_type="remover",
model_features="",
one_hot_features="",
n_estimators=10,
criterion="mse",
max_depth=None,
max_features="auto",
),
)
def test_notebooks_advanced(isolated_filesystem):
notebooks = glob.glob("advanced/*.ipynb")
notebooks += glob.glob("advanced/Split Neural Network/*.ipynb")
for notebook in notebooks:
list_name = Path("examples/tutorials/") / notebook
if list_name in not_excluded_notebooks:
not_excluded_notebooks.remove(list_name)
res = pm.execute_notebook(notebook,
"/dev/null",
parameters={"epochs": 1},
timeout=300)
assert isinstance(res, nbformat.notebooknode.NotebookNode)
def test_fastai_smoke(notebooks, output_notebook, kernel_name):
notebook_path = notebooks["fastai"]
pm.execute_notebook(
notebook_path,
output_notebook,
kernel_name=kernel_name,
parameters=dict(TOP_K=10, MOVIELENS_DATA_SIZE="100k", EPOCHS=1),
)
results = sb.read_notebook(output_notebook).scraps.dataframe.set_index(
"name")["data"]
assert results["rmse"] == pytest.approx(0.959352, rel=TOL, abs=ABS_TOL)
assert results["mae"] == pytest.approx(0.766504, rel=TOL, abs=ABS_TOL)
assert results["rsquared"] == pytest.approx(0.287902, rel=TOL, abs=ABS_TOL)
assert results["exp_var"] == pytest.approx(0.289008, rel=TOL, abs=ABS_TOL)
assert results["map"] == pytest.approx(0.024379, rel=TOL, abs=ABS_TOL)
assert results["ndcg"] == pytest.approx(0.148380, rel=TOL, abs=ABS_TOL)
assert results["precision"] == pytest.approx(0.138494,
rel=TOL,
abs=ABS_TOL)
assert results["recall"] == pytest.approx(0.058747, rel=TOL, abs=ABS_TOL)
def test_notebooks_basic(isolated_filesystem, notebook):
"""Test Notebooks in the tutorial root folder."""
notebook = notebook.split("/")[-1]
list_name = Path("examples/tutorials/") / notebook
tested_notebooks.append(str(list_name))
res = pm.execute_notebook(
notebook,
"/dev/null",
parameters={"epochs": 1, "n_test_batches": 5, "n_train_items": 64, "n_test_items": 64},
timeout=300,
)
assert isinstance(res, nbformat.notebooknode.NotebookNode)
def test_experiment_titanic(self):
notebook_path = "tasks/mlp-classifier/Experiment.ipynb"
papermill.execute_notebook(
notebook_path,
"/dev/null",
parameters=dict(
dataset="/tmp/data/titanic.csv",
target="Survived",
filter_type="remover",
model_features="",
one_hot_features="",
hidden_layer_sizes=100,
activation="relu",
solver="adam",
learning_rate="constant",
max_iter=200,
shuffle=True,
method="predict_proba",
),
)
def test_dkn_quickstart_integration(notebooks):
notebook_path = notebooks["dkn_quickstart"]
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
parameters=dict(epochs=5, batch_size=500),
)
results = pm.read_notebook(OUTPUT_NOTEBOOK).dataframe.set_index(
"name")["value"]
assert results["res"]["auc"] == pytest.approx(0.5651, rel=TOL, abs=ABS_TOL)
assert results["res"]["mean_mrr"] == pytest.approx(0.1639,
rel=TOL,
abs=ABS_TOL)
assert results["res"]["ndcg@5"] == pytest.approx(0.1735,
rel=TOL,
abs=ABS_TOL)
assert results["res"]["ndcg@10"] == pytest.approx(0.2301,
rel=TOL,
abs=ABS_TOL)
def test_execute_script(tmpdir):
tmp_py = str(tmpdir.join('script.py'))
tmp_ipynb = str(tmpdir.join('executed.ipynb'))
with open(tmp_py, 'w') as fp:
fp.write("""# %% {"tags": ["parameters"]}
# This cell defines the default value for parameters
integer = 1
text = "default"
# %%
print('Parameters are {}'.format({'integer': integer, 'text': text}))
""")
pm.execute_notebook('txt://' + tmp_py,
tmp_ipynb,
parameters=dict(integer=2))
assert os.path.isfile(tmp_ipynb)
nb = nbformat.read(tmp_ipynb, as_version=4)
assert len(nb.cells) == 3
def test_experiment_titanic(self):
notebook_path = "tasks/svc/Experiment.ipynb"
papermill.execute_notebook(
notebook_path,
"/dev/null",
parameters=dict(
dataset="/tmp/data/titanic.csv",
target="Survived",
filter_type="remover",
model_features="",
one_hot_features="",
C=1.0,
kernel="rbf",
degree=3,
gamma="auto",
probability=True,
max_iter=-1,
method="predict_proba",
),
)
def execute(self, **kwargs) -> Any:
"""
TODO: Figure out how to share FlyteContext ExecutionParameters with the notebook kernel (as notebook kernel
is executed in a separate python process)
For Spark, the notebooks today need to use the new_session or just getOrCreate session and get a handle to the
singleton
"""
logger.info(
f"Hijacking the call for task-type {self.task_type}, to call notebook."
)
# Execute Notebook via Papermill.
pm.execute_notebook(self._notebook_path,
self.output_notebook_path,
parameters=kwargs) # type: ignore
outputs = self.extract_outputs(self.output_notebook_path)
self.render_nb_html(self.output_notebook_path,
self.rendered_output_path)
m = {}
if outputs:
m = outputs.literals
output_list = []
for k, type_v in self.python_interface.outputs.items():
if k == self._IMPLICIT_OP_NOTEBOOK:
output_list.append(self.output_notebook_path)
elif k == self._IMPLICIT_RENDERED_NOTEBOOK:
output_list.append(self.rendered_output_path)
elif k in m:
v = TypeEngine.to_python_value(
ctx=FlyteContext.current_context(),
lv=m[k],
expected_python_type=type_v)
output_list.append(v)
else:
raise RuntimeError(
f"Expected output {k} of type {v} not found in the notebook outputs"
)
return tuple(output_list)
def test_wide_deep(notebooks, tmp):
notebook_path = notebooks["wide_deep"]
model_dir = os.path.join(tmp, "wide_deep_0")
os.mkdir(model_dir)
params = {
'MOVIELENS_DATA_SIZE': '100k',
'EPOCHS': 0,
'EVALUATE_WHILE_TRAINING': False,
'MODEL_DIR': model_dir,
'EXPORT_DIR_BASE': model_dir,
'RATING_METRICS': ['rmse'],
'RANKING_METRICS': ['ndcg_at_k'],
}
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
parameters=params,
)
# Test w/o item features
model_dir = os.path.join(tmp, "wide_deep_1")
os.mkdir(model_dir)
params = {
'MOVIELENS_DATA_SIZE': '100k',
'EPOCHS': 0,
'ITEM_FEAT_COL': None,
'EVALUATE_WHILE_TRAINING': True,
'MODEL_DIR': model_dir,
'EXPORT_DIR_BASE': model_dir,
'RATING_METRICS': ['rsquared'],
'RANKING_METRICS': ['map_at_k'],
}
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
parameters=params,
)
def drive_hovm_difference(settings, analysis_name):
driver_settings = settings[analysis_name].copy()
current_params = create_current_params(settings)
if analysis_name != 'hovm_difference_clim':
check_num_paths(settings, min_number=2)
current_params = fill_input(current_params, settings, fill_type = 'reference')
else:
check_num_paths(settings, min_number=1)
current_params = fill_input(current_params, settings, fill_type = 'climatology')
webpage = {}
image_count = 0
for region_name, region in driver_settings.items():
for variable_name, variable in region.items():
current_params["region"] = region_name
current_params["variable"] = variable_name
current_params.update(variable)
region_name_underscore = region_name.replace(' ', '_')
ofile = f"{settings['workflow_name']}_{analysis_name}_{region_name_underscore}_{variable_name}.png"
ofile_nb = f"{settings['workflow_name']}_{analysis_name}_{region_name_underscore}_{variable_name}.ipynb"
current_params["ofile"] = os.path.join(settings['ofolder_figures'], ofile)
pm.execute_notebook(
f"{templates_nb_path}/hovm_difference.ipynb",
os.path.join(settings['ofolder_notebooks'], ofile_nb),
parameters=current_params,
nest_asyncio=True,
)
webpage[f"image_{image_count}"] = {}
webpage[f"image_{image_count}"][
"name"
] = f"{variable_name.capitalize()} for {region_name}"
webpage[f"image_{image_count}"]["path"] = os.path.join('./figures/', ofile)
webpage[f"image_{image_count}"]["path_nb"] = os.path.join('./notebooks/', ofile_nb)
webpage[f"image_{image_count}"][
"short_name"
] = f"{settings['workflow_name']}_{analysis_name}_{region_name_underscore}_{variable_name}"
image_count += 1
return webpage
def test_wide_deep(notebooks, tmp):
notebook_path = notebooks["wide_deep"]
model_dir = os.path.join(tmp, "wide_deep_0")
os.mkdir(model_dir)
params = {
'MOVIELENS_DATA_SIZE': '100k',
'EPOCHS': 0,
'EVALUATE_WHILE_TRAINING': False,
'MODEL_DIR': model_dir,
'EXPORT_DIR_BASE': model_dir,
'RATING_METRICS': ['rmse'],
'RANKING_METRICS': ['ndcg_at_k'],
}
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
parameters=params,
)
# Test w/o item features
model_dir = os.path.join(tmp, "wide_deep_1")
os.mkdir(model_dir)
params = {
'MOVIELENS_DATA_SIZE': '100k',
'EPOCHS': 0,
'ITEM_FEAT_COL': None,
'EVALUATE_WHILE_TRAINING': True,
'MODEL_DIR': model_dir,
'EXPORT_DIR_BASE': model_dir,
'RATING_METRICS': ['rsquared'],
'RANKING_METRICS': ['map_at_k'],
}
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
parameters=params,
)
def test_experiment_hymenoptera(self):
papermill.execute_notebook(
"Experiment.ipynb",
"/dev/null",
parameters=dict(
dataset="/tmp/data/hymenoptera.zip",
arch_list=["resnet18", "resnet50", "vgg16"],
aug_polices=["fa_reduced_cifar10",
"fa_resnet50_rimagenet", "fa_reduced_svhn"],
dataset_id="hymenoptera",
checkpoint_path="/tmp/data/models-output/",
output_graphs="/tmp/data/eval-images/",
top_predictions=1,
batch=12,
epochs=1,
lr=0.001,
gamma=0.1,
step_size=7,
momentum=0.1,
),
)
def test_deep_and_unified_understanding(notebooks):
notebook_path = notebooks["deep_and_unified_understanding"]
pm.execute_notebook(notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME)
result = sb.read_notebook(OUTPUT_NOTEBOOK).scraps.data_dict
sigma_numbers = [
0.00317593, 0.00172284, 0.00634005, 0.00164305, 0.00317159
]
sigma_bert = [
0.1735696, 0.14028822, 0.14590865, 0.2263149, 0.20640415, 0.21249843,
0.18685372, 0.14112663, 0.25824168, 0.22399105, 0.2393731, 0.12868434,
0.27386534, 0.35876372
]
np.testing.assert_array_almost_equal(result["sigma_numbers"],
sigma_numbers,
decimal=3)
np.testing.assert_array_almost_equal(result["sigma_bert"],
sigma_bert,
decimal=1)
def exec(self, context: Dict, rundir: str):
"""Execute the notebook using papermill in the given workflow context.
Parameters
----------
context: dict
Mapping of parameter names to their current value in the workflow
executon state. These are the global variables in the execution
context.
rundir: string
Directory for the workflow run that contains all the run files.
"""
import papermill as pm
# Prepare parameters for running the notebook using papermill.
kwargs = self._get_parameters(context=context)
# Change working directory temporarily to the given rundir.
cwd = os.getcwd()
os.chdir(rundir)
try:
pm.execute_notebook(self.notebook, self.output, parameters=kwargs)
finally:
os.chdir(cwd)
def _compile_sample(self):
os.chdir(self._work_dir)
print('Run the sample tests...')
# For presubmit check, do not do any image injection as for now.
# Notebook samples need to be papermilled first.
if self._test_name == 'lightweight_component':
pm.execute_notebook(
input_path='Lightweight Python components - basics.ipynb',
output_path='%s.ipynb' % self._test_name,
parameters=dict(EXPERIMENT_NAME='%s-test' % self._test_name))
elif self._test_name == 'dsl_static_type_checking':
pm.execute_notebook(input_path='DSL Static Type Checking.ipynb',
output_path='%s.ipynb' % self._test_name,
parameters={})
else:
subprocess.call([
'dsl-compile', '--py',
'%s.py' % self._test_name, '--output',
'%s.yaml' % self._test_name
])
def test_experiment_hotel_bookings(self):
papermill.execute_notebook(
"Experiment.ipynb",
"/dev/null",
parameters=dict(dataset="/tmp/data/hotel_bookings.csv",
features_to_filter=[
"reservation_status_date", "arrival_date_year"
]),
)
papermill.execute_notebook(
"Deployment.ipynb",
"/dev/null",
)
data = datasets.hotel_bookings_testdata_full()
with server.Server() as s:
response = s.test(data=data)
names = response["names"]
ndarray = response["ndarray"]
self.assertEqual(len(ndarray[0]), 30) # 32 features - 2 removed
self.assertEqual(len(names), 30)
def test_tc_dac_bert_ar(notebooks, tmp):
notebook_path = notebooks["tc_dac_bert_ar"]
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
parameters=dict(
NUM_GPUS=1,
DATA_FOLDER=tmp,
BERT_CACHE_DIR=tmp,
BATCH_SIZE=32,
NUM_EPOCHS=1,
TRAIN_SIZE=0.8,
NUM_ROWS=15000,
RANDOM_STATE=0,
),
)
result = sb.read_notebook(OUTPUT_NOTEBOOK).scraps.data_dict
assert pytest.approx(result["accuracy"], 0.93, abs=ABS_TOL)
assert pytest.approx(result["precision"], 0.91, abs=ABS_TOL)
assert pytest.approx(result["recall"], 0.91, abs=ABS_TOL)
assert pytest.approx(result["f1"], 0.91, abs=ABS_TOL)
def test_wide_deep_integration(notebooks, size, steps, expected_values, seed, tmp):
notebook_path = notebooks["wide_deep"]
params = {
"MOVIELENS_DATA_SIZE": size,
"STEPS": steps,
"EVALUATE_WHILE_TRAINING": False,
"MODEL_DIR": tmp,
"EXPORT_DIR_BASE": tmp,
"RATING_METRICS": ["rmse", "mae", "rsquared", "exp_var"],
"RANKING_METRICS": ["ndcg_at_k", "map_at_k", "precision_at_k", "recall_at_k"],
"RANDOM_SEED": seed,
}
pm.execute_notebook(
notebook_path, OUTPUT_NOTEBOOK, kernel_name=KERNEL_NAME, parameters=params
)
results = sb.read_notebook(OUTPUT_NOTEBOOK).scraps.dataframe.set_index("name")[
"data"
]
for key, value in expected_values.items():
assert results[key] == pytest.approx(value, rel=TOL, abs=ABS_TOL)
def test_entailment_multinli_bert(notebooks, tmp):
notebook_path = notebooks["entailment_multinli_transformers"]
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
parameters={
"MODEL_NAME": "bert-base-uncased",
"TO_LOWER": True,
"TRAIN_DATA_USED_FRACTION": 0.05,
"DEV_DATA_USED_FRACTION": 0.05,
"NUM_EPOCHS": 1,
"CACHE_DIR": tmp
},
kernel_name=KERNEL_NAME,
)
result = sb.read_notebook(OUTPUT_NOTEBOOK).scraps.data_dict
assert pytest.approx(result["matched_precision"], 0.76, abs=ABS_TOL)
assert pytest.approx(result["matched_recall"], 0.76, abs=ABS_TOL)
assert pytest.approx(result["matched_f1"], 0.76, abs=ABS_TOL)
assert pytest.approx(result["mismatched_precision"], 0.76, abs=ABS_TOL)
assert pytest.approx(result["mismatched_recall"], 0.76, abs=ABS_TOL)
assert pytest.approx(result["mismatched_f1"], 0.76, abs=ABS_TOL)
def test_surprise_svd_smoke(notebooks):
notebook_path = notebooks["surprise_svd_deep_dive"]
pm.execute_notebook(notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME)
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
parameters=dict(MOVIELENS_DATA_SIZE="100k"),
)
results = pm.read_notebook(OUTPUT_NOTEBOOK).dataframe.set_index(
"name")["value"]
assert results["rmse"] == pytest.approx(0.96, rel=TOL, abs=ABS_TOL)
assert results["mae"] == pytest.approx(0.75, rel=TOL, abs=ABS_TOL)
assert results["rsquared"] == pytest.approx(0.29, rel=TOL, abs=ABS_TOL)
assert results["exp_var"] == pytest.approx(0.29, rel=TOL, abs=ABS_TOL)
assert results["map"] == pytest.approx(0.013, rel=TOL, abs=ABS_TOL)
assert results["ndcg"] == pytest.approx(0.1, rel=TOL, abs=ABS_TOL)
assert results["precision"] == pytest.approx(0.095, rel=TOL, abs=ABS_TOL)
assert results["recall"] == pytest.approx(0.032, rel=TOL, abs=ABS_TOL)
def skip_test_21_notebook_run(classification_notebooks, tiny_ic_data_path):
""" NOTE - this function is intentionally prefixed with 'skip' so that
pytests bypasses this function
"""
notebook_path = classification_notebooks[
"21_deployment_on_azure_container_instances"]
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
parameters=dict(PM_VERSION=pm.__version__,
DATA_PATH=tiny_ic_data_path),
kernel_name=KERNEL_NAME,
)
try:
os.remove("myenv.yml")
except OSError:
pass
try:
os.remove("score.py")
except OSError:
pass
try:
os.remove("output.ipynb")
except OSError:
pass
# There should be only one file, but the name may be changed
file_list = glob.glob("./*.pkl")
for filePath in file_list:
try:
os.remove(filePath)
except OSError:
pass
# TODO should use temp folder for safe cleanup. Notebook should accept the folder paths via papermill param.
shutil.rmtree(os.path.join(os.getcwd(), "azureml-models"))
shutil.rmtree(os.path.join(os.getcwd(), "models"))
shutil.rmtree(os.path.join(os.getcwd(), "outputs"))
def test_notebooks_mnist_02():
notebook_mnist_02 = data_centric_mnist_path.joinpath(
"02-FL-mnist-train-model.ipynb")
res = pm.execute_notebook(
str(notebook_mnist_02),
os.devnull,
dict(grid_address="http://localhost:" + GRID_NETWORK_PORT,
N_EPOCHS=2,
N_TEST=2),
)
assert isinstance(res, nbformat.notebooknode.NotebookNode)
def test_sar_single_node_smoke(notebooks):
notebook_path = notebooks["sar_single_node"]
pm.execute_notebook(notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME)
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
parameters=dict(TOP_K=10, MOVIELENS_DATA_SIZE="100k"),
)
results = pm.read_notebook(OUTPUT_NOTEBOOK).dataframe.set_index(
"name")["value"]
assert results["map"] == pytest.approx(0.105815262, rel=TOL, abs=ABS_TOL)
assert results["ndcg"] == pytest.approx(0.373197255, rel=TOL, abs=ABS_TOL)
assert results["precision"] == pytest.approx(0.326617179,
rel=TOL,
abs=ABS_TOL)
assert results["recall"] == pytest.approx(0.175956743,
rel=TOL,
abs=ABS_TOL)
def test_bidaf_deep_dive(
notebooks, subscription_id, resource_group, workspace_name, workspace_region
):
notebook_path = notebooks["bidaf_deep_dive"]
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
parameters={
"NUM_EPOCHS": 1,
"config_path": None,
"PROJECT_FOLDER": "examples/question_answering/bidaf-question-answering",
"SQUAD_FOLDER": "examples/question_answering/squad",
"LOGS_FOLDER": "examples/question_answering/",
"BIDAF_CONFIG_PATH": "examples/question_answering/",
"subscription_id": subscription_id,
"resource_group": resource_group,
"workspace_name": workspace_name,
"workspace_region": workspace_region,
},
)
result = sb.read_notebook(OUTPUT_NOTEBOOK).scraps.data_dict["validation_EM"]
assert result == pytest.approx(0.5, abs=ABS_TOL)
def test_notebooks_basic_translations(isolated_filesystem, translated_notebook): # pragma: no cover
"""Test Notebooks in the tutorial translations folder."""
notebook = "/".join(translated_notebook.split("/")[-2:])
notebook = f"translations/{notebook}"
list_name = Path(f"examples/tutorials/{notebook}")
tested_notebooks.append(str(list_name))
res = pm.execute_notebook(
notebook,
"/dev/null",
parameters={"epochs": 1, "n_test_batches": 5, "n_train_items": 64, "n_test_items": 64},
timeout=400,
)
assert isinstance(res, nbformat.notebooknode.NotebookNode)
def test_ncf_deep_dive_integration(
notebooks, size, epochs, batch_size, expected_values, seed
):
notebook_path = notebooks["ncf_deep_dive"]
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
parameters=dict(
TOP_K=10,
MOVIELENS_DATA_SIZE=size,
EPOCHS=epochs,
BATCH_SIZE=batch_size,
SEED=seed,
),
)
results = sb.read_notebook(OUTPUT_NOTEBOOK).scraps.dataframe.set_index("name")[
"data"
]
for key, value in expected_values.items():
assert results[key] == pytest.approx(value, rel=TOL, abs=ABS_TOL)
def test_fastai(notebooks):
notebook_path = notebooks["fastai"]
pm.execute_notebook(notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME)
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
parameters=dict(TOP_K=10, MOVIELENS_DATA_SIZE="100k", EPOCHS=1),
)
results = pm.read_notebook(OUTPUT_NOTEBOOK).dataframe.set_index(
"name")["value"]
assert results["rmse"] == pytest.approx(0.959352, TOL)
assert results["mae"] == pytest.approx(0.766504, TOL)
assert results["rsquared"] == pytest.approx(0.287902, TOL)
assert results["exp_var"] == pytest.approx(0.289008, TOL)
assert results["map"] == pytest.approx(0.024379, TOL)
assert results["ndcg"] == pytest.approx(0.148380, TOL)
assert results["precision"] == pytest.approx(0.138494, TOL)
assert results["recall"] == pytest.approx(0.058747, TOL)
def test_minilm_abstractive_summarization(notebooks, tmp):
notebook_path = notebooks["minilm_abstractive_summarization"]
pm.execute_notebook(
notebook_path,
OUTPUT_NOTEBOOK,
kernel_name=KERNEL_NAME,
parameters=dict(
QUICK_RUN=True,
NUM_GPUS=0,
TOP_N=2,
WARMUP_STEPS=5,
MAX_STEPS=50,
GRADIENT_ACCUMULATION_STEPS=1,
TEST_PER_GPU_BATCH_SIZE=2,
BEAM_SIZE=3,
CLEANUP_RESULTS=True,
),
)
result = sb.read_notebook(OUTPUT_NOTEBOOK).scraps.data_dict
assert pytest.approx(result["rouge_1_f_score"], 0.1, abs=ABS_TOL)
assert pytest.approx(result["rouge_2_f_score"], 0.05, abs=ABS_TOL)
assert pytest.approx(result["rouge_l_f_score"], 0.1, abs=ABS_TOL)
def test_experiment_boston(self):
papermill.execute_notebook(
"Experiment.ipynb",
"/dev/null",
parameters=dict(
dataset="/tmp/data/boston.csv",
target="medv",
norm="l2",
),
)
papermill.execute_notebook(
"Deployment.ipynb",
"/dev/null",
)
data = datasets.boston_testdata()
with server.Server() as s:
response = s.test(data=data)
names = response["names"]
ndarray = response["ndarray"]
self.assertEqual(len(ndarray[0]), 13) # 13 features
self.assertEqual(len(names), 13)
def test_vw_deep_dive_runs(notebooks):
notebook_path = notebooks["vowpal_wabbit_deep_dive"]
pm.execute_notebook(notebook_path, OUTPUT_NOTEBOOK, kernel_name=KERNEL_NAME)
def test_als_pyspark_runs(notebooks):
notebook_path = notebooks["als_pyspark"]
pm.execute_notebook(notebook_path, OUTPUT_NOTEBOOK, kernel_name=KERNEL_NAME)
def test_sar_single_node_runs(notebooks):
notebook_path = notebooks["sar_single_node"]
pm.execute_notebook(notebook_path, OUTPUT_NOTEBOOK, kernel_name=KERNEL_NAME)
def test_data_split_runs(notebooks):
notebook_path = notebooks["data_split"]
pm.execute_notebook(notebook_path, OUTPUT_NOTEBOOK, kernel_name=KERNEL_NAME)
def test_evaluation_runs(notebooks):
notebook_path = notebooks["evaluation"]
pm.execute_notebook(notebook_path, OUTPUT_NOTEBOOK, kernel_name=KERNEL_NAME)
if params['TOP_K'] <= 0:
raise ValueError("Top K should be larger than 0")
if params['MODEL_TYPE'] not in {'wide', 'deep', 'wide_deep'}:
raise ValueError("Model type should be either 'wide', 'deep', or 'wide_deep'")
if params['DATA_DIR'] is None:
raise ValueError("Datastore path should be given")
print("Args:")
for k, v in params.items():
_log(k, v)
print("Run", NOTEBOOK_NAME)
pm.execute_notebook(
NOTEBOOK_NAME,
OUTPUT_NOTEBOOK,
parameters=params,
kernel_name='python3'
)
nb = pm.read_notebook(OUTPUT_NOTEBOOK)
for m, v in nb.data.items():
_log(m, v)
# clean-up
os.remove(OUTPUT_NOTEBOOK)
shutil.rmtree(params['MODEL_DIR'], ignore_errors=True)
def test_surprise_deep_dive_runs(notebooks):
notebook_path = notebooks["surprise_svd_deep_dive"]
pm.execute_notebook(notebook_path, OUTPUT_NOTEBOOK, kernel_name=KERNEL_NAME)
def test_baseline_deep_dive_runs(notebooks):
notebook_path = notebooks["baseline_deep_dive"]
pm.execute_notebook(notebook_path, OUTPUT_NOTEBOOK, kernel_name=KERNEL_NAME)
