def run_mnist_estimator_data_layer_test(tf2: bool, storage_type: str) -> None:
config = conf.load_config(
conf.features_examples_path("data_layer_mnist_estimator/const.yaml"))
config = conf.set_max_length(config, {"batches": 200})
config = conf.set_tf2_image(config) if tf2 else conf.set_tf1_image(config)
if storage_type == "lfs":
config = conf.set_shared_fs_data_layer(config)
else:
config = conf.set_s3_data_layer(config)
exp.run_basic_test_with_temp_config(
config, conf.features_examples_path("data_layer_mnist_estimator"), 1)
def test_tf_keras_mnist_data_layer_parallel(tf2: bool, storage_type: str,
secrets: Dict[str, str]) -> None:
config = conf.load_config(
conf.features_examples_path("data_layer_mnist_tf_keras/const.yaml"))
config = conf.set_max_length(config, {"batches": 200})
config = conf.set_slots_per_trial(config, 8)
config = conf.set_tf2_image(config) if tf2 else conf.set_tf1_image(config)
if storage_type == "lfs":
config = conf.set_shared_fs_data_layer(config)
else:
config = conf.set_s3_data_layer(config)
exp.run_basic_test_with_temp_config(
config, conf.features_examples_path("data_layer_mnist_tf_keras"), 1)
def test_mnist_estimator_adaptive_with_data_layer() -> None:
config = conf.load_config(
conf.fixtures_path("mnist_estimator/adaptive.yaml"))
config = conf.set_tf2_image(config)
config = conf.set_shared_fs_data_layer(config)
exp.run_basic_test_with_temp_config(
config, conf.features_examples_path("data_layer_mnist_estimator"),
None)
def test_data_layer_mnist_estimator_accuracy() -> None:
config = conf.load_config(
conf.features_examples_path("data_layer_mnist_estimator/const.yaml"))
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.features_examples_path("data_layer_mnist_estimator"), 1)
trials = exp.experiment_trials(experiment_id)
trial_metrics = exp.trial_metrics(trials[0]["id"])
validation_accuracies = [
step["validation"]["metrics"]["validation_metrics"]["accuracy"]
for step in trial_metrics["steps"] if step.get("validation")
]
target_accuracy = 0.92
assert max(validation_accuracies) > target_accuracy, (
"data_layer_mnist_estimator did not reach minimum target accuracy {} in {} steps."
" full validation accuracy history: {}".format(
target_accuracy, len(trial_metrics["steps"]),
validation_accuracies))
def test_text_classification_tf_keras_accuracy() -> None:
config = conf.load_config(
conf.features_examples_path("text_classification_tf_keras/const.yaml")
)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.features_examples_path("text_classification_tf_keras"), 1
)
trials = exp.experiment_trials(experiment_id)
trial_metrics = exp.trial_metrics(trials[0]["id"])
validation_accuracies = [
step["validation"]["metrics"]["validation_metrics"]["val_sparse_categorical_accuracy"]
for step in trial_metrics["steps"]
if step.get("validation")
]
target_accuracy = 0.95
assert max(validation_accuracies) > target_accuracy, (
"text_classification_tf_keras did not reach minimum target accuracy {} in {} steps."
" full validation accuracy history: {}".format(
target_accuracy, len(trial_metrics["steps"]), validation_accuracies
)
)