def test_cifar10_byol_pytorch_accuracy() -> None:
config = conf.load_config(
conf.cv_examples_path("byol_pytorch/const-cifar10.yaml"))
# Limit convergence time, since was running over 30 minute limit.
config["searcher"]["max_length"]["epochs"] = 20
config["hyperparameters"]["classifier"]["train_epochs"] = 1
config = conf.set_random_seed(config, 1591280374)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.cv_examples_path("byol_pytorch"), 1)
trials = exp.experiment_trials(experiment_id)
trial_metrics = exp.trial_metrics(trials[0].trial.id)
validation_accuracies = [
step["validation"]["metrics"]["validation_metrics"]["test_accuracy"]
for step in trial_metrics["steps"] if step.get("validation")
]
# Accuracy reachable within limited convergence time -- goes higher given full training.
target_accuracy = 0.40
assert max(validation_accuracies) > target_accuracy, (
"cifar10_byol_pytorch did not reach minimum target accuracy {} in {} steps."
" full validation accuracy history: {}".format(
target_accuracy, len(trial_metrics["steps"]),
validation_accuracies))
def test_fashion_mnist_tf_keras() -> None:
config = conf.load_config(
conf.official_examples_path("trial/fashion_mnist_tf_keras/const.yaml"))
config = conf.set_random_seed(config, 1591110586)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("trial/fashion_mnist_tf_keras"), 1)
trials = exp.experiment_trials(experiment_id)
trial_metrics = exp.trial_metrics(trials[0]["id"])
validation_errors = [
step["validation"]["metrics"]["validation_metrics"]["val_accuracy"]
for step in trial_metrics["steps"] if step.get("validation")
]
accuracy = 0.85
assert max(validation_errors) > accuracy, (
"fashion_mnist_tf_keras did not reach minimum target accuracy {} in {} steps."
" full validation error history: {}".format(
accuracy, len(trial_metrics["steps"]), validation_errors))
def test_fasterrcnn_coco_pytorch_accuracy() -> None:
config = conf.load_config(
conf.cv_examples_path("fasterrcnn_coco_pytorch/const.yaml"))
config = conf.set_random_seed(config, 1590497309)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.cv_examples_path("fasterrcnn_coco_pytorch"), 1)
trials = exp.experiment_trials(experiment_id)
trial_metrics = exp.trial_metrics(trials[0]["id"])
validation_iou = [
step["validation"]["metrics"]["validation_metrics"]["val_avg_iou"]
for step in trial_metrics["steps"] if step.get("validation")
]
target_iou = 0.42
assert max(validation_iou) > target_iou, (
"fasterrcnn_coco_pytorch did not reach minimum target accuracy {} in {} steps."
" full validation avg_iou history: {}".format(
target_iou, len(trial_metrics["steps"]), validation_iou))
def test_unets_tf_keras_accuracy() -> None:
config = conf.load_config(
conf.cv_examples_path("unets_tf_keras/const.yaml"))
config = conf.set_random_seed(config, 1591280374)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.cv_examples_path("unets_tf_keras"), 1)
trials = exp.experiment_trials(experiment_id)
trial_metrics = exp.trial_metrics(trials[0]["id"])
validation_accuracies = [
step["validation"]["metrics"]["validation_metrics"]["val_accuracy"]
for step in trial_metrics["steps"] if step.get("validation")
]
target_accuracy = 0.85
assert max(validation_accuracies) > target_accuracy, (
"unets_tf_keras did not reach minimum target accuracy {} in {} steps."
" full validation accuracy history: {}".format(
target_accuracy, len(trial_metrics["steps"]),
validation_accuracies))
def test_cifar10_tf_keras_accuracy() -> None:
config = conf.load_config(
conf.cv_examples_path("cifar10_tf_keras/const.yaml"))
config = conf.set_random_seed(config, 1591110586)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.cv_examples_path("cifar10_tf_keras"), 1, None, 6000)
trials = exp.experiment_trials(experiment_id)
trial_metrics = exp.trial_metrics(trials[0].trial.id)
validation_accuracies = [
step["validation"]["metrics"]["validation_metrics"]
["val_categorical_accuracy"] for step in trial_metrics["steps"]
if step.get("validation")
]
target_accuracy = 0.73
assert max(validation_accuracies) > target_accuracy, (
"cifar10_pytorch did not reach minimum target accuracy {} in {} steps."
" full validation accuracy history: {}".format(
target_accuracy, len(trial_metrics["steps"]),
validation_accuracies))
