def test_imagenet_nas() -> None: config = conf.load_config( conf.experimental_path("trial/gaea_nas/eval/const.yaml")) config = conf.set_max_length(config, {"batches": 200}) exp.run_basic_test_with_temp_config( config, conf.experimental_path("trial/gaea_nas/eval"), 1)
def test_mnist_pytorch_multi_output() -> None: config = conf.load_config( conf.experimental_path("trial/mnist_pytorch_multi_output/const.yaml")) config = conf.set_max_length(config, {"batches": 200}) exp.run_basic_test_with_temp_config( config, conf.experimental_path("trial/mnist_pytorch_multi_output"), 1)
def test_resnet50() -> None: config = conf.load_config( conf.experimental_path("trial/resnet50_tf_keras/const.yaml")) config = conf.set_max_length(config, {"batches": 200}) exp.run_basic_test_with_temp_config( config, conf.experimental_path("trial/resnet50_tf_keras"), 1)
def test_bert_glue() -> None: config = conf.load_config( conf.experimental_path("trial/bert_glue_pytorch/const.yaml")) config = conf.set_max_length(config, {"batches": 200}) exp.run_basic_test_with_temp_config( config, conf.experimental_path("trial/bert_glue_pytorch/"), 1)
def test_faster_rcnn() -> None: config = conf.load_config(conf.experimental_path("trial/FasterRCNN_tp/16-gpus.yaml")) config = conf.set_max_length(config, {"batches": 128}) config = conf.set_slots_per_trial(config, 1) exp.run_basic_test_with_temp_config( config, conf.experimental_path("trial/FasterRCNN_tp"), 1, max_wait_secs=4800 )
def test_nas_search() -> None: config = conf.load_config( conf.experimental_path("nas_search/train_one_arch.yaml")) config = conf.set_max_steps(config, 2) exp.run_basic_test_with_temp_config(config, conf.experimental_path("nas_search"), 1)
def run_mnist_estimator_data_layer_test(tf2: bool, storage_type: str) -> None: config = conf.load_config( conf.experimental_path("data_layer_mnist_estimator/const.yaml")) config = conf.set_max_steps(config, 2) config = conf.set_tf2_image(config) if tf2 else conf.set_tf1_image(config) if storage_type == "lfs": config = conf.set_shared_fs_data_layer(config) else: config = conf.set_s3_data_layer(config) exp.run_basic_test_with_temp_config( config, conf.experimental_path("data_layer_mnist_estimator"), 1)
def test_mnist_estimator_data_layer_parallel(storage_type: str) -> None: config = conf.load_config( conf.experimental_path("data_layer_mnist_estimator/const.yaml")) config = conf.set_max_steps(config, 2) config = conf.set_slots_per_trial(config, 8) config = conf.set_tf1_image(config) if storage_type == "lfs": config = conf.set_shared_fs_data_layer(config) else: config = conf.set_s3_data_layer(config) exp.run_basic_test_with_temp_config( config, conf.experimental_path("data_layer_mnist_estimator"), 1)
def test_mnist_estimator_data_layer_parallel(storage_type: str, secrets: Dict[str, str]) -> None: config = conf.load_config(conf.experimental_path("trial/data_layer_mnist_estimator/const.yaml")) config = conf.set_max_length(config, {"batches": 200}) config = conf.set_slots_per_trial(config, 8) config = conf.set_tf1_image(config) if storage_type == "lfs": config = conf.set_shared_fs_data_layer(config) else: config = conf.set_s3_data_layer(config) exp.run_basic_test_with_temp_config( config, conf.experimental_path("trial/data_layer_mnist_estimator"), 1 )
def run_tf_keras_mnist_data_layer_test(tf2: bool, storage_type: str) -> None: config = conf.load_config( conf.experimental_path("trial/data_layer_mnist_tf_keras/const.yaml")) config = conf.set_max_length(config, {"batches": 200}) config = conf.set_min_validation_period(config, {"batches": 1000}) config = conf.set_tf2_image(config) if tf2 else conf.set_tf1_image(config) if storage_type == "lfs": config = conf.set_shared_fs_data_layer(config) else: config = conf.set_s3_data_layer(config) exp.run_basic_test_with_temp_config( config, conf.experimental_path("trial/data_layer_mnist_tf_keras"), 1)
def test_mnist_estimator_adaptive_with_data_layer() -> None: config = conf.load_config( conf.fixtures_path("mnist_estimator/adaptive.yaml")) config = conf.set_tf2_image(config) config = conf.set_shared_fs_data_layer(config) exp.run_basic_test_with_temp_config( config, conf.experimental_path("data_layer_mnist_estimator"), None)
def test_gbt_estimator() -> None: config = conf.load_config(conf.experimental_path("trial/gbt_estimator/const.yaml")) config = conf.set_max_length(config, {"batches": 200}) exp.run_basic_test_with_temp_config(config, conf.experimental_path("trial/gbt_estimator"), 1)
def test_nas_search() -> None: config = conf.load_config(conf.experimental_path("trial/rsws_nas/train_one_arch.yaml")) config = conf.set_max_length(config, {"batches": 200}) exp.run_basic_test_with_temp_config(config, conf.experimental_path("trial/nas_search"), 1)
class NativeImplementations: PytorchMNISTCNNSingleGeneric = NativeImplementation( cwd=conf.experimental_path("native_mnist_pytorch"), command=[ "python", conf.experimental_path("native_mnist_pytorch/trial_impl.py") ], configuration={ "checkpoint_storage": experiment.shared_fs_checkpoint_config(), "searcher": { "name": "single", "max_steps": 1, "metric": "validation_error" }, "max_restarts": 0, }, num_expected_steps_per_trial=1, num_expected_trials=1, ) TFEstimatorMNISTCNNSingle = NativeImplementation( cwd=conf.experimental_path("native_mnist_estimator"), command=[ "python", conf.experimental_path("native_mnist_estimator/native_impl.py") ], configuration={ "batches_per_step": 4, "checkpoint_storage": experiment.shared_fs_checkpoint_config(), "searcher": { "name": "single", "max_steps": 1, "metric": "accuracy" }, "max_restarts": 0, }, num_expected_steps_per_trial=1, num_expected_trials=1, ) TFEstimatorMNISTCNNSingleGeneric = NativeImplementation( cwd=conf.experimental_path("native_mnist_estimator"), command=[ "python", conf.experimental_path("native_mnist_estimator/trial_impl.py") ], configuration={ "batches_per_step": 4, "checkpoint_storage": experiment.shared_fs_checkpoint_config(), "searcher": { "name": "single", "max_steps": 1, "metric": "accuracy" }, "max_restarts": 0, }, num_expected_steps_per_trial=1, num_expected_trials=1, ) # Train a single tf.keras model using fit(). TFKerasMNISTCNNSingleFit = NativeImplementation( cwd=conf.experimental_path("native_fashion_mnist_tf_keras"), command=[ "python", conf.experimental_path( "native_fashion_mnist_tf_keras/native_impl.py"), "--use-fit", ], configuration={ "batches_per_step": 4, "checkpoint_storage": experiment.shared_fs_checkpoint_config(), "searcher": { "name": "single", "max_steps": 1, "metric": "val_accuracy" }, "max_restarts": 2, }, num_expected_steps_per_trial=1, num_expected_trials=1, ) # Train a single tf.keras model using fit_generator(). TFKerasMNISTCNNSingleFitGenerator = NativeImplementation( cwd=conf.experimental_path("native_fashion_mnist_tf_keras"), command=[ "python", conf.experimental_path( "native_fashion_mnist_tf_keras/native_impl.py") ], configuration={ "batches_per_step": 4, "checkpoint_storage": experiment.shared_fs_checkpoint_config(), "searcher": { "name": "single", "max_steps": 1, "metric": "val_accuracy" }, "max_restarts": 2, }, num_expected_steps_per_trial=1, num_expected_trials=1, ) TFKerasMNISTCNNSingleGeneric = NativeImplementation( cwd=conf.experimental_path("native_fashion_mnist_tf_keras"), command=[ "python", conf.experimental_path( "native_fashion_mnist_tf_keras/trial_impl.py") ], configuration={ "batches_per_step": 4, "checkpoint_storage": experiment.shared_fs_checkpoint_config(), "searcher": { "name": "single", "max_steps": 1, "metric": "val_accuracy" }, "max_restarts": 2, }, num_expected_steps_per_trial=1, num_expected_trials=1, )