def test_tf_keras_const_warm_start(tf2: bool) -> None:
config = conf.load_config(conf.official_examples_path("cifar10_cnn_tf_keras/const.yaml"))
config = conf.set_max_steps(config, 2)
config = conf.set_tf2_image(config) if tf2 else conf.set_tf1_image(config)
experiment_id1 = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("cifar10_cnn_tf_keras"), 1
)
trials = exp.experiment_trials(experiment_id1)
assert len(trials) == 1
first_trial = trials[0]
first_trial_id = first_trial["id"]
assert len(first_trial["steps"]) == 2
first_checkpoint_id = first_trial["steps"][1]["checkpoint"]["id"]
# Add a source trial ID to warm start from.
config["searcher"]["source_trial_id"] = first_trial_id
experiment_id2 = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("cifar10_cnn_tf_keras"), 1
)
# The new trials should have a warm start checkpoint ID.
trials = exp.experiment_trials(experiment_id2)
assert len(trials) == 1
for trial in trials:
assert trial["warm_start_checkpoint_id"] == first_checkpoint_id
def test_nas_search() -> None:
config = conf.load_config(
conf.experimental_path("trial/nas_search/train_one_arch.yaml"))
config = conf.set_max_steps(config, 2)
exp.run_basic_test_with_temp_config(
config, conf.experimental_path("trial/nas_search"), 1)
def test_pytorch_const_multi_output() -> None:
config = conf.load_config(
conf.experimental_path("mnist_pytorch_multi_output/const.yaml"))
config = conf.set_max_steps(config, 2)
exp.run_basic_test_with_temp_config(
config, conf.experimental_path("mnist_pytorch_multi_output"), 1)
def test_resnet50() -> None:
config = conf.load_config(
conf.experimental_path("resnet50_tf_keras/const.yaml"))
config = conf.set_max_steps(config, 2)
exp.run_basic_test_with_temp_config(
config, conf.experimental_path("resnet50_tf_keras"), 1)
def test_imagenet_nas() -> None:
config = conf.load_config(
conf.experimental_path("imagenet_nas_arch_pytorch/const.yaml"))
config = conf.set_max_steps(config, 2)
exp.run_basic_test_with_temp_config(
config, conf.experimental_path("imagenet_nas_arch_pytorch"), 1)
def test_bert_glue() -> None:
config = conf.load_config(
conf.experimental_path("bert_glue_pytorch/const.yaml"))
config = conf.set_max_steps(config, 2)
exp.run_basic_test_with_temp_config(
config, conf.experimental_path("bert_glue_pytorch/"), 1)
def test_mnist_tp_to_estimator() -> None:
config = conf.load_config(
conf.experimental_path("mnist_tp_to_estimator/const.yaml"))
config = conf.set_max_steps(config, 2)
exp.run_basic_test_with_temp_config(
config, conf.experimental_path("mnist_tp_to_estimator"), 1)
def test_faster_rcnn() -> None:
config = conf.load_config(
conf.experimental_path("FasterRCNN_tp/16-gpus.yaml"))
config = conf.set_max_steps(config, 2)
config = conf.set_slots_per_trial(config, 1)
exp.run_basic_test_with_temp_config(
config, conf.experimental_path("FasterRCNN_tp"), 1, max_wait_secs=4800)
def test_pytorch_const_with_amp() -> None:
config = conf.load_config(
conf.official_examples_path("mnist_pytorch/const.yaml"))
config = conf.set_max_steps(config, 2)
config = conf.set_amp_level(config, "O1")
exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("mnist_pytorch"), 1)
def test_pytorch_const_native_parallel() -> None:
config = conf.load_config(
conf.official_examples_path("mnist_pytorch/const.yaml"))
config = conf.set_slots_per_trial(config, 8)
config = conf.set_native_parallel(config, True)
config = conf.set_max_steps(config, 2)
exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("mnist_pytorch"), 1)
def test_iris() -> None:
config = conf.load_config(
conf.official_examples_path("iris_tf_keras/const.yaml"))
config = conf.set_max_steps(config, 2)
exp_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("iris_tf_keras"), 1)
exp_ref = Determined(conf.make_master_url()).get_experiment(exp_id)
model = exp_ref.top_checkpoint().load()
model.summary()
def test_tf_keras_mnist_parallel() -> None:
config = conf.load_config(conf.official_examples_path("fashion_mnist_tf_keras/const.yaml"))
config = conf.set_slots_per_trial(config, 8)
config = conf.set_native_parallel(config, False)
config = conf.set_max_steps(config, 2)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("fashion_mnist_tf_keras"), 1
)
trials = exp.experiment_trials(experiment_id)
assert len(trials) == 1
def test_tf_keras_single_gpu(tf2: bool) -> None:
config = conf.load_config(conf.official_examples_path("cifar10_cnn_tf_keras/const.yaml"))
config = conf.set_slots_per_trial(config, 1)
config = conf.set_max_steps(config, 2)
config = conf.set_tf2_image(config) if tf2 else conf.set_tf1_image(config)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("cifar10_cnn_tf_keras"), 1
)
trials = exp.experiment_trials(experiment_id)
assert len(trials) == 1
def test_tensorpack_native_parallel() -> None:
config = conf.load_config(
conf.official_examples_path("mnist_tp/const.yaml"))
config = conf.set_slots_per_trial(config, 8)
config = conf.set_native_parallel(config, True)
config = conf.set_max_steps(config, 2)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("mnist_tp"), 1)
trials = exp.experiment_trials(experiment_id)
assert len(trials) == 1
def run_mnist_estimator_data_layer_test(tf2: bool, storage_type: str) -> None:
config = conf.load_config(
conf.experimental_path("data_layer_mnist_estimator/const.yaml"))
config = conf.set_max_steps(config, 2)
config = conf.set_tf2_image(config) if tf2 else conf.set_tf1_image(config)
if storage_type == "lfs":
config = conf.set_shared_fs_data_layer(config)
else:
config = conf.set_s3_data_layer(config)
exp.run_basic_test_with_temp_config(
config, conf.experimental_path("data_layer_mnist_estimator"), 1)
def test_pytorch_cifar10_parallel() -> None:
config = conf.load_config(
conf.official_examples_path("trial/cifar10_cnn_pytorch/const.yaml"))
config = conf.set_max_steps(config, 2)
config = conf.set_slots_per_trial(config, 8)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("trial/cifar10_cnn_pytorch"), 1)
trials = exp.experiment_trials(experiment_id)
nn = (Determined(conf.make_master_url()).get_trial(
trials[0]["id"]).select_checkpoint(latest=True).load())
assert isinstance(nn, torch.nn.Module)
def test_pytorch_cifar10_const() -> None:
config = conf.load_config(
conf.official_examples_path("cifar10_cnn_pytorch/const.yaml"))
config = conf.set_max_steps(config, 2)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("cifar10_cnn_pytorch"), 1)
trials = exp.experiment_trials(experiment_id)
nn = (Determined(conf.make_master_url()).get_trial(
trials[0].id).select_checkpoint(latest=True).load(
map_location=torch.device("cpu")))
assert isinstance(nn, torch.nn.Module)
def test_tensorpack_parallel(aggregation_frequency: int) -> None:
config = conf.load_config(
conf.official_examples_path("trial/mnist_tp/const.yaml"))
config = conf.set_slots_per_trial(config, 8)
config = conf.set_native_parallel(config, False)
config = conf.set_max_steps(config, 2)
config = conf.set_aggregation_frequency(config, aggregation_frequency)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("trial/mnist_tp"), 1)
trials = exp.experiment_trials(experiment_id)
assert len(trials) == 1
def test_tf_keras_parallel(aggregation_frequency: int, tf2: bool) -> None:
config = conf.load_config(conf.official_examples_path("cifar10_cnn_tf_keras/const.yaml"))
config = conf.set_slots_per_trial(config, 8)
config = conf.set_native_parallel(config, False)
config = conf.set_max_steps(config, 2)
config = conf.set_aggregation_frequency(config, aggregation_frequency)
config = conf.set_tf2_image(config) if tf2 else conf.set_tf1_image(config)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("cifar10_cnn_tf_keras"), 1
)
trials = exp.experiment_trials(experiment_id)
assert len(trials) == 1
def test_mnist_estimator_data_layer_parallel(storage_type: str) -> None:
config = conf.load_config(
conf.experimental_path("data_layer_mnist_estimator/const.yaml"))
config = conf.set_max_steps(config, 2)
config = conf.set_slots_per_trial(config, 8)
config = conf.set_tf1_image(config)
if storage_type == "lfs":
config = conf.set_shared_fs_data_layer(config)
else:
config = conf.set_s3_data_layer(config)
exp.run_basic_test_with_temp_config(
config, conf.experimental_path("data_layer_mnist_estimator"), 1)
def test_pytorch_const_parallel(aggregation_frequency: int,
use_amp: bool) -> None:
config = conf.load_config(
conf.official_examples_path("mnist_pytorch/const.yaml"))
config = conf.set_slots_per_trial(config, 8)
config = conf.set_native_parallel(config, False)
config = conf.set_max_steps(config, 2)
config = conf.set_aggregation_frequency(config, aggregation_frequency)
if use_amp:
config = conf.set_amp_level(config, "O1")
exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("mnist_pytorch"), 1)
def test_mnist_estimmator_const_parallel(native_parallel: bool,
tf2: bool) -> None:
if tf2 and native_parallel:
pytest.skip("TF2 native parallel training is not currently supported.")
config = conf.load_config(
conf.fixtures_path("mnist_estimator/single-multi-slot.yaml"))
config = conf.set_slots_per_trial(config, 8)
config = conf.set_native_parallel(config, native_parallel)
config = conf.set_max_steps(config, 2)
config = conf.set_tf2_image(config) if tf2 else conf.set_tf1_image(config)
exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("mnist_estimator"), 1)
def test_pytorch_const_parallel(aggregation_frequency: int,
use_amp: bool) -> None:
if use_amp and aggregation_frequency > 1:
pytest.skip(
"Mixed precision is not support with aggregation frequency > 1.")
config = conf.load_config(
conf.official_examples_path("trial/mnist_pytorch/const.yaml"))
config = conf.set_slots_per_trial(config, 8)
config = conf.set_native_parallel(config, False)
config = conf.set_max_steps(config, 2)
config = conf.set_aggregation_frequency(config, aggregation_frequency)
if use_amp:
config = conf.set_amp_level(config, "O1")
exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("trial/mnist_pytorch"), 1)
def test_custom_reducer_distributed(secrets: Dict[str, str],
tf2: bool) -> None:
config = conf.load_config(
conf.fixtures_path("estimator_dataset/distributed.yaml"))
# Run with multiple steps to verify we are resetting reducers right.
config = conf.set_max_steps(config, 2)
config = conf.set_slots_per_trial(config, 8)
config = conf.set_tf2_image(config) if tf2 else conf.set_tf1_image(config)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.fixtures_path("estimator_dataset"), 1)
trial = exp.experiment_trials(experiment_id)[0]
last_validation = trial["steps"][len(trial["steps"]) - 1]["validation"]
metrics = last_validation["metrics"]["validation_metrics"]
label_sum = 2 * sum(range(16))
assert metrics["label_sum_fn"] == label_sum
assert metrics["label_sum_cls"] == label_sum
def test_pytorch_const_warm_start() -> None:
"""
Test that specifying an earlier trial checkpoint to warm-start from
correctly populates the later trials' `warm_start_checkpoint_id` fields.
"""
config = conf.load_config(
conf.official_examples_path("mnist_pytorch/const.yaml"))
config = conf.set_max_steps(config, 2)
experiment_id1 = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("mnist_pytorch"), 1)
trials = exp.experiment_trials(experiment_id1)
assert len(trials) == 1
first_trial = trials[0]
first_trial_id = first_trial["id"]
assert len(first_trial["steps"]) == 2
first_checkpoint_id = first_trial["steps"][-1]["checkpoint"]["id"]
config_obj = conf.load_config(
conf.official_examples_path("mnist_pytorch/const.yaml"))
# Change the search method to random, and add a source trial ID to warm
# start from.
config_obj["searcher"]["source_trial_id"] = first_trial_id
config_obj["searcher"]["name"] = "random"
config_obj["searcher"]["max_steps"] = 1
config_obj["searcher"]["max_trials"] = 3
experiment_id2 = exp.run_basic_test_with_temp_config(
config_obj, conf.official_examples_path("mnist_pytorch"), 3)
trials = exp.experiment_trials(experiment_id2)
assert len(trials) == 3
for trial in trials:
assert trial["warm_start_checkpoint_id"] == first_checkpoint_id
