def test_tf_keras_const_warm_start(tf2: bool) -> None:
config = conf.load_config(
conf.official_examples_path("cifar10_cnn_tf_keras/const.yaml"))
config["checkpoint_storage"] = exp.shared_fs_checkpoint_config()
config.setdefault("bind_mounts",
[]).append(exp.root_user_home_bind_mount())
config = conf.set_max_steps(config, 2)
config = conf.set_tf2_image(config) if tf2 else conf.set_tf1_image(config)
experiment_id1 = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("cifar10_cnn_tf_keras"), 1)
trials = exp.experiment_trials(experiment_id1)
assert len(trials) == 1
first_trial = trials[0]
first_trial_id = first_trial["id"]
assert len(first_trial["steps"]) == 2
first_checkpoint_id = first_trial["steps"][1]["checkpoint"]["id"]
# Add a source trial ID to warm start from.
config["searcher"]["source_trial_id"] = first_trial_id
experiment_id2 = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("cifar10_cnn_tf_keras"), 1)
# The new trials should have a warm start checkpoint ID.
trials = exp.experiment_trials(experiment_id2)
assert len(trials) == 1
for trial in trials:
assert trial["warm_start_checkpoint_id"] == first_checkpoint_id
def test_tensorpack_const() -> None:
config = conf.load_config(
conf.official_examples_path("mnist_tp/const.yaml"))
config["checkpoint_storage"] = exp.shared_fs_checkpoint_config()
config.get("bind_mounts", []).append(exp.root_user_home_bind_mount())
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("mnist_tp"), 1)
trials = exp.experiment_trials(experiment_id)
assert len(trials) == 1
def test_tf_keras_single_gpu(tf2: bool) -> None:
config = conf.load_config(
conf.official_examples_path("cifar10_cnn_tf_keras/const.yaml"))
config["checkpoint_storage"] = exp.shared_fs_checkpoint_config()
config.get("bind_mounts", []).append(exp.root_user_home_bind_mount())
config = conf.set_slots_per_trial(config, 1)
config = conf.set_max_steps(config, 2)
config = conf.set_tf2_image(config) if tf2 else conf.set_tf1_image(config)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("cifar10_cnn_tf_keras"), 1)
trials = exp.experiment_trials(experiment_id)
assert len(trials) == 1
def test_tf_keras_mnist_parallel() -> None:
config = conf.load_config(
conf.official_examples_path("fashion_mnist_tf_keras/const.yaml"))
config["checkpoint_storage"] = exp.shared_fs_checkpoint_config()
config.get("bind_mounts", []).append(exp.root_user_home_bind_mount())
config = conf.set_slots_per_trial(config, 8)
config = conf.set_native_parallel(config, False)
config = conf.set_max_steps(config, 2)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("fashion_mnist_tf_keras"), 1)
trials = exp.experiment_trials(experiment_id)
assert len(trials) == 1
def test_tensorpack_parallel(aggregation_frequency: int) -> None:
config = conf.load_config(
conf.official_examples_path("mnist_tp/const.yaml"))
config["checkpoint_storage"] = exp.shared_fs_checkpoint_config()
config.get("bind_mounts", []).append(exp.root_user_home_bind_mount())
config = conf.set_slots_per_trial(config, 8)
config = conf.set_native_parallel(config, False)
config = conf.set_max_steps(config, 2)
config = conf.set_aggregation_frequency(config, aggregation_frequency)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("mnist_tp"), 1)
trials = exp.experiment_trials(experiment_id)
assert len(trials) == 1
def test_gc_checkpoints_lfs() -> None:
run_gc_checkpoints_test(exp.shared_fs_checkpoint_config())