def test_mnist_estimator_warm_start(tf2: bool) -> None:
config = conf.load_config(
conf.fixtures_path("mnist_estimator/single.yaml"))
config = conf.set_tf2_image(config) if tf2 else conf.set_tf1_image(config)
experiment_id1 = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("mnist_estimator"), 1)
trials = exp.experiment_trials(experiment_id1)
assert len(trials) == 1
first_trial = trials[0]
first_trial_id = first_trial.id
assert len(first_trial.steps) == 1
first_checkpoint_id = first_trial.steps[0].checkpoint.id
config_obj = conf.load_config(
conf.fixtures_path("mnist_estimator/single.yaml"))
config_obj["searcher"]["source_trial_id"] = first_trial_id
config_obj = conf.set_tf2_image(config_obj) if tf2 else conf.set_tf1_image(
config_obj)
experiment_id2 = exp.run_basic_test_with_temp_config(
config_obj, conf.official_examples_path("mnist_estimator"), 1)
trials = exp.experiment_trials(experiment_id2)
assert len(trials) == 1
assert trials[0].warm_start_checkpoint_id == first_checkpoint_id
def test_mnist_estimator_const(tf2: bool) -> None:
config = conf.load_config(
conf.fixtures_path("mnist_estimator/single.yaml"))
config = conf.set_tf2_image(config) if tf2 else conf.set_tf1_image(config)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("mnist_estimator"), 1)
trials = exp.experiment_trials(experiment_id)
assert len(trials) == 1
# Check validation metrics.
steps = trials[0].steps
assert len(steps) == 1
step = steps[0]
assert "validation" in step
v_metrics = step.validation.metrics["validation_metrics"]
# GPU training is non-deterministic, but on CPU we can validate that we
# reach a consistent result.
if not cluster.running_on_gpu():
assert v_metrics["accuracy"] == 0.9125999808311462
# Check training metrics.
full_trial_metrics = exp.trial_metrics(trials[0].id)
for step in full_trial_metrics.steps:
metrics = step.metrics
batch_metrics = metrics["batch_metrics"]
assert len(batch_metrics) == 100
for batch_metric in batch_metrics:
assert batch_metric["loss"] > 0
def test_tf_keras_const_warm_start(tf2: bool) -> None:
config = conf.load_config(
conf.official_examples_path("cifar10_cnn_tf_keras/const.yaml"))
config = conf.set_max_steps(config, 2)
config = conf.set_tf2_image(config) if tf2 else conf.set_tf1_image(config)
experiment_id1 = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("cifar10_cnn_tf_keras"), 1)
trials = exp.experiment_trials(experiment_id1)
assert len(trials) == 1
first_trial = trials[0]
first_trial_id = first_trial["id"]
assert len(first_trial["steps"]) == 2
first_checkpoint_id = first_trial["steps"][1]["checkpoint"]["id"]
# Add a source trial ID to warm start from.
config["searcher"]["source_trial_id"] = first_trial_id
experiment_id2 = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("cifar10_cnn_tf_keras"), 1)
# The new trials should have a warm start checkpoint ID.
trials = exp.experiment_trials(experiment_id2)
assert len(trials) == 1
for trial in trials:
assert trial["warm_start_checkpoint_id"] == first_checkpoint_id
def test_pytorch_const_multi_output() -> None:
config = conf.load_config(
conf.experimental_path("mnist_pytorch_multi_output/const.yaml"))
config = conf.set_max_steps(config, 2)
exp.run_basic_test_with_temp_config(
config, conf.experimental_path("mnist_pytorch_multi_output"), 1)
def run_dataset_experiment(
searcher_max_steps: int,
batches_per_step: int,
secrets: Dict[str, str],
tf2: bool,
slots_per_trial: int = 1,
source_trial_id: Optional[str] = None,
) -> List[gql.trials]:
config = conf.load_config(
conf.fixtures_path("estimator_dataset/const.yaml"))
config.setdefault("searcher", {})
config["searcher"]["max_steps"] = searcher_max_steps
config["batches_per_step"] = batches_per_step
config = conf.set_tf2_image(config) if tf2 else conf.set_tf1_image(config)
if source_trial_id is not None:
config["searcher"]["source_trial_id"] = source_trial_id
config.setdefault("resources", {})
config["resources"]["slots_per_trial"] = slots_per_trial
if cluster.num_agents() > 1:
config["checkpoint_storage"] = exp.s3_checkpoint_config(secrets)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.fixtures_path("estimator_dataset"), 1)
return exp.experiment_trials(experiment_id)
def test_mnist_tp_to_estimator() -> None:
config = conf.load_config(
conf.experimental_path("mnist_tp_to_estimator/const.yaml"))
config = conf.set_max_steps(config, 2)
exp.run_basic_test_with_temp_config(
config, conf.experimental_path("mnist_tp_to_estimator"), 1)
def test_bert_glue() -> None:
config = conf.load_config(
conf.experimental_path("bert_glue_pytorch/const.yaml"))
config = conf.set_max_steps(config, 2)
exp.run_basic_test_with_temp_config(
config, conf.experimental_path("bert_glue_pytorch/"), 1)
def test_resnet50() -> None:
config = conf.load_config(
conf.experimental_path("resnet50_tf_keras/const.yaml"))
config = conf.set_max_steps(config, 2)
exp.run_basic_test_with_temp_config(
config, conf.experimental_path("resnet50_tf_keras"), 1)
def test_pytorch_11_const(aggregation_frequency: int) -> None:
config = conf.load_config(
conf.fixtures_path("mnist_pytorch/const-pytorch11.yaml"))
config = conf.set_aggregation_frequency(config, aggregation_frequency)
exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("mnist_pytorch"), 1)
def test_tensorpack_const() -> None:
config = conf.load_config(
conf.official_examples_path("mnist_tp/const.yaml"))
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("mnist_tp"), 1)
trials = exp.experiment_trials(experiment_id)
assert len(trials) == 1
def test_mnist_estimator_adaptive_with_data_layer() -> None:
config = conf.load_config(
conf.fixtures_path("mnist_estimator/adaptive.yaml"))
config = conf.set_tf2_image(config)
config = conf.set_shared_fs_data_layer(config)
exp.run_basic_test_with_temp_config(
config, conf.experimental_path("data_layer_mnist_estimator"), None)
def test_faster_rcnn() -> None:
config = conf.load_config(
conf.experimental_path("FasterRCNN_tp/16-gpus.yaml"))
config = conf.set_max_steps(config, 2)
config = conf.set_slots_per_trial(config, 1)
exp.run_basic_test_with_temp_config(
config, conf.experimental_path("FasterRCNN_tp"), 1, max_wait_secs=4800)
def test_nas_search() -> None:
config = conf.load_config(
conf.experimental_path("nas_search/train_one_arch.yaml"))
config = conf.set_max_steps(config, 2)
exp.run_basic_test_with_temp_config(config,
conf.experimental_path("nas_search"),
1)
def test_pytorch_parallel() -> None:
config = conf.load_config(conf.official_examples_path("mnist_pytorch/const.yaml"))
config = conf.set_slots_per_trial(config, 8)
config = conf.set_native_parallel(config, False)
config = conf.set_max_steps(config, 2)
config = conf.set_tensor_auto_tuning(config, True)
exp.run_basic_test_with_temp_config(config, conf.official_examples_path("mnist_pytorch"), 1)
def test_pytorch_const_with_amp() -> None:
config = conf.load_config(
conf.official_examples_path("mnist_pytorch/const.yaml"))
config = conf.set_max_steps(config, 2)
config = conf.set_amp_level(config, "O1")
exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("mnist_pytorch"), 1)
def test_mnist_estimator_adaptive(tf2: bool) -> None:
# Only test tf1 here, because a tf2 test would add no extra coverage.
config = conf.load_config(
conf.fixtures_path("mnist_estimator/adaptive.yaml"))
config = conf.set_tf2_image(config) if tf2 else conf.set_tf1_image(config)
exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("mnist_estimator"), None)
def test_pytorch_load() -> None:
config = conf.load_config(
conf.fixtures_path("mnist_pytorch/const-pytorch11.yaml"))
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("mnist_pytorch"), 1)
nn = Determined().get_experiment(experiment_id).top_checkpoint().load()
assert isinstance(nn, torch.nn.Module)
def test_iris() -> None:
config = conf.load_config(
conf.official_examples_path("iris_tf_keras/const.yaml"))
config = conf.set_max_steps(config, 2)
exp_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("iris_tf_keras"), 1)
exp_ref = Determined(conf.make_master_url()).get_experiment(exp_id)
model = exp_ref.top_checkpoint().load()
model.summary()
def test_tensorpack_const() -> None:
config = conf.load_config(
conf.official_examples_path("mnist_tp/const.yaml"))
config["checkpoint_storage"] = exp.shared_fs_checkpoint_config()
config.get("bind_mounts", []).append(exp.root_user_home_bind_mount())
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("mnist_tp"), 1)
trials = exp.experiment_trials(experiment_id)
assert len(trials) == 1
def test_mnist_estimator_load() -> None:
config = conf.load_config(
conf.fixtures_path("mnist_estimator/single.yaml"))
config = conf.set_tf1_image(config)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("mnist_estimator"), 1)
trials = exp.experiment_trials(experiment_id)
model = Determined().get_trial(trials[0].id).top_checkpoint().load()
assert isinstance(model, AutoTrackable)
def test_tensorpack_native_parallel() -> None:
config = conf.load_config(
conf.official_examples_path("mnist_tp/const.yaml"))
config = conf.set_slots_per_trial(config, 8)
config = conf.set_native_parallel(config, True)
config = conf.set_max_steps(config, 2)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("mnist_tp"), 1)
trials = exp.experiment_trials(experiment_id)
assert len(trials) == 1
def test_pytorch_cifar10_const() -> None:
config = conf.load_config(
conf.official_examples_path("cifar10_cnn_pytorch/const.yaml"))
config = conf.set_max_steps(config, 2)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("cifar10_cnn_pytorch"), 1)
trials = exp.experiment_trials(experiment_id)
nn = Determined().get_trial(
trials[0].id).select_checkpoint(latest=True).load()
assert isinstance(nn, torch.nn.Module)
def test_log_null_bytes() -> None:
config_obj = conf.load_config(conf.fixtures_path("no_op/single.yaml"))
config_obj["hyperparameters"]["write_null"] = True
config_obj["max_restarts"] = 0
config_obj["searcher"]["max_steps"] = 1
experiment_id = exp.run_basic_test_with_temp_config(config_obj, conf.fixtures_path("no_op"), 1)
trials = exp.experiment_trials(experiment_id)
assert len(trials) == 1
logs = exp.trial_logs(trials[0].id)
assert len(logs) > 0
def test_tf_keras_mnist_parallel() -> None:
config = conf.load_config(
conf.official_examples_path("fashion_mnist_tf_keras/const.yaml"))
config = conf.set_slots_per_trial(config, 8)
config = conf.set_native_parallel(config, False)
config = conf.set_max_steps(config, 2)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("fashion_mnist_tf_keras"), 1)
trials = exp.experiment_trials(experiment_id)
assert len(trials) == 1
def test_s3_no_creds(secrets: Dict[str, str]) -> None:
pytest.skip("Temporarily skipping this until we find a more secure way of testing this.")
config = conf.load_config(conf.official_examples_path("mnist_pytorch/const.yaml"))
config["checkpoint_storage"] = exp.s3_checkpoint_config_no_creds()
config.setdefault("environment", {})
config["environment"].setdefault("environment_variables", [])
config["environment"]["environment_variables"] += [
f"AWS_ACCESS_KEY_ID={secrets['INTEGRATIONS_S3_ACCESS_KEY']}",
f"AWS_SECRET_ACCESS_KEY={secrets['INTEGRATIONS_S3_SECRET_KEY']}",
]
exp.run_basic_test_with_temp_config(config, conf.official_examples_path("mnist_pytorch"), 1)
def test_tf_keras_single_gpu(tf2: bool) -> None:
config = conf.load_config(
conf.official_examples_path("cifar10_cnn_tf_keras/const.yaml"))
config = conf.set_slots_per_trial(config, 1)
config = conf.set_max_steps(config, 2)
config = conf.set_tf2_image(config) if tf2 else conf.set_tf1_image(config)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("cifar10_cnn_tf_keras"), 1)
trials = exp.experiment_trials(experiment_id)
assert len(trials) == 1
def run_tf_keras_mnist_data_layer_test(tf2: bool, storage_type: str) -> None:
config = conf.load_config(
conf.experimental_path("data_layer_mnist_tf_keras/const.yaml"))
config = conf.set_max_steps(config, 2)
config = conf.set_tf2_image(config) if tf2 else conf.set_tf1_image(config)
if storage_type == "lfs":
config = conf.set_shared_fs_data_layer(config)
else:
config = conf.set_s3_data_layer(config)
exp.run_basic_test_with_temp_config(
config, conf.experimental_path("data_layer_mnist_tf_keras"), 1)
def test_tf_keras_single_gpu(tf2: bool) -> None:
config = conf.load_config(
conf.official_examples_path("cifar10_cnn_tf_keras/const.yaml"))
config["checkpoint_storage"] = exp.shared_fs_checkpoint_config()
config.get("bind_mounts", []).append(exp.root_user_home_bind_mount())
config = conf.set_slots_per_trial(config, 1)
config = conf.set_max_steps(config, 2)
config = conf.set_tf2_image(config) if tf2 else conf.set_tf1_image(config)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("cifar10_cnn_tf_keras"), 1)
trials = exp.experiment_trials(experiment_id)
assert len(trials) == 1
def test_pytorch_const_parallel(aggregation_frequency: int,
use_amp: bool) -> None:
config = conf.load_config(
conf.official_examples_path("mnist_pytorch/const.yaml"))
config = conf.set_slots_per_trial(config, 8)
config = conf.set_native_parallel(config, False)
config = conf.set_max_steps(config, 2)
config = conf.set_aggregation_frequency(config, aggregation_frequency)
if use_amp:
config = conf.set_amp_level(config, "O1")
exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("mnist_pytorch"), 1)
def test_tf_keras_parallel(aggregation_frequency: int, tf2: bool) -> None:
config = conf.load_config(
conf.official_examples_path("cifar10_cnn_tf_keras/const.yaml"))
config = conf.set_slots_per_trial(config, 8)
config = conf.set_native_parallel(config, False)
config = conf.set_max_steps(config, 2)
config = conf.set_aggregation_frequency(config, aggregation_frequency)
config = conf.set_tf2_image(config) if tf2 else conf.set_tf1_image(config)
experiment_id = exp.run_basic_test_with_temp_config(
config, conf.official_examples_path("cifar10_cnn_tf_keras"), 1)
trials = exp.experiment_trials(experiment_id)
assert len(trials) == 1
