def test_worker_failure(ray_start_2_cpus):
config = TestConfig()
e = BackendExecutor(config, num_workers=2)
e.start()
def train_fail():
ray.actor.exit_actor()
new_execute_func = gen_execute_special(train_fail)
with patch.object(WorkerGroup, "execute_async", new_execute_func):
with pytest.raises(TrainingWorkerError):
e.start_training(lambda: 1, dataset_spec=EMPTY_RAY_DATASET_SPEC)
e.finish_training()
def test_train(ray_start_2_cpus):
config = TestConfig()
e = BackendExecutor(config, num_workers=2)
e.start()
e.start_training(lambda: 1, dataset_spec=EMPTY_RAY_DATASET_SPEC)
assert e.finish_training() == [1, 1]
def test_torch_start_shutdown(ray_start_2_cpus, init_method):
torch_config = TorchConfig(backend="gloo", init_method=init_method)
e = BackendExecutor(torch_config, num_workers=2)
e.start()
def check_process_group():
import torch
return (torch.distributed.is_initialized()
and torch.distributed.get_world_size() == 2)
e.start_training(check_process_group, dataset_spec=EMPTY_RAY_DATASET_SPEC)
assert all(e.finish_training())
e._backend.on_shutdown(e.worker_group, e._backend_config)
e.start_training(check_process_group, dataset_spec=EMPTY_RAY_DATASET_SPEC)
assert not any(e.finish_training())
def test_local_ranks(ray_start_2_cpus):
config = TestConfig()
e = BackendExecutor(config, num_workers=2)
e.start()
def train_func():
return train.local_rank()
e.start_training(train_func, dataset_spec=EMPTY_RAY_DATASET_SPEC)
assert set(e.finish_training()) == {0, 1}
def test_train_failure(ray_start_2_cpus):
config = TestConfig()
e = BackendExecutor(config, num_workers=2)
e.start()
with pytest.raises(TrainBackendError):
e.get_next_results()
with pytest.raises(TrainBackendError):
e.pause_reporting()
with pytest.raises(TrainBackendError):
e.finish_training()
e.start_training(lambda: 1, dataset_spec=EMPTY_RAY_DATASET_SPEC)
with pytest.raises(TrainBackendError):
e.start_training(lambda: 2, dataset_spec=EMPTY_RAY_DATASET_SPEC)
assert e.finish_training() == [1, 1]
def test_cuda_visible_devices_multiple(ray_2_node_4_gpu, worker_results):
config = TestConfig()
def get_resources():
return os.environ["CUDA_VISIBLE_DEVICES"]
num_workers, expected_results = worker_results
os.environ[ENABLE_SHARE_CUDA_VISIBLE_DEVICES_ENV] = "1"
e = BackendExecutor(
config, num_workers=num_workers, num_cpus_per_worker=0, num_gpus_per_worker=2
)
e.start()
e.start_training(get_resources, dataset_spec=EMPTY_RAY_DATASET_SPEC)
results = e.finish_training()
results.sort()
assert results == expected_results
def test_initialization_hook(ray_start_2_cpus):
config = TestConfig()
e = BackendExecutor(config, num_workers=2)
def init_hook():
import os
os.environ["TEST"] = "1"
e.start(initialization_hook=init_hook)
def check():
import os
return os.getenv("TEST", "0")
e.start_training(check, dataset_spec=EMPTY_RAY_DATASET_SPEC)
assert e.finish_training() == ["1", "1"]
def test_tensorflow_start(ray_start_2_cpus):
num_workers = 2
tensorflow_config = TensorflowConfig()
e = BackendExecutor(tensorflow_config, num_workers=num_workers)
e.start()
def get_tf_config():
import json
import os
return json.loads(os.environ["TF_CONFIG"])
e.start_training(get_tf_config, dataset_spec=EMPTY_RAY_DATASET_SPEC)
results = e.finish_training()
assert len(results) == num_workers
workers = [result["cluster"]["worker"] for result in results]
assert all(worker == workers[0] for worker in workers)
indexes = [result["task"]["index"] for result in results]
assert len(set(indexes)) == num_workers
def test():
config = TestConfig()
e = BackendExecutor(config, num_workers=2)
e.start()
e.start_training(train_func, dataset_spec=EMPTY_RAY_DATASET_SPEC)
return e.finish_training()