def _test__hvd_dist_model_create_from_context_no_dist(true_backend,
true_device):
with pytest.raises(ValueError, match=r"Horovod has not been initialized"):
hvd.rank()
assert _HorovodDistModel.create_from_context() is None
hvd.init()
true_conf = {
"device": true_device,
"local_rank": 0,
"rank": 0,
"world_size": 1,
"node_index": 0,
"nnodes": 1,
"nproc_per_node": 1,
}
model = _HorovodDistModel.create_from_context()
assert model.backend() == true_backend
_assert_model(model, true_conf)
hvd.shutdown()
def _test__hvd_dist_model_warning_index_less_localrank():
assert torch.cuda.is_available()
assert _HorovodDistModel.create_from_context() is None
hvd.init()
# We deliberately incorrectly set cuda device to 0
torch.cuda.set_device(0)
model = _HorovodDistModel.create_from_context()
assert isinstance(model, _HorovodDistModel), f"{type(model)} vs _HorovodDistModel"
if hvd.local_rank() == 1:
with pytest.warns(UserWarning, match=r"Current device index is less than current local rank."):
model.device()
hvd.shutdown()
def _test__hvd_dist_model_create_from_context_dist(true_backend, true_device):
assert _HorovodDistModel.create_from_context() is None
hvd.init()
true_conf = {
"device": true_device,
"local_rank": hvd.local_rank(),
"rank": hvd.rank(),
"world_size": hvd.size(),
"node_index": 0,
"nnodes": 1,
"nproc_per_node": hvd.local_size(),
}
model = _HorovodDistModel.create_from_context()
_assert_model(model, true_conf)
hvd.shutdown()
def _test__hvd_dist_model_create_from_context_dist(true_backend, true_device):
assert _HorovodDistModel.create_from_context() is None
hvd.init()
lrank = hvd.local_rank()
if torch.cuda.is_available():
torch.cuda.set_device(lrank)
true_conf = {
"device": true_device,
"local_rank": lrank,
"rank": hvd.rank(),
"world_size": hvd.size(),
"node_index": 0,
"nnodes": 1,
"nproc_per_node": hvd.local_size(),
}
model = _HorovodDistModel.create_from_context()
assert model.backend() == true_backend
_assert_model(model, true_conf)
hvd.shutdown()
