def _sanity_check():
from ignite.distributed.utils import _model
assert _model.get_world_size() == _model.get_nnodes() * _model.get_nproc_per_node()
assert _model.get_local_rank() < _model.get_nproc_per_node()
assert _model.get_rank() < _model.get_world_size()
assert _model.get_node_rank() < _model.get_nnodes()
def _test_dist_spawn_fn(local_rank, backend, world_size, device):
from ignite.distributed.utils import _model
assert dist.is_available() and dist.is_initialized()
assert dist.get_backend() == backend
assert isinstance(_model,
_NativeDistModel), f"{type(_model)} vs _NativeDistModel"
assert _model.get_local_rank() == local_rank
assert _model.get_world_size() == world_size
assert _model.device().type == torch.device(device).type
def _test_xla_spawn_fn(local_rank, world_size, device):
from ignite.distributed.utils import _model
assert isinstance(_model, _XlaDistModel), f"{type(_model)} vs _XlaDistModel"
assert _model.get_local_rank() == local_rank
assert _model.get_world_size() == world_size
d = _model.device()
assert isinstance(d, torch.device) and d.type == device
assert _model.get_rank() == local_rank
assert _model.get_nproc_per_node() == world_size
assert _model.get_node_rank() == 0
assert _model.get_nnodes() == 1
def _test_dist_spawn_fn(local_rank, backend, world_size, device):
from ignite.distributed.utils import _model
assert dist.is_available() and dist.is_initialized()
assert dist.get_backend() == backend
assert isinstance(_model, _NativeDistModel), "{} vs _NativeDistModel".format(type(_model))
assert _model.get_local_rank() == local_rank
assert _model.get_world_size() == world_size
if backend == "nccl":
assert _model.device() == torch.device("{}:{}".format(device, local_rank))
elif backend == "gloo":
assert _model.device() == torch.device(device)
def _test_dist_spawn_fn(local_rank, backend, world_size, device):
from ignite.distributed.utils import _model
assert hvd.rank() > -1
assert isinstance(_model, _HorovodDistModel), f"{type(_model)} vs _HorovodDistModel"
assert _model.get_local_rank() == local_rank
assert _model.get_world_size() == world_size
assert _model.backend() == backend
if "cuda" in device:
assert _model.device() == torch.device(f"{device}:{local_rank}")
else:
assert _model.device() == torch.device(device)
def _test_dist_spawn_fn(local_rank, backend, world_size, device, **kwargs):
from ignite.distributed.utils import _model
assert dist.is_available() and dist.is_initialized()
assert dist.get_backend() == backend
assert isinstance(_model,
_NativeDistModel), f"{type(_model)} vs _NativeDistModel"
assert _model.get_local_rank() == local_rank
assert _model.get_world_size() == world_size
assert _model.device().type == torch.device(device).type
if "master_addr" in kwargs:
assert os.environ["MASTER_ADDR"] == kwargs["master_addr"]
if "master_port" in kwargs:
assert os.environ["MASTER_PORT"] == str(kwargs["master_port"])
