def _sanity_check(): from ignite.distributed.utils import _model assert _model.get_world_size() == _model.get_nnodes() * _model.get_nproc_per_node() assert _model.get_local_rank() < _model.get_nproc_per_node() assert _model.get_rank() < _model.get_world_size() assert _model.get_node_rank() < _model.get_nnodes()
def _test_dist_spawn_fn(local_rank, backend, world_size, device): from ignite.distributed.utils import _model assert dist.is_available() and dist.is_initialized() assert dist.get_backend() == backend assert isinstance(_model, _NativeDistModel), f"{type(_model)} vs _NativeDistModel" assert _model.get_local_rank() == local_rank assert _model.get_world_size() == world_size assert _model.device().type == torch.device(device).type
def _test_xla_spawn_fn(local_rank, world_size, device): from ignite.distributed.utils import _model assert isinstance(_model, _XlaDistModel), f"{type(_model)} vs _XlaDistModel" assert _model.get_local_rank() == local_rank assert _model.get_world_size() == world_size d = _model.device() assert isinstance(d, torch.device) and d.type == device assert _model.get_rank() == local_rank assert _model.get_nproc_per_node() == world_size assert _model.get_node_rank() == 0 assert _model.get_nnodes() == 1
def _test_dist_spawn_fn(local_rank, backend, world_size, device): from ignite.distributed.utils import _model assert dist.is_available() and dist.is_initialized() assert dist.get_backend() == backend assert isinstance(_model, _NativeDistModel), "{} vs _NativeDistModel".format(type(_model)) assert _model.get_local_rank() == local_rank assert _model.get_world_size() == world_size if backend == "nccl": assert _model.device() == torch.device("{}:{}".format(device, local_rank)) elif backend == "gloo": assert _model.device() == torch.device(device)
def _test_dist_spawn_fn(local_rank, backend, world_size, device): from ignite.distributed.utils import _model assert hvd.rank() > -1 assert isinstance(_model, _HorovodDistModel), f"{type(_model)} vs _HorovodDistModel" assert _model.get_local_rank() == local_rank assert _model.get_world_size() == world_size assert _model.backend() == backend if "cuda" in device: assert _model.device() == torch.device(f"{device}:{local_rank}") else: assert _model.device() == torch.device(device)
def _test_dist_spawn_fn(local_rank, backend, world_size, device, **kwargs): from ignite.distributed.utils import _model assert dist.is_available() and dist.is_initialized() assert dist.get_backend() == backend assert isinstance(_model, _NativeDistModel), f"{type(_model)} vs _NativeDistModel" assert _model.get_local_rank() == local_rank assert _model.get_world_size() == world_size assert _model.device().type == torch.device(device).type if "master_addr" in kwargs: assert os.environ["MASTER_ADDR"] == kwargs["master_addr"] if "master_port" in kwargs: assert os.environ["MASTER_PORT"] == str(kwargs["master_port"])