def _dist_worker_task_fn(backend, fn, args, kwargs_dict):
from ignite.distributed.utils import _set_model, finalize
model = _HorovodDistModel.create_from_backend(backend)
_set_model(model)
fn(model.get_local_rank(), *args, **kwargs_dict)
finalize()
def _dist_worker_task_fn(
local_rank: int,
backend: str,
fn: Callable,
args: Tuple,
kw_dict: Mapping,
world_size: int,
nprocs_per_node: int,
node_rank: int,
master_addr: str,
master_port: str,
kw: Any,
) -> None:
from ignite.distributed.utils import _set_model, finalize
copy_env_vars = os.environ.copy()
os.environ["LOCAL_RANK"] = str(local_rank)
os.environ["RANK"] = str(node_rank * nprocs_per_node + local_rank)
os.environ["WORLD_SIZE"] = str(world_size)
os.environ["MASTER_ADDR"] = str(master_addr)
os.environ["MASTER_PORT"] = str(master_port)
model = _NativeDistModel.create_from_backend(backend, **kw)
_set_model(model)
fn(local_rank, *args, **kw_dict)
finalize()
os.environ.clear()
os.environ.update(copy_env_vars)
def _dist_worker_task_fn(backend: str, fn: Callable, args: Tuple, kwargs_dict: Mapping) -> None:
from ignite.distributed.utils import _set_model, finalize
model = _HorovodDistModel.create_from_backend(backend)
_set_model(model)
fn(model.get_local_rank(), *args, **kwargs_dict)
finalize()
def _dist_worker_task_fn(
local_rank: int,
backend: str,
fn: Callable,
args: Tuple,
kw_dict: Mapping,
world_size: int,
nprocs_per_node: int,
node_rank: int,
master_addr: Optional[str],
master_port: Optional[str],
init_method: str,
kw: Any,
) -> None:
from ignite.distributed.utils import _set_model, finalize
copy_env_vars = os.environ.copy()
rank = node_rank * nprocs_per_node + local_rank
os.environ["LOCAL_RANK"] = str(local_rank)
os.environ["RANK"] = str(rank)
os.environ["WORLD_SIZE"] = str(world_size)
arg_world_size = world_size # type: Optional[int]
arg_rank = rank # type: Optional[int]
if init_method == "env://":
os.environ["MASTER_ADDR"] = str(master_addr)
os.environ["MASTER_PORT"] = str(master_port)
arg_world_size = None
arg_rank = None
model = _NativeDistModel.create_from_backend(
backend,
init_method=init_method,
world_size=arg_world_size,
rank=arg_rank,
**kw)
_set_model(model)
fn(local_rank, *args, **kw_dict)
finalize()
os.environ.clear()
os.environ.update(copy_env_vars)
def _dist_worker_task_fn(local_rank, backend, fn, args, kw_dict,
world_size, nprocs_per_node, node_rank,
master_addr, master_port, kw):
from ignite.distributed.utils import _set_model, finalize
copy_env_vars = dict(os.environ)
os.environ["LOCAL_RANK"] = str(local_rank)
os.environ["RANK"] = str(node_rank * nprocs_per_node + local_rank)
os.environ["WORLD_SIZE"] = str(world_size)
os.environ["MASTER_ADDR"] = str(master_addr)
os.environ["MASTER_PORT"] = str(master_port)
model = _NativeDistModel.create_from_backend(backend, **kw)
_set_model(model)
fn(local_rank, *args, **kw_dict)
finalize()
os.environ = copy_env_vars
def __exit__(self, *args, **kwargs):
if (self.backend is not None) and self._spawn_params is None:
self.logger.info(
"Finalized processing group with backend: '{}'".format(
self.backend))
idist.finalize()
def __exit__(self, *args: Any, **kwargs: Any) -> None:
if (self.backend is not None) and self._spawn_params is None:
self.logger.info(
f"Finalized processing group with backend: '{self.backend}'")
idist.finalize()
