def _test_idist_methods_overhead(ok_factor, sync_model):
import time
import horovod.torch as hvd
if sync_model:
idist.sync()
from ignite.distributed.utils import _model
from ignite.distributed.comp_models.horovod import _HorovodDistModel
assert isinstance(_model, _HorovodDistModel)
n = 100000
m = 5
t2 = 0.0
t1 = 0.0
for j in range(m):
start = time.time()
for _ in range(n):
_ = hvd.size()
_ = hvd.rank()
elapsed = time.time() - start
t2 += elapsed / n / m
start = time.time()
for _ in range(n):
_ = idist.get_world_size()
_ = idist.get_rank()
elapsed = time.time() - start
t1 += elapsed / n / m
overhead_factor = t1 / t2
assert overhead_factor < ok_factor, "{} vs {} | {} vs {}".format(
overhead_factor, ok_factor, t2, t1)
def test_idist_methods_overhead_nccl(distributed_context_single_node_nccl):
_test_idist_methods_overhead(2.5)
idist.sync()
from ignite.distributed.utils import _model
from ignite.distributed.comp_models.native import _NativeDistModel
assert isinstance(_model, _NativeDistModel)
_test_idist_methods_overhead(1.7)