def test_bluefog_size(self):
"""Test that the size returned by bf.size() is correct."""
_, true_size = mpi_env_rank_and_size()
bf.init()
size = bf.size()
# print("Size: ", true_size, size)
assert true_size == size
def test_bluefog_rank(self):
"""Test that the rank returned by bf.rank() is correct."""
true_rank, _ = mpi_env_rank_and_size()
bf.init()
rank = bf.rank()
# print("Rank: ", true_rank, rank)
assert true_rank == rank
def test_in_out_neighbors_expo2(self):
rank, size = mpi_env_rank_and_size()
bf.init()
bf.set_topology(ExponentialGraph(size))
in_neighobrs = bf.in_neighbor_ranks()
out_neighbors = bf.out_neighbor_ranks()
degree = int(np.ceil(np.log2(size)))
expected_in_neighbors = sorted([(rank - 2**i) % size
for i in range(degree)])
expected_out_neighbors = sorted([(rank + 2**i) % size
for i in range(degree)])
assert sorted(in_neighobrs) == expected_in_neighbors
assert sorted(out_neighbors) == expected_out_neighbors
def test_horovod_size(self):
"""Test that the size returned by hvd.size() is correct."""
_, mpi_size = mpi_env_rank_and_size()
gloo_size = int(os.getenv('HOROVOD_SIZE', -1))
# The mpi size does not match gloo size, we need to figure which one
# we are using to run the test.
is_mpi = gloo_size == -1
hvd.init()
size = hvd.size()
if is_mpi:
assert mpi_size == size
else:
assert gloo_size == size
def test_horovod_rank(self):
"""Test that the rank returned by hvd.rank() is correct."""
mpi_rank, _ = mpi_env_rank_and_size()
gloo_rank = int(os.getenv('HOROVOD_RANK', -1))
# The mpi rank does not match gloo rank, we need to figure which one
# we are using to run the test.
is_mpi = gloo_rank == -1
hvd.init()
rank = hvd.rank()
if is_mpi:
assert mpi_rank == rank
else:
assert gloo_rank == rank
def test_set_and_load_topology(self):
_, size = mpi_env_rank_and_size()
if size == 4:
expected_topology = nx.DiGraph(
np.array([[1 / 3., 1 / 3., 1 / 3., 0.],
[0., 1 / 3., 1 / 3., 1 / 3.],
[1 / 3., 0., 1 / 3., 1 / 3.],
[1 / 3., 1 / 3., 0., 1 / 3.]]))
elif size == 1:
expected_topology = nx.DiGraph(np.array([[1.0]]))
else:
expected_topology = ExponentialGraph(size)
bf.init()
topology = bf.load_topology()
assert isinstance(topology, nx.DiGraph)
np.testing.assert_array_equal(nx.to_numpy_array(expected_topology),
nx.to_numpy_array(topology))
def test_in_out_neighbors_biring(self):
rank, size = mpi_env_rank_and_size()
bf.init()
bf.set_topology(RingGraph(size))
in_neighobrs = bf.in_neighbor_ranks()
out_neighbors = bf.out_neighbor_ranks()
expected_in_neighbors = list(
set(map(lambda x: x % size, [rank - 1, rank + 1])))
expected_out_neighbors = list(
set(map(lambda x: x % size, [rank - 1, rank + 1])))
if size <= 1:
expected_in_neighbors = []
expected_out_neighbors = []
assert sorted(in_neighobrs) == expected_in_neighbors
assert sorted(out_neighbors) == expected_out_neighbors
def test_horovod_size(self):
"""Test that the size returned by hvd.size() is correct."""
_, true_size = mpi_env_rank_and_size()
hvd.init()
size = hvd.size()
assert true_size == size
def test_horovod_rank(self):
"""Test that the rank returned by hvd.rank() is correct."""
true_rank, _ = mpi_env_rank_and_size()
hvd.init()
rank = hvd.rank()
assert true_rank == rank
def test_static(self):
mpi_rank, mpi_size = mpi_env_rank_and_size()
gloo_rank = int(os.getenv('HOROVOD_RANK', -1))
gloo_size = int(os.getenv('HOROVOD_SIZE', -1))
is_mpi = gloo_rank == -1
rank = max(mpi_rank, gloo_rank)
size = max(mpi_size, gloo_size)
# This test does not apply if there is only one worker.
if size == 1:
self.skipTest("Only one worker available")
if is_mpi:
try:
import mpi4py
mpi4py.rc.initialize = False
except ImportError:
pass
if rank == 0:
my_process_sets = [
hvd.ProcessSet([0]),
hvd.ProcessSet(range(1, size)),
hvd.ProcessSet(range(size - 1, -1, -1)), # duplicate
hvd.ProcessSet([0]) # duplicate
]
else:
my_process_sets = [
hvd.ProcessSet([0]),
hvd.ProcessSet(reversed(range(
1, size))), # permuting a process set does not matter
hvd.ProcessSet(range(size - 1, -1, -1)), # duplicate
hvd.ProcessSet([0]) # duplicate
]
with self.assertRaises(ValueError):
hvd.init(process_sets=my_process_sets)
if rank == 0:
my_process_sets = [
hvd.ProcessSet([0]),
hvd.ProcessSet(range(1, size)),
]
else:
my_process_sets = [
hvd.ProcessSet([0]),
hvd.ProcessSet(reversed(range(
1, size))), # permuting a process set does not matter
]
hvd.init(process_sets=my_process_sets)
self.assertEqual(hvd.global_process_set.process_set_id, 0)
self.assertListEqual(hvd.global_process_set.ranks, list(range(size)))
# Here we test some implementation details (numeric process set id values) using an internal function.
ps = hvd.mpi_ops._basics._get_process_set_ids_and_ranks()
self.assertDictEqual(ps, {
0: list(range(size)),
1: [0],
2: list(range(1, size))
})
# If another process initiates shutdown while this process is still processing _get_process_set_ids_and_ranks(),
# a race condition may be triggered. Avoid with a barrier.
try:
if is_mpi:
# barrier before shutdown
from mpi4py import MPI
MPI.COMM_WORLD.barrier()
else:
time.sleep(0.1)
except ImportError:
time.sleep(0.1)
hvd.shutdown()
def test_bluefog_local_rank(hier_setup):
true_rank, true_size = mpi_env_rank_and_size()
local_rank = bf.local_rank()
assert true_rank % min(2, true_size) == local_rank
def test_bluefog_local_size(hier_setup):
_, true_size = mpi_env_rank_and_size()
local_size = bf.local_size()
assert local_size == min(2, true_size)
