def test_scatter_large_dataset_naive():
mpi_comm = mpi4py.MPI.COMM_WORLD
communicator = NaiveCommunicator(mpi_comm)
# This test only runs when comm.size >= 2.
if communicator.size == 1:
pytest.skip('This test is for multinode')
scatter_large_data(communicator)
class TestDataset(unittest.TestCase):
def setUp(self):
self.mpi_comm = mpi4py.MPI.COMM_WORLD
self.communicator = NaiveCommunicator(self.mpi_comm)
def check_scatter_dataset(self, original_dataset, shuffle=False, root=0):
if self.communicator.rank != root:
original_dataset = None
my_dataset = chainermn.scatter_dataset(original_dataset,
self.communicator,
shuffle=shuffle,
root=root)
sub_datasets = self.communicator.gather_obj(my_dataset, root=root)
if self.communicator.rank == root:
# Test the sizes
sub_sizes = [len(sub_dataset) for sub_dataset in sub_datasets]
self.assertEqual(len(set(sub_sizes)), 1)
sub_size = sub_sizes[0]
self.assertLessEqual(len(original_dataset),
sub_size * self.mpi_comm.size)
self.assertGreater(len(original_dataset),
(sub_size - 1) * self.mpi_comm.size)
# Test the content of scattered datasets
joined_dataset = sum(
(sub_dataset[:] for sub_dataset in sub_datasets), [])
self.assertEqual(set(joined_dataset), set(original_dataset))
def test_scatter_dataset(self):
n = self.communicator.size
for shuffle in [True, False]:
for root in range(self.communicator.size):
self.check_scatter_dataset([], shuffle, root)
self.check_scatter_dataset([0], shuffle, root)
self.check_scatter_dataset(list(range(n)), shuffle, root)
self.check_scatter_dataset(list(range(n * 5 - 1)), shuffle,
root)
self.check_scatter_dataset(np.array([]), shuffle, root)
self.check_scatter_dataset(np.array([0]), shuffle, root)
self.check_scatter_dataset(np.arange(n), shuffle, root)
self.check_scatter_dataset(np.arange(n * 5 - 1), shuffle, root)
class TestDataset(unittest.TestCase):
def setUp(self):
self.mpi_comm = mpi4py.MPI.COMM_WORLD
self.communicator = NaiveCommunicator(self.mpi_comm)
def check_scatter_dataset(self, original_dataset, shuffle=False, root=0):
my_dataset = chainermn.scatter_dataset(
original_dataset, self.communicator,
shuffle=shuffle, root=root)
sub_datasets = self.communicator.gather_obj(my_dataset, root=root)
if self.communicator.rank == root:
# Test the sizes
sub_sizes = [len(sub_dataset) for sub_dataset in sub_datasets]
self.assertEqual(len(set(sub_sizes)), 1)
sub_size = sub_sizes[0]
self.assertLessEqual(
len(original_dataset), sub_size * self.mpi_comm.size)
self.assertGreater(
len(original_dataset), (sub_size - 1) * self.mpi_comm.size)
# Test the content of scattered datasets
joined_dataset = sum((sub_dataset[:]
for sub_dataset in sub_datasets), [])
self.assertEqual(set(joined_dataset), set(original_dataset))
def test_scatter_dataset(self):
n = self.communicator.size
for shuffle in [True, False]:
for root in range(self.communicator.size):
self.check_scatter_dataset([], shuffle, root)
self.check_scatter_dataset([0], shuffle, root)
self.check_scatter_dataset(list(range(n)), shuffle, root)
self.check_scatter_dataset(list(range(n * 5 - 1)),
shuffle, root)
self.check_scatter_dataset(np.array([]), shuffle, root)
self.check_scatter_dataset(np.array([0]), shuffle, root)
self.check_scatter_dataset(np.arange(n), shuffle, root)
self.check_scatter_dataset(np.arange(n * 5 - 1), shuffle, root)
def create_communicator(communicator_name='hierarchical',
mpi_comm=None,
allreduce_grad_dtype=None):
"""Create a ChainerMN communicator.
Different communicators provide different approaches of communication, so
they have different performance charasteristics. The default communicator
``hierarchical`` is expected to generally perform well on a variety of
environments, so one need not to change communicators in most cases.
However, choosing proper communicator may give better performance.
The following communicators are available.
+---------------+---+---+--------+--------------------------------------+
|Name |CPU|GPU|NCCL |Recommended Use Cases |
+===============+===+===+========+======================================+
|pure_nccl | |OK |Required|``pure_nccl`` is recommended when |
| | | |(>= v2) |NCCL2 is available in the environment.|
+---------------+---+---+--------+--------------------------------------+
|hierarchical | |OK |Required|Each node has a single NIC or HCA |
+---------------+---+---+--------+--------------------------------------+
|two_dimensional| |OK |Required|Each node has multiple NICs or HCAs |
+---------------+---+---+--------+--------------------------------------+
|single_node | |OK |Required|Single node with multiple GPUs |
+---------------+---+---+--------+--------------------------------------+
|flat | |OK | |N/A |
+---------------+---+---+--------+--------------------------------------+
|naive |OK |OK | |Testing on CPU mode |
+---------------+---+---+--------+--------------------------------------+
Args:
communicator_name: The name of communicator (``naive``, ``flat``,
``hierarchical``, ``two_dimensional``, ``pure_nccl``, or
``single_node``)
mpi_comm: MPI4py communicator
allreduce_grad_dtype: Data type of gradient used in All-Reduce.
If ``None``, the dtype of a model is used.
Returns:
ChainerMN communicator that implements methods defined in
:class:`chainermn.CommunicatorBase`
"""
if mpi_comm is None:
import mpi4py.MPI
mpi_comm = mpi4py.MPI.COMM_WORLD
if communicator_name != 'pure_nccl' and allreduce_grad_dtype is not None:
raise ValueError('allreduce_grad_dtype is only available'
'at \'pure_nccl\' communicator.')
if communicator_name == 'naive':
from chainermn.communicators.naive_communicator \
import NaiveCommunicator
return NaiveCommunicator(mpi_comm=mpi_comm)
elif communicator_name == 'flat':
from chainermn.communicators.flat_communicator \
import FlatCommunicator
return FlatCommunicator(mpi_comm=mpi_comm)
elif communicator_name == 'hierarchical':
from chainermn.communicators.hierarchical_communicator \
import HierarchicalCommunicator
return HierarchicalCommunicator(mpi_comm=mpi_comm)
elif communicator_name == 'two_dimensional':
from chainermn.communicators.two_dimensional_communicator \
import TwoDimensionalCommunicator
return TwoDimensionalCommunicator(mpi_comm=mpi_comm)
elif communicator_name == 'single_node':
from chainermn.communicators.single_node_communicator \
import SingleNodeCommunicator
return SingleNodeCommunicator(mpi_comm=mpi_comm)
elif communicator_name == 'non_cuda_aware':
from chainermn.communicators.non_cuda_aware_communicator \
import NonCudaAwareCommunicator
return NonCudaAwareCommunicator(mpi_comm=mpi_comm)
elif communicator_name == 'pure_nccl':
from chainermn.communicators.pure_nccl_communicator \
import PureNcclCommunicator
return PureNcclCommunicator(mpi_comm=mpi_comm,
allreduce_grad_dtype=allreduce_grad_dtype)
elif communicator_name == 'dummy':
from chainermn.communicators.dummy_communicator \
import DummyCommunicator
return DummyCommunicator(mpi_comm=mpi_comm)
else:
raise ValueError(
'Unrecognized communicator: "{}"'.format(communicator_name))
def setUp(self):
self.mpi_comm = mpi4py.MPI.COMM_WORLD
self.communicator = NaiveCommunicator(self.mpi_comm)
def setUp(self):
self.mpi_comm = mpi4py.MPI.COMM_WORLD
self.communicator = NaiveCommunicator(self.mpi_comm)
def create_communicator(communicator_name='pure_nccl',
mpi_comm=None,
allreduce_grad_dtype=None,
batched_copy=False):
"""Create a ChainerMN communicator.
Different communicators provide different approaches of communication, so
they have different performance charasteristics. The default communicator
``pure_nccl`` is expected to generally perform well on a variety of
environments, so one need not to change communicators in most cases.
However, you may need to choose other communicators depending on
your computing platform and the availability of NCCL library.
The following communicators are available.
+---------------+---+---+--------+--------------------------------------+
|Name |CPU|GPU|NCCL |Recommended Use Cases |
+===============+===+===+========+======================================+
|pure_nccl | |OK |Required|``pure_nccl`` is recommended when |
| | | |(>= v2) |NCCL2 is available in the environment.|
+---------------+---+---+--------+--------------------------------------+
|flat | |OK | |N/A |
+---------------+---+---+--------+--------------------------------------+
|naive |OK |OK | |Testing on CPU mode |
+---------------+---+---+--------+--------------------------------------+
pure_nccl communicator supports multiple data types, FP32 and FP16,
in gradient exchange. The communication data type is determined based on
`chainer.global_config.dtype` and `allreduce_grad_dtype`.
When `allreduce_grad_dtype` is the default value `None`,
FP32 is used when `chainer.global_config.dtype` is `numpy.float32` and
FP16 otherwise.
`allreduce_grad_dtype` parameter,
which is either `numpy.float16` or `numpy.float32`,
overwrites the `chainer.global_config.dtype`.
The table blow summarizes the data type selection in gradient exchange.
+---------------------+--------------------------------------------+
| | allreduce_grad_dtype |
+---------------------+---------+------------------+---------------+
| global_config.dtype | None | numpy.float16 | numpy.float32 |
+=====================+=========+==================+===============+
| chainer.mixed16 | FP16 | FP16 | FP32 |
+---------------------+---------+------------------+---------------+
| numpy.float16 | FP16 | FP16 | FP32 |
+---------------------+---------+------------------+---------------+
| numpy.float32 | FP32 | FP16 | FP32 |
+---------------------+---------+------------------+---------------+
Other communicators, namely ``flat`` and ``naive``, support only
float32 communication, no matter what the model is. This is due to
MPI's limited support of float16.
Args:
communicator_name: The name of communicator (``naive``, ``flat``,
or ``pure_nccl``)
mpi_comm: MPI4py communicator
allreduce_grad_dtype: Data type of gradient used in All-Reduce.
If ``None``, the dtype of a model is used.
Returns:
ChainerMN communicator that implements methods defined in
:class:`chainermn.CommunicatorBase`
"""
if mpi_comm is None:
try:
import mpi4py.MPI
except ImportError as e:
raise ImportError(
str(e) + ': '
'ChainerMN requires mpi4py for '
'distributed training. '
'Please read the Chainer official document '
'and setup MPI and mpi4py.')
mpi_comm = mpi4py.MPI.COMM_WORLD
if communicator_name != 'pure_nccl' and allreduce_grad_dtype is not None:
raise ValueError('allreduce_grad_dtype is only available'
'at \'pure_nccl\' communicator.')
if communicator_name == 'naive':
from chainermn.communicators.naive_communicator \
import NaiveCommunicator
return NaiveCommunicator(mpi_comm=mpi_comm, batched_copy=batched_copy)
elif communicator_name == 'flat':
from chainermn.communicators.flat_communicator \
import FlatCommunicator
return FlatCommunicator(mpi_comm=mpi_comm, batched_copy=batched_copy)
elif communicator_name == 'non_cuda_aware':
from chainermn.communicators.non_cuda_aware_communicator \
import NonCudaAwareCommunicator
return NonCudaAwareCommunicator(mpi_comm=mpi_comm,
batched_copy=batched_copy)
elif communicator_name == 'pure_nccl':
from chainermn.communicators.pure_nccl_communicator \
import PureNcclCommunicator
return PureNcclCommunicator(mpi_comm=mpi_comm,
allreduce_grad_dtype=allreduce_grad_dtype,
batched_copy=batched_copy)
elif communicator_name == 'dummy':
from chainermn.communicators.dummy_communicator \
import DummyCommunicator
return DummyCommunicator(mpi_comm=mpi_comm, batched_copy=batched_copy)
else:
raise ValueError(
'Unrecognized communicator: "{}"'.format(communicator_name))
def create_communicator(communicator_name='hierarchical', mpi_comm=None):
"""Create a ChainerMN communicator.
Different communicators provide different approaches of communication, so
they have different performance charasteristics. The default communicator
``hierarchical`` is expected to generally perform well on a variety of
environments, so one need not to change communicators in most cases.
However, choosing proper communicator may give better performance.
The following communicators are available.
=============== === === ======== =======================================
Name CPU GPU NCCL Recommended Use Cases
=============== === === ======== =======================================
naive OK OK Testing on CPU mode
hierarchical OK Required Each node has a single NIC or HCA
two_dimensional OK Required Each node has multiple NICs or HCAs
single_node OK Required Single node with multiple GPUs
flat OK N/A
=============== === === ======== =======================================
Args:
communicator_name: The name of communicator (``naive``, ``flat``,
``hierarchical``, ``two_dimensional``, or ``single_node``)
mpi_comm: MPI4py communicator
Returns:
ChainerMN communicator
"""
if mpi_comm is None:
import mpi4py.MPI
mpi_comm = mpi4py.MPI.COMM_WORLD
if communicator_name == 'naive':
from chainermn.communicators.naive_communicator \
import NaiveCommunicator
return NaiveCommunicator(mpi_comm=mpi_comm)
elif communicator_name == 'flat':
from chainermn.communicators.flat_communicator \
import FlatCommunicator
return FlatCommunicator(mpi_comm=mpi_comm)
elif communicator_name == 'hierarchical':
from chainermn.communicators.hierarchical_communicator \
import HierarchicalCommunicator
return HierarchicalCommunicator(mpi_comm=mpi_comm)
elif communicator_name == 'two_dimensional':
from chainermn.communicators.two_dimensional_communicator \
import TwoDimensionalCommunicator
return TwoDimensionalCommunicator(mpi_comm=mpi_comm)
elif communicator_name == 'single_node':
from chainermn.communicators.single_node_communicator \
import SingleNodeCommunicator
return SingleNodeCommunicator(mpi_comm=mpi_comm)
elif communicator_name == 'dummy':
from chainermn.communicators.dummy_communicator \
import DummyCommunicator
return DummyCommunicator(mpi_comm=mpi_comm)
else:
raise ValueError(
'Unrecognized communicator: "{}"'.format(communicator_name))
class TestDataset(unittest.TestCase):
def setUp(self):
self.mpi_comm = mpi4py.MPI.COMM_WORLD
self.communicator = NaiveCommunicator(self.mpi_comm)
def check_scatter_dataset(self, original_dataset, shuffle=False, root=0):
if self.communicator.rank != root:
original_dataset = None
my_dataset = chainermn.scatter_dataset(original_dataset,
self.communicator,
shuffle=shuffle,
root=root)
sub_datasets = self.communicator.gather_obj(my_dataset, root=root)
if self.communicator.rank == root:
# Test the sizes
sub_sizes = [len(sub_dataset) for sub_dataset in sub_datasets]
self.assertEqual(len(set(sub_sizes)), 1)
sub_size = sub_sizes[0]
self.assertLessEqual(len(original_dataset),
sub_size * self.mpi_comm.size)
self.assertGreater(len(original_dataset),
(sub_size - 1) * self.mpi_comm.size)
# Test the content of scattered datasets
joined_dataset = sum(
(sub_dataset[:] for sub_dataset in sub_datasets), [])
# NOTE: The values in `original_dataset` and
# `joined_dataset` must be casted to int to compare.
# There are 2 backgrounds on this issue.
#
# (1) numpy and cupy/chainerx have different behaviours on
# 1-element array. Numpy implicitly converts a 1-element array to
# a scalar value.
# type(numpy.array([1])[0])
# => <class 'numpy.int64'> # Scalar
# type(chainerx.array([1])[0])
# => <class 'chainerx.ndarray'> # array of one element
#
# (2) Two different ChainerX arrays are never identical in the
# context of `set()`.
# set([chainerx.array([0]), chainerx.array([0])])
# => {array([0], shape=(1,), dtype=int64, device='native:0'),
# array([0], shape=(1,), dtype=int64, device='native:0')}
joined_dataset = [int(e) for e in joined_dataset]
original_dataset = [int(e) for e in original_dataset]
self.assertEqual(set(joined_dataset), set(original_dataset))
def test_scatter_dataset(self):
n = self.communicator.size
for shuffle in [True, False]:
for root in range(self.communicator.size):
self.check_scatter_dataset([], shuffle, root)
self.check_scatter_dataset([0], shuffle, root)
self.check_scatter_dataset(list(range(n)), shuffle, root)
self.check_scatter_dataset(list(range(n * 5 - 1)), shuffle,
root)
self.check_scatter_dataset(np.array([]), shuffle, root)
self.check_scatter_dataset(np.array([0]), shuffle, root)
self.check_scatter_dataset(np.arange(n), shuffle, root)
self.check_scatter_dataset(np.arange(n * 5 - 1), shuffle, root)
self.check_scatter_dataset(chx.array([]), shuffle, root)
self.check_scatter_dataset(chx.array([0]), shuffle, root)
self.check_scatter_dataset(chx.arange(n), shuffle, root)
self.check_scatter_dataset(chx.arange(n * 5 - 1), shuffle,
root)
