def test_invalid_allreduce_grad_dtype(self): with self.assertRaises(ValueError): PureNcclCommunicator(self.mpi_comm, allreduce_grad_dtype=np.int32)
def create_communicator(communicator_name='hierarchical', mpi_comm=None, allreduce_grad_dtype=None): """Create a ChainerMN communicator. Different communicators provide different approaches of communication, so they have different performance charasteristics. The default communicator ``hierarchical`` is expected to generally perform well on a variety of environments, so one need not to change communicators in most cases. However, choosing proper communicator may give better performance. The following communicators are available. +---------------+---+---+--------+--------------------------------------+ |Name |CPU|GPU|NCCL |Recommended Use Cases | +===============+===+===+========+======================================+ |pure_nccl | |OK |Required|``pure_nccl`` is recommended when | | | | |(>= v2) |NCCL2 is available in the environment.| +---------------+---+---+--------+--------------------------------------+ |hierarchical | |OK |Required|Each node has a single NIC or HCA | +---------------+---+---+--------+--------------------------------------+ |two_dimensional| |OK |Required|Each node has multiple NICs or HCAs | +---------------+---+---+--------+--------------------------------------+ |single_node | |OK |Required|Single node with multiple GPUs | +---------------+---+---+--------+--------------------------------------+ |flat | |OK | |N/A | +---------------+---+---+--------+--------------------------------------+ |naive |OK |OK | |Testing on CPU mode | +---------------+---+---+--------+--------------------------------------+ Args: communicator_name: The name of communicator (``naive``, ``flat``, ``hierarchical``, ``two_dimensional``, ``pure_nccl``, or ``single_node``) mpi_comm: MPI4py communicator allreduce_grad_dtype: Data type of gradient used in All-Reduce. If ``None``, the dtype of a model is used. Returns: ChainerMN communicator that implements methods defined in :class:`chainermn.CommunicatorBase` """ if mpi_comm is None: import mpi4py.MPI mpi_comm = mpi4py.MPI.COMM_WORLD if communicator_name != 'pure_nccl' and allreduce_grad_dtype is not None: raise ValueError('allreduce_grad_dtype is only available' 'at \'pure_nccl\' communicator.') if communicator_name == 'naive': from chainermn.communicators.naive_communicator \ import NaiveCommunicator return NaiveCommunicator(mpi_comm=mpi_comm) elif communicator_name == 'flat': from chainermn.communicators.flat_communicator \ import FlatCommunicator return FlatCommunicator(mpi_comm=mpi_comm) elif communicator_name == 'hierarchical': from chainermn.communicators.hierarchical_communicator \ import HierarchicalCommunicator return HierarchicalCommunicator(mpi_comm=mpi_comm) elif communicator_name == 'two_dimensional': from chainermn.communicators.two_dimensional_communicator \ import TwoDimensionalCommunicator return TwoDimensionalCommunicator(mpi_comm=mpi_comm) elif communicator_name == 'single_node': from chainermn.communicators.single_node_communicator \ import SingleNodeCommunicator return SingleNodeCommunicator(mpi_comm=mpi_comm) elif communicator_name == 'non_cuda_aware': from chainermn.communicators.non_cuda_aware_communicator \ import NonCudaAwareCommunicator return NonCudaAwareCommunicator(mpi_comm=mpi_comm) elif communicator_name == 'pure_nccl': from chainermn.communicators.pure_nccl_communicator \ import PureNcclCommunicator return PureNcclCommunicator(mpi_comm=mpi_comm, allreduce_grad_dtype=allreduce_grad_dtype) elif communicator_name == 'dummy': from chainermn.communicators.dummy_communicator \ import DummyCommunicator return DummyCommunicator(mpi_comm=mpi_comm) else: raise ValueError( 'Unrecognized communicator: "{}"'.format(communicator_name))
def create_communicator(communicator_name='pure_nccl', mpi_comm=None, allreduce_grad_dtype=None, batched_copy=False): """Create a ChainerMN communicator. Different communicators provide different approaches of communication, so they have different performance charasteristics. The default communicator ``pure_nccl`` is expected to generally perform well on a variety of environments, so one need not to change communicators in most cases. However, you may need to choose other communicators depending on your computing platform and the availability of NCCL library. The following communicators are available. +---------------+---+---+--------+--------------------------------------+ |Name |CPU|GPU|NCCL |Recommended Use Cases | +===============+===+===+========+======================================+ |pure_nccl | |OK |Required|``pure_nccl`` is recommended when | | | | |(>= v2) |NCCL2 is available in the environment.| +---------------+---+---+--------+--------------------------------------+ |flat | |OK | |N/A | +---------------+---+---+--------+--------------------------------------+ |naive |OK |OK | |Testing on CPU mode | +---------------+---+---+--------+--------------------------------------+ pure_nccl communicator supports multiple data types, FP32 and FP16, in gradient exchange. The communication data type is determined based on `chainer.global_config.dtype` and `allreduce_grad_dtype`. When `allreduce_grad_dtype` is the default value `None`, FP32 is used when `chainer.global_config.dtype` is `numpy.float32` and FP16 otherwise. `allreduce_grad_dtype` parameter, which is either `numpy.float16` or `numpy.float32`, overwrites the `chainer.global_config.dtype`. The table blow summarizes the data type selection in gradient exchange. +---------------------+--------------------------------------------+ | | allreduce_grad_dtype | +---------------------+---------+------------------+---------------+ | global_config.dtype | None | numpy.float16 | numpy.float32 | +=====================+=========+==================+===============+ | chainer.mixed16 | FP16 | FP16 | FP32 | +---------------------+---------+------------------+---------------+ | numpy.float16 | FP16 | FP16 | FP32 | +---------------------+---------+------------------+---------------+ | numpy.float32 | FP32 | FP16 | FP32 | +---------------------+---------+------------------+---------------+ Other communicators, namely ``flat`` and ``naive``, support only float32 communication, no matter what the model is. This is due to MPI's limited support of float16. Args: communicator_name: The name of communicator (``naive``, ``flat``, or ``pure_nccl``) mpi_comm: MPI4py communicator allreduce_grad_dtype: Data type of gradient used in All-Reduce. If ``None``, the dtype of a model is used. Returns: ChainerMN communicator that implements methods defined in :class:`chainermn.CommunicatorBase` """ if mpi_comm is None: try: import mpi4py.MPI except ImportError as e: raise ImportError( str(e) + ': ' 'ChainerMN requires mpi4py for ' 'distributed training. ' 'Please read the Chainer official document ' 'and setup MPI and mpi4py.') mpi_comm = mpi4py.MPI.COMM_WORLD if communicator_name != 'pure_nccl' and allreduce_grad_dtype is not None: raise ValueError('allreduce_grad_dtype is only available' 'at \'pure_nccl\' communicator.') if communicator_name == 'naive': from chainermn.communicators.naive_communicator \ import NaiveCommunicator return NaiveCommunicator(mpi_comm=mpi_comm, batched_copy=batched_copy) elif communicator_name == 'flat': from chainermn.communicators.flat_communicator \ import FlatCommunicator return FlatCommunicator(mpi_comm=mpi_comm, batched_copy=batched_copy) elif communicator_name == 'non_cuda_aware': from chainermn.communicators.non_cuda_aware_communicator \ import NonCudaAwareCommunicator return NonCudaAwareCommunicator(mpi_comm=mpi_comm, batched_copy=batched_copy) elif communicator_name == 'pure_nccl': from chainermn.communicators.pure_nccl_communicator \ import PureNcclCommunicator return PureNcclCommunicator(mpi_comm=mpi_comm, allreduce_grad_dtype=allreduce_grad_dtype, batched_copy=batched_copy) elif communicator_name == 'dummy': from chainermn.communicators.dummy_communicator \ import DummyCommunicator return DummyCommunicator(mpi_comm=mpi_comm, batched_copy=batched_copy) else: raise ValueError( 'Unrecognized communicator: "{}"'.format(communicator_name))
def test_finalize(self): communicator = PureNcclCommunicator(self.mpi_comm) communicator._init_comms() communicator.finalize() self.assertIsNone(communicator.nccl_comm)
def test_invalid_allreduce_grad_dtype(self): with self.assertRaises(ValueError): comm = PureNcclCommunicator(self.mpi_comm) comm.set_config('allreduce_grad_dtype', np.int32)