# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== from __future__ import absolute_import from __future__ import division from __future__ import print_function from horovod.common import check_extension try: check_extension('horovod.torch', 'HOROVOD_WITH_PYTORCH', __file__, 'mpi_lib_v2') except: check_extension('horovod.torch', 'HOROVOD_WITH_PYTORCH', __file__, 'mpi_lib', '_mpi_lib') from horovod.torch.compression import Compression from horovod.torch.mpi_ops import allreduce, allreduce_async, allreduce_, allreduce_async_ from horovod.torch.mpi_ops import allgather, allgather_async from horovod.torch.mpi_ops import broadcast, broadcast_async, broadcast_, broadcast_async_ from horovod.torch.mpi_ops import poll, synchronize from horovod.torch.mpi_ops import init, shutdown from horovod.torch.mpi_ops import size, local_size, rank, local_rank from horovod.torch.mpi_ops import mpi_threads_supported import torch import collections
""" from __future__ import absolute_import from __future__ import division from __future__ import print_function from horovod.common import init from horovod.common import shutdown from horovod.common import size from horovod.common import local_size from horovod.common import rank from horovod.common import local_rank from horovod.common import mpi_threads_supported from horovod.common import check_extension check_extension('horovod.tensorflow', 'HOROVOD_WITH_TENSORFLOW', __file__, 'mpi_lib') from horovod.tensorflow.mpi_ops import allgather from horovod.tensorflow.mpi_ops import broadcast from horovod.tensorflow.mpi_ops import _allreduce import tensorflow as tf def allreduce(tensor, average=True, device_dense='', device_sparse=''): """Perform an allreduce on a tf.Tensor or tf.IndexedSlices. Arguments: tensor: tf.Tensor, tf.Variable, or tf.IndexedSlices to reduce. The shape of the input must be identical across all ranks. average: If True, computes the average over all ranks.
# limitations under the License. # ============================================================================== from __future__ import absolute_import from __future__ import division from __future__ import print_function from horovod.common import init from horovod.common import size from horovod.common import local_size from horovod.common import rank from horovod.common import local_rank from horovod.common import mpi_threads_supported from horovod.common import check_extension check_extension('horovod.torch', 'HOROVOD_WITH_PYTORCH', __file__, 'mpi_lib', '_mpi_lib') from horovod.torch.mpi_ops import allreduce, allreduce_async, allreduce_, allreduce_async_ from horovod.torch.mpi_ops import allgather, allgather_async from horovod.torch.mpi_ops import broadcast, broadcast_async, broadcast_, broadcast_async_ from horovod.torch.mpi_ops import poll, synchronize import torch import collections class _DistributedOptimizer(torch.optim.Optimizer): def __init__(self, params, named_parameters=None): super(self.__class__, self).__init__(params) if named_parameters is not None:
# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== from __future__ import absolute_import from __future__ import division from __future__ import print_function from horovod.common import check_extension check_extension('horovod.mxnet', 'HOROVOD_WITH_MXNET', __file__, 'mpi_lib') from horovod.mxnet.mpi_ops import allgather from horovod.mxnet.mpi_ops import allreduce, allreduce_ from horovod.mxnet.mpi_ops import broadcast, broadcast_ from horovod.mxnet.mpi_ops import init, shutdown from horovod.mxnet.mpi_ops import size, local_size, rank, local_rank from horovod.mxnet.mpi_ops import mpi_threads_supported import mxnet as mx # This is where Horovod's DistributedOptimizer wrapper for MXNet goes class DistributedOptimizer(mx.optimizer.Optimizer): def __init__(self, optimizer): self._optimizer = optimizer
advantage of hardware-specific MPI libraries for efficient communication. """ from __future__ import absolute_import from __future__ import division from __future__ import print_function from horovod.common import init from horovod.common import size from horovod.common import local_size from horovod.common import rank from horovod.common import local_rank from horovod.common import mpi_threads_supported from horovod.common import check_extension check_extension('horovod.tensorflow', 'HOROVOD_WITH_TENSORFLOW', __file__, 'mpi_lib') from horovod.tensorflow.mpi_ops import allgather from horovod.tensorflow.mpi_ops import broadcast from horovod.tensorflow.mpi_ops import _allreduce import tensorflow as tf def allreduce(tensor, average=True, device_dense='', device_sparse=''): """Perform an allreduce on a tf.Tensor or tf.IndexedSlices. Arguments: tensor: tf.Tensor, tf.Variable, or tf.IndexedSlices to reduce. The shape of the input must be identical across all ranks. average: If True, computes the average over all ranks.