def __init__(self, **kwargs):
self._name = name or "Distributed%s" % self.__class__.__base__.__name__
self._aggregated_gradients = False
self._allreduce_grads = hvd._make_allreduce_grads_fn(
self._name, device_dense, device_sparse, compression,
sparse_as_dense, op, gradient_predivide_factor)
self._agg_helper = None
if backward_passes_per_step > 1:
if hvd._executing_eagerly():
self._agg_helper = LocalGradientAggregationHelperEager(
backward_passes_per_step=backward_passes_per_step,
allreduce_func=self._allreduce_grads,
sparse_as_dense=sparse_as_dense,
average_aggregated_gradients=
average_aggregated_gradients,
)
else:
self._agg_helper = LocalGradientAggregationHelper(
backward_passes_per_step=backward_passes_per_step,
allreduce_func=self._allreduce_grads,
sparse_as_dense=sparse_as_dense,
average_aggregated_gradients=
average_aggregated_gradients,
rank=rank(),
optimizer_type=LocalGradientAggregationHelper.
_OPTIMIZER_TYPE_KERAS,
)
super(self.__class__, self).__init__(**kwargs)
def __init__(self, name, device_dense, device_sparse, compression, sparse_as_dense,
config):
if name is None:
name = "Distributed%s" % self.__class__.__base__.__name__
self._allreduce_grads = hvd._make_allreduce_grads_fn(
name, device_dense, device_sparse, compression, sparse_as_dense)
super(self.__class__, self).__init__(**config)
def __init__(self, **kwargs):
self._name = name or "Distributed%s" % self.__class__.__base__.__name__
self._aggregated_gradients = False
self._allreduce_grads = hvd._make_allreduce_grads_fn(
self._name, device_dense, device_sparse, compression,
sparse_as_dense, hvd.Average, gradient_predivide_factor)
super(self.__class__, self).__init__(**kwargs)
def __init__(self,
compression=None,
sparse_as_dense=False,
device_dense="",
device_sparse="",
hvd_backend="horovod",
**kwargs):
super(self.__class__, self).__init__(**kwargs)
# using revised loss scale
self._loss_scale = RevisedDynamicLossScale(initial_loss_scale=2**15,
growth_steps=2000,
multiplier=2)
self._track_trackable(self._loss_scale, "loss_scale", overwrite=True)
self._device_dense = device_dense
self._device_sparse = device_sparse
self._hvd_backend = hvd_backend
self._compression = compression
self._sparse_as_dense = sparse_as_dense
self._aggregated_gradients = False
if hvd_backend == "horovod":
import horovod.tensorflow as hvd
self._allreduce_grads = hvd._make_allreduce_grads_fn(
"DistributedLossScaleOptimizer", self._device_dense,
self._device_sparse, compression, sparse_as_dense, hvd.Average,
1.0, 0)
else:
assert hvd_backend == "byteps", f"Unknown `hvd_backend`={hvd_backend}"
def __init__(
self,
optimizer,
name=None,
use_locking=False,
device_dense="",
device_sparse="",
compression=hvd.Compression.none,
sparse_as_dense=False,
op=hvd.Average,
gradient_predivide_factor=1.0,
backward_passes_per_step=1,
average_aggregated_gradients=False,
num_groups=0,
global_batch_count_per_step=None,
):
if name is None:
name = "Distributed{}".format(type(optimizer).__name__)
super(_DistributedOptimizer, self).__init__(name=name,
use_locking=use_locking)
self._optimizer = optimizer
self._allreduce_grads = hvd._make_allreduce_grads_fn(
name,
device_dense,
device_sparse,
compression,
sparse_as_dense,
op,
gradient_predivide_factor,
num_groups,
)
# *ElasticDL Update*: Always create LocalGradientAggregationHelper
# for Elastic training with fixed global batch size.
self._agg_helper = LocalGradientAggregationHelper(
backward_passes_per_step=backward_passes_per_step,
allreduce_func=self._allreduce_grads,
sparse_as_dense=sparse_as_dense,
average_aggregated_gradients=average_aggregated_gradients,
rank=hvd.rank(),
optimizer_type=LocalGradientAggregationHelper.
_OPTIMIZER_TYPE_LEGACY, # noqa: ignore=E501
global_batch_count_per_step=global_batch_count_per_step,
op=op,
)