def __init__(self,
optimizer,
nccl=False,
nccl_fusion=True,
name=None,
use_locking=False):
super(SynchronousSGDOptimizer, self).__init__(optimizer,
name,
use_locking=use_locking)
_rank, np = peer_info()
# FIXME: use type of gradient
self._num_workers = tf.cast(np, tf.float32)
self._nccl = nccl
self._nccl_fusion = nccl_fusion
def __init__(self,
optimizer,
name=None,
monitor_interval=1,
use_locking=False):
super(SyncSGDWithGradVarianceOptimizer,
self).__init__(optimizer, name, use_locking=use_locking)
_rank, np = peer_info()
# FIXME: use type of gradient
self._num_workers = tf.cast(np, tf.float32)
self._step = tf.Variable(0, trainable=False, dtype=tf.int32)
self._interval = monitor_interval
self._summed_variance = None
self._variances = None
def __init__(self,
optimizer,
device_batch_size,
name=None,
monitor_interval=1,
use_locking=False):
super(SyncSGDWithGradNoiseScaleOptimizer,
self).__init__(optimizer, name, use_locking=use_locking)
_rank, np = peer_info()
# FIXME: use type of gradient
self._num_workers = tf.cast(np, tf.float32)
self._step = tf.Variable(0, trainable=False, dtype=tf.int32)
self._interval = monitor_interval
self._device_batch_size = tf.cast(device_batch_size, dtype=tf.float32)
self._global_batch_size = self._device_batch_size * self._num_workers
self._noise_op = None
def test_peer_info():
info = peer_info()
with tf.Session() as sess:
rank, np = sess.run(info)
print('rank=%d, np=%d' % (rank, np))
