def __init__(self, optimizer, nccl=False, nccl_fusion=True, name=None, use_locking=False): super(SynchronousSGDOptimizer, self).__init__(optimizer, name, use_locking=use_locking) _rank, np = peer_info() # FIXME: use type of gradient self._num_workers = tf.cast(np, tf.float32) self._nccl = nccl self._nccl_fusion = nccl_fusion
def __init__(self, optimizer, name=None, monitor_interval=1, use_locking=False): super(SyncSGDWithGradVarianceOptimizer, self).__init__(optimizer, name, use_locking=use_locking) _rank, np = peer_info() # FIXME: use type of gradient self._num_workers = tf.cast(np, tf.float32) self._step = tf.Variable(0, trainable=False, dtype=tf.int32) self._interval = monitor_interval self._summed_variance = None self._variances = None
def __init__(self, optimizer, device_batch_size, name=None, monitor_interval=1, use_locking=False): super(SyncSGDWithGradNoiseScaleOptimizer, self).__init__(optimizer, name, use_locking=use_locking) _rank, np = peer_info() # FIXME: use type of gradient self._num_workers = tf.cast(np, tf.float32) self._step = tf.Variable(0, trainable=False, dtype=tf.int32) self._interval = monitor_interval self._device_batch_size = tf.cast(device_batch_size, dtype=tf.float32) self._global_batch_size = self._device_batch_size * self._num_workers self._noise_op = None
def test_peer_info(): info = peer_info() with tf.Session() as sess: rank, np = sess.run(info) print('rank=%d, np=%d' % (rank, np))