def test_gradients_are_computed(self):
with self.test_session() as session:
tower_specs = replicate_model_fn._get_loss_towers(
self.model_fn,
mode=None,
features=[[0.6], [1.6]],
labels=[[0.6], [0.6]],
params=None,
config=None,
devices=['/gpu:0', '/gpu:1'],
local_ps_device='/gpu:0',
name_scope_pattern='test_tower_{}')
session.run(variables.global_variables_initializer())
self.assertEqual(len(tower_specs), 2)
self.assertEqual('/device:GPU:0', tower_specs[0].loss.device)
self.assertEqual('Sum:0', tower_specs[0].loss.name)
self.assertEqual(1.0, session.run(tower_specs[0].loss))
self.assertEqual('/device:GPU:1', tower_specs[1].loss.device)
self.assertEqual('test_tower_1/Sum:0', tower_specs[1].loss.name)
# The input batch for the second tower had a loss that is 1.0
# bigger: 0.6 vs 1.6.
self.assertEqual(2.0, session.run(tower_specs[1].loss))
self.assertEqual(1, len(variables.global_variables()))
self.assertEqual(1, len(variables.trainable_variables()))
with variable_scope.variable_scope('', reuse=True):
c = variable_scope.get_variable('c', dtype=dtypes.float64)
self.assertEqual(0.25, session.run(c))
def test_gradients_are_computed_with_mean_reduction(self):
with self.test_session() as session:
tower_specs = replicate_model_fn._get_loss_towers(
self.model_fn,
mode=model_fn_lib.ModeKeys.EVAL,
features=[[0.6], [1.6]],
labels=[[0.6], [0.6]],
params=None,
loss_reduction=losses.Reduction.MEAN,
config=None,
devices=['/gpu:0', '/gpu:1'],
local_ps_devices=['/gpu:0'],
name_scope_pattern='test_tower_{}')
session.run(variables.global_variables_initializer())
self.assertEqual(len(tower_specs), 2)
self.assertEqual('/device:GPU:0', tower_specs[0].loss.device)
self.assertEqual('averaged_loss:0', tower_specs[0].loss.name)
self.assertEqual(0.5, session.run(tower_specs[0].loss))
self.assertEqual('/device:GPU:1', tower_specs[1].loss.device)
self.assertEqual('test_tower_1/averaged_loss:0', tower_specs[1].loss.name)
# The input batch for the second tower had a loss that is 1.0
# bigger: 0.6 vs 1.6.
self.assertEqual(1.0, session.run(tower_specs[1].loss))
self.assertEqual(1, len(variables.global_variables()))
self.assertEqual(1, len(variables.trainable_variables()))
with variable_scope.variable_scope('', reuse=True):
c = variable_scope.get_variable('c', dtype=dtypes.float64)
self.assertEqual(0.25, session.run(c))
def test_variables_are_round_robined_correctly(self):
"""Test that creates multiple variables and tests round-robin placement."""
def model_fn(mode, features, labels, params):
del params
for variable_name in ['a', 'b', 'c', 'd']:
c = variable_scope.get_variable(
variable_name,
initializer=constant_op.constant(0.25, dtype=dtypes.float64),
dtype=dtypes.float64)
predictions = math_ops.add(np.array([0.1, 0.2, 0.3, features[0]]), c)
labels = np.array([0.1, 0.2, 0.3, labels[0]])
loss = losses.absolute_difference(
labels=labels,
predictions=predictions,
reduction=losses.Reduction.SUM)
return model_fn_lib.EstimatorSpec(
mode=mode, loss=math_ops.reduce_sum(loss))
with self.test_session() as session:
tower_specs = replicate_model_fn._get_loss_towers(
model_fn,
mode=None,
features=[[0.6], [1.6], [2.6]],
labels=[[0.6], [0.6], [2.6]],
params=None,
config=None,
devices=['/gpu:0', '/gpu:1', '/gpu:3'],
local_ps_devices=['/gpu:0', '/gpu:1', '/gpu:3'],
name_scope_pattern='test_tower_{}')
session.run(variables.global_variables_initializer())
self.assertEqual(len(tower_specs), 3)
self.assertEqual('/device:GPU:0', tower_specs[0].loss.device)
self.assertEqual('/device:GPU:1', tower_specs[1].loss.device)
self.assertEqual('/device:GPU:3', tower_specs[2].loss.device)
with variable_scope.variable_scope('', reuse=True):
a = variable_scope.get_variable('a', dtype=dtypes.float64)
self.assertEqual('/device:GPU:0', a.device)
b = variable_scope.get_variable('b', dtype=dtypes.float64)
self.assertEqual('/device:GPU:1', b.device)
c = variable_scope.get_variable('c', dtype=dtypes.float64)
self.assertEqual('/device:GPU:3', c.device)
d = variable_scope.get_variable('d', dtype=dtypes.float64)
self.assertEqual('/device:GPU:0', d.device)
def test_variables_are_round_robined_correctly(self):
"""Test that creates multiple variables and tests round-robin placement."""
def model_fn(mode, features, labels, params):
del params
for variable_name in ['a', 'b', 'c', 'd']:
c = variable_scope.get_variable(
variable_name,
initializer=constant_op.constant(0.25,
dtype=dtypes.float64),
dtype=dtypes.float64)
predictions = math_ops.add(np.array([0.1, 0.2, 0.3, features[0]]),
c)
labels = np.array([0.1, 0.2, 0.3, labels[0]])
loss = losses.absolute_difference(labels=labels,
predictions=predictions,
reduction=losses.Reduction.SUM)
return model_fn_lib.EstimatorSpec(mode=mode,
loss=math_ops.reduce_sum(loss))
with self.test_session() as session:
tower_specs = replicate_model_fn._get_loss_towers(
model_fn,
mode=None,
features=[[0.6], [1.6], [2.6]],
labels=[[0.6], [0.6], [2.6]],
params=None,
config=None,
devices=['/gpu:0', '/gpu:1', '/gpu:3'],
local_ps_devices=['/gpu:0', '/gpu:1', '/gpu:3'],
name_scope_pattern='test_tower_{}')
session.run(variables.global_variables_initializer())
self.assertEqual(len(tower_specs), 3)
self.assertEqual('/device:GPU:0', tower_specs[0].loss.device)
self.assertEqual('/device:GPU:1', tower_specs[1].loss.device)
self.assertEqual('/device:GPU:3', tower_specs[2].loss.device)
with variable_scope.variable_scope('', reuse=True):
a = variable_scope.get_variable('a', dtype=dtypes.float64)
self.assertEqual('/device:GPU:0', a.device)
b = variable_scope.get_variable('b', dtype=dtypes.float64)
self.assertEqual('/device:GPU:1', b.device)
c = variable_scope.get_variable('c', dtype=dtypes.float64)
self.assertEqual('/device:GPU:3', c.device)
d = variable_scope.get_variable('d', dtype=dtypes.float64)
self.assertEqual('/device:GPU:0', d.device)
def test_example(self):
with self.test_session() as session:
tower_specs = replicate_model_fn._get_loss_towers(
self.model_fn,
mode=None,
features=[[0.1], [0.2]],
labels=[[], []],
params=None,
config=None,
devices=['/gpu:0', '/gpu:1'],
local_ps_devices=['/gpu:0'],
)
session.run(variables.global_variables_initializer())
estimator_spec = replicate_model_fn._predict_spec(
tower_specs, aggregation_device='/gpu:0')
self.assertEqual('/device:GPU:0',
estimator_spec.predictions['probabilities'].device)
self.assertAllClose({
'probabilities': np.array([0.35, 0.35, 0.45, 0.45])
}, session.run(estimator_spec.predictions))
def test_example(self):
with self.test_session() as session:
tower_specs = replicate_model_fn._get_loss_towers(
self.model_fn,
mode=None,
features=[[0.1], [0.2]],
labels=[[], []],
params=None,
config=None,
devices=['/gpu:0', '/gpu:1'],
local_ps_device='/gpu:0',
)
session.run(variables.global_variables_initializer())
estimator_spec = replicate_model_fn._predict_spec(
tower_specs, aggregation_device='/gpu:0')
self.assertEqual('/device:GPU:0',
estimator_spec.predictions['probabilities'].device)
self.assertAllClose({
'probabilities': np.array([0.35, 0.35, 0.45, 0.45])
}, session.run(estimator_spec.predictions))
