def test_gradients_are_computed(self): with self.test_session() as session: tower_specs = replicate_model_fn._get_loss_towers( self.model_fn, mode=None, features=[[0.6], [1.6]], labels=[[0.6], [0.6]], params=None, config=None, devices=['/gpu:0', '/gpu:1'], local_ps_device='/gpu:0', name_scope_pattern='test_tower_{}') session.run(variables.global_variables_initializer()) self.assertEqual(len(tower_specs), 2) self.assertEqual('/device:GPU:0', tower_specs[0].loss.device) self.assertEqual('Sum:0', tower_specs[0].loss.name) self.assertEqual(1.0, session.run(tower_specs[0].loss)) self.assertEqual('/device:GPU:1', tower_specs[1].loss.device) self.assertEqual('test_tower_1/Sum:0', tower_specs[1].loss.name) # The input batch for the second tower had a loss that is 1.0 # bigger: 0.6 vs 1.6. self.assertEqual(2.0, session.run(tower_specs[1].loss)) self.assertEqual(1, len(variables.global_variables())) self.assertEqual(1, len(variables.trainable_variables())) with variable_scope.variable_scope('', reuse=True): c = variable_scope.get_variable('c', dtype=dtypes.float64) self.assertEqual(0.25, session.run(c))
def test_gradients_are_computed_with_mean_reduction(self): with self.test_session() as session: tower_specs = replicate_model_fn._get_loss_towers( self.model_fn, mode=model_fn_lib.ModeKeys.EVAL, features=[[0.6], [1.6]], labels=[[0.6], [0.6]], params=None, loss_reduction=losses.Reduction.MEAN, config=None, devices=['/gpu:0', '/gpu:1'], local_ps_devices=['/gpu:0'], name_scope_pattern='test_tower_{}') session.run(variables.global_variables_initializer()) self.assertEqual(len(tower_specs), 2) self.assertEqual('/device:GPU:0', tower_specs[0].loss.device) self.assertEqual('averaged_loss:0', tower_specs[0].loss.name) self.assertEqual(0.5, session.run(tower_specs[0].loss)) self.assertEqual('/device:GPU:1', tower_specs[1].loss.device) self.assertEqual('test_tower_1/averaged_loss:0', tower_specs[1].loss.name) # The input batch for the second tower had a loss that is 1.0 # bigger: 0.6 vs 1.6. self.assertEqual(1.0, session.run(tower_specs[1].loss)) self.assertEqual(1, len(variables.global_variables())) self.assertEqual(1, len(variables.trainable_variables())) with variable_scope.variable_scope('', reuse=True): c = variable_scope.get_variable('c', dtype=dtypes.float64) self.assertEqual(0.25, session.run(c))
def test_variables_are_round_robined_correctly(self): """Test that creates multiple variables and tests round-robin placement.""" def model_fn(mode, features, labels, params): del params for variable_name in ['a', 'b', 'c', 'd']: c = variable_scope.get_variable( variable_name, initializer=constant_op.constant(0.25, dtype=dtypes.float64), dtype=dtypes.float64) predictions = math_ops.add(np.array([0.1, 0.2, 0.3, features[0]]), c) labels = np.array([0.1, 0.2, 0.3, labels[0]]) loss = losses.absolute_difference( labels=labels, predictions=predictions, reduction=losses.Reduction.SUM) return model_fn_lib.EstimatorSpec( mode=mode, loss=math_ops.reduce_sum(loss)) with self.test_session() as session: tower_specs = replicate_model_fn._get_loss_towers( model_fn, mode=None, features=[[0.6], [1.6], [2.6]], labels=[[0.6], [0.6], [2.6]], params=None, config=None, devices=['/gpu:0', '/gpu:1', '/gpu:3'], local_ps_devices=['/gpu:0', '/gpu:1', '/gpu:3'], name_scope_pattern='test_tower_{}') session.run(variables.global_variables_initializer()) self.assertEqual(len(tower_specs), 3) self.assertEqual('/device:GPU:0', tower_specs[0].loss.device) self.assertEqual('/device:GPU:1', tower_specs[1].loss.device) self.assertEqual('/device:GPU:3', tower_specs[2].loss.device) with variable_scope.variable_scope('', reuse=True): a = variable_scope.get_variable('a', dtype=dtypes.float64) self.assertEqual('/device:GPU:0', a.device) b = variable_scope.get_variable('b', dtype=dtypes.float64) self.assertEqual('/device:GPU:1', b.device) c = variable_scope.get_variable('c', dtype=dtypes.float64) self.assertEqual('/device:GPU:3', c.device) d = variable_scope.get_variable('d', dtype=dtypes.float64) self.assertEqual('/device:GPU:0', d.device)
def test_variables_are_round_robined_correctly(self): """Test that creates multiple variables and tests round-robin placement.""" def model_fn(mode, features, labels, params): del params for variable_name in ['a', 'b', 'c', 'd']: c = variable_scope.get_variable( variable_name, initializer=constant_op.constant(0.25, dtype=dtypes.float64), dtype=dtypes.float64) predictions = math_ops.add(np.array([0.1, 0.2, 0.3, features[0]]), c) labels = np.array([0.1, 0.2, 0.3, labels[0]]) loss = losses.absolute_difference(labels=labels, predictions=predictions, reduction=losses.Reduction.SUM) return model_fn_lib.EstimatorSpec(mode=mode, loss=math_ops.reduce_sum(loss)) with self.test_session() as session: tower_specs = replicate_model_fn._get_loss_towers( model_fn, mode=None, features=[[0.6], [1.6], [2.6]], labels=[[0.6], [0.6], [2.6]], params=None, config=None, devices=['/gpu:0', '/gpu:1', '/gpu:3'], local_ps_devices=['/gpu:0', '/gpu:1', '/gpu:3'], name_scope_pattern='test_tower_{}') session.run(variables.global_variables_initializer()) self.assertEqual(len(tower_specs), 3) self.assertEqual('/device:GPU:0', tower_specs[0].loss.device) self.assertEqual('/device:GPU:1', tower_specs[1].loss.device) self.assertEqual('/device:GPU:3', tower_specs[2].loss.device) with variable_scope.variable_scope('', reuse=True): a = variable_scope.get_variable('a', dtype=dtypes.float64) self.assertEqual('/device:GPU:0', a.device) b = variable_scope.get_variable('b', dtype=dtypes.float64) self.assertEqual('/device:GPU:1', b.device) c = variable_scope.get_variable('c', dtype=dtypes.float64) self.assertEqual('/device:GPU:3', c.device) d = variable_scope.get_variable('d', dtype=dtypes.float64) self.assertEqual('/device:GPU:0', d.device)
def test_example(self): with self.test_session() as session: tower_specs = replicate_model_fn._get_loss_towers( self.model_fn, mode=None, features=[[0.1], [0.2]], labels=[[], []], params=None, config=None, devices=['/gpu:0', '/gpu:1'], local_ps_devices=['/gpu:0'], ) session.run(variables.global_variables_initializer()) estimator_spec = replicate_model_fn._predict_spec( tower_specs, aggregation_device='/gpu:0') self.assertEqual('/device:GPU:0', estimator_spec.predictions['probabilities'].device) self.assertAllClose({ 'probabilities': np.array([0.35, 0.35, 0.45, 0.45]) }, session.run(estimator_spec.predictions))
def test_example(self): with self.test_session() as session: tower_specs = replicate_model_fn._get_loss_towers( self.model_fn, mode=None, features=[[0.1], [0.2]], labels=[[], []], params=None, config=None, devices=['/gpu:0', '/gpu:1'], local_ps_device='/gpu:0', ) session.run(variables.global_variables_initializer()) estimator_spec = replicate_model_fn._predict_spec( tower_specs, aggregation_device='/gpu:0') self.assertEqual('/device:GPU:0', estimator_spec.predictions['probabilities'].device) self.assertAllClose({ 'probabilities': np.array([0.35, 0.35, 0.45, 0.45]) }, session.run(estimator_spec.predictions))