def test_eval_single_tower(self): features = np.array([[0.01], [0.002]]) labels = np.array([[0.01], [0.02]]) with self.test_session() as session: replicated_model_fn = replicate_model_fn.replicate_model_fn( self.model_fn, self.optimizer_fn, devices=['/gpu:0']) estimator_spec = replicated_model_fn(model_fn_lib.ModeKeys.EVAL, features, labels, self.params) session.run(variables.local_variables_initializer()) session.run(variables.global_variables_initializer()) accuracy, a = estimator_spec.eval_metric_ops['accuracy'] auc, b = estimator_spec.eval_metric_ops['auc'] session.run([a, b]) accuracy = session.run(accuracy) auc = session.run(auc) # Accuracy is 0.0 (no match) in the first tower. # Accuracy is 1.0 (match) in the second tower, since the feature # times weight "c" happened to be equal to the label. total_loss = ((0.01 * 10 - 0.01) + (0.002 * 10 - 0.02)) self.assertNear((0.0 + 1.0) / 2.0, accuracy, 0.01) self.assertEqual(0, auc) self.assertNear(total_loss, session.run(estimator_spec.loss), 0.01)
def test_eval_single_tower(self): features = np.array([[0.01], [0.002]]) labels = np.array([[0.01], [0.02]]) with self.test_session() as session: replicated_model_fn = replicate_model_fn.replicate_model_fn( self.model_fn, self.optimizer_fn, devices=['/gpu:0']) estimator_spec = replicated_model_fn( features, labels, model_fn_lib.ModeKeys.EVAL, self.params) session.run(variables.local_variables_initializer()) session.run(variables.global_variables_initializer()) accuracy, a = estimator_spec.eval_metric_ops['accuracy'] auc, b = estimator_spec.eval_metric_ops['auc'] session.run([a, b]) accuracy = session.run(accuracy) auc = session.run(auc) # Accuracy is 0.0 (no match) in the first tower. # Accuracy is 1.0 (match) in the second tower, since the feature # times weight "c" happened to be equal to the label. total_loss = ((0.01 * 10 - 0.01) + (0.002 * 10 - 0.02)) self.assertNear((0.0 + 1.0) / 2.0, accuracy, 0.01) self.assertEqual(0, auc) self.assertNear(total_loss, session.run(estimator_spec.loss), 0.01)
def test_train_with_mean_reduction(self): features = np.array([[1.0], [2.0]]) labels = np.array([[1.0], [2.0]]) with self.test_session() as session: replicated_model_fn = replicate_model_fn.replicate_model_fn( self.model_fn, self.optimizer_fn, losses.Reduction.MEAN, devices=['/gpu:0', '/gpu:1']) estimator_spec = replicated_model_fn( features, labels, model_fn_lib.ModeKeys.TRAIN, self.params) session.run(variables.global_variables_initializer()) # loss = feature * c - label total_loss = ((1.0 * 10 - 1.0) + (2.0 * 10 - 2.0)) / 2.0 self.assertEqual(total_loss, session.run(estimator_spec.loss)) # derivative of loss = (1*c - 1)/2 + (2*c - 2)/2 is 1.5. # It's the same computation as without mean reduction, but the # loss from every tower is scaled by 1/<number of towers>. # new value of c = 10 - learning rate * 1.5 = 8.5 session.run(estimator_spec.train_op) with variable_scope.variable_scope('', reuse=True): c = variable_scope.get_variable('c', dtype=dtypes.float64) self.assertEqual(8.5, session.run(c))
def replicate_estimator_spec(self, session): features = np.array([0.01, 0.002]) labels = np.array([0.01, 0.02]) replicated_model_fn = replicate_model_fn.replicate_model_fn( self.model_fn, self.optimizer_fn, devices=['/gpu:0', '/gpu:1']) estimator_spec = replicated_model_fn(model_fn_lib.ModeKeys.PREDICT, features, labels, {}) session.run(variables.global_variables_initializer()) return estimator_spec
def replicate_estimator_spec(self, session): features = np.array([0.01, 0.002]) labels = np.array([0.01, 0.02]) replicated_model_fn = replicate_model_fn.replicate_model_fn( self.model_fn, self.optimizer_fn, devices=['/gpu:0', '/gpu:1']) estimator_spec = replicated_model_fn(features, labels, model_fn_lib.ModeKeys.PREDICT, {}) session.run(variables.global_variables_initializer()) return estimator_spec
def test_predict_single_tower(self): features = np.array([[0.01], [0.002]]) labels = np.array([[0.01], [0.02]]) with self.test_session() as session: replicated_model_fn = replicate_model_fn.replicate_model_fn( self.model_fn, self.optimizer_fn, devices=['/gpu:0']) estimator_spec = replicated_model_fn(model_fn_lib.ModeKeys.PREDICT, features, labels, self.params) session.run(variables.global_variables_initializer()) self.assertAllClose({'probabilities': np.array([[0.1], [0.02]])}, session.run(estimator_spec.predictions))
def test_predict_single_tower(self): features = np.array([[0.01], [0.002]]) labels = np.array([[0.01], [0.02]]) with self.test_session() as session: replicated_model_fn = replicate_model_fn.replicate_model_fn( self.model_fn, self.optimizer_fn, devices=['/gpu:0']) estimator_spec = replicated_model_fn( features, labels, model_fn_lib.ModeKeys.PREDICT, self.params) session.run(variables.global_variables_initializer()) self.assertAllClose({ 'probabilities': np.array([[0.1], [0.02]]) }, session.run(estimator_spec.predictions))
def test_train_spec_with_optimizer_without_params(self): def optimizer_fn_without_params(): return gradient_descent.GradientDescentOptimizer(learning_rate=1.0) features = np.array([[1.0], [2.0]]) labels = np.array([[1.0], [2.0]]) with self.test_session() as session: # pylint: disable=unused-variable replicated_model_fn = replicate_model_fn.replicate_model_fn( self.model_fn, optimizer_fn_without_params, devices=['/gpu:0', '/gpu:1']) # This call is going to fail if `replicated_model_fn` is still passing # `params` inside `optimizer_fn`, even though the latter doesn't take any: estimator_spec = replicated_model_fn(model_fn_lib.ModeKeys.TRAIN, features, labels, self.params) del estimator_spec
def test_train_spec_with_optimizer_without_params(self): def optimizer_fn_without_params(): return gradient_descent.GradientDescentOptimizer(learning_rate=1.0) features = np.array([[1.0], [2.0]]) labels = np.array([[1.0], [2.0]]) with self.test_session() as session: # pylint: disable=unused-variable replicated_model_fn = replicate_model_fn.replicate_model_fn( self.model_fn, optimizer_fn_without_params, devices=['/gpu:0', '/gpu:1']) # This call is going to fail if `replicated_model_fn` is still passing # `params` inside `optimizer_fn`, even though the latter doesn't take any: estimator_spec = replicated_model_fn( features, labels, model_fn_lib.ModeKeys.TRAIN, self.params) del estimator_spec
def test_train_single_tower(self): features = np.array([[1.0], [2.0]]) labels = np.array([[1.0], [2.0]]) with self.test_session() as session: replicated_model_fn = replicate_model_fn.replicate_model_fn( self.model_fn, self.optimizer_fn) estimator_spec = replicated_model_fn(model_fn_lib.ModeKeys.TRAIN, features, labels, self.params) session.run(variables.global_variables_initializer()) # loss = feature * c - label total_loss = (1.0 * 10 - 1.0) + (2.0 * 10 - 2.0) self.assertEqual(total_loss, session.run(estimator_spec.loss)) # loss' of c is 3. # new value of c = 10 - learning rate * 3 = 7.0. session.run(estimator_spec.train_op) with variable_scope.variable_scope('', reuse=True): c = variable_scope.get_variable('c', dtype=dtypes.float64) self.assertEqual(7.0, session.run(c))
def test_train_single_tower(self): features = np.array([[1.0], [2.0]]) labels = np.array([[1.0], [2.0]]) with self.test_session() as session: replicated_model_fn = replicate_model_fn.replicate_model_fn( self.model_fn, self.optimizer_fn, devices=['/gpu:0']) estimator_spec = replicated_model_fn( features, labels, model_fn_lib.ModeKeys.TRAIN, self.params) session.run(variables.global_variables_initializer()) # loss = feature * c - label total_loss = (1.0 * 10 - 1.0) + (2.0 * 10 - 2.0) self.assertEqual(total_loss, session.run(estimator_spec.loss)) # loss' of c is 3. # new value of c = 10 - learning rate * 3 = 7.0. session.run(estimator_spec.train_op) with variable_scope.variable_scope('', reuse=True): c = variable_scope.get_variable('c', dtype=dtypes.float64) self.assertEqual(7.0, session.run(c))
def test_complete_flow(self): n_classes = 3 input_dimension = 2 batch_size = 12 data = np.linspace(0., n_classes - 1., batch_size * input_dimension, dtype=np.float32) x_data = data.reshape(batch_size, input_dimension) categorical_data = np.random.random_integers(0, len(x_data), size=len(x_data)) y_data = np.reshape(self._as_label(data[:batch_size]), (batch_size, 1)) train_input_fn = numpy_io.numpy_input_fn(x={ 'x': x_data, 'categories': categorical_data }, y=y_data, batch_size=batch_size, num_epochs=None, shuffle=True) eval_input_fn = numpy_io.numpy_input_fn(x={ 'x': x_data, 'categories': categorical_data }, y=y_data, batch_size=batch_size, shuffle=False) predict_input_fn = numpy_io.numpy_input_fn(x={ 'x': x_data, 'categories': categorical_data }, batch_size=batch_size, shuffle=False) feature_columns = [ feature_column.numeric_column('x', shape=(input_dimension, )), feature_column.embedding_column( feature_column.categorical_column_with_vocabulary_list( 'categories', vocabulary_list=np.linspace(0., len(x_data), len(x_data), dtype=np.int64)), 1) ] estimator = dnn.DNNClassifier(hidden_units=(2, 2), feature_columns=feature_columns, n_classes=n_classes, model_dir=self._model_dir) def optimizer_fn(): return optimizers.get_optimizer_instance('Adagrad', learning_rate=0.05) estimator = estimator_lib.Estimator( model_fn=replicate_model_fn.replicate_model_fn( estimator.model_fn, optimizer_fn, devices=['/gpu:0', '/gpu:1', '/gpu:2']), model_dir=estimator.model_dir, config=estimator.config, params=estimator.params) num_steps = 10 estimator.train(train_input_fn, steps=num_steps) scores = estimator.evaluate(eval_input_fn) self.assertEqual(num_steps, scores[ops_lib.GraphKeys.GLOBAL_STEP]) self.assertIn('loss', six.iterkeys(scores)) predicted_proba = np.array([ x[prediction_keys.PredictionKeys.PROBABILITIES] for x in estimator.predict(predict_input_fn) ]) self.assertAllEqual((batch_size, n_classes), predicted_proba.shape) feature_spec = feature_column.make_parse_example_spec(feature_columns) serving_input_receiver_fn = export.build_parsing_serving_input_receiver_fn( feature_spec) export_dir = estimator.export_savedmodel(tempfile.mkdtemp(), serving_input_receiver_fn) self.assertTrue(gfile.Exists(export_dir))
def _complete_flow_with_mode(self, mode): n_classes = 3 input_dimension = 2 batch_size = 12 data = np.linspace( 0., n_classes - 1., batch_size * input_dimension, dtype=np.float32) x_data = data.reshape(batch_size, input_dimension) categorical_data = np.random.random_integers( 0, len(x_data), size=len(x_data)) y_data = np.reshape(self._as_label(data[:batch_size]), (batch_size, 1)) train_input_fn = numpy_io.numpy_input_fn( x={'x': x_data, 'categories': categorical_data}, y=y_data, batch_size=batch_size, num_epochs=None, shuffle=True) eval_input_fn = numpy_io.numpy_input_fn( x={'x': x_data, 'categories': categorical_data}, y=y_data, batch_size=batch_size, shuffle=False) predict_input_fn = numpy_io.numpy_input_fn( x={'x': x_data, 'categories': categorical_data}, batch_size=batch_size, shuffle=False) feature_columns = [ feature_column.numeric_column('x', shape=(input_dimension,)), feature_column.embedding_column( feature_column.categorical_column_with_vocabulary_list( 'categories', vocabulary_list=np.linspace( 0., len(x_data), len(x_data), dtype=np.int64)), 1) ] estimator = dnn.DNNClassifier( hidden_units=(2, 2), feature_columns=feature_columns, n_classes=n_classes, model_dir=self._model_dir) def optimizer_fn(): return optimizers.get_optimizer_instance('Adagrad', learning_rate=0.05) if not mode: # Use the public `replicate_model_fn`. model_fn = replicate_model_fn.replicate_model_fn( estimator.model_fn, optimizer_fn, devices=['/gpu:0', '/gpu:1', '/gpu:2']) else: model_fn = replicate_model_fn._replicate_model_fn_with_mode( estimator.model_fn, optimizer_fn, devices=['/gpu:0', '/gpu:1', '/gpu:2'], mode=mode) estimator = estimator_lib.Estimator( model_fn=model_fn, model_dir=estimator.model_dir, config=estimator.config, params=estimator.params) num_steps = 10 estimator.train(train_input_fn, steps=num_steps) scores = estimator.evaluate(eval_input_fn) self.assertEqual(num_steps, scores[ops_lib.GraphKeys.GLOBAL_STEP]) self.assertIn('loss', six.iterkeys(scores)) predicted_proba = np.array([ x[prediction_keys.PredictionKeys.PROBABILITIES] for x in estimator.predict(predict_input_fn) ]) self.assertAllEqual((batch_size, n_classes), predicted_proba.shape) feature_spec = feature_column.make_parse_example_spec(feature_columns) serving_input_receiver_fn = export.build_parsing_serving_input_receiver_fn( feature_spec) export_dir = estimator.export_savedmodel(tempfile.mkdtemp(), serving_input_receiver_fn) self.assertTrue(gfile.Exists(export_dir))
def test_unsupported_loss_reduction(self): with self.assertRaisesRegexp(ValueError, ''): _ = replicate_model_fn.replicate_model_fn( self.model_fn, self.optimizer_fn, losses.Reduction.NONE)
def test_complete_flow(self): n_classes = 3 input_dimension = 2 batch_size = 12 data = np.linspace(0., n_classes - 1., batch_size * input_dimension, dtype=np.float32) x_data = data.reshape(batch_size, input_dimension) y_data = np.reshape(self._as_label(data[:batch_size]), (batch_size, 1)) train_input_fn = numpy_io.numpy_input_fn(x={'x': x_data}, y=y_data, batch_size=batch_size, num_epochs=None, shuffle=True) eval_input_fn = numpy_io.numpy_input_fn(x={'x': x_data}, y=y_data, batch_size=batch_size, shuffle=False) predict_input_fn = numpy_io.numpy_input_fn(x={'x': x_data}, batch_size=batch_size, shuffle=False) feature_columns = [ feature_column.numeric_column('x', shape=(input_dimension, )) ] estimator = dnn.DNNClassifier(hidden_units=(2, 2), feature_columns=feature_columns, n_classes=n_classes, model_dir=self._model_dir) def optimizer_fn(): return optimizers.get_optimizer_instance('Adagrad', learning_rate=0.05) # TODO(isaprykin): Switch Estimator to use allow_soft_placement=True # during export_savedmodel and then switch this test to replicate over # GPUs instead of CPUs. estimator = estimator_lib.Estimator( model_fn=replicate_model_fn.replicate_model_fn( estimator.model_fn, optimizer_fn, devices=['/cpu:0', '/cpu:0', '/cpu:0']), model_dir=estimator.model_dir, config=estimator.config, params=estimator.params) num_steps = 10 estimator.train(train_input_fn, steps=num_steps) scores = estimator.evaluate(eval_input_fn) self.assertEqual(num_steps, scores[ops_lib.GraphKeys.GLOBAL_STEP]) self.assertIn('loss', six.iterkeys(scores)) predicted_proba = np.array([ x[prediction_keys.PredictionKeys.PROBABILITIES] for x in estimator.predict(predict_input_fn) ]) self.assertAllEqual((batch_size, n_classes), predicted_proba.shape) feature_spec = feature_column.make_parse_example_spec(feature_columns) serving_input_receiver_fn = export.build_parsing_serving_input_receiver_fn( feature_spec) export_dir = estimator.export_savedmodel(tempfile.mkdtemp(), serving_input_receiver_fn) self.assertTrue(gfile.Exists(export_dir))