def test_eval_single_tower(self):
features = np.array([[0.01], [0.002]])
labels = np.array([[0.01], [0.02]])
with self.test_session() as session:
replicated_model_fn = replicate_model_fn.replicate_model_fn(
self.model_fn, self.optimizer_fn, devices=['/gpu:0'])
estimator_spec = replicated_model_fn(model_fn_lib.ModeKeys.EVAL,
features, labels, self.params)
session.run(variables.local_variables_initializer())
session.run(variables.global_variables_initializer())
accuracy, a = estimator_spec.eval_metric_ops['accuracy']
auc, b = estimator_spec.eval_metric_ops['auc']
session.run([a, b])
accuracy = session.run(accuracy)
auc = session.run(auc)
# Accuracy is 0.0 (no match) in the first tower.
# Accuracy is 1.0 (match) in the second tower, since the feature
# times weight "c" happened to be equal to the label.
total_loss = ((0.01 * 10 - 0.01) + (0.002 * 10 - 0.02))
self.assertNear((0.0 + 1.0) / 2.0, accuracy, 0.01)
self.assertEqual(0, auc)
self.assertNear(total_loss, session.run(estimator_spec.loss), 0.01)
def test_eval_single_tower(self):
features = np.array([[0.01], [0.002]])
labels = np.array([[0.01], [0.02]])
with self.test_session() as session:
replicated_model_fn = replicate_model_fn.replicate_model_fn(
self.model_fn, self.optimizer_fn, devices=['/gpu:0'])
estimator_spec = replicated_model_fn(
features, labels, model_fn_lib.ModeKeys.EVAL, self.params)
session.run(variables.local_variables_initializer())
session.run(variables.global_variables_initializer())
accuracy, a = estimator_spec.eval_metric_ops['accuracy']
auc, b = estimator_spec.eval_metric_ops['auc']
session.run([a, b])
accuracy = session.run(accuracy)
auc = session.run(auc)
# Accuracy is 0.0 (no match) in the first tower.
# Accuracy is 1.0 (match) in the second tower, since the feature
# times weight "c" happened to be equal to the label.
total_loss = ((0.01 * 10 - 0.01) + (0.002 * 10 - 0.02))
self.assertNear((0.0 + 1.0) / 2.0, accuracy, 0.01)
self.assertEqual(0, auc)
self.assertNear(total_loss, session.run(estimator_spec.loss), 0.01)
def test_train_with_mean_reduction(self):
features = np.array([[1.0], [2.0]])
labels = np.array([[1.0], [2.0]])
with self.test_session() as session:
replicated_model_fn = replicate_model_fn.replicate_model_fn(
self.model_fn,
self.optimizer_fn,
losses.Reduction.MEAN,
devices=['/gpu:0', '/gpu:1'])
estimator_spec = replicated_model_fn(
features, labels, model_fn_lib.ModeKeys.TRAIN, self.params)
session.run(variables.global_variables_initializer())
# loss = feature * c - label
total_loss = ((1.0 * 10 - 1.0) + (2.0 * 10 - 2.0)) / 2.0
self.assertEqual(total_loss, session.run(estimator_spec.loss))
# derivative of loss = (1*c - 1)/2 + (2*c - 2)/2 is 1.5.
# It's the same computation as without mean reduction, but the
# loss from every tower is scaled by 1/<number of towers>.
# new value of c = 10 - learning rate * 1.5 = 8.5
session.run(estimator_spec.train_op)
with variable_scope.variable_scope('', reuse=True):
c = variable_scope.get_variable('c', dtype=dtypes.float64)
self.assertEqual(8.5, session.run(c))
def replicate_estimator_spec(self, session):
features = np.array([0.01, 0.002])
labels = np.array([0.01, 0.02])
replicated_model_fn = replicate_model_fn.replicate_model_fn(
self.model_fn, self.optimizer_fn, devices=['/gpu:0', '/gpu:1'])
estimator_spec = replicated_model_fn(model_fn_lib.ModeKeys.PREDICT,
features, labels, {})
session.run(variables.global_variables_initializer())
return estimator_spec
def replicate_estimator_spec(self, session):
features = np.array([0.01, 0.002])
labels = np.array([0.01, 0.02])
replicated_model_fn = replicate_model_fn.replicate_model_fn(
self.model_fn, self.optimizer_fn, devices=['/gpu:0', '/gpu:1'])
estimator_spec = replicated_model_fn(features, labels,
model_fn_lib.ModeKeys.PREDICT, {})
session.run(variables.global_variables_initializer())
return estimator_spec
def test_predict_single_tower(self):
features = np.array([[0.01], [0.002]])
labels = np.array([[0.01], [0.02]])
with self.test_session() as session:
replicated_model_fn = replicate_model_fn.replicate_model_fn(
self.model_fn, self.optimizer_fn, devices=['/gpu:0'])
estimator_spec = replicated_model_fn(model_fn_lib.ModeKeys.PREDICT,
features, labels, self.params)
session.run(variables.global_variables_initializer())
self.assertAllClose({'probabilities': np.array([[0.1], [0.02]])},
session.run(estimator_spec.predictions))
def test_predict_single_tower(self):
features = np.array([[0.01], [0.002]])
labels = np.array([[0.01], [0.02]])
with self.test_session() as session:
replicated_model_fn = replicate_model_fn.replicate_model_fn(
self.model_fn, self.optimizer_fn, devices=['/gpu:0'])
estimator_spec = replicated_model_fn(
features, labels, model_fn_lib.ModeKeys.PREDICT, self.params)
session.run(variables.global_variables_initializer())
self.assertAllClose({
'probabilities': np.array([[0.1], [0.02]])
}, session.run(estimator_spec.predictions))
def test_train_spec_with_optimizer_without_params(self):
def optimizer_fn_without_params():
return gradient_descent.GradientDescentOptimizer(learning_rate=1.0)
features = np.array([[1.0], [2.0]])
labels = np.array([[1.0], [2.0]])
with self.test_session() as session: # pylint: disable=unused-variable
replicated_model_fn = replicate_model_fn.replicate_model_fn(
self.model_fn,
optimizer_fn_without_params,
devices=['/gpu:0', '/gpu:1'])
# This call is going to fail if `replicated_model_fn` is still passing
# `params` inside `optimizer_fn`, even though the latter doesn't take any:
estimator_spec = replicated_model_fn(model_fn_lib.ModeKeys.TRAIN,
features, labels, self.params)
del estimator_spec
def test_train_spec_with_optimizer_without_params(self):
def optimizer_fn_without_params():
return gradient_descent.GradientDescentOptimizer(learning_rate=1.0)
features = np.array([[1.0], [2.0]])
labels = np.array([[1.0], [2.0]])
with self.test_session() as session: # pylint: disable=unused-variable
replicated_model_fn = replicate_model_fn.replicate_model_fn(
self.model_fn,
optimizer_fn_without_params,
devices=['/gpu:0', '/gpu:1'])
# This call is going to fail if `replicated_model_fn` is still passing
# `params` inside `optimizer_fn`, even though the latter doesn't take any:
estimator_spec = replicated_model_fn(
features, labels, model_fn_lib.ModeKeys.TRAIN, self.params)
del estimator_spec
def test_train_single_tower(self):
features = np.array([[1.0], [2.0]])
labels = np.array([[1.0], [2.0]])
with self.test_session() as session:
replicated_model_fn = replicate_model_fn.replicate_model_fn(
self.model_fn, self.optimizer_fn)
estimator_spec = replicated_model_fn(model_fn_lib.ModeKeys.TRAIN,
features, labels, self.params)
session.run(variables.global_variables_initializer())
# loss = feature * c - label
total_loss = (1.0 * 10 - 1.0) + (2.0 * 10 - 2.0)
self.assertEqual(total_loss, session.run(estimator_spec.loss))
# loss' of c is 3.
# new value of c = 10 - learning rate * 3 = 7.0.
session.run(estimator_spec.train_op)
with variable_scope.variable_scope('', reuse=True):
c = variable_scope.get_variable('c', dtype=dtypes.float64)
self.assertEqual(7.0, session.run(c))
def test_train_single_tower(self):
features = np.array([[1.0], [2.0]])
labels = np.array([[1.0], [2.0]])
with self.test_session() as session:
replicated_model_fn = replicate_model_fn.replicate_model_fn(
self.model_fn, self.optimizer_fn, devices=['/gpu:0'])
estimator_spec = replicated_model_fn(
features, labels, model_fn_lib.ModeKeys.TRAIN, self.params)
session.run(variables.global_variables_initializer())
# loss = feature * c - label
total_loss = (1.0 * 10 - 1.0) + (2.0 * 10 - 2.0)
self.assertEqual(total_loss, session.run(estimator_spec.loss))
# loss' of c is 3.
# new value of c = 10 - learning rate * 3 = 7.0.
session.run(estimator_spec.train_op)
with variable_scope.variable_scope('', reuse=True):
c = variable_scope.get_variable('c', dtype=dtypes.float64)
self.assertEqual(7.0, session.run(c))
def test_complete_flow(self):
n_classes = 3
input_dimension = 2
batch_size = 12
data = np.linspace(0.,
n_classes - 1.,
batch_size * input_dimension,
dtype=np.float32)
x_data = data.reshape(batch_size, input_dimension)
categorical_data = np.random.random_integers(0,
len(x_data),
size=len(x_data))
y_data = np.reshape(self._as_label(data[:batch_size]), (batch_size, 1))
train_input_fn = numpy_io.numpy_input_fn(x={
'x': x_data,
'categories': categorical_data
},
y=y_data,
batch_size=batch_size,
num_epochs=None,
shuffle=True)
eval_input_fn = numpy_io.numpy_input_fn(x={
'x': x_data,
'categories': categorical_data
},
y=y_data,
batch_size=batch_size,
shuffle=False)
predict_input_fn = numpy_io.numpy_input_fn(x={
'x':
x_data,
'categories':
categorical_data
},
batch_size=batch_size,
shuffle=False)
feature_columns = [
feature_column.numeric_column('x', shape=(input_dimension, )),
feature_column.embedding_column(
feature_column.categorical_column_with_vocabulary_list(
'categories',
vocabulary_list=np.linspace(0.,
len(x_data),
len(x_data),
dtype=np.int64)), 1)
]
estimator = dnn.DNNClassifier(hidden_units=(2, 2),
feature_columns=feature_columns,
n_classes=n_classes,
model_dir=self._model_dir)
def optimizer_fn():
return optimizers.get_optimizer_instance('Adagrad',
learning_rate=0.05)
estimator = estimator_lib.Estimator(
model_fn=replicate_model_fn.replicate_model_fn(
estimator.model_fn,
optimizer_fn,
devices=['/gpu:0', '/gpu:1', '/gpu:2']),
model_dir=estimator.model_dir,
config=estimator.config,
params=estimator.params)
num_steps = 10
estimator.train(train_input_fn, steps=num_steps)
scores = estimator.evaluate(eval_input_fn)
self.assertEqual(num_steps, scores[ops_lib.GraphKeys.GLOBAL_STEP])
self.assertIn('loss', six.iterkeys(scores))
predicted_proba = np.array([
x[prediction_keys.PredictionKeys.PROBABILITIES]
for x in estimator.predict(predict_input_fn)
])
self.assertAllEqual((batch_size, n_classes), predicted_proba.shape)
feature_spec = feature_column.make_parse_example_spec(feature_columns)
serving_input_receiver_fn = export.build_parsing_serving_input_receiver_fn(
feature_spec)
export_dir = estimator.export_savedmodel(tempfile.mkdtemp(),
serving_input_receiver_fn)
self.assertTrue(gfile.Exists(export_dir))
def _complete_flow_with_mode(self, mode):
n_classes = 3
input_dimension = 2
batch_size = 12
data = np.linspace(
0., n_classes - 1., batch_size * input_dimension, dtype=np.float32)
x_data = data.reshape(batch_size, input_dimension)
categorical_data = np.random.random_integers(
0, len(x_data), size=len(x_data))
y_data = np.reshape(self._as_label(data[:batch_size]), (batch_size, 1))
train_input_fn = numpy_io.numpy_input_fn(
x={'x': x_data,
'categories': categorical_data},
y=y_data,
batch_size=batch_size,
num_epochs=None,
shuffle=True)
eval_input_fn = numpy_io.numpy_input_fn(
x={'x': x_data,
'categories': categorical_data},
y=y_data,
batch_size=batch_size,
shuffle=False)
predict_input_fn = numpy_io.numpy_input_fn(
x={'x': x_data,
'categories': categorical_data},
batch_size=batch_size,
shuffle=False)
feature_columns = [
feature_column.numeric_column('x', shape=(input_dimension,)),
feature_column.embedding_column(
feature_column.categorical_column_with_vocabulary_list(
'categories',
vocabulary_list=np.linspace(
0., len(x_data), len(x_data), dtype=np.int64)), 1)
]
estimator = dnn.DNNClassifier(
hidden_units=(2, 2),
feature_columns=feature_columns,
n_classes=n_classes,
model_dir=self._model_dir)
def optimizer_fn():
return optimizers.get_optimizer_instance('Adagrad', learning_rate=0.05)
if not mode: # Use the public `replicate_model_fn`.
model_fn = replicate_model_fn.replicate_model_fn(
estimator.model_fn,
optimizer_fn,
devices=['/gpu:0', '/gpu:1', '/gpu:2'])
else:
model_fn = replicate_model_fn._replicate_model_fn_with_mode(
estimator.model_fn,
optimizer_fn,
devices=['/gpu:0', '/gpu:1', '/gpu:2'],
mode=mode)
estimator = estimator_lib.Estimator(
model_fn=model_fn,
model_dir=estimator.model_dir,
config=estimator.config,
params=estimator.params)
num_steps = 10
estimator.train(train_input_fn, steps=num_steps)
scores = estimator.evaluate(eval_input_fn)
self.assertEqual(num_steps, scores[ops_lib.GraphKeys.GLOBAL_STEP])
self.assertIn('loss', six.iterkeys(scores))
predicted_proba = np.array([
x[prediction_keys.PredictionKeys.PROBABILITIES]
for x in estimator.predict(predict_input_fn)
])
self.assertAllEqual((batch_size, n_classes), predicted_proba.shape)
feature_spec = feature_column.make_parse_example_spec(feature_columns)
serving_input_receiver_fn = export.build_parsing_serving_input_receiver_fn(
feature_spec)
export_dir = estimator.export_savedmodel(tempfile.mkdtemp(),
serving_input_receiver_fn)
self.assertTrue(gfile.Exists(export_dir))
def test_unsupported_loss_reduction(self):
with self.assertRaisesRegexp(ValueError, ''):
_ = replicate_model_fn.replicate_model_fn(
self.model_fn, self.optimizer_fn, losses.Reduction.NONE)
def test_complete_flow(self):
n_classes = 3
input_dimension = 2
batch_size = 12
data = np.linspace(0.,
n_classes - 1.,
batch_size * input_dimension,
dtype=np.float32)
x_data = data.reshape(batch_size, input_dimension)
y_data = np.reshape(self._as_label(data[:batch_size]), (batch_size, 1))
train_input_fn = numpy_io.numpy_input_fn(x={'x': x_data},
y=y_data,
batch_size=batch_size,
num_epochs=None,
shuffle=True)
eval_input_fn = numpy_io.numpy_input_fn(x={'x': x_data},
y=y_data,
batch_size=batch_size,
shuffle=False)
predict_input_fn = numpy_io.numpy_input_fn(x={'x': x_data},
batch_size=batch_size,
shuffle=False)
feature_columns = [
feature_column.numeric_column('x', shape=(input_dimension, ))
]
estimator = dnn.DNNClassifier(hidden_units=(2, 2),
feature_columns=feature_columns,
n_classes=n_classes,
model_dir=self._model_dir)
def optimizer_fn():
return optimizers.get_optimizer_instance('Adagrad',
learning_rate=0.05)
# TODO(isaprykin): Switch Estimator to use allow_soft_placement=True
# during export_savedmodel and then switch this test to replicate over
# GPUs instead of CPUs.
estimator = estimator_lib.Estimator(
model_fn=replicate_model_fn.replicate_model_fn(
estimator.model_fn,
optimizer_fn,
devices=['/cpu:0', '/cpu:0', '/cpu:0']),
model_dir=estimator.model_dir,
config=estimator.config,
params=estimator.params)
num_steps = 10
estimator.train(train_input_fn, steps=num_steps)
scores = estimator.evaluate(eval_input_fn)
self.assertEqual(num_steps, scores[ops_lib.GraphKeys.GLOBAL_STEP])
self.assertIn('loss', six.iterkeys(scores))
predicted_proba = np.array([
x[prediction_keys.PredictionKeys.PROBABILITIES]
for x in estimator.predict(predict_input_fn)
])
self.assertAllEqual((batch_size, n_classes), predicted_proba.shape)
feature_spec = feature_column.make_parse_example_spec(feature_columns)
serving_input_receiver_fn = export.build_parsing_serving_input_receiver_fn(
feature_spec)
export_dir = estimator.export_savedmodel(tempfile.mkdtemp(),
serving_input_receiver_fn)
self.assertTrue(gfile.Exists(export_dir))
