def testFromScratch(self):
n_classes = self._n_classes
label = 1
age = 17
# For binary classifer:
# loss = sigmoid_cross_entropy(logits, label) where logits=0 (weights are
# all zero initially) and label = 1 so,
# loss = 1 * -log ( sigmoid(logits) ) = 0.69315
# For multi class classifer:
# loss = cross_entropy(logits, label) where logits are all 0s (weights are
# all zero initially) and label = 1 so,
# loss = 1 * -log ( 1.0 / n_classes )
# For this particular test case, as logits are same, the formular
# 1 * -log ( 1.0 / n_classes ) covers both binary and multi class cases.
mock_optimizer = self._mock_optimizer(
expected_loss=-1 * math.log(1.0/n_classes))
est = linear.LinearClassifier(
feature_columns=(feature_column_lib.numeric_column('age'),),
n_classes=n_classes,
optimizer=mock_optimizer,
model_dir=self._model_dir)
self.assertEqual(0, mock_optimizer.minimize.call_count)
# Train for a few steps, and validate optimizer and final checkpoint.
num_steps = 10
est.train(
input_fn=lambda: ({'age': ((age,),)}, ((label,),)), steps=num_steps)
self.assertEqual(1, mock_optimizer.minimize.call_count)
self._assert_checkpoint(
expected_global_step=num_steps,
expected_age_weight=[[0.]] if n_classes == 2 else [[0.] * n_classes],
expected_bias=[0.] if n_classes == 2 else [.0] * n_classes)
def testFromCheckpoint(self):
# Create initial checkpoint.
n_classes = self._n_classes
label = 1
age = 17
# For binary case, the expected weight has shape (1,1). For multi class
# case, the shape is (1, n_classes). In order to test the weights, set
# weights as 2.0 * range(n_classes).
age_weight = [[2.0]] if n_classes == 2 else (np.reshape(
2.0 * np.array(list(range(n_classes)), dtype=np.float32),
(1, n_classes)))
bias = [-35.0] if n_classes == 2 else [-35.0] * n_classes
initial_global_step = 100
with ops.Graph().as_default():
variables.Variable(age_weight,
name=linear_testing_utils.AGE_WEIGHT_NAME)
variables.Variable(bias, name=linear_testing_utils.BIAS_NAME)
variables.Variable(initial_global_step,
name=ops.GraphKeys.GLOBAL_STEP,
dtype=dtypes.int64)
linear_testing_utils.save_variables_to_ckpt(self._model_dir)
# For binary classifer:
# logits = age * age_weight + bias = 17 * 2. - 35. = -1.
# loss = sigmoid_cross_entropy(logits, label)
# so, loss = 1 * -log ( sigmoid(-1) ) = 1.3133
# For multi class classifer:
# loss = cross_entropy(logits, label)
# where logits = 17 * age_weight + bias and label = 1
# so, loss = 1 * -log ( soft_max(logits)[1] )
if n_classes == 2:
expected_loss = 1.3133
else:
logits = age_weight * age + bias
logits_exp = np.exp(logits)
softmax = logits_exp / logits_exp.sum()
expected_loss = -1 * math.log(softmax[0, label])
mock_optimizer = self._mock_optimizer(expected_loss=expected_loss)
est = linear.LinearClassifier(
feature_columns=(feature_column_lib.numeric_column('age'), ),
n_classes=n_classes,
optimizer=mock_optimizer,
model_dir=self._model_dir)
self.assertEqual(0, mock_optimizer.minimize.call_count)
# Train for a few steps, and validate optimizer and final checkpoint.
num_steps = 10
est.train(input_fn=lambda: ({
'age': ((age, ), )
}, ((label, ), )),
steps=num_steps)
self.assertEqual(1, mock_optimizer.minimize.call_count)
self._assert_checkpoint(expected_global_step=initial_global_step +
num_steps,
expected_age_weight=age_weight,
expected_bias=bias)
def test_should_error_out_for_not_recognized_args(self):
estimator = linear.LinearClassifier([fc.numeric_column('x')])
def metric_fn(features, not_recognized):
_, _ = features, not_recognized
return {}
with self.assertRaisesRegexp(ValueError, 'not_recognized'):
estimator = extenders.add_metrics(estimator, metric_fn)
def test_all_args_are_optional(self):
input_fn = get_input_fn(x=[[[0.]]], y=[[[1]]])
estimator = linear.LinearClassifier([fc.numeric_column('x')])
def metric_fn():
return {'two': metrics_lib.mean(constant_op.constant([2.]))}
estimator = extenders.add_metrics(estimator, metric_fn)
estimator.train(input_fn=input_fn)
metrics = estimator.evaluate(input_fn=input_fn)
self.assertEqual(2., metrics['two'])
def test_overrides_existing_metrics(self):
input_fn = get_input_fn(x=[[[0.]]], y=[[[1]]])
estimator = linear.LinearClassifier([fc.numeric_column('x')])
estimator.train(input_fn=input_fn)
metrics = estimator.evaluate(input_fn=input_fn)
self.assertNotEqual(2., metrics['auc'])
def metric_fn():
return {'auc': metrics_lib.mean(constant_op.constant([2.]))}
estimator = extenders.add_metrics(estimator, metric_fn)
metrics = estimator.evaluate(input_fn=input_fn)
self.assertEqual(2., metrics['auc'])
def testFromScratchWithDefaultOptimizer(self):
n_classes = self._n_classes
label = 0
age = 17
est = linear.LinearClassifier(
feature_columns=(feature_column_lib.numeric_column('age'),),
n_classes=n_classes,
model_dir=self._model_dir)
# Train for a few steps, and validate final checkpoint.
num_steps = 10
est.train(
input_fn=lambda: ({'age': ((age,),)}, ((label,),)), steps=num_steps)
self._assert_checkpoint(num_steps)
def test_forward_in_exported_sparse(self):
features_columns = [
fc.indicator_column(
fc.categorical_column_with_vocabulary_list('x', range(10)))
]
classifier = linear.LinearClassifier(feature_columns=features_columns)
def train_input_fn():
dataset = dataset_ops.Dataset.from_tensors({
'x':
sparse_tensor.SparseTensor(values=[1, 2, 3],
indices=[[0, 0], [1, 0], [1, 1]],
dense_shape=[2, 2]),
'labels': [[0], [1]]
})
def _split(x):
labels = x.pop('labels')
return x, labels
dataset = dataset.map(_split)
return dataset
classifier.train(train_input_fn, max_steps=1)
classifier = extenders.forward_features(classifier,
keys=['x'],
sparse_default_values={'x': 0})
def serving_input_fn():
features_ph = array_ops.placeholder(dtype=dtypes.int32,
name='x',
shape=[None])
features = {'x': layers.dense_to_sparse(features_ph)}
return estimator_lib.export.ServingInputReceiver(
features, {'x': features_ph})
export_dir, tmpdir = self._export_estimator(classifier,
serving_input_fn)
prediction_fn = from_saved_model(export_dir,
signature_def_key='predict')
features = (0, 2)
prediction = prediction_fn({'x': features})
self.assertIn('x', prediction)
self.assertEqual(features, tuple(prediction['x']))
gfile.DeleteRecursively(tmpdir)
def test_all_supported_args_in_different_order(self):
input_fn = get_input_fn(x=[[[0.]]], y=[[[1]]])
estimator = linear.LinearClassifier([fc.numeric_column('x')])
def metric_fn(labels, config, features, predictions):
self.assertIn('x', features)
self.assertIsNotNone(labels)
self.assertIn('logistic', predictions)
self.assertTrue(isinstance(config, run_config.RunConfig))
return {}
estimator = extenders.add_metrics(estimator, metric_fn)
estimator.train(input_fn=input_fn)
estimator.evaluate(input_fn=input_fn)
def test_should_add_metrics(self):
input_fn = get_input_fn(
x=np.arange(4)[:, None, None], y=np.ones(4)[:, None])
estimator = linear.LinearClassifier([fc.numeric_column('x')])
def metric_fn(features):
return {'mean_x': metrics_lib.mean(features['x'])}
estimator = extenders.add_metrics(estimator, metric_fn)
estimator.train(input_fn=input_fn)
metrics = estimator.evaluate(input_fn=input_fn)
self.assertIn('mean_x', metrics)
self.assertEqual(1.5, metrics['mean_x'])
# assert that it keeps original estimators metrics
self.assertIn('auc', metrics)
def testTrainWithOneDimLabel(self):
n_classes = self._n_classes
batch_size = 20
est = linear.LinearClassifier(
feature_columns=(feature_column_lib.numeric_column('age'), ),
n_classes=n_classes,
model_dir=self._model_dir)
data_rank_1 = np.array([0, 1])
self.assertEqual((2, ), data_rank_1.shape)
train_input_fn = numpy_io.numpy_input_fn(x={'age': data_rank_1},
y=data_rank_1,
batch_size=batch_size,
num_epochs=None,
shuffle=True)
est.train(train_input_fn, steps=200)
self._assert_checkpoint(200)
def _LinearClassifier(self, feature_columns):
# Can be used for baseline.
return linear_estimator.LinearClassifier(
n_classes=2, feature_columns=feature_columns)
def _linear_classifier_fn(*args, **kwargs):
return linear.LinearClassifier(*args, **kwargs)
def testFromCheckpointMultiBatch(self):
# Create initial checkpoint.
n_classes = self._n_classes
label = [1, 0]
age = [17, 18.5]
# For binary case, the expected weight has shape (1,1). For multi class
# case, the shape is (1, n_classes). In order to test the weights, set
# weights as 2.0 * range(n_classes).
age_weight = [[2.0]] if n_classes == 2 else (
np.reshape(2.0 * np.array(list(range(n_classes)), dtype=np.float32),
(1, n_classes)))
bias = [-35.0] if n_classes == 2 else [-35.0] * n_classes
initial_global_step = 100
with ops.Graph().as_default():
variables.Variable(age_weight, name=_AGE_WEIGHT_NAME)
variables.Variable(bias, name=_BIAS_NAME)
variables.Variable(
initial_global_step, name=ops.GraphKeys.GLOBAL_STEP,
dtype=dtypes.int64)
_save_variables_to_ckpt(self._model_dir)
# For binary classifer:
# logits = age * age_weight + bias
# logits[0] = 17 * 2. - 35. = -1.
# logits[1] = 18.5 * 2. - 35. = 2.
# loss = sigmoid_cross_entropy(logits, label)
# so, loss[0] = 1 * -log ( sigmoid(-1) ) = 1.3133
# loss[1] = (1 - 0) * -log ( 1- sigmoid(2) ) = 2.1269
# For multi class classifer:
# loss = cross_entropy(logits, label)
# where logits = [17, 18.5] * age_weight + bias and label = [1, 0]
# so, loss = 1 * -log ( soft_max(logits)[label] )
if n_classes == 2:
expected_loss = (1.3133 + 2.1269)
else:
logits = age_weight * np.reshape(age, (2, 1)) + bias
logits_exp = np.exp(logits)
softmax_row_0 = logits_exp[0] / logits_exp[0].sum()
softmax_row_1 = logits_exp[1] / logits_exp[1].sum()
expected_loss_0 = -1 * math.log(softmax_row_0[label[0]])
expected_loss_1 = -1 * math.log(softmax_row_1[label[1]])
expected_loss = expected_loss_0 + expected_loss_1
mock_optimizer = self._mock_optimizer(expected_loss=expected_loss)
est = linear.LinearClassifier(
feature_columns=(feature_column_lib.numeric_column('age'),),
n_classes=n_classes,
optimizer=mock_optimizer,
model_dir=self._model_dir)
self.assertEqual(0, mock_optimizer.minimize.call_count)
# Train for a few steps, and validate optimizer and final checkpoint.
num_steps = 10
est.train(
input_fn=lambda: ({'age': (age)}, (label)),
steps=num_steps)
self.assertEqual(1, mock_optimizer.minimize.call_count)
self._assert_checkpoint(
expected_global_step=initial_global_step + num_steps,
expected_age_weight=age_weight,
expected_bias=bias)