def test_get_metadata(self):
classifier_dnn = tf.estimator.DNNClassifier(
hidden_units=[4],
feature_columns=[
self.age, self.language_indicator, self.language_embedding
])
classifier_dnn.train(input_fn=self._train_input_fn, steps=5)
md_builder = estimator_metadata_builder.EstimatorMetadataBuilder(
classifier_dnn, [self.age], self._get_json_serving_input_fn,
'logits')
expected_md = {
'outputs': {
'logits': {
'output_tensor_name': 'dnn/logits/BiasAdd:0'
}
},
'inputs': {
'age': {
'input_tensor_name':
'dnn/input_from_feature_columns/input_layer/age/'
'ExpandDims:0',
'encoding':
'identity'
}
},
'framework': 'Tensorflow',
'tags': ['explainable_ai_sdk']
}
self._assertMetadataEqualsWithPrefix(expected_md,
md_builder.get_metadata())
def test_save_model_with_metadata_single_encoding_with_crossed(self):
columns = [self.age, self.language_indicator, self.language_x_class]
classifier_linear = tf.estimator.LinearClassifier(columns)
classifier_linear.train(input_fn=self._train_input_fn, steps=5)
md_builder = estimator_metadata_builder.EstimatorMetadataBuilder(
classifier_linear, [self.age, self.language, self.class_identity],
self._get_json_serving_input_fn, 'probabilities')
saved_path = md_builder.save_model_with_metadata(
tf.test.get_temp_dir())
metadata_file_path = os.path.join(saved_path,
b'explanation_metadata.json')
self.assertTrue(os.path.isfile(metadata_file_path))
with tf.io.gfile.GFile(metadata_file_path, 'r') as f:
generated_md = json.load(f)
expected_md = {
'outputs': {
'probabilities': {
'output_tensor_name':
'linear/head/predictions/probabilities:0'
}
},
'inputs': {
'age': {
'input_tensor_name':
'linear/linear_model/linear_model/linear_model/age/'
'ExpandDims:0',
'encoding':
'identity'
},
'language': {
'input_tensor_name':
'linear/linear_model/linear_model/linear_model/class_'
'identity_X_language/to_sparse_input',
'encoding':
'identity',
'indices_tensor_name':
'linear/linear_model/linear_model/linear_model/class_'
'identity_X_language/to_sparse_input',
'dense_shape_tensor_name':
'linear/linear_model/linear_model/linear_model/class_'
'identity_X_language/to_sparse_input'
},
'class_identity': {
'input_tensor_name':
'linear/linear_model/linear_model/linear_model/class_'
'identity_X_language/to_sparse_input_1',
'encoding':
'identity',
'indices_tensor_name':
'linear/linear_model/linear_model/linear_model/class_'
'identity_X_language/to_sparse_input_1',
'dense_shape_tensor_name':
'linear/linear_model/linear_model/linear_model/class_'
'identity_X_language/to_sparse_input_1'
}
},
'framework': 'Tensorflow',
'tags': ['explainable_ai_sdk']
}
self._assertMetadataEqualsWithPrefix(expected_md, generated_md)
def test_save_model_with_metadata_no_encodings(self):
columns = [self.age]
boosted_tree = tf.estimator.BoostedTreesClassifier(columns, 5)
boosted_tree.train(input_fn=self._train_input_fn, steps=5)
md_builder = estimator_metadata_builder.EstimatorMetadataBuilder(
boosted_tree, columns, self._get_json_serving_input_fn, 'logits')
saved_path = md_builder.save_model_with_metadata(
tf.test.get_temp_dir())
metadata_file_path = os.path.join(saved_path,
b'explanation_metadata.json')
self.assertTrue(os.path.isfile(metadata_file_path))
with tf.io.gfile.GFile(metadata_file_path, 'r') as f:
generated_md = json.load(f)
expected_md = {
'outputs': {
'logits': {
'output_tensor_name': 'boosted_trees/BoostedTreesPredict:0'
}
},
'inputs': {
'age': {
'input_tensor_name':
'boosted_trees/transform_features/age/ExpandDims:0',
'encoding': 'identity'
}
},
'framework': 'Tensorflow',
'tags': ['explainable_ai_sdk']
}
self._assertMetadataEqualsWithPrefix(expected_md, generated_md)
def test_save_model_with_metadata_selected_columns(self):
classifier_dnn = tf.estimator.DNNClassifier(
hidden_units=[4],
feature_columns=[
self.age, self.language_indicator, self.language_embedding
])
classifier_dnn.train(input_fn=self._train_input_fn, steps=5)
md_builder = estimator_metadata_builder.EstimatorMetadataBuilder(
classifier_dnn, [self.age],
self._get_json_serving_input_fn, 'logits')
saved_path = md_builder.save_model_with_metadata(tf.test.get_temp_dir())
metadata_file_path = os.path.join(saved_path, b'explanation_metadata.json')
self.assertTrue(os.path.isfile(metadata_file_path))
with tf.io.gfile.GFile(metadata_file_path, 'r') as f:
generated_md = json.load(f)
expected_md = {
'outputs': {
'logits': {
'output_tensor_name': 'dnn/logits/BiasAdd:0'
}
},
'inputs': {
'age': {
'input_tensor_name':
'dnn/input_from_feature_columns/input_layer/age/'
'ExpandDims:0',
'encoding':
'identity'
}
},
'framework': 'Tensorflow',
'tags': ['explainable_ai_sdk']
}
self._assertMetadataEqualsWithPrefix(expected_md, generated_md)
def test_save_model_with_metadata_weighted_column(self):
columns = [self.age, self.language_weighted]
classifier_linear = tf.estimator.LinearClassifier(columns)
classifier_linear.train(input_fn=self._train_input_fn, steps=5)
md_builder = estimator_metadata_builder.EstimatorMetadataBuilder(
classifier_linear, [self.age, self.language],
self._get_weighted_json_serving_input_fn, 'logits')
saved_path = md_builder.save_model_with_metadata(
tf.test.get_temp_dir())
metadata_file_path = os.path.join(saved_path,
b'explanation_metadata.json')
with tf.io.gfile.GFile(metadata_file_path, 'r') as f:
generated_md = json.load(f)
expected_md = {
'outputs': {
'logits': {
'output_tensor_name':
'linear/linear_model/linear_model/linear_model/'
'weighted_sum:0'
}
},
'inputs': {
'age': {
'input_tensor_name':
'linear/linear_model/linear_model/linear_model/age/'
'ExpandDims:0',
'encoding':
'identity'
},
'language': {
'input_tensor_name':
'linear/linear_model/linear_model/linear_model/language_'
'weighted_by_language_weights/to_sparse_input',
'encoding':
'combined_embedding',
'indices_tensor_name':
'linear/linear_model/linear_model/linear_model/language_'
'weighted_by_language_weights/to_sparse_input',
'dense_shape_tensor_name':
'linear/linear_model/linear_model/linear_model/language_'
'weighted_by_language_weights/to_sparse_input',
'encoded_tensor_name':
'linear/linear_model/linear_model/linear_model/language_'
'weighted_by_language_weights/weighted_sum',
'weight_values_name':
'linear/linear_model/linear_model/linear_model/language_'
'weighted_by_language_weights/cond',
'weight_indices_name':
'linear/linear_model/linear_model/linear_model/language_'
'weighted_by_language_weights/cond',
'weight_dense_shape_name':
'linear/linear_model/linear_model/linear_model/language_'
'weighted_by_language_weights/cond'
}
},
'framework': 'Tensorflow',
'tags': ['explainable_ai_sdk']
}
self._assertMetadataEqualsWithPrefix(expected_md, generated_md)
def test_constructor_invalid_duplicate_treatment(self):
with self.assertRaisesRegex(
ValueError, 'Unrecognized treatment option for duplicates'):
estimator_metadata_builder.EstimatorMetadataBuilder(
tf.estimator.LinearClassifier([self.age, self.language]),
[self.age, self.language],
self._get_json_serving_input_fn,
'logits',
duplicate_feature_treatment='invalid')
def test_save_model_with_metadata_estimator_saved(self):
classifier_linear = tf.estimator.LinearClassifier([self.age, self.language])
classifier_linear.train(input_fn=self._train_input_fn, steps=5)
md_builder = estimator_metadata_builder.EstimatorMetadataBuilder(
classifier_linear, [self.age, self.language],
self._get_weighted_json_serving_input_fn, 'logits')
saved_path = md_builder.save_model_with_metadata(tf.test.get_temp_dir())
proto_file_path = os.path.join(saved_path, b'saved_model.pb')
self.assertTrue(os.path.isfile(proto_file_path))
variables_dir_path = os.path.join(saved_path, b'variables')
self.assertTrue(os.path.isdir(variables_dir_path))
def test_save_model_with_metadata_duplica_treatment(
self, treatment, expected_md):
dnn_columns = [self.age, self.class_id_indicator]
wide_columns = [self.age, self.class_identity]
wide_deep_classifier = tf.estimator.DNNLinearCombinedClassifier(
linear_feature_columns=wide_columns,
dnn_feature_columns=dnn_columns,
dnn_hidden_units=[4, 2])
wide_deep_classifier.train(input_fn=self._train_input_fn, steps=5)
md_builder = estimator_metadata_builder.EstimatorMetadataBuilder(
wide_deep_classifier,
wide_columns,
self._get_weighted_json_serving_input_fn,
'logits',
duplicate_feature_treatment=treatment)
saved_path = md_builder.save_model_with_metadata(
tf.test.get_temp_dir())
metadata_file_path = os.path.join(saved_path,
b'explanation_metadata.json')
self.assertTrue(os.path.isfile(metadata_file_path))
with tf.io.gfile.GFile(metadata_file_path, 'r') as f:
generated_md = json.load(f)
self._assertMetadataEqualsWithPrefix(expected_md, generated_md)
def test_constructor_none_estimator(self):
with self.assertRaisesRegex(ValueError,
'A valid estimator needs to be provided'):
estimator_metadata_builder.EstimatorMetadataBuilder(
None, [], self._get_json_serving_input_fn, 'logits')
def test_constructor_empty_feature_columns(self):
with self.assertRaisesRegex(ValueError,
'feature_columns cannot be empty'):
estimator_metadata_builder.EstimatorMetadataBuilder(
tf.estimator.LinearClassifier([self.age, self.language]), [],
self._get_json_serving_input_fn, 'logits')
def test_save_model_with_metadata_multiple_tensor_groups(self):
dnn_columns = [self.age, self.class_id_indicator]
wide_columns = [self.age, self.class_identity]
wide_deep_classifier = tf.estimator.DNNLinearCombinedClassifier(
linear_feature_columns=wide_columns,
dnn_feature_columns=dnn_columns,
dnn_hidden_units=[4, 2])
wide_deep_classifier.train(input_fn=self._train_input_fn, steps=5)
md_builder = estimator_metadata_builder.EstimatorMetadataBuilder(
wide_deep_classifier, wide_columns,
self._get_weighted_json_serving_input_fn, 'logits')
saved_path = md_builder.save_model_with_metadata(
tf.test.get_temp_dir())
metadata_file_path = os.path.join(saved_path,
b'explanation_metadata.json')
self.assertTrue(os.path.isfile(metadata_file_path))
with tf.io.gfile.GFile(metadata_file_path, 'r') as f:
generated_md = json.load(f)
expected_md = {
'outputs': {
'logits': {
'output_tensor_name': 'add:0'
}
},
'inputs': {
'age_dnn': {
'input_tensor_name':
'dnn/input_from_feature_columns/input_layer/age/'
'ExpandDims:0',
'encoding':
'identity'
},
'age_linear': {
'input_tensor_name':
'linear/linear_model/linear_model/linear_model/age/'
'ExpandDims:0',
'encoding':
'identity'
},
'class_identity_dnn': {
'input_tensor_name':
'dnn/input_from_feature_columns/input_layer/class_identity_'
'indicator/',
'encoding':
'combined_embedding',
'indices_tensor_name':
'dnn/input_from_feature_columns/input_layer/class_identity_'
'indicator/to_sparse_input/',
'dense_shape_tensor_name':
'dnn/input_from_feature_columns/input_layer/class_identity_'
'indicator/to_sparse_input/',
'encoded_tensor_name':
'dnn/input_from_feature_columns/input_layer/class_identity_'
'indicator/'
},
'class_identity_linear': {
'input_tensor_name':
'linear/linear_model/linear_model/linear_model/class_'
'identity/',
'encoding':
'combined_embedding',
'indices_tensor_name':
'linear/linear_model/linear_model/linear_model/class_'
'identity/to_sparse_input/',
'dense_shape_tensor_name':
'linear/linear_model/linear_model/linear_model/class_'
'identity/to_sparse_input/',
'encoded_tensor_name':
'linear/linear_model/linear_model/linear_model/class_'
'identity/weighted_sum/'
}
},
'framework': 'Tensorflow',
'tags': ['explainable_ai_sdk']
}
self._assertMetadataEqualsWithPrefix(expected_md, generated_md)
def test_save_model_with_metadata_groups_duplicates(self):
dnn_columns = [self.age, self.class_id_indicator]
wide_columns = [self.age, self.class_identity]
wide_deep_classifier = tf.estimator.DNNLinearCombinedClassifier(
linear_feature_columns=wide_columns,
dnn_feature_columns=dnn_columns,
dnn_hidden_units=[4, 2])
wide_deep_classifier.train(input_fn=self._train_input_fn, steps=5)
md_builder = estimator_metadata_builder.EstimatorMetadataBuilder(
wide_deep_classifier, wide_columns,
self._get_weighted_json_serving_input_fn, 'logits')
saved_path = md_builder.save_model_with_metadata(
tf.test.get_temp_dir(), group_duplicate_features=True)
metadata_file_path = os.path.join(saved_path,
b'explanation_metadata.json')
self.assertTrue(os.path.isfile(metadata_file_path))
with tf.io.gfile.GFile(metadata_file_path, 'r') as f:
generated_md = json.load(f)
# Note that age and class_identity are single feature inputs
# here in the metadata, unlike in the test
# 'test_save_model_with_metadata_multiple_tensor_groups' where we don't
# drop duplicate inputs for the feature columns 'age' and 'class_identity'.
# Specifically, we have 'age' and 'class_identity' singular entries here
# below, instead of 'age_dnn', 'age_linear', 'class_identity_dnn' and
# 'class_identity_linear' when the parameter drop_duplicate_features is set
# to False.
expected_md = {
'outputs': {
'logits': {
'output_tensor_name': 'add:0'
}
},
'inputs': {
'age_dnn': {
'input_tensor_name':
'dnn/input_from_feature_columns/input_layer/age/'
'ExpandDims:0',
'encoding':
'identity',
'group_name':
'age'
},
'age_linear': {
'input_tensor_name':
'linear/linear_model/linear_model/linear_model/age/'
'ExpandDims:0',
'encoding':
'identity',
'group_name':
'age'
},
'class_identity_dnn': {
'input_tensor_name':
'dnn/input_from_feature_columns/input_layer/class_identity_'
'indicator/',
'encoding':
'combined_embedding',
'indices_tensor_name':
'dnn/input_from_feature_columns/input_layer/class_identity_'
'indicator/to_sparse_input/',
'dense_shape_tensor_name':
'dnn/input_from_feature_columns/input_layer/class_identity_'
'indicator/to_sparse_input/',
'encoded_tensor_name':
'dnn/input_from_feature_columns/input_layer/class_identity_'
'indicator/',
'group_name':
'class_identity'
},
'class_identity_linear': {
'input_tensor_name':
'linear/linear_model/linear_model/linear_model/class_'
'identity/',
'encoding':
'combined_embedding',
'indices_tensor_name':
'linear/linear_model/linear_model/linear_model/class_'
'identity/to_sparse_input/',
'dense_shape_tensor_name':
'linear/linear_model/linear_model/linear_model/class_'
'identity/to_sparse_input/',
'encoded_tensor_name':
'linear/linear_model/linear_model/linear_model/class_'
'identity/weighted_sum/',
'group_name':
'class_identity'
}
},
'framework': 'Tensorflow',
'tags': ['explainable_ai_sdk']
}
self._assertMetadataEqualsWithPrefix(expected_md, generated_md)
