def test_get_metadata(self): classifier_dnn = tf.estimator.DNNClassifier( hidden_units=[4], feature_columns=[ self.age, self.language_indicator, self.language_embedding ]) classifier_dnn.train(input_fn=self._train_input_fn, steps=5) md_builder = estimator_metadata_builder.EstimatorMetadataBuilder( classifier_dnn, [self.age], self._get_json_serving_input_fn, 'logits') expected_md = { 'outputs': { 'logits': { 'output_tensor_name': 'dnn/logits/BiasAdd:0' } }, 'inputs': { 'age': { 'input_tensor_name': 'dnn/input_from_feature_columns/input_layer/age/' 'ExpandDims:0', 'encoding': 'identity' } }, 'framework': 'Tensorflow', 'tags': ['explainable_ai_sdk'] } self._assertMetadataEqualsWithPrefix(expected_md, md_builder.get_metadata())
def test_save_model_with_metadata_single_encoding_with_crossed(self): columns = [self.age, self.language_indicator, self.language_x_class] classifier_linear = tf.estimator.LinearClassifier(columns) classifier_linear.train(input_fn=self._train_input_fn, steps=5) md_builder = estimator_metadata_builder.EstimatorMetadataBuilder( classifier_linear, [self.age, self.language, self.class_identity], self._get_json_serving_input_fn, 'probabilities') saved_path = md_builder.save_model_with_metadata( tf.test.get_temp_dir()) metadata_file_path = os.path.join(saved_path, b'explanation_metadata.json') self.assertTrue(os.path.isfile(metadata_file_path)) with tf.io.gfile.GFile(metadata_file_path, 'r') as f: generated_md = json.load(f) expected_md = { 'outputs': { 'probabilities': { 'output_tensor_name': 'linear/head/predictions/probabilities:0' } }, 'inputs': { 'age': { 'input_tensor_name': 'linear/linear_model/linear_model/linear_model/age/' 'ExpandDims:0', 'encoding': 'identity' }, 'language': { 'input_tensor_name': 'linear/linear_model/linear_model/linear_model/class_' 'identity_X_language/to_sparse_input', 'encoding': 'identity', 'indices_tensor_name': 'linear/linear_model/linear_model/linear_model/class_' 'identity_X_language/to_sparse_input', 'dense_shape_tensor_name': 'linear/linear_model/linear_model/linear_model/class_' 'identity_X_language/to_sparse_input' }, 'class_identity': { 'input_tensor_name': 'linear/linear_model/linear_model/linear_model/class_' 'identity_X_language/to_sparse_input_1', 'encoding': 'identity', 'indices_tensor_name': 'linear/linear_model/linear_model/linear_model/class_' 'identity_X_language/to_sparse_input_1', 'dense_shape_tensor_name': 'linear/linear_model/linear_model/linear_model/class_' 'identity_X_language/to_sparse_input_1' } }, 'framework': 'Tensorflow', 'tags': ['explainable_ai_sdk'] } self._assertMetadataEqualsWithPrefix(expected_md, generated_md)
def test_save_model_with_metadata_no_encodings(self): columns = [self.age] boosted_tree = tf.estimator.BoostedTreesClassifier(columns, 5) boosted_tree.train(input_fn=self._train_input_fn, steps=5) md_builder = estimator_metadata_builder.EstimatorMetadataBuilder( boosted_tree, columns, self._get_json_serving_input_fn, 'logits') saved_path = md_builder.save_model_with_metadata( tf.test.get_temp_dir()) metadata_file_path = os.path.join(saved_path, b'explanation_metadata.json') self.assertTrue(os.path.isfile(metadata_file_path)) with tf.io.gfile.GFile(metadata_file_path, 'r') as f: generated_md = json.load(f) expected_md = { 'outputs': { 'logits': { 'output_tensor_name': 'boosted_trees/BoostedTreesPredict:0' } }, 'inputs': { 'age': { 'input_tensor_name': 'boosted_trees/transform_features/age/ExpandDims:0', 'encoding': 'identity' } }, 'framework': 'Tensorflow', 'tags': ['explainable_ai_sdk'] } self._assertMetadataEqualsWithPrefix(expected_md, generated_md)
def test_save_model_with_metadata_selected_columns(self): classifier_dnn = tf.estimator.DNNClassifier( hidden_units=[4], feature_columns=[ self.age, self.language_indicator, self.language_embedding ]) classifier_dnn.train(input_fn=self._train_input_fn, steps=5) md_builder = estimator_metadata_builder.EstimatorMetadataBuilder( classifier_dnn, [self.age], self._get_json_serving_input_fn, 'logits') saved_path = md_builder.save_model_with_metadata(tf.test.get_temp_dir()) metadata_file_path = os.path.join(saved_path, b'explanation_metadata.json') self.assertTrue(os.path.isfile(metadata_file_path)) with tf.io.gfile.GFile(metadata_file_path, 'r') as f: generated_md = json.load(f) expected_md = { 'outputs': { 'logits': { 'output_tensor_name': 'dnn/logits/BiasAdd:0' } }, 'inputs': { 'age': { 'input_tensor_name': 'dnn/input_from_feature_columns/input_layer/age/' 'ExpandDims:0', 'encoding': 'identity' } }, 'framework': 'Tensorflow', 'tags': ['explainable_ai_sdk'] } self._assertMetadataEqualsWithPrefix(expected_md, generated_md)
def test_save_model_with_metadata_weighted_column(self): columns = [self.age, self.language_weighted] classifier_linear = tf.estimator.LinearClassifier(columns) classifier_linear.train(input_fn=self._train_input_fn, steps=5) md_builder = estimator_metadata_builder.EstimatorMetadataBuilder( classifier_linear, [self.age, self.language], self._get_weighted_json_serving_input_fn, 'logits') saved_path = md_builder.save_model_with_metadata( tf.test.get_temp_dir()) metadata_file_path = os.path.join(saved_path, b'explanation_metadata.json') with tf.io.gfile.GFile(metadata_file_path, 'r') as f: generated_md = json.load(f) expected_md = { 'outputs': { 'logits': { 'output_tensor_name': 'linear/linear_model/linear_model/linear_model/' 'weighted_sum:0' } }, 'inputs': { 'age': { 'input_tensor_name': 'linear/linear_model/linear_model/linear_model/age/' 'ExpandDims:0', 'encoding': 'identity' }, 'language': { 'input_tensor_name': 'linear/linear_model/linear_model/linear_model/language_' 'weighted_by_language_weights/to_sparse_input', 'encoding': 'combined_embedding', 'indices_tensor_name': 'linear/linear_model/linear_model/linear_model/language_' 'weighted_by_language_weights/to_sparse_input', 'dense_shape_tensor_name': 'linear/linear_model/linear_model/linear_model/language_' 'weighted_by_language_weights/to_sparse_input', 'encoded_tensor_name': 'linear/linear_model/linear_model/linear_model/language_' 'weighted_by_language_weights/weighted_sum', 'weight_values_name': 'linear/linear_model/linear_model/linear_model/language_' 'weighted_by_language_weights/cond', 'weight_indices_name': 'linear/linear_model/linear_model/linear_model/language_' 'weighted_by_language_weights/cond', 'weight_dense_shape_name': 'linear/linear_model/linear_model/linear_model/language_' 'weighted_by_language_weights/cond' } }, 'framework': 'Tensorflow', 'tags': ['explainable_ai_sdk'] } self._assertMetadataEqualsWithPrefix(expected_md, generated_md)
def test_constructor_invalid_duplicate_treatment(self): with self.assertRaisesRegex( ValueError, 'Unrecognized treatment option for duplicates'): estimator_metadata_builder.EstimatorMetadataBuilder( tf.estimator.LinearClassifier([self.age, self.language]), [self.age, self.language], self._get_json_serving_input_fn, 'logits', duplicate_feature_treatment='invalid')
def test_save_model_with_metadata_estimator_saved(self): classifier_linear = tf.estimator.LinearClassifier([self.age, self.language]) classifier_linear.train(input_fn=self._train_input_fn, steps=5) md_builder = estimator_metadata_builder.EstimatorMetadataBuilder( classifier_linear, [self.age, self.language], self._get_weighted_json_serving_input_fn, 'logits') saved_path = md_builder.save_model_with_metadata(tf.test.get_temp_dir()) proto_file_path = os.path.join(saved_path, b'saved_model.pb') self.assertTrue(os.path.isfile(proto_file_path)) variables_dir_path = os.path.join(saved_path, b'variables') self.assertTrue(os.path.isdir(variables_dir_path))
def test_save_model_with_metadata_duplica_treatment( self, treatment, expected_md): dnn_columns = [self.age, self.class_id_indicator] wide_columns = [self.age, self.class_identity] wide_deep_classifier = tf.estimator.DNNLinearCombinedClassifier( linear_feature_columns=wide_columns, dnn_feature_columns=dnn_columns, dnn_hidden_units=[4, 2]) wide_deep_classifier.train(input_fn=self._train_input_fn, steps=5) md_builder = estimator_metadata_builder.EstimatorMetadataBuilder( wide_deep_classifier, wide_columns, self._get_weighted_json_serving_input_fn, 'logits', duplicate_feature_treatment=treatment) saved_path = md_builder.save_model_with_metadata( tf.test.get_temp_dir()) metadata_file_path = os.path.join(saved_path, b'explanation_metadata.json') self.assertTrue(os.path.isfile(metadata_file_path)) with tf.io.gfile.GFile(metadata_file_path, 'r') as f: generated_md = json.load(f) self._assertMetadataEqualsWithPrefix(expected_md, generated_md)
def test_constructor_none_estimator(self): with self.assertRaisesRegex(ValueError, 'A valid estimator needs to be provided'): estimator_metadata_builder.EstimatorMetadataBuilder( None, [], self._get_json_serving_input_fn, 'logits')
def test_constructor_empty_feature_columns(self): with self.assertRaisesRegex(ValueError, 'feature_columns cannot be empty'): estimator_metadata_builder.EstimatorMetadataBuilder( tf.estimator.LinearClassifier([self.age, self.language]), [], self._get_json_serving_input_fn, 'logits')
def test_save_model_with_metadata_multiple_tensor_groups(self): dnn_columns = [self.age, self.class_id_indicator] wide_columns = [self.age, self.class_identity] wide_deep_classifier = tf.estimator.DNNLinearCombinedClassifier( linear_feature_columns=wide_columns, dnn_feature_columns=dnn_columns, dnn_hidden_units=[4, 2]) wide_deep_classifier.train(input_fn=self._train_input_fn, steps=5) md_builder = estimator_metadata_builder.EstimatorMetadataBuilder( wide_deep_classifier, wide_columns, self._get_weighted_json_serving_input_fn, 'logits') saved_path = md_builder.save_model_with_metadata( tf.test.get_temp_dir()) metadata_file_path = os.path.join(saved_path, b'explanation_metadata.json') self.assertTrue(os.path.isfile(metadata_file_path)) with tf.io.gfile.GFile(metadata_file_path, 'r') as f: generated_md = json.load(f) expected_md = { 'outputs': { 'logits': { 'output_tensor_name': 'add:0' } }, 'inputs': { 'age_dnn': { 'input_tensor_name': 'dnn/input_from_feature_columns/input_layer/age/' 'ExpandDims:0', 'encoding': 'identity' }, 'age_linear': { 'input_tensor_name': 'linear/linear_model/linear_model/linear_model/age/' 'ExpandDims:0', 'encoding': 'identity' }, 'class_identity_dnn': { 'input_tensor_name': 'dnn/input_from_feature_columns/input_layer/class_identity_' 'indicator/', 'encoding': 'combined_embedding', 'indices_tensor_name': 'dnn/input_from_feature_columns/input_layer/class_identity_' 'indicator/to_sparse_input/', 'dense_shape_tensor_name': 'dnn/input_from_feature_columns/input_layer/class_identity_' 'indicator/to_sparse_input/', 'encoded_tensor_name': 'dnn/input_from_feature_columns/input_layer/class_identity_' 'indicator/' }, 'class_identity_linear': { 'input_tensor_name': 'linear/linear_model/linear_model/linear_model/class_' 'identity/', 'encoding': 'combined_embedding', 'indices_tensor_name': 'linear/linear_model/linear_model/linear_model/class_' 'identity/to_sparse_input/', 'dense_shape_tensor_name': 'linear/linear_model/linear_model/linear_model/class_' 'identity/to_sparse_input/', 'encoded_tensor_name': 'linear/linear_model/linear_model/linear_model/class_' 'identity/weighted_sum/' } }, 'framework': 'Tensorflow', 'tags': ['explainable_ai_sdk'] } self._assertMetadataEqualsWithPrefix(expected_md, generated_md)
def test_save_model_with_metadata_groups_duplicates(self): dnn_columns = [self.age, self.class_id_indicator] wide_columns = [self.age, self.class_identity] wide_deep_classifier = tf.estimator.DNNLinearCombinedClassifier( linear_feature_columns=wide_columns, dnn_feature_columns=dnn_columns, dnn_hidden_units=[4, 2]) wide_deep_classifier.train(input_fn=self._train_input_fn, steps=5) md_builder = estimator_metadata_builder.EstimatorMetadataBuilder( wide_deep_classifier, wide_columns, self._get_weighted_json_serving_input_fn, 'logits') saved_path = md_builder.save_model_with_metadata( tf.test.get_temp_dir(), group_duplicate_features=True) metadata_file_path = os.path.join(saved_path, b'explanation_metadata.json') self.assertTrue(os.path.isfile(metadata_file_path)) with tf.io.gfile.GFile(metadata_file_path, 'r') as f: generated_md = json.load(f) # Note that age and class_identity are single feature inputs # here in the metadata, unlike in the test # 'test_save_model_with_metadata_multiple_tensor_groups' where we don't # drop duplicate inputs for the feature columns 'age' and 'class_identity'. # Specifically, we have 'age' and 'class_identity' singular entries here # below, instead of 'age_dnn', 'age_linear', 'class_identity_dnn' and # 'class_identity_linear' when the parameter drop_duplicate_features is set # to False. expected_md = { 'outputs': { 'logits': { 'output_tensor_name': 'add:0' } }, 'inputs': { 'age_dnn': { 'input_tensor_name': 'dnn/input_from_feature_columns/input_layer/age/' 'ExpandDims:0', 'encoding': 'identity', 'group_name': 'age' }, 'age_linear': { 'input_tensor_name': 'linear/linear_model/linear_model/linear_model/age/' 'ExpandDims:0', 'encoding': 'identity', 'group_name': 'age' }, 'class_identity_dnn': { 'input_tensor_name': 'dnn/input_from_feature_columns/input_layer/class_identity_' 'indicator/', 'encoding': 'combined_embedding', 'indices_tensor_name': 'dnn/input_from_feature_columns/input_layer/class_identity_' 'indicator/to_sparse_input/', 'dense_shape_tensor_name': 'dnn/input_from_feature_columns/input_layer/class_identity_' 'indicator/to_sparse_input/', 'encoded_tensor_name': 'dnn/input_from_feature_columns/input_layer/class_identity_' 'indicator/', 'group_name': 'class_identity' }, 'class_identity_linear': { 'input_tensor_name': 'linear/linear_model/linear_model/linear_model/class_' 'identity/', 'encoding': 'combined_embedding', 'indices_tensor_name': 'linear/linear_model/linear_model/linear_model/class_' 'identity/to_sparse_input/', 'dense_shape_tensor_name': 'linear/linear_model/linear_model/linear_model/class_' 'identity/to_sparse_input/', 'encoded_tensor_name': 'linear/linear_model/linear_model/linear_model/class_' 'identity/weighted_sum/', 'group_name': 'class_identity' } }, 'framework': 'Tensorflow', 'tags': ['explainable_ai_sdk'] } self._assertMetadataEqualsWithPrefix(expected_md, generated_md)