def test_set_numeric_metadata_no_input(self): builder = keras_metadata_builder.KerasGraphMetadataBuilder( self.seq_model) with self.assertRaisesRegex(ValueError, 'Input .* does not exist.'): builder.set_numeric_metadata(tf.placeholder(shape=(1, ), dtype=tf.int32), name='not_input')
def test_get_metadata_functional(self): inputs1 = keras.Input(shape=(10, ), name='model_input1') inputs2 = keras.Input(shape=(10, ), name='model_input2') x = keras.layers.Dense(32, activation='relu')(inputs1) x = keras.layers.Dense(32, activation='relu')(x) x = keras.layers.concatenate([x, inputs2]) outputs = keras.layers.Dense(1, activation='sigmoid')(x) fun_model = keras.Model(inputs=[inputs1, inputs2], outputs=outputs, name='fun') builder = keras_metadata_builder.KerasGraphMetadataBuilder(fun_model) generated_md = builder.get_metadata() expected_md = { 'inputs': { 'model_input1': { 'input_tensor_name': 'model_input1:0', 'modality': 'numeric', 'encoding': 'identity' }, 'model_input2': { 'input_tensor_name': 'model_input2:0', 'modality': 'numeric', 'encoding': 'identity' } }, 'outputs': { 'dense_2/Sigmoid': { 'output_tensor_name': 'dense_2/Sigmoid:0' } }, 'framework': 'Tensorflow', 'tags': ['explainable_ai_sdk'] } self.assertDictEqual(expected_md, generated_md)
def test_set_image_metadata(self): builder = keras_metadata_builder.KerasGraphMetadataBuilder( self.seq_model) builder.set_image_metadata( self.seq_model.inputs[0], name='image_input', visualization=parameters.VisualizationParameters( parameters.VisualizationType.PIXELS), domain=parameters.DomainInfo(0.1, 0.2, 0.3)) generated_md = builder.get_metadata() expected_inputs = { 'image_input': { 'input_tensor_name': 'dense_input:0', 'encoding': 'identity', 'modality': 'image', 'visualization': { 'type': 'pixels' }, 'domain': { 'min': 0.1, 'max': 0.2, 'original_mean': 0.3 } } } self.assertDictEqual(expected_inputs, generated_md['inputs'])
def test_add_output_metadata_already_exists(self): builder = keras_metadata_builder.KerasGraphMetadataBuilder( self.seq_model) with self.assertRaisesRegex(ValueError, 'Only one output can be added.'): builder.add_output_metadata(tf.placeholder(shape=(1, ), dtype=tf.int32), name='output')
def test_set_output_metadata(self): builder = keras_metadata_builder.KerasGraphMetadataBuilder( self.seq_model) builder.set_output_metadata(self.seq_model.outputs[0], name='logits') generated_md = builder.get_metadata() expected_outputs = { 'logits': { 'output_tensor_name': 'dense_2/Sigmoid:0', } } self.assertDictEqual(expected_outputs, generated_md['outputs'])
def test_get_metadata_multiple_outputs_incorrect_output(self): inputs1 = keras.Input(shape=(10, ), name='model_input') x = keras.layers.Dense(32, activation='relu')(inputs1) x = keras.layers.Dense(32, activation='relu')(x) outputs1 = keras.layers.Dense(1, activation='sigmoid')(x) outputs2 = keras.layers.Dense(1, activation='relu')(x) fun_model = keras.Model(inputs=[inputs1], outputs=[outputs1, outputs2], name='fun') with self.assertRaisesRegex( ValueError, 'Provided output is not one of model outputs'): keras_metadata_builder.KerasGraphMetadataBuilder( fun_model, outputs_to_explain=[fun_model.layers[0].output])
def test_set_text_metadata(self): builder = keras_metadata_builder.KerasGraphMetadataBuilder( self.seq_model) builder.set_text_metadata(self.seq_model.inputs[0], name='text_input', encoded_layer=self.seq_model.layers[1], encoding='concat_embedding') generated_md = builder.get_metadata() expected_inputs = { 'text_input': { 'input_tensor_name': 'dense_input:0', 'encoding': 'concat_embedding', 'encoded_tensor_name': 'dense_1/Relu:0', 'modality': 'text' } } self.assertDictEqual(expected_inputs, generated_md['inputs'])
def test_set_numeric_metadata(self): builder = keras_metadata_builder.KerasGraphMetadataBuilder( self.seq_model) builder.set_numeric_metadata(self.seq_model.inputs[0], name='numeric_input', input_baselines=[1], index_feature_mapping=['feat1', 'feat2']) generated_md = builder.get_metadata() expected_inputs = { 'numeric_input': { 'input_tensor_name': 'dense_input:0', 'input_baselines': [1], 'encoding': 'bag_of_features', 'modality': 'numeric', 'index_feature_mapping': ['feat1', 'feat2'] } } self.assertDictEqual(expected_inputs, generated_md['inputs'])
def test_save_model_with_metadata(self): tf.reset_default_graph() model = keras.models.Sequential() model.add(keras.layers.Dense(32, activation='relu', input_dim=10)) model.add(keras.layers.Dense(1, activation='sigmoid')) builder = keras_metadata_builder.KerasGraphMetadataBuilder(model) model_path = os.path.join(tf.test.get_temp_dir(), 'keras_model') builder.save_model_with_metadata(model_path) metadata_file_path = os.path.join(model_path, 'explanation_metadata.json') saved_model_file_path = os.path.join(model_path, 'saved_model.pb') self.assertTrue(os.path.isfile(metadata_file_path)) with tf.io.gfile.GFile(metadata_file_path, 'r') as f: written_md = json.load(f) self.assertDictEqual(builder.get_metadata(), written_md) self.assertTrue(os.path.isfile(saved_model_file_path))
def test_set_image_metadata(self): builder = keras_metadata_builder.KerasGraphMetadataBuilder( self.seq_model) builder.set_image_metadata(self.seq_model.inputs[0], name='image_input', visualization={'type': 'Pixels'}) generated_md = builder.get_metadata() expected_inputs = { 'image_input': { 'input_tensor_name': 'dense_input:0', 'encoding': 'identity', 'modality': 'image', 'visualization': { 'type': 'Pixels' } } } self.assertDictEqual(expected_inputs, generated_md['inputs'])
def test_get_metadata_multiple_outputs(self): inputs1 = keras.Input(shape=(10, ), name='model_input') x = keras.layers.Dense(32, activation='relu')(inputs1) x = keras.layers.Dense(32, activation='relu')(x) outputs1 = keras.layers.Dense(1, activation='sigmoid')(x) outputs2 = keras.layers.Dense(1, activation='relu')(x) fun_model = keras.Model(inputs=[inputs1], outputs=[outputs1, outputs2], name='fun') builder = keras_metadata_builder.KerasGraphMetadataBuilder( fun_model, outputs_to_explain=[fun_model.outputs[0]]) generated_md = builder.get_metadata() expected_outputs = { 'dense_2/Sigmoid': { 'output_tensor_name': 'dense_2/Sigmoid:0' } } self.assertDictEqual(expected_outputs, generated_md['outputs'])
def test_add_numeric_metadata(self): builder = keras_metadata_builder.KerasGraphMetadataBuilder( self.seq_model) builder.add_numeric_metadata(tf.placeholder(shape=(1, ), dtype=tf.int32), name='new_input') generated_md = builder.get_metadata() expected_inputs = { 'dense_input': { 'input_tensor_name': 'dense_input:0', 'modality': 'numeric', 'encoding': 'identity' }, 'new_input': { 'input_tensor_name': 'Placeholder:0', 'modality': 'numeric', 'encoding': 'identity' } } self.assertDictEqual(expected_inputs, generated_md['inputs'])
def test_get_metadata_sequential(self): builder = keras_metadata_builder.KerasGraphMetadataBuilder( self.seq_model) generated_md = builder.get_metadata() expected_md = { 'outputs': { 'dense_2/Sigmoid': { 'output_tensor_name': 'dense_2/Sigmoid:0' } }, 'inputs': { 'dense_input': { 'input_tensor_name': 'dense_input:0', 'modality': 'numeric', 'encoding': 'identity' } }, 'framework': 'Tensorflow', 'tags': ['explainable_ai_sdk'] } self.assertDictEqual(expected_md, generated_md)
def test_get_metadata_subclassed_model(self): class MyModel(keras.Model): def __init__(self, num_classes=2): super(MyModel, self).__init__(name='my_model') self.num_classes = num_classes self.dense_1 = keras.layers.Dense(32, activation='relu') self.dense_2 = keras.layers.Dense(num_classes, activation='sigmoid') def call(self, inputs): x = self.dense_1(inputs) return self.dense_2(x) subclassed_model = MyModel() subclassed_model.compile(loss='categorical_crossentropy') x_train = np.random.random((1, 100)) y_train = np.random.randint(2, size=(1, 2)) subclassed_model.fit(x_train, y_train, batch_size=1, epochs=1) builder = keras_metadata_builder.KerasGraphMetadataBuilder( subclassed_model) generated_md = builder.get_metadata() expected_md = { 'inputs': { 'input_1': { 'input_tensor_name': 'input_1:0', 'modality': 'numeric', 'encoding': 'identity' } }, 'outputs': { 'my_model/dense_1/Sigmoid': { 'output_tensor_name': 'my_model/dense_1/Sigmoid:0' } }, 'framework': 'Tensorflow', 'tags': ['explainable_ai_sdk'] } self.assertDictEqual(expected_md, generated_md)
def test_get_metadata_not_inferred(self): builder = keras_metadata_builder.KerasGraphMetadataBuilder( self.seq_model, auto_infer=False) builder.add_numeric_metadata(self.seq_model.layers[1].input) builder.add_output_metadata(self.seq_model.layers[2].output) generated_md = builder.get_metadata() expected_md = { 'inputs': { 'dense/Relu': { 'encoding': 'identity', 'input_tensor_name': 'dense/Relu:0', 'modality': 'numeric' } }, 'outputs': { 'dense_2/Sigmoid': { 'output_tensor_name': 'dense_2/Sigmoid:0' } }, 'framework': 'Tensorflow', 'tags': ['explainable_ai_sdk'] } self.assertDictEqual(expected_md, generated_md)
def test_remove_output_metadata_valid(self): builder = keras_metadata_builder.KerasGraphMetadataBuilder( self.seq_model) builder.remove_output_metadata(self.seq_model.outputs[0]) generated_md = builder.get_metadata() self.assertEmpty(generated_md['outputs'])
def test_remove_output_metadata_non_existent(self): builder = keras_metadata_builder.KerasGraphMetadataBuilder( self.seq_model) with self.assertRaisesRegex(ValueError, 'Output .* does not exist.'): builder.remove_output_metadata( tf.placeholder(shape=(1, ), dtype=tf.int32))