def test_restore(self):
# Create a test model.
test_model_path = os.path.join(tempfile.mkdtemp(dir=FLAGS.test_tmpdir),
'checkpoint')
tf.keras.backend.set_learning_phase(0)
test_model = tf.keras.Sequential([
tf.keras.layers.Dense(units=3,
input_shape=(3, ),
activation='relu')
])
test_model.compile(optimizer=tf.keras.optimizers.Adam(1e-3),
loss=tf.keras.losses.CategoricalCrossentropy(),
metrics=['accuracy'])
# The model saved using simple_save is used to create the frozen_graph
tf.saved_model.simple_save(session=tf.keras.backend.get_session(),
export_dir=test_model_path,
inputs={'input': test_model.inputs[0]},
outputs={'output': test_model.outputs[0]})
session = utils.restore_model(model_path=os.path.join(test_model_path))
np.testing.assert_allclose(
session.run('dense/BiasAdd:0',
feed_dict={
'dense_input:0': np.ones((1, 3))
}).shape, (1, 3))
session.close()
def test_formulate_smt_constraints_convolution_layer_text(self):
with self.test_session():
# Temporary graphs should be created inside a session. Notice multiple
# graphs are being created in this particular code. So, if each graph
# isn't created inside a separate session, the tensor names will have
# unwanted integer suffices, which then would cause problems while
# accessing tensors by name.
_create_temporary_tf_graph_text_cnn(self.test_model_path)
# The 1st convolution layer has 12 neurons.
image = np.ones(5)
tensor_names = {
'input': 'input_1:0',
'embedding': 'embedding/embedding_lookup/Identity_1:0',
'first_layer': 'conv1d/BiasAdd:0',
'first_layer_relu': 'conv1d/Relu:0',
'logits': 'dense/BiasAdd:0',
'softmax': 'dense/Sigmoid:0',
'weights_layer_1': 'conv1d/conv1d/ExpandDims_1:0',
'biases_layer_1': 'conv1d/BiasAdd/ReadVariableOp:0'
}
session = utils.restore_model(self.test_model_path)
cnn_predictions = session.run(
tensor_names,
feed_dict={tensor_names['input']: image.reshape(1, 5)})
text_embedding = masking._remove_batch_axis(
cnn_predictions['embedding'])
z3_mask = [
z3.Int('mask_%d' % i) for i in range(text_embedding.shape[0])
]
masked_input = []
for mask_bit, embedding_row in zip(z3_mask, text_embedding):
masked_input.append(
[z3.ToReal(mask_bit) * i for i in embedding_row])
first_layer_activations = masking._reorder(
masking._remove_batch_axis(
cnn_predictions['first_layer'])).reshape(-1)
z3_optimizer = masking._formulate_smt_constraints_convolution_layer(
z3_optimizer=utils.TextOptimizer(z3_mask=z3_mask),
kernels=masking._reshape_kernels(
kernels=cnn_predictions['weights_layer_1'],
model_type='text_cnn'),
biases=cnn_predictions['biases_layer_1'],
chosen_indices=first_layer_activations.argsort()[-5:],
conv_activations=first_layer_activations,
input_activation_maps=[masked_input],
output_activation_map_shape=masking._get_activation_map_shape(
activation_maps_shape=cnn_predictions['first_layer'].shape,
model_type='text_cnn'),
strides=1,
padding=(0, 0),
gamma=0.5)
mask, result = z3_optimizer.generate_mask()
self.assertEqual(result, 'sat')
self.assertEqual(mask.shape, (5, ))
session.close()
def test_formulate_smt_constraints_convolution_layer(self):
with self.test_session():
# Temporary graphs should be created inside a session. Notice multiple
# graphs are being created in this particular code. So, if each graph
# isn't created inside a separate session, the tensor names will have
# unwanted integer suffices, which then would cause problems while
# accessing tensors by name.
_create_temporary_tf_graph_cnn(self.test_model_path)
image_edge_length = 4
image_channels = 3
# The 1st convolution layer has 48 neurons.
top_k = np.random.randint(low=1, high=48)
image = np.ones((image_edge_length, image_edge_length, image_channels))
tensor_names = {
'input': 'conv2d_input:0',
'first_layer': 'conv2d/BiasAdd:0',
'first_layer_relu': 'conv2d/Relu:0',
'logits': 'dense/BiasAdd:0',
'softmax': 'dense/Softmax:0',
'weights_layer_1': 'conv2d/Conv2D/ReadVariableOp:0',
'biases_layer_1': 'conv2d/bias/Read/ReadVariableOp:0'}
session = utils.restore_model(self.test_model_path)
cnn_predictions = session.run(
tensor_names,
feed_dict={
tensor_names['input']: image.reshape(
(1, image_edge_length, image_edge_length, image_channels))})
z3_mask = [_get_z3_var(index=i) for i in range(image_edge_length ** 2)]
first_layer_activations = masking._reorder(masking._remove_batch_axis(
cnn_predictions['first_layer'])).reshape(-1)
masked_input = masking._encode_input(
image=image, z3_mask=z3_mask, window_size=1)
z3_optimizer = masking._formulate_smt_constraints_convolution_layer(
z3_optimizer=utils.ImageOptimizer(
z3_mask=z3_mask,
window_size=1,
edge_length=image_edge_length),
kernels=masking._reorder(cnn_predictions['weights_layer_1']),
biases=cnn_predictions['biases_layer_1'],
chosen_indices=first_layer_activations.argsort()[-top_k:],
conv_activations=first_layer_activations,
input_activation_maps=masked_input,
output_activation_map_shape=(image_edge_length, image_edge_length),
strides=1,
padding=(0, 1),
gamma=0.5)
mask, result = z3_optimizer.generate_mask()
self.assertEqual(result, 'sat')
self.assertEqual(mask.shape, (image_edge_length, image_edge_length))
session.close()
def _process_text(image, run_params):
"""Generates the masked embedding and does a forward pass of the image.
Args:
image: float numpy array with shape (num_words,), text to be masked.
run_params: RunParams with model_type, model_path, image_placeholder_shape,
activations, tensor_names, input, first_layer, logits.
Returns:
masked_input: nested list of z3.ExprRef with dimensions
(1, num_words, num_latent_dimensions).
unmasked_predictions: dict,
* input: float numpy array, the input tensor to the neural network.
* first_layer: float numpy array, the first layer tensor in the neural
network.
* first_layer_relu: str, the first layer relu activation
tensor in the neural network.
* logits: str, the logits tensor in the neural network.
* softmax: float numpy array, the softmax tensor in the neural network.
* weights_layer_1: float numpy array, the first layer fc / conv weights.
* biases_layer_1: float numpy array, the first layer fc / conv biases.
* (text only) embedding: float numpy array with shape (num_words,
num_latent_dimensions), the embedding layer.
session: tf.Session, tensorflow session with the loaded neural network.
optimizer: utils.TextOptimizer, z3 optimizer for image.
Raises:
ValueError: Raises an error if the text isn't a 1D array.
"""
if image.ndim != 1:
raise ValueError('The text input should be a 1D numpy array. '
'Shape of the received input: %s' % str(image.shape))
session = utils.restore_model(run_params.model_path)
unmasked_predictions = session.run(
run_params.tensor_names, feed_dict={
run_params.tensor_names['input']: image.reshape(
run_params.image_placeholder_shape)})
text_embedding = _remove_batch_axis(unmasked_predictions['embedding'])
# text_embedding has a shape (num_words, num_latent_dimensions)
z3_mask = [z3.Int('mask_%d' % i) for i in range(text_embedding.shape[0])]
# masked_input has a shape (num_words, num_latent_dimensions)
masked_input = []
for mask_bit, embedding_row in zip(z3_mask, text_embedding):
masked_input.append([z3.ToReal(mask_bit) * i for i in embedding_row])
return ([masked_input], unmasked_predictions, session,
utils.TextOptimizer(z3_mask=z3_mask))
def test_get_saliency_map(self, saliency_method, saliency_map_shape):
with self.test_session():
# Temporary graphs should be created inside a session. Notice multiple
# graphs are being created in this particular code. So, if each graph
# isn't created inside a separate session, the tensor names will have
# unwanted integer suffices, which then would cause problems while
# accessing tensors by name.
_create_temporary_tf_graph_cnn(self.test_model_path)
self.assertEqual(
masking.get_saliency_map(
session=utils.restore_model(self.test_model_path),
features=np.random.rand(4, 4, 3),
saliency_method=saliency_method,
label=0,
input_tensor_name='conv2d_input:0',
output_tensor_name='dense/Softmax:0',
).shape, saliency_map_shape)
def _process_image(image, run_params, window_size):
"""Generates the masked input and does a forward pass of the image.
Args:
image: float numpy array with shape (image_edge_length,
image_edge_length, image_channels), image to be masked. For MNIST,
the pixel values are between [0, 1] and for Imagenet, the pixel
values are between [-117, 138].
run_params: RunParams with model_type, model_path, image_placeholder_shape,
activations, tensor_names, input, first_layer, logits.
window_size: int, side length of the square mask.
Returns:
masked_input: nested list of z3.ExprRef with dimensions
(image_channels, image_edge_length, image_edge_length)
unmasked_predictions: dict,
* input: float numpy array, the input tensor to the neural network.
* first_layer: float numpy array, the first layer tensor in the neural
network.
* first_layer_relu: str, the first layer relu activation
tensor in the neural network.
* logits: str, the logits tensor in the neural network.
* softmax: float numpy array, the softmax tensor in the neural network.
* weights_layer_1: float numpy array, the first layer fc / conv weights.
* biases_layer_1: float numpy array, the first layer fc / conv biases.
session: tf.Session, tensorflow session with the loaded neural network.
optimizer: utils.ImageOptimizer, z3 optimizer for image.
"""
_verify_image_dimensions(image)
image_edge_length, _, _ = image.shape
num_masks_along_row = image_edge_length // window_size
# We always find a 2d mask irrespective of the number of image channels.
z3_mask = [z3.Int('mask_%d' % i) for i in range(num_masks_along_row ** 2)]
session = utils.restore_model(run_params.model_path)
unmasked_predictions = session.run(
run_params.tensor_names,
feed_dict={run_params.tensor_names['input']: image.reshape(
run_params.image_placeholder_shape)})
# _encode_input generates a masked_input with a shape
# (image_channels, image_edge_length, image_edge_length)
return (_encode_input(image=image, z3_mask=z3_mask, window_size=window_size),
unmasked_predictions, session,
utils.ImageOptimizer(z3_mask=z3_mask, window_size=window_size,
edge_length=image_edge_length))
def get_no_minimization_mask(image, label_index, top_k, run_params,
sum_attributions=False):
"""Generates a no minimization mask for a given image.
Args:
image:
* image: float numpy array with shape (image_edge_length,
image_edge_length, image_channels), image to be masked. For MNIST,
the pixel values are between [0, 1] and for Imagenet, the pixel
values are between [-117, 138].
* text: float numpy array with shape (num_words,), text to be masked.
label_index: int, index of the label of the training image.
top_k: int, constrain the nodes with top k activations in the first hidden
layer.
run_params: RunParams with model_type, model_path, image_placeholder_shape,
padding, strides, tensor_names.
sum_attributions: bool, If true, the attribution of a pixel is the sum of
all the attributions of the equations in which the masking variable
appears. Otherwise, it is the max of all the attributions of the
equations in which the masking variable appears.
Returns:
float numpy array with shape (num_rows, num_cols), no minimization mask.
"""
session = utils.restore_model(run_params.model_path)
unmasked_predictions = session.run(
run_params.tensor_names,
feed_dict={run_params.tensor_names['input']: image.reshape(
run_params.image_placeholder_shape)})
chosen_indices = _sort_indices(
unmasked_predictions=unmasked_predictions,
score_method='integrated_gradients',
session=session,
image=image,
run_params=run_params,
label_index=label_index)
scores = _reorder(get_saliency_map(
session=session,
features=_remove_batch_axis(
unmasked_predictions['first_layer_relu']),
saliency_method='integrated_gradients',
input_tensor_name=run_params.tensor_names['first_layer_relu'],
output_tensor_name=run_params.tensor_names['softmax'],
label=label_index)).reshape(-1)
if run_params.model_type == 'text_cnn':
mask = np.zeros(image.shape[0] + sum(run_params.padding))
else:
mask = np.zeros((image.shape[0] + sum(run_params.padding),
image.shape[1] + sum(run_params.padding)))
num_rows, num_columns = _get_activation_map_shape(
activation_maps_shape=unmasked_predictions['first_layer'].shape,
model_type=run_params.model_type)
kernel_size = unmasked_predictions['weights_layer_1'].shape[1]
for index in chosen_indices[-top_k:]:
_, row, column = _get_hidden_node_location(
flattened_index=index, num_rows=num_rows, num_columns=num_columns)
top = row * run_params.strides
left = column * run_params.strides
if sum_attributions:
if run_params.model_type == 'text_cnn':
mask[top: top + kernel_size] += scores[index]
else:
mask[top: top + kernel_size, left: left + kernel_size] += scores[index]
else:
# The importance of a masking variable is always over-written with
# a higher value as the indices are sorted.
if run_params.model_type == 'text_cnn':
mask[top: top + kernel_size] = scores[index]
else:
mask[top: top + kernel_size, left: left + kernel_size] = scores[index]
session.close()
left_padding = run_params.padding[0]
if run_params.model_type == 'text_cnn':
return mask[left_padding: image.shape[0] + left_padding]
else:
return mask[left_padding: image.shape[0] + left_padding,
left_padding: image.shape[1] + left_padding]
def find_mask_full_encoding(image,
weights,
biases,
run_params,
window_size,
label_index,
delta=0,
timeout=600,
num_unique_solutions=1):
"""Finds a binary mask for a given image and a trained Neural Network.
Args:
image: float numpy array with shape (image_edge_length, image_edge_length,
image_channels), image to be masked. For MNIST, the pixel values are
between [0, 1] and for Imagenet, the pixel values are between
[-117, 138].
weights: list of num_layers float numpy arrays with shape
(output_dim, input_dim), weights of the neural network.
biases: list of num_layers float numpy arrays with shape (output_dim,),
biases of the neural network.
run_params: RunParams with model_type, model_path, image_placeholder_shape,
activations, tensor_names.
window_size: int, side length of the square mask.
label_index: int, index of the label of the training image.
delta: float, logit of the correct label is greater than the rest of the
logit by an amount delta. Its value is always >= 0. It is only used when
constrain_final_layer is True.
timeout: int, solver timeout in seconds.
num_unique_solutions: int, number of unique solutions you want to sample.
Returns:
result: dictionary,
* image: float numpy array with shape
(image_edge_length * image_edge_length * image_channels,)
* combined_solver_runtime: float, time taken by the solver to find all
the solutions.
* unmasked_logits: float numpy array with shape (num_outputs,)
* unmasked_first_layer: float numpy array with shape
(num_hidden_nodes_first_layer,)
* masked_first_layer: list with length num_sols, contains float numpy
array with shape (num_hidden_nodes_first_layer,)
* inv_masked_first_layer: list with length num_sols, contains float numpy
array with shape (num_hidden_nodes_first_layer,)
* masks: list with length num_sols, contains float numpy array
with shape (image_edge_length ** 2,)
* masked_images: list with length num_sols, contains float numpy array
with shape (image_edge_length ** 2,)
* inv_masked_images: list with length num_sols, contains float numpy
array with shape (image_edge_length ** 2,)
* masked_logits: list with length num_sols, contains float numpy array
with shape (num_outputs,)
* inv_masked_logits: list with length num_sols, contains float numpy
array with shape (num_outputs,)
* solver_outputs: list with length num_sols, contains strings
corresponding to every sampled solution saying 'sat', 'unsat' or
'unknown'.
"""
_verify_image_dimensions(image)
image_placeholder_shape = run_params.image_placeholder_shape
tensor_names = run_params.tensor_names
# z3's timeout is in milliseconds
z3.set_option('timeout', timeout * 1000)
image_edge_length, _, _ = image.shape
num_masks_along_row = image_edge_length // window_size
session = utils.restore_model(run_params.model_path)
z3_mask = [z3.Int('mask_%d' % i) for i in range(num_masks_along_row ** 2)]
unmasked_predictions = session.run(
tensor_names,
feed_dict={
tensor_names['input']: image.reshape(image_placeholder_shape)})
smt_output, _ = utils.smt_forward(
features=utils.flatten_nested_lists(_encode_input(
image=image,
z3_mask=z3_mask,
window_size=window_size)),
weights=weights,
biases=biases,
activations=run_params.activations)
z3_optimizer = _formulate_smt_constraints_final_layer(
z3_optimizer=utils.ImageOptimizer(
z3_mask=z3_mask,
window_size=window_size,
edge_length=image_edge_length),
smt_output=smt_output,
delta=delta,
label_index=label_index)
solver_start_time = time.time()
result = collections.defaultdict(list)
# All the masks found in each call of z3_optimizer.generator() is guarranteed
# to be unique since duplicated solutions are blocked. For more details
# refer z3_optimizer.generator().
for mask, solver_output in z3_optimizer.generator(num_unique_solutions):
_record_solution(result=result,
mask=mask,
solver_output=solver_output,
image=image,
session=session,
run_params=run_params)
result.update({
'image': image.reshape(-1),
'combined_solver_runtime': time.time() - solver_start_time,
'unmasked_logits': np.squeeze(unmasked_predictions['logits']),
'unmasked_first_layer': np.squeeze(unmasked_predictions['first_layer'])})
session.close()
return result
