def _construct_optimizer(z3_mask, optimizer_type):
if optimizer_type == 'image':
return utils.ImageOptimizer(z3_mask=z3_mask,
window_size=1,
edge_length=2)
else:
return utils.TextOptimizer(z3_mask=z3_mask)
def test_formulate_smt_constraints_convolution_layer_text(self):
with self.test_session():
# Temporary graphs should be created inside a session. Notice multiple
# graphs are being created in this particular code. So, if each graph
# isn't created inside a separate session, the tensor names will have
# unwanted integer suffices, which then would cause problems while
# accessing tensors by name.
_create_temporary_tf_graph_text_cnn(self.test_model_path)
# The 1st convolution layer has 12 neurons.
image = np.ones(5)
tensor_names = {
'input': 'input_1:0',
'embedding': 'embedding/embedding_lookup/Identity_1:0',
'first_layer': 'conv1d/BiasAdd:0',
'first_layer_relu': 'conv1d/Relu:0',
'logits': 'dense/BiasAdd:0',
'softmax': 'dense/Sigmoid:0',
'weights_layer_1': 'conv1d/conv1d/ExpandDims_1:0',
'biases_layer_1': 'conv1d/BiasAdd/ReadVariableOp:0'
}
session = utils.restore_model(self.test_model_path)
cnn_predictions = session.run(
tensor_names,
feed_dict={tensor_names['input']: image.reshape(1, 5)})
text_embedding = masking._remove_batch_axis(
cnn_predictions['embedding'])
z3_mask = [
z3.Int('mask_%d' % i) for i in range(text_embedding.shape[0])
]
masked_input = []
for mask_bit, embedding_row in zip(z3_mask, text_embedding):
masked_input.append(
[z3.ToReal(mask_bit) * i for i in embedding_row])
first_layer_activations = masking._reorder(
masking._remove_batch_axis(
cnn_predictions['first_layer'])).reshape(-1)
z3_optimizer = masking._formulate_smt_constraints_convolution_layer(
z3_optimizer=utils.TextOptimizer(z3_mask=z3_mask),
kernels=masking._reshape_kernels(
kernels=cnn_predictions['weights_layer_1'],
model_type='text_cnn'),
biases=cnn_predictions['biases_layer_1'],
chosen_indices=first_layer_activations.argsort()[-5:],
conv_activations=first_layer_activations,
input_activation_maps=[masked_input],
output_activation_map_shape=masking._get_activation_map_shape(
activation_maps_shape=cnn_predictions['first_layer'].shape,
model_type='text_cnn'),
strides=1,
padding=(0, 0),
gamma=0.5)
mask, result = z3_optimizer.generate_mask()
self.assertEqual(result, 'sat')
self.assertEqual(mask.shape, (5, ))
session.close()
def _process_text(image, run_params):
"""Generates the masked embedding and does a forward pass of the image.
Args:
image: float numpy array with shape (num_words,), text to be masked.
run_params: RunParams with model_type, model_path, image_placeholder_shape,
activations, tensor_names, input, first_layer, logits.
Returns:
masked_input: nested list of z3.ExprRef with dimensions
(1, num_words, num_latent_dimensions).
unmasked_predictions: dict,
* input: float numpy array, the input tensor to the neural network.
* first_layer: float numpy array, the first layer tensor in the neural
network.
* first_layer_relu: str, the first layer relu activation
tensor in the neural network.
* logits: str, the logits tensor in the neural network.
* softmax: float numpy array, the softmax tensor in the neural network.
* weights_layer_1: float numpy array, the first layer fc / conv weights.
* biases_layer_1: float numpy array, the first layer fc / conv biases.
* (text only) embedding: float numpy array with shape (num_words,
num_latent_dimensions), the embedding layer.
session: tf.Session, tensorflow session with the loaded neural network.
optimizer: utils.TextOptimizer, z3 optimizer for image.
Raises:
ValueError: Raises an error if the text isn't a 1D array.
"""
if image.ndim != 1:
raise ValueError('The text input should be a 1D numpy array. '
'Shape of the received input: %s' % str(image.shape))
session = utils.restore_model(run_params.model_path)
unmasked_predictions = session.run(
run_params.tensor_names, feed_dict={
run_params.tensor_names['input']: image.reshape(
run_params.image_placeholder_shape)})
text_embedding = _remove_batch_axis(unmasked_predictions['embedding'])
# text_embedding has a shape (num_words, num_latent_dimensions)
z3_mask = [z3.Int('mask_%d' % i) for i in range(text_embedding.shape[0])]
# masked_input has a shape (num_words, num_latent_dimensions)
masked_input = []
for mask_bit, embedding_row in zip(z3_mask, text_embedding):
masked_input.append([z3.ToReal(mask_bit) * i for i in embedding_row])
return ([masked_input], unmasked_predictions, session,
utils.TextOptimizer(z3_mask=z3_mask))