def _delete_channels(self,
node,
inputs,
input_masks,
channels=None,
layer_name=None):
"""Delete selected channels of node.outbound_layer. Add it to the graph.
"""
old_layer = node.outbound_layer
old_layer_output = utils.single_element(node.output_tensors)
# Create a mask to propagate the deleted channels to downstream layers
new_delete_mask = self._make_delete_mask(old_layer, channels)
if len(set(channels)) == getattr(old_layer,
utils.get_channels_attr(old_layer)):
self._replace_tensors[old_layer_output] = (None, new_delete_mask)
return None
# If this layer has already been operated on, use the cached copy of
# the new layer. Otherwise, apply the inbound delete mask and
# delete channels to obtain the new layer
if old_layer in self._new_layers_map.keys():
new_layer = self._new_layers_map[old_layer]
else:
temp_layer, new_mask = self._apply_delete_mask(node, input_masks)
# This call is needed to initialise input_shape and output_shape
temp_layer(utils.single_element(inputs))
new_layer = self._delete_channel_weights(temp_layer, channels)
if layer_name:
new_layer.name = layer_name
self._new_layers_map[old_layer] = new_layer
new_output = new_layer(utils.single_element(inputs))
# Replace the original layer's output with the modified layer's output
self._replace_tensors[old_layer_output] = (new_output, new_delete_mask)
def _rebuild_rec(node):
"""Rebuild the graph up to `node` recursively.
Args:
node(Node): Node to rebuild up to.
Returns:
(tuple) containing :
The output tensor of the rebuilt `node`
The output mask of the rebuilt `node`
"""
# TODO: What happens if nodes have multiple output tensors?
# Does that ever happen?
#pdb.set_trace()
print(node, "rebuild node val")
layer = node.outbound_layer
#logging.debug('getting inputs for: {0}'.format(layer.name))
node_output = utils.single_element(node.output_tensors)
# First check for conditions to bottom out the recursion
# Check for replaced tensors before any other checks:
# these are created by the surgery methods.
if node_output in self._replace_tensors:
'''logging.debug('bottomed out at replaced output: {0}'.format(
node_output))'''
output, output_mask = self._replace_tensors[node_output]
#pdb.set_trace()
return output, output_mask
# Next check if the current node has already been rebuilt.
elif node in self._finished_nodes:
#logging.debug('reached finished node: {0}'.format(node))
#pdb.set_trace()
return self._finished_nodes[node]
# Next check if one of the graph_inputs has been reached.
elif node_output in graph_inputs:
#logging.debug('bottomed out at a model input')
output_mask = graph_input_masks[graph_inputs.index(
node_output)]
#pdb.set_trace()
return node_output, output_mask
# Otherwise recursively call this method on the inbound nodes.
else:
inbound_nodes = utils.get_inbound_nodes(node)
# Recursively rebuild the model up to `node`s inbound nodes to
# obtain its inputs and input masks
inputs, input_masks = zip(
*[_rebuild_rec(n) for n in inbound_nodes])
# Apply masks to the node's layer's weights and call the layer
# on the inputs
new_layer, output_mask = self._apply_delete_mask(
node, input_masks)
output = new_layer(utils.single_element(list(inputs)))
# Record that this node has been rebuild
self._finished_nodes[node] = (output, output_mask)
#logging.debug('layer complete: {0}'.format(layer.name))
#pdb.set_trace()
return output, output_mask
def _replace_layer(self, node, inputs, input_masks, new_layer=None):
"""Replace node.outbound_layer with new_layer. Add it to the graph."""
# Call the new layer on the rebuild submodel's inputs
new_output = new_layer(utils.single_element(inputs))
# Replace the original layer's output with the new layer's output
replaced_layer_output = utils.single_element(node.output_tensors)
input_masks = utils.single_element(input_masks)
self._replace_tensors[replaced_layer_output] = (new_output, input_masks)
def _delete_layer(self, node, inputs, input_masks):
"""Skip adding node.outbound_layer when building the graph."""
# Skip the deleted layer by replacing its outputs with it inputs
if len(inputs) >= 2:
raise ValueError('Cannot insert new layer at node with multiple '
'inbound layers.')
inputs = utils.single_element(inputs)
input_masks = utils.single_element(input_masks)
deleted_layer_output = utils.single_element(node.output_tensors)
self._replace_tensors[deleted_layer_output] = (inputs, input_masks)
def _insert_layer(self, node, inputs, input_masks, new_layer=None):
"""Insert new_layer into the graph before node.outbound_layer."""
# This will not work for nodes with multiple inbound layers
if len(inputs) >= 2:
raise ValueError('Cannot insert new layer at node with multiple '
'inbound layers.')
# Call the new layer on the inbound layer's output
new_output = new_layer(utils.single_element(inputs))
# Replace the inbound layer's output with the new layer's output
old_output = node.input_tensors[0]
input_masks = utils.single_element(input_masks)
self._replace_tensors[old_output] = (new_output, input_masks)
def _delete_channels(self, node, inputs, input_masks, channels=None):
"""Delete selected channels of node.outbound_layer. Add it to the graph.
"""
old_layer = node.outbound_layer
#pdb.set_trace()
# If this layer has already been operated on, use the cached copy of
# the new layer. Otherwise, apply the inbound delete mask and
# delete channels to obtain the new layer
if old_layer in self._new_layers_map.iterkeys():
new_layer = self._new_layers_map[old_layer]
else:
temp_layer, new_mask = self._apply_delete_mask(node, input_masks)
# This call is needed to initialise input_shape and output_shape
temp_layer(utils.single_element(inputs))
new_layer = self._delete_channel_weights(temp_layer, channels)
self._new_layers_map[old_layer] = new_layer
# Create a mask to propagate the deleted channels to downstream layers
new_delete_mask = self._make_delete_mask(old_layer, channels)
# Call the new layer on the rebuild submodel's inputs
new_output = new_layer(utils.single_element(inputs))
# Replace the original layer's output with the modified layer's output
old_layer_output = utils.single_element(node.output_tensors)
self._replace_tensors[old_layer_output] = (new_output, new_delete_mask)
def get_apoz(model, layer, x_val, node_indices=None):
if isinstance(layer, str):
layer = model.get_layer(name=layer)
# Check that layer is in the model
if layer not in model.layers:
raise ValueError('layer is not a valid Layer in model.')
layer_node_indices = utils.find_nodes_in_model(model, layer)
# If no nodes are specified, all of the layer's inbound nodes which are
# in model are selected.
if not node_indices:
node_indices = layer_node_indices
# Check for duplicate node indices
elif len(node_indices) != len(set(node_indices)):
raise ValueError('`node_indices` contains duplicate values.')
# Check that all of the selected nodes are in the layer
elif not set(node_indices).issubset(layer_node_indices):
raise ValueError('One or more nodes specified by `layer` and '
'`node_indices` are not in `model`.')
data_format = getattr(layer, 'data_format', 'channels_last')
# Perform the forward pass and get the activations of the layer.
mean_calculator = utils.MeanCalculator(sum_axis=0)
for node_index in node_indices:
act_layer, act_index = utils.find_activation_layer(layer, node_index)
# Get activations
if hasattr(x_val, "__iter__"):
temp_model = Model(model.inputs,
act_layer.get_output_at(act_index))
a = temp_model.predict(x_val)
else:
get_activations = k.function(
[utils.single_element(model.inputs),
k.learning_phase()], [act_layer.get_output_at(act_index)])
a = get_activations([x_val, 0])[0]
# Ensure that the channels axis is last
if data_format == 'channels_first':
a = np.swapaxes(a, 1, -1)
# Flatten all except channels axis
activations = np.reshape(a, [-1, a.shape[-1]])
zeros = (activations == 0).astype(int)
mean_calculator.add(zeros)
return mean_calculator.calculate()
def get_apoz(model, layer, x_val, node_indices=None, batch_size=1):
"""Identify neurons with high Average Percentage of Zeros (APoZ).
The APoZ a.k.a. (A)verage (P)ercentage (o)f activations equal to (Z)ero,
is a metric for the usefulness of a channel defined in this paper:
"Network Trimming: A Data-Driven Neuron Pruning Approach towards Efficient
Deep Architectures" - [Hu et al. (2016)][]
`high_apoz()` enables the pruning methodology described in this paper to be
replicated.
If node_indices are not specified and the layer is shared within the model
the APoZ will be calculated over all instances of the shared layer.
Args:
model: A Keras model.
layer: The layer whose channels will be evaluated for pruning.
x_val: The input of the validation set. This will be used to calculate
the activations of the layer of interest.
node_indices(list[int]): (optional) A list of node indices.
Returns:
List of the APoZ values for each channel in the layer.
"""
if isinstance(layer, str):
layer = model.get_layer(name=layer)
# Check that layer is in the model
if layer not in model.layers:
raise ValueError('layer is not a valid Layer in model.')
layer_node_indices = utils.find_nodes_in_model(model, layer)
# If no nodes are specified, all of the layer's inbound nodes which are
# in model are selected.
if not node_indices:
node_indices = layer_node_indices
# Check for duplicate node indices
elif len(node_indices) != len(set(node_indices)):
raise ValueError('`node_indices` contains duplicate values.')
# Check that all of the selected nodes are in the layer
elif not set(node_indices).issubset(layer_node_indices):
raise ValueError('One or more nodes specified by `layer` and '
'`node_indices` are not in `model`.')
data_format = getattr(layer, 'data_format', 'channels_last')
# Perform the forward pass and get the activations of the layer.
mean_calculator = utils.MeanCalculator(sum_axis=0)
for node_index in node_indices:
act_layer, act_index = utils.find_activation_layer(layer, node_index)
# Get activations
if isinstance(x_val, np.ndarray):
temp_model = Model(model.inputs,
act_layer.get_output_at(act_index))
a = temp_model.predict_generator(
x_val, x_val.shape[0] // batch_size)
elif hasattr(x_val, "__iter__"):
temp_model = Model(model.inputs,
act_layer.get_output_at(act_index))
a = temp_model.predict_generator(
x_val, x_val.n // batch_size)
else:
get_activations = k.function(
[utils.single_element(model.inputs), k.learning_phase()],
[act_layer.get_output_at(act_index)])
a = get_activations([x_val, 0])[0]
# Ensure that the channels axis is last
if data_format == 'channels_first':
a = np.swapaxes(a, 1, -1)
# Flatten all except channels axis
activations = np.reshape(a, [-1, a.shape[-1]])
zeros = (activations == 0).astype(int)
mean_calculator.add(zeros)
return mean_calculator.calculate()
def _apply_delete_mask(self, node, inbound_masks):
"""Apply the inbound delete mask and return the outbound delete mask
When specific channels in a layer or layer instance are deleted, the
mask propagates information about which channels are affected to
downstream layers.
If the layer contains weights, those which were previously connected
to the deleted channels are deleted and outbound masks are set to None
since further downstream layers aren't affected.
If the layer does not contain weights, its output mask is calculated to
reflect any transformations performed by the layer to ensure that
information about the deleted channels is propagated downstream.
Arguments:
node(Node): The node where the delete mask is applied.
inbound_masks: Mask(s) from inbound node(s).
Returns:
new_layer: Pass through `layer` if it has no weights, otherwise a
new `Layer` object with weights corresponding to the
inbound mask deleted.
outbound_mask: Mask corresponding to `new_layer`.
"""
# TODO: This breaks layer sharing. Write a test for this.
# if delete_mask is None or all values are True, it does not affect
# this layer or any layers above/downstream from it
print(node, "node")
print(inbound_masks, "inbound_masks")
layer = node.outbound_layer
print(layer, "outbound layer")
if all(mask is None for mask in inbound_masks):
new_layer = layer
outbound_mask = None
print(node, "node if return")
return new_layer, outbound_mask
elif any(mask is None for mask in inbound_masks):
inbound_masks = [
np.ones(shape[1:], dtype=bool)
if inbound_masks[i] is None else inbound_masks[i]
for i, shape in enumerate(node.input_shapes)
]
output_shape = utils.single_element(node.output_shapes)
input_shape = utils.single_element(node.input_shapes)
data_format = getattr(layer, 'data_format', 'channels_last')
inbound_masks = utils.single_element(inbound_masks)
print(node, "node")
# otherwise, delete_mask.shape should be: layer.input_shape[1:]
layer_class = layer.__class__.__name__
if layer_class == 'InputLayer':
raise RuntimeError('This should never get here!')
elif layer_class == 'Dense':
if np.all(inbound_masks):
new_layer = layer
else:
weights = layer.get_weights()
weights[0] = weights[0][np.where(inbound_masks)[0], :]
config = layer.get_config()
config['weights'] = weights
new_layer = type(layer).from_config(config)
outbound_mask = None
elif layer_class == 'Flatten':
outbound_mask = np.reshape(inbound_masks, [
-1,
])
new_layer = layer
elif layer_class in ('Conv1D', 'Conv2D', 'Conv3D'):
if np.all(inbound_masks):
new_layer = layer
print("npall conv1d")
else:
if data_format == 'channels_first':
inbound_masks = np.swapaxes(inbound_masks, 0, -1)
# Conv layer: trim down inbound_masks to filter shape
k_size = layer.kernel_size
index = [slice(None, dim_size, None) for dim_size in k_size]
inbound_masks = inbound_masks[index + [slice(None)]]
# Delete unused weights to obtain new_weights
weights = layer.get_weights()
# Each deleted channel was connected to all of the channels
# in layer; therefore, the mask must be repeated for each
# channel.
# `delete_mask`'s size: size(inbound_mask)+[layer.filters]
# TODO: replace repeat with tile
delete_mask = np.repeat(inbound_masks[..., np.newaxis],
weights[0].shape[-1],
axis=-1)
new_shape = list(weights[0].shape)
new_shape[-2] = -1 # Weights always have channels_last
print(weights[0].shape, "weights[0]")
print(delete_mask.shape, "deletemask")
print(new_shape, "new_shape")
weights[0] = np.reshape(weights[0][delete_mask], new_shape)
# Instantiate new layer with new_weights
config = layer.get_config()
config['weights'] = weights
new_layer = type(layer).from_config(config)
outbound_mask = None
elif layer_class in ('Cropping1D', 'Cropping2D', 'Cropping3D',
'MaxPooling1D', 'MaxPooling2D', 'MaxPooling3D',
'AveragePooling1D', 'AveragePooling2D',
'AveragePooling3D'):
index = [slice(None, x, None) for x in output_shape[1:]]
if data_format == 'channels_first':
index[0] = slice(None)
elif data_format == 'channels_last':
index[-1] = slice(None)
else:
raise ValueError('Invalid data format')
outbound_mask = inbound_masks[index]
new_layer = layer
elif layer_class in ('UpSampling1D', 'UpSampling2D', 'UpSampling3D',
'ZeroPadding1D', 'ZeroPadding2D',
'ZeroPadding3D'):
# Get slice of mask with all singleton dimensions except
# channels dimension
index = [slice(1)] * (len(input_shape) - 1)
tile_shape = list(output_shape[1:])
if data_format == 'channels_first':
index[0] = slice(None)
tile_shape[0] = 1
elif data_format == 'channels_last':
index[-1] = slice(None)
tile_shape[-1] = 1
else:
raise ValueError('Invalid data format')
channels_vector = inbound_masks[index]
# Tile this slice to create the outbound mask
outbound_mask = np.tile(channels_vector, tile_shape)
new_layer = layer
elif layer_class in ('GlobalMaxPooling1D', 'GlobalMaxPooling2D',
'GlobalAveragePooling1D',
'GlobalAveragePooling2D'):
# Get slice of mask with all singleton dimensions except
# channels dimension
index = [0] * (len(input_shape) - 1)
if data_format == 'channels_first':
index[0] = slice(None)
elif data_format == 'channels_last':
index[-1] = slice(None)
else:
raise ValueError('Invalid data format')
channels_vector = inbound_masks[index]
# Tile this slice to create the outbound mask
outbound_mask = channels_vector
new_layer = layer
elif layer_class in ('Dropout', 'Activation', 'SpatialDropout1D',
'SpatialDropout2D', 'SpatialDropout3D',
'ActivityRegularization', 'Masking', 'LeakyReLU',
'ELU', 'ThresholdedReLU', 'GaussianNoise',
'GaussianDropout', 'AlphaDropout'):
# Pass-through layers
outbound_mask = inbound_masks
new_layer = layer
elif layer_class == 'Reshape':
outbound_mask = np.reshape(inbound_masks, layer.target_shape)
new_layer = layer
elif layer_class == 'Permute':
outbound_mask = np.transpose(inbound_masks,
[x - 1 for x in layer.dims])
new_layer = layer
elif layer_class == 'RepeatVector':
outbound_mask = np.repeat(np.expand_dims(inbound_masks, 0),
layer.n,
axis=0)
new_layer = layer
elif layer_class == 'Embedding':
# Embedding will always be the first layer so it doesn't need
# to consider the inbound_delete_mask
if inbound_masks is not None:
raise ValueError('Channels cannot be deleted bedore Embedding '
'layers because they change the number of '
'channels.')
outbound_mask = None
new_layer = layer
elif layer_class in ('Add', 'Multiply', 'Average', 'Maximum'):
# The inputs must be the same size
if not utils.all_equal(inbound_masks):
ValueError(
'{0} layers must have the same size inputs. All '
'inbound nodes must have the same channels deleted'.format(
layer_class))
outbound_mask = inbound_masks[1]
new_layer = layer
elif layer_class == 'Concatenate':
axis = layer.axis
if layer.axis < 0:
axis = axis % len(layer.input_shape[0])
# Below: axis=axis-1 because the mask excludes the batch dimension
outbound_mask = np.concatenate(inbound_masks, axis=axis - 1)
new_layer = layer
elif layer_class in ('SimpleRNN', 'GRU', 'LSTM'):
if np.all(inbound_masks):
new_layer = layer
else:
weights = layer.get_weights()
weights[0] = weights[0][np.where(inbound_masks[0, :])[0], :]
config = layer.get_config()
config['weights'] = weights
new_layer = type(layer).from_config(config)
outbound_mask = None
elif layer_class == 'BatchNormalization':
outbound_mask = inbound_masks
# Get slice of mask with all singleton dimensions except
# channels dimension
index = [0] * (len(input_shape))
index[layer.axis] = slice(None)
index = index[1:]
# TODO: Maybe use channel indices everywhere instead of masks?
channel_indices = np.where(inbound_masks[index] == False)[0]
weights = [
np.delete(w, channel_indices, axis=-1)
for w in layer.get_weights()
]
new_layer = BatchNormalization.from_config(layer.get_config())
new_input_shape = list(input_shape)
new_input_shape[new_layer.axis] -= len(channel_indices)
new_layer.build(new_input_shape)
new_layer.set_weights(weights)
else:
# Not implemented:
# - Lambda
# - SeparableConv2D
# - Conv2DTranspose
# - LocallyConnected1D
# - LocallyConnected2D
# - TimeDistributed
# - Bidirectional
# - Dot
# - PReLU
# Warning/error needed for Reshape if channels axis is split
raise ValueError('"{0}" layers are currently '
'unsupported.'.format(layer_class))
print(node, "node return")
return new_layer, outbound_mask
def _apply_delete_mask(self, node, inbound_masks):
"""Apply the inbound delete mask and return the outbound delete mask
When specific channels in a layer or layer instance are deleted, the
mask propagates information about which channels are affected to
downstream layers.
If the layer contains weights, those which were previously connected
to the deleted channels are deleted and outbound masks are set to None
since further downstream layers aren't affected.
If the layer does not contain weights, its output mask is calculated to
reflect any transformations performed by the layer to ensure that
information about the deleted channels is propagated downstream.
Arguments:
node(Node): The node where the delete mask is applied.
inbound_masks: Mask(s) from inbound node(s).
Returns:
new_layer: Pass through `layer` if it has no weights, otherwise a
new `Layer` object with weights corresponding to the
inbound mask deleted.
outbound_mask: Mask corresponding to `new_layer`.
"""
# if delete_mask is None or all values are True, it does not affect
# this layer or any layers above/downstream from it
layer = node.outbound_layer
if all(mask is None for mask in inbound_masks):
new_layer = layer
outbound_mask = None
return new_layer, outbound_mask
# If one or more of the masks are None, replace them with ones.
if any(mask is None for mask in inbound_masks):
inbound_masks = [
np.ones(shape[1:], dtype=bool)
if inbound_masks[i] is None else inbound_masks[i]
for i, shape in enumerate(node.input_shapes)
]
# If the layer is shared and has already been affected by this
# operation, use the cached new layer.
if (len(get_inbound_nodes(layer)) > 1
and layer in self._replace_layers_map.keys()):
return self._replace_layers_map[layer]
output_shape = utils.single_element(node.output_shapes)
input_shape = utils.single_element(node.input_shapes)
data_format = getattr(layer, "data_format", "channels_last")
inbound_masks = utils.single_element(inbound_masks)
# otherwise, delete_mask.shape should be: layer.input_shape[1:]
layer_class = layer.__class__.__name__
if layer_class == "InputLayer":
raise RuntimeError("This should never get here!")
elif layer_class == "Dense":
if np.all(inbound_masks):
new_layer = layer
else:
weights = layer.get_weights()
weights[0] = weights[0][np.where(inbound_masks)[0], :]
config = layer.get_config()
config["weights"] = weights
new_layer = type(layer).from_config(config)
outbound_mask = None
elif layer_class == "Flatten":
outbound_mask = np.reshape(inbound_masks, [
-1,
])
new_layer = layer
elif layer_class in ("Conv1D", "Conv2D", "Conv3D"):
if np.all(inbound_masks):
new_layer = layer
else:
if data_format == "channels_first":
inbound_masks = np.swapaxes(inbound_masks, 0, -1)
# Conv layer: trim down inbound_masks to filter shape
k_size = layer.kernel_size
index = [slice(None, 1, None) for _ in k_size]
inbound_masks = inbound_masks[tuple(index + [slice(None)])]
weights = layer.get_weights()
# Delete unused weights to obtain new_weights
# Each deleted channel was connected to all of the channels
# in layer; therefore, the mask must be repeated for each
# channel.
# `delete_mask`'s size: size(weights[0])
delete_mask = np.tile(
inbound_masks[..., np.newaxis],
list(k_size) + [1, weights[0].shape[-1]],
)
new_shape = list(weights[0].shape)
new_shape[-2] = -1 # Weights always have channels_last
weights[0] = np.reshape(weights[0][delete_mask], new_shape)
# Instantiate new layer with new_weights
config = layer.get_config()
config["weights"] = weights
new_layer = type(layer).from_config(config)
outbound_mask = None
elif layer_class in (
"Cropping1D",
"Cropping2D",
"Cropping3D",
"MaxPooling1D",
"MaxPooling2D",
"MaxPooling3D",
"AveragePooling1D",
"AveragePooling2D",
"AveragePooling3D",
):
index = [slice(None, x, None) for x in output_shape[1:]]
if data_format == "channels_first":
index[0] = slice(None)
elif data_format == "channels_last":
index[-1] = slice(None)
else:
raise ValueError("Invalid data format")
outbound_mask = inbound_masks[tuple(index)]
new_layer = layer
elif layer_class in (
"UpSampling1D",
"UpSampling2D",
"UpSampling3D",
"ZeroPadding1D",
"ZeroPadding2D",
"ZeroPadding3D",
):
# Get slice of mask with all singleton dimensions except
# channels dimension
index = [slice(1)] * (len(input_shape) - 1)
tile_shape = list(output_shape[1:])
if data_format == "channels_first":
index[0] = slice(None)
tile_shape[0] = 1
elif data_format == "channels_last":
index[-1] = slice(None)
tile_shape[-1] = 1
else:
raise ValueError("Invalid data format")
channels_vector = inbound_masks[tuple(index)]
# Tile this slice to create the outbound mask
outbound_mask = np.tile(channels_vector, tile_shape)
new_layer = layer
elif layer_class in (
"GlobalMaxPooling1D",
"GlobalMaxPooling2D",
"GlobalAveragePooling1D",
"GlobalAveragePooling2D",
):
# Get slice of mask with all singleton dimensions except
# channels dimension
index = [0] * (len(input_shape) - 1)
if data_format == "channels_first":
index[0] = slice(None)
elif data_format == "channels_last":
index[-1] = slice(None)
else:
raise ValueError("Invalid data format")
channels_vector = inbound_masks[tuple(index)]
# Tile this slice to create the outbound mask
outbound_mask = channels_vector
new_layer = layer
elif layer_class in (
"Dropout",
"Activation",
"SpatialDropout1D",
"SpatialDropout2D",
"SpatialDropout3D",
"ActivityRegularization",
"Masking",
"LeakyReLU",
"ELU",
"ThresholdedReLU",
"GaussianNoise",
"GaussianDropout",
"AlphaDropout",
"Rename",
) or (layer_class == "TensorFlowOpLayer"
and layer.node_def.op == "ResizeBilinear"):
# Pass-through layers
outbound_mask = inbound_masks
new_layer = layer
elif layer_class == "Reshape":
outbound_mask = np.reshape(inbound_masks, layer.target_shape)
new_layer = layer
elif layer_class == "Permute":
outbound_mask = np.transpose(inbound_masks,
[x - 1 for x in layer.dims])
new_layer = layer
elif layer_class == "RepeatVector":
outbound_mask = np.repeat(np.expand_dims(inbound_masks, 0),
layer.n,
axis=0)
new_layer = layer
elif layer_class == "Embedding":
# Embedding will always be the first layer so it doesn't need
# to consider the inbound_delete_mask
if inbound_masks is not None:
raise ValueError("Channels cannot be deleted bedore Embedding "
"layers because they change the number of "
"channels.")
outbound_mask = None
new_layer = layer
elif layer_class in ("Add", "Multiply", "Average", "Maximum"):
# The inputs must be the same size
if not utils.all_equal(inbound_masks):
ValueError(
"{0} layers must have the same size inputs. All "
"inbound nodes must have the same channels deleted".format(
layer_class))
outbound_mask = inbound_masks[1]
new_layer = layer
elif layer_class == "Concatenate":
axis = layer.axis
if layer.axis < 0:
axis = axis % len(layer.input_shape[0])
# Below: axis=axis-1 because the mask excludes the batch dimension
outbound_mask = np.concatenate(inbound_masks, axis=axis - 1)
new_layer = layer
elif layer_class in ("SimpleRNN", "GRU", "LSTM"):
if np.all(inbound_masks):
new_layer = layer
else:
weights = layer.get_weights()
weights[0] = weights[0][np.where(inbound_masks[0, :])[0], :]
config = layer.get_config()
config["weights"] = weights
new_layer = type(layer).from_config(config)
outbound_mask = None
elif layer_class == "BatchNormalization":
outbound_mask = inbound_masks
# Get slice of mask with all singleton dimensions except
# channels dimension
index = [0] * (len(input_shape))
index[layer.axis] = slice(None)
index = index[1:]
# TODO: Maybe use channel indices everywhere instead of masks?
channel_indices = np.where(inbound_masks[tuple(index)] == False)[0]
weights = [
np.delete(w, channel_indices, axis=-1)
for w in layer.get_weights()
]
new_layer = BatchNormalization.from_config(layer.get_config())
new_input_shape = list(input_shape)
new_input_shape[new_layer.axis] -= len(channel_indices)
new_layer.build(new_input_shape)
new_layer.set_weights(weights)
else:
# Not implemented:
# - Lambda
# - SeparableConv2D
# - Conv2DTranspose
# - LocallyConnected1D
# - LocallyConnected2D
# - TimeDistributed
# - Bidirectional
# - Dot
# - PReLU
# Warning/error needed for Reshape if channels axis is split
raise ValueError('"{0}" layers are currently '
"unsupported.".format(layer_class))
if len(get_inbound_nodes(layer)) > 1 and new_layer != layer:
self._replace_layers_map[layer] = (new_layer, outbound_mask)
return new_layer, outbound_mask
def _rebuild_rec(node):
"""Rebuild the graph up to `node` recursively.
Args:
node(Node): Node to rebuild up to.
Returns:
(tuple) containing :
The output tensor of the rebuilt `node`
The output mask of the rebuilt `node`
"""
# TODO: What happens if nodes have multiple output tensors?
# Does that ever happen?
layer = node.outbound_layer
logger.debug("getting inputs for: {0}".format(layer.name))
node_output = utils.single_element(node.output_tensors)
# First check for conditions to bottom out the recursion
# Check for replaced tensors before any other checks:
# these are created by the surgery methods.
if node_output.name in self._replace_tensors.keys():
logger.debug(
"bottomed out at replaced output: {0}".format(node_output))
output, output_mask = self._replace_tensors[node_output.name]
return output, output_mask
# Next check if the current node has already been rebuilt.
elif node in self._finished_nodes.keys():
logger.debug("reached finished node: {0}".format(node))
return self._finished_nodes[node]
# Next check if one of the graph_inputs has been reached.
elif node_output.name in names(graph_inputs):
logger.debug("bottomed out at a model input")
output_mask = graph_input_masks[graph_inputs.index(
node_output)]
return node_output, output_mask
# Otherwise recursively call this method on the inbound nodes.
else:
inbound_nodes = utils.get_node_inbound_nodes(node)
logger.debug("inbound_layers: {0}".format(
[node.outbound_layer.name for node in inbound_nodes]))
# Recursively rebuild the model up to `node`s inbound nodes to
# obtain its inputs and input masks
inputs, input_masks = zip(
*[_rebuild_rec(n) for n in inbound_nodes])
if all(i is None for i in inputs):
output = None
output_mask = np.zeros(node.output_shapes[0][1:],
dtype=bool)
elif any(i is None for i in inputs):
if node.outbound_layer.__class__.__name__ != "Concatenate":
TypeError(
"Inputs can only be missing for concatenate layers."
)
# remove Nones from inputs list
inputs = [i for i in inputs if i is not None]
new_layer, output_mask = self._apply_delete_mask(
node, input_masks)
if len(inputs) == 1:
output = utils.single_element(list(inputs))
else:
output = new_layer(utils.single_element(list(inputs)))
else:
new_layer, output_mask = self._apply_delete_mask(
node, input_masks)
output = new_layer(utils.single_element(list(inputs)))
# Record that this node has been rebuild
self._finished_nodes[node] = (output, output_mask)
logger.debug("layer complete: {0}".format(layer.name))
return output, output_mask
def _rebuild_rec(node):
"""Rebuild the graph up to `node` recursively.
Args:
node(Node): Node to rebuild up to.
Returns:
(tuple) containing :
The output tensor of the rebuilt `node`
The output mask of the rebuilt `node`
"""
# TODO: What happens if nodes have multiple output tensors?
# Does that ever happen?
layer = node.outbound_layer
logging.debug('getting inputs for: {0}'.format(layer.name))
node_output = utils.single_element(node.output_tensors)
# First check for conditions to bottom out the recursion
# Check for replaced tensors before any other checks:
# these are created by the surgery methods.
if node_output in self._replace_tensors.keys():
logging.debug(
'bottomed out at replaced output: {0}'.format(node_output))
output, output_mask = self._replace_tensors[node_output]
return output, output_mask
# Next check if the current node has already been rebuilt.
elif node in self._finished_nodes.keys():
logging.debug('reached finished node: {0}'.format(node))
return self._finished_nodes[node]
# Next check if one of the graph_inputs has been reached.
mask_map = TensorDict()
for input, mask in zip(graph_inputs, graph_input_masks):
mask_map[input] = mask
try:
output_mask = mask_map[node_output]
logging.debug('bottomed out at a model input')
return node_output, output_mask
except KeyError:
# Otherwise recursively call this method on the inbound nodes.
inbound_nodes = node_utils.parent_nodes(node)
logging.debug('inbound_layers: {0}'.format(
[node.outbound_layer.name for node in inbound_nodes]))
# Recursively rebuild the model up to `node`s inbound nodes to
# obtain its inputs and input masks
inputs, input_masks = zip(
*[_rebuild_rec(n) for n in inbound_nodes])
if all(i is None for i in inputs):
output = None
try:
assert len(node.output_tensors) <= 1
except AssertionError as e:
raise e
except:
pass
output_mask = np.zeros(node.output_tensors.shape[1:],
dtype=bool)
elif any(i is None for i in inputs):
if node.outbound_layer.__class__.__name__ != 'Concatenate':
TypeError(
'Inputs can only be missing for concatenate layers.'
)
# remove Nones from inputs list
inputs = [i for i in inputs if i is not None]
new_layer, output_mask = self._apply_delete_mask(
node, input_masks)
if len(inputs) == 1:
output = utils.single_element(list(inputs))
else:
output = new_layer(utils.single_element(list(inputs)))
else:
new_layer, output_mask = self._apply_delete_mask(
node, input_masks)
output = new_layer(utils.single_element(list(inputs)))
# Record that this node has been rebuild
self._finished_nodes[node] = (output, output_mask)
logging.debug('layer complete: {0}'.format(layer.name))
return output, output_mask
def get_output_sum(model, layer, x_val, node_indices=None, steps=None):
"""
Args:
model: A Keras model.
layer: The layer whose channels will be evaluated for pruning.
x_val: The input of the validation set. This will be used to calculate
the activations of the layer of interest.
node_indices(list[int]): (optional) A list of node indices.
steps: number of steps for a generator
Returns:
total: total output given a dataset from each kernel
"""
if isinstance(layer, str):
layer = model.get_layer(name=layer)
# Check that layer is in the model
if layer not in model.layers:
raise ValueError('layer is not a valid Layer in model.')
layer_node_indices = utils.find_nodes_in_model(model, layer)
# If no nodes are specified, all of the layer's inbound nodes which are
# in model are selected.
if not node_indices:
node_indices = layer_node_indices
# Check for duplicate node indices
elif len(node_indices) != len(set(node_indices)):
raise ValueError('`node_indices` contains duplicate values.')
# Check that all of the selected nodes are in the layer
elif not set(node_indices).issubset(layer_node_indices):
raise ValueError('One or more nodes specified by `layer` and '
'`node_indices` are not in `model`.')
data_format = getattr(layer, 'data_format', 'channels_last')
# Perform the forward pass and get the activations of the layer.
total_sum = None
for node_index in node_indices:
act_layer, act_index = utils.find_activation_layer(layer, node_index)
# Get activations
if isinstance(x_val, types.GeneratorType):
# temp_model = Model(model.inputs, act_layer.get_output_at(act_index))
temp_model = Model(model.inputs, layer.get_output_at(node_index))
a = temp_model.predict_generator_intermediate(x_val, steps=steps)
else:
get_activations = k.function(
[utils.single_element(model.inputs),
k.learning_phase()], [act_layer.get_output_at(act_index)])
a = get_activations([x_val, 0])[0]
# Ensure that the channels axis is last
if data_format == 'channels_first':
a = np.swapaxes(a, 1, -1)
numAxes = len(a.shape) - 1
total = np.sum(a, axis=numAxes)
for n in range(numAxes - 1, -1, -1):
total = np.sum(total, axis=n)
# previous - for time distributed convolution
# a = np.abs(a)
# total = np.sum(a, axis=2)
# total = np.sum(total, axis=1)
# total = np.sum(total, axis=0)
if total_sum is None:
total_sum = total
else:
total_sum += total
return total_sum
