def test_find_activation_layer():
conv1_filters = 1
conv2_filters = 1
dense_units = 1
model = Sequential()
model.add(
Conv2D(conv1_filters, [3, 3],
input_shape=(28, 28, 1),
data_format="channels_last",
name='conv_1'))
model.add(Activation('relu', name='act_1'))
model.add(MaxPool2D((2, 2), name='pool_1'))
model.add(
Conv2D(conv2_filters, [3, 3],
data_format="channels_last",
name='conv_2'))
model.add(Activation('relu', name='act_2'))
model.add(MaxPool2D((2, 2), name='pool_2'))
model.add(Flatten(name='flat_1'))
model.add(Dense(dense_units, name='dense_1'))
model.add(Activation('relu', name='act_3'))
model.add(Dense(10, name='dense_2'))
model.add(Activation('softmax', name='act_4'))
assert find_activation_layer(model.get_layer('conv_1'),
0) == (model.get_layer('act_1'), 0)
assert find_activation_layer(model.get_layer('conv_2'),
0) == (model.get_layer('act_2'), 0)
assert find_activation_layer(model.get_layer('dense_1'),
0) == (model.get_layer('act_3'), 0)
assert find_activation_layer(model.get_layer('dense_2'),
0) == (model.get_layer('act_4'), 0)
def test_find_activation_layer():
conv1_filters = 1
conv2_filters = 1
dense_units = 1
model = Sequential()
model.add(
Conv2D(
conv1_filters,
[3, 3],
input_shape=(28, 28, 1),
data_format="channels_last",
name="conv_1",
))
model.add(Activation("relu", name="act_1"))
model.add(MaxPool2D((2, 2), name="pool_1"))
model.add(
Conv2D(conv2_filters, [3, 3],
data_format="channels_last",
name="conv_2"))
model.add(Activation("relu", name="act_2"))
model.add(MaxPool2D((2, 2), name="pool_2"))
model.add(Flatten(name="flat_1"))
model.add(Dense(dense_units, name="dense_1"))
model.add(Activation("relu", name="act_3"))
model.add(Dense(10, name="dense_2"))
model.add(Activation("softmax", name="act_4"))
assert find_activation_layer(model.get_layer("conv_1"), 0) == (
model.get_layer("act_1"),
0,
)
assert find_activation_layer(model.get_layer("conv_2"), 0) == (
model.get_layer("act_2"),
0,
)
assert find_activation_layer(model.get_layer("dense_1"), 0) == (
model.get_layer("act_3"),
0,
)
assert find_activation_layer(model.get_layer("dense_2"), 0) == (
model.get_layer("act_4"),
0,
)
def get_channels_apoz_importance(model, layer, x_val, node_indices=None):
if isinstance(layer, str):
layer = model.get_layer(name=layer)
# Check that layer is in the model
if layer not in model.layers:
raise ValueError('layer is not a valid Layer in model.')
layer_node_indices = utils.find_nodes_in_model(model, layer)
# If no nodes are specified, all of the layer's inbound nodes which are
# in model are selected.
if not node_indices:
node_indices = layer_node_indices
# Check for duplicate node indices
elif len(node_indices) != len(set(node_indices)):
raise ValueError('`node_indices` contains duplicate values.')
# Check that all of the selected nodes are in the layer
elif not set(node_indices).issubset(layer_node_indices):
raise ValueError('One or more nodes specified by `layer` and '
'`node_indices` are not in `model`.')
data_format = getattr(layer, 'data_format', 'channels_last')
# Perform the forward pass and get the activations of the layer.
mean_calculator = utils.MeanCalculator(sum_axis=0)
print('layer:', layer, layer_node_indices, node_indices)
for node_index in node_indices:
act_layer, act_index = utils.find_activation_layer(layer, node_index)
print('act layer', act_layer, act_index)
# Get activations
if hasattr(x_val, "__iter__"):
temp_model = Model(model.inputs,
act_layer.get_output_at(act_index))
print('before: act output', act_layer.get_output_at(act_index))
a = temp_model.predict(x_val)
#a=temp_model.predict_generator(x_val, x_val.n // x_val.batch_size)
print('after:', layer, a.shape)
else:
get_activations = K.function(
[single_element(model.inputs),
K.learning_phase()], [act_layer.get_output_at(act_index)])
a = get_activations([x_val, 0])[0]
# Ensure that the channels axis is last
if data_format == 'channels_first':
a = np.swapaxes(a, 1, -1)
# Flatten all except channels axis
activations = np.reshape(a, [-1, a.shape[-1]])
zeros = (activations == 0).astype(int)
mean_calculator.add(zeros)
return mean_calculator.calculate()
def get_apoz(model, layer, x_val, node_indices=None, batch_size=1):
"""Identify neurons with high Average Percentage of Zeros (APoZ).
The APoZ a.k.a. (A)verage (P)ercentage (o)f activations equal to (Z)ero,
is a metric for the usefulness of a channel defined in this paper:
"Network Trimming: A Data-Driven Neuron Pruning Approach towards Efficient
Deep Architectures" - [Hu et al. (2016)][]
`high_apoz()` enables the pruning methodology described in this paper to be
replicated.
If node_indices are not specified and the layer is shared within the model
the APoZ will be calculated over all instances of the shared layer.
Args:
model: A Keras model.
layer: The layer whose channels will be evaluated for pruning.
x_val: The input of the validation set. This will be used to calculate
the activations of the layer of interest.
node_indices(list[int]): (optional) A list of node indices.
Returns:
List of the APoZ values for each channel in the layer.
"""
if isinstance(layer, str):
layer = model.get_layer(name=layer)
# Check that layer is in the model
if layer not in model.layers:
raise ValueError('layer is not a valid Layer in model.')
layer_node_indices = utils.find_nodes_in_model(model, layer)
# If no nodes are specified, all of the layer's inbound nodes which are
# in model are selected.
if not node_indices:
node_indices = layer_node_indices
# Check for duplicate node indices
elif len(node_indices) != len(set(node_indices)):
raise ValueError('`node_indices` contains duplicate values.')
# Check that all of the selected nodes are in the layer
elif not set(node_indices).issubset(layer_node_indices):
raise ValueError('One or more nodes specified by `layer` and '
'`node_indices` are not in `model`.')
data_format = getattr(layer, 'data_format', 'channels_last')
# Perform the forward pass and get the activations of the layer.
mean_calculator = utils.MeanCalculator(sum_axis=0)
for node_index in node_indices:
act_layer, act_index = utils.find_activation_layer(layer, node_index)
# Get activations
if isinstance(x_val, np.ndarray):
temp_model = Model(model.inputs,
act_layer.get_output_at(act_index))
a = temp_model.predict_generator(
x_val, x_val.shape[0] // batch_size)
elif hasattr(x_val, "__iter__"):
temp_model = Model(model.inputs,
act_layer.get_output_at(act_index))
a = temp_model.predict_generator(
x_val, x_val.n // batch_size)
else:
get_activations = k.function(
[utils.single_element(model.inputs), k.learning_phase()],
[act_layer.get_output_at(act_index)])
a = get_activations([x_val, 0])[0]
# Ensure that the channels axis is last
if data_format == 'channels_first':
a = np.swapaxes(a, 1, -1)
# Flatten all except channels axis
activations = np.reshape(a, [-1, a.shape[-1]])
zeros = (activations == 0).astype(int)
mean_calculator.add(zeros)
return mean_calculator.calculate()
def get_activations(model, layer, x_val):
act_layer, act_index = utils.find_activation_layer(layer, 0)
temp_model = Model(inputs=model.inputs,
outputs=act_layer.get_output_at(act_index))
return temp_model.predict(x_val)
def get_output_sum(model, layer, x_val, node_indices=None, steps=None):
"""
Args:
model: A Keras model.
layer: The layer whose channels will be evaluated for pruning.
x_val: The input of the validation set. This will be used to calculate
the activations of the layer of interest.
node_indices(list[int]): (optional) A list of node indices.
steps: number of steps for a generator
Returns:
total: total output given a dataset from each kernel
"""
if isinstance(layer, str):
layer = model.get_layer(name=layer)
# Check that layer is in the model
if layer not in model.layers:
raise ValueError('layer is not a valid Layer in model.')
layer_node_indices = utils.find_nodes_in_model(model, layer)
# If no nodes are specified, all of the layer's inbound nodes which are
# in model are selected.
if not node_indices:
node_indices = layer_node_indices
# Check for duplicate node indices
elif len(node_indices) != len(set(node_indices)):
raise ValueError('`node_indices` contains duplicate values.')
# Check that all of the selected nodes are in the layer
elif not set(node_indices).issubset(layer_node_indices):
raise ValueError('One or more nodes specified by `layer` and '
'`node_indices` are not in `model`.')
data_format = getattr(layer, 'data_format', 'channels_last')
# Perform the forward pass and get the activations of the layer.
total_sum = None
for node_index in node_indices:
act_layer, act_index = utils.find_activation_layer(layer, node_index)
# Get activations
if isinstance(x_val, types.GeneratorType):
# temp_model = Model(model.inputs, act_layer.get_output_at(act_index))
temp_model = Model(model.inputs, layer.get_output_at(node_index))
a = temp_model.predict_generator_intermediate(x_val, steps=steps)
else:
get_activations = k.function(
[utils.single_element(model.inputs),
k.learning_phase()], [act_layer.get_output_at(act_index)])
a = get_activations([x_val, 0])[0]
# Ensure that the channels axis is last
if data_format == 'channels_first':
a = np.swapaxes(a, 1, -1)
numAxes = len(a.shape) - 1
total = np.sum(a, axis=numAxes)
for n in range(numAxes - 1, -1, -1):
total = np.sum(total, axis=n)
# previous - for time distributed convolution
# a = np.abs(a)
# total = np.sum(a, axis=2)
# total = np.sum(total, axis=1)
# total = np.sum(total, axis=0)
if total_sum is None:
total_sum = total
else:
total_sum += total
return total_sum