def replacement(self, match_layer):
relu_layer_node = match_layer
add_layer_node = relu_layer_node.input_layers[0]
add_layer_node.metadata['quantize_config'] = (
default_8bit_quantize_configs.NoOpQuantizeConfig())
return match_layer
def _replace(self, relu_layer_node, bn_layer_node, conv_layer_node):
if _has_custom_quantize_config(
relu_layer_node, bn_layer_node, conv_layer_node):
return relu_layer_node
conv_layer_node.layer['config']['activation'] = \
keras.activations.serialize(quantize_aware_activation.NoOpActivation())
bn_layer_node.metadata['quantize_config'] = \
default_8bit_quantize_configs.NoOpQuantizeConfig()
return relu_layer_node
def replacement(self, match_layer):
relu_layer_node = match_layer
bn_layer_node = relu_layer_node.input_layers[0]
conv_layer_node = bn_layer_node.input_layers[0]
if self._has_custom_quantize_config(relu_layer_node, bn_layer_node,
conv_layer_node):
return match_layer
conv_layer_node.layer['config']['activation'] = \
keras.activations.serialize(quantize_aware_activation.NoOpActivation())
bn_layer_node.metadata['quantize_config'] = \
default_8bit_quantize_configs.NoOpQuantizeConfig()
return match_layer
def replacement(self, match_layer):
concat_layer_node = match_layer
feeding_layer_nodes = match_layer.input_layers
default_registry = (
default_8bit_quantize_registry.Default8BitQuantizeRegistry())
feed_quantize_configs = []
for feed_layer_node in feeding_layer_nodes:
quantize_config = feed_layer_node.metadata.get('quantize_config')
if not quantize_config:
layer_class = self._get_layer_type(
feed_layer_node.layer['class_name'])
if layer_class is None:
# Concat has an input layer we don't recognize. Return.
return match_layer
if layer_class == keras.layers.Concatenate:
# Input layer to Concat is also Concat. Don't quantize it.
feed_layer_node.metadata['quantize_config'] = (
default_8bit_quantize_configs.NoOpQuantizeConfig())
continue
if not default_registry._is_supported_layer(layer_class):
# Feeding layer is not supported by registry
return match_layer
quantize_config = default_registry._get_quantize_config(
layer_class)
feed_layer_node.metadata['quantize_config'] = quantize_config
feed_quantize_configs.append(quantize_config)
# TODO(pulkitb): this currently only disables output quantize config, but
# cannot properly handle if the FQ was added to the activation. Hand this
# properly.
for quantize_config in feed_quantize_configs:
self._disable_output_quantize(quantize_config)
if not concat_layer_node.metadata.get('quantize_config'):
concat_layer_node.metadata['quantize_config'] = (
default_8bit_quantize_configs.Default8BitOutputQuantizeConfig(
))
return concat_layer_node
def __init__(self, label_count, apply_quantization, **kwargs):
super(ConvModel, self).__init__(**kwargs)
# create layers
self.input_quant = quantize_layer.QuantizeLayer(
AllValuesQuantizer(num_bits=8,
per_axis=False,
symmetric=False,
narrow_range=False))
self.conv1 = quantize.quantize_layer(
tf.keras.layers.Conv2D(filters=2,
kernel_size=[1, 3],
padding='SAME'))
self.bn1 = quantize.quantize_layer(
tf.keras.layers.BatchNormalization())
self.relu1 = quantize.quantize_layer(tf.keras.layers.ReLU())
self.conv2 = ring_buffer.RingBuffer(
quantize.quantize_layer(
tf.keras.layers.Conv2D(filters=2,
kernel_size=(3, 1),
dilation_rate=1,
strides=2,
use_bias=False), apply_quantization,
quantize.NoOpActivationConfig(['kernel'], ['activation'],
False)),
use_one_step=False,
inference_batch_size=self.inference_batch_size,
pad_time_dim='causal')
self.bn2 = quantize.quantize_layer(
tf.keras.layers.BatchNormalization(),
default_8bit_quantize_configs.NoOpQuantizeConfig())
self.relu2 = quantize.quantize_layer(tf.keras.layers.ReLU())
self.flatten = ring_buffer.RingBuffer(
quantize.quantize_layer(tf.keras.layers.Flatten(),
apply_quantization),
use_one_step=True,
inference_batch_size=self.inference_batch_size)
self.dense = quantize.quantize_layer(
tf.keras.layers.Dense(label_count,
activation='softmax',
use_bias=False), apply_quantization)
def replacement(self, match_layer):
if _has_custom_quantize_config(match_layer):
return match_layer
sepconv1d_layer = match_layer.layer
sepconv1d_config = sepconv1d_layer['config']
sepconv1d_weights = list(match_layer.weights.values())
padding = sepconv1d_config['padding']
# SepConv2D does not accept causal padding, and SepConv1D has some special
# handling for it.
# TODO(pulkitb): Add support for causal padding.
if padding == 'causal':
raise ValueError(
'SeparableConv1D with causal padding is not supported.')
# TODO(pulkitb): Handle other base_layer args such as dtype, input_dim etc.
sepconv2d_layer = tf.keras.layers.SeparableConv2D(
filters=sepconv1d_config['filters'],
kernel_size=(1, ) +
_normalize_tuple(sepconv1d_config['kernel_size']),
strides=_normalize_tuple(sepconv1d_config['strides']) * 2,
padding=padding,
data_format=sepconv1d_config['data_format'],
dilation_rate=(1, ) +
_normalize_tuple(sepconv1d_config['dilation_rate']),
depth_multiplier=sepconv1d_config['depth_multiplier'],
activation=sepconv1d_config['activation'],
use_bias=sepconv1d_config['use_bias'],
depthwise_initializer=sepconv1d_config['depthwise_initializer'],
pointwise_initializer=sepconv1d_config['pointwise_initializer'],
bias_initializer=sepconv1d_config['bias_initializer'],
depthwise_regularizer=sepconv1d_config['depthwise_regularizer'],
pointwise_regularizer=sepconv1d_config['pointwise_regularizer'],
bias_regularizer=sepconv1d_config['bias_regularizer'],
activity_regularizer=sepconv1d_config['activity_regularizer'],
depthwise_constraint=sepconv1d_config['depthwise_constraint'],
pointwise_constraint=sepconv1d_config['pointwise_constraint'],
bias_constraint=sepconv1d_config['bias_constraint'],
# TODO(pulkitb): Rethink what to do for name. Using the same name leads
# to confusion, since it's typically separable_conv1d
name=sepconv1d_config['name'] + '_QAT_SepConv2D',
trainable=sepconv1d_config['trainable'])
sepconv2d_weights = collections.OrderedDict()
sepconv2d_weights['depthwise_kernel:0'] = np.expand_dims(
sepconv1d_weights[0], 0)
sepconv2d_weights['pointwise_kernel:0'] = np.expand_dims(
sepconv1d_weights[1], 0)
if sepconv1d_config['use_bias']:
sepconv2d_weights['bias:0'] = sepconv1d_weights[2]
if sepconv1d_config['data_format'] == 'channels_last':
spatial_dim = 1
else:
spatial_dim = 2
sepconv2d_layer_config = keras.layers.serialize(sepconv2d_layer)
sepconv2d_layer_config['name'] = sepconv2d_layer.name
# Needed to ensure these new layers are considered for quantization.
sepconv2d_metadata = {'quantize_config': None}
# TODO(pulkitb): Consider moving from Lambda to custom ExpandDims/Squeeze.
# Layer before SeparableConv2D which expands input tensors to match 2D.
expand_layer = tf.keras.layers.Lambda(
lambda x: tf.expand_dims(x, spatial_dim),
name=self._get_name('sepconv1d_expand'))
expand_layer_config = keras.layers.serialize(expand_layer)
expand_layer_config['name'] = expand_layer.name
expand_layer_metadata = {
'quantize_config':
default_8bit_quantize_configs.NoOpQuantizeConfig()
}
squeeze_layer = tf.keras.layers.Lambda(
lambda x: tf.squeeze(x, [spatial_dim]),
name=self._get_name('sepconv1d_squeeze'))
squeeze_layer_config = keras.layers.serialize(squeeze_layer)
squeeze_layer_config['name'] = squeeze_layer.name
squeeze_layer_metadata = {
'quantize_config':
default_8bit_quantize_configs.NoOpQuantizeConfig()
}
return LayerNode(squeeze_layer_config,
metadata=squeeze_layer_metadata,
input_layers=[
LayerNode(sepconv2d_layer_config,
weights=sepconv2d_weights,
metadata=sepconv2d_metadata,
input_layers=[
LayerNode(
expand_layer_config,
metadata=expand_layer_metadata)
])
])
def model(flags):
"""CNN model.
It is based on paper:
Convolutional Neural Networks for Small-footprint Keyword Spotting
http://www.isca-speech.org/archive/interspeech_2015/papers/i15_1478.pdf
Model topology is similar with "Hello Edge: Keyword Spotting on
Microcontrollers" https://arxiv.org/pdf/1711.07128.pdf
Args:
flags: data/model parameters
Returns:
Keras model for training
"""
input_audio = tf.keras.layers.Input(shape=modes.get_input_data_shape(
flags, modes.Modes.TRAINING),
batch_size=flags.batch_size)
net = input_audio
if flags.preprocess == 'raw':
# it is a self contained model, user need to feed raw audio only
net = speech_features.SpeechFeatures(
speech_features.SpeechFeatures.get_params(flags))(net)
if flags.quantize:
net = quantize_layer.QuantizeLayer(
AllValuesQuantizer(num_bits=8,
per_axis=False,
symmetric=False,
narrow_range=False))(net)
net = tf.keras.backend.expand_dims(net)
for filters, kernel_size, activation, dilation_rate, strides in zip(
utils.parse(flags.cnn_filters), utils.parse(flags.cnn_kernel_size),
utils.parse(flags.cnn_act), utils.parse(flags.cnn_dilation_rate),
utils.parse(flags.cnn_strides)):
net = stream.Stream(cell=quantize.quantize_layer(
tf.keras.layers.Conv2D(filters=filters,
kernel_size=kernel_size,
dilation_rate=dilation_rate,
activation='linear',
strides=strides), flags.quantize,
quantize.NoOpActivationConfig(['kernel'], ['activation'], False)),
pad_time_dim='causal',
use_one_step=False)(net)
net = quantize.quantize_layer(
tf.keras.layers.BatchNormalization(),
default_8bit_quantize_configs.NoOpQuantizeConfig())(net)
net = quantize.quantize_layer(
tf.keras.layers.Activation(activation))(net)
net = stream.Stream(cell=quantize.quantize_layer(
tf.keras.layers.Flatten(), apply_quantization=flags.quantize))(net)
net = tf.keras.layers.Dropout(rate=flags.dropout1)(net)
for units, activation in zip(utils.parse(flags.units2),
utils.parse(flags.act2)):
net = quantize.quantize_layer(tf.keras.layers.Dense(
units=units, activation=activation),
apply_quantization=flags.quantize)(net)
net = quantize.quantize_layer(
tf.keras.layers.Dense(units=flags.label_count),
apply_quantization=flags.quantize)(net)
if flags.return_softmax:
net = quantize.quantize_layer(tf.keras.layers.Activation('softmax'),
apply_quantization=flags.quantize)(net)
return tf.keras.Model(input_audio, net)
