def get_output_quantizers(self, layer):
self._assert_activation_layer(layer)
if not hasattr(layer.activation, '__name__'):
raise ValueError('Activation {} not supported by '
'DefaultNBitActivationQuantizeConfig.'.format(
layer.activation))
if layer.activation.__name__ in ['relu', 'swish']:
# 'relu' should generally get fused into the previous layer.
return [
quantizers.MovingAverageQuantizer(
num_bits=self._num_bits_activation,
per_axis=False,
symmetric=False,
narrow_range=False)
] # activation/output
elif layer.activation.__name__ in [
'linear', 'softmax', 'sigmoid', 'tanh'
]:
return []
raise ValueError('Activation {} not supported by '
'DefaultNBitActivationQuantizeConfig.'.format(
layer.activation))
def get_output_quantizers(self, layer):
return [
quantizers.MovingAverageQuantizer(num_bits=8,
per_axis=False,
symmetric=False,
narrow_range=False)
]
def __init__(self, weight_attrs, activation_attrs, quantize_output):
self.weight_attrs = weight_attrs
self.activation_attrs = activation_attrs
self.quantize_output = quantize_output
# TODO(pulkitb): For some layers such as Conv2D, per_axis should be True.
# Add mapping for which layers support per_axis.
self.weight_quantizer = quantizers.LastValueQuantizer(
num_bits=8, per_axis=False, symmetric=True, narrow_range=True)
self.activation_quantizer = quantizers.MovingAverageQuantizer(
num_bits=8, per_axis=False, symmetric=False, narrow_range=False)
def replacement(self, match_layer):
# TODO(pulkitb): Replace quantizer with InputLayer specific quantizer.
quant_layer = quantize_layer.QuantizeLayer(
quantizers.MovingAverageQuantizer(
num_bits=8, per_axis=False, symmetric=False, narrow_range=False))
layer_config = keras.layers.serialize(quant_layer)
layer_config['name'] = quant_layer.name
quant_layer_node = LayerNode(
layer_config,
input_layers=[match_layer])
return quant_layer_node
def __init__(
self,
# Conv2D params
filters,
kernel_size,
strides=(1, 1),
padding='valid',
data_format=None,
dilation_rate=(1, 1),
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None,
name=None,
# BatchNormalization params
axis=-1,
momentum=0.99,
epsilon=1e-3,
center=True,
scale=True,
beta_initializer='zeros',
gamma_initializer='ones',
moving_mean_initializer='zeros',
moving_variance_initializer='ones',
beta_regularizer=None,
gamma_regularizer=None,
beta_constraint=None,
gamma_constraint=None,
renorm=False,
renorm_clipping=None,
renorm_momentum=0.99,
fused=None,
trainable=True,
virtual_batch_size=None,
adjustment=None,
# Post-batchnorm activation.
post_activation=None,
# quantization params
is_quantized=True,
**kwargs):
super(_ConvBatchNorm2D,
self).__init__(filters,
kernel_size,
strides=strides,
padding=padding,
data_format=data_format,
dilation_rate=dilation_rate,
use_bias=False,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
activity_regularizer=activity_regularizer,
kernel_constraint=kernel_constraint,
bias_constraint=bias_constraint,
name=name,
**kwargs)
self.batchnorm = BatchNormalization(
axis=axis,
momentum=momentum,
epsilon=epsilon,
center=center,
scale=scale,
beta_initializer=beta_initializer,
gamma_initializer=gamma_initializer,
moving_mean_initializer=moving_mean_initializer,
moving_variance_initializer=moving_variance_initializer,
beta_regularizer=beta_regularizer,
gamma_regularizer=gamma_regularizer,
beta_constraint=beta_constraint,
gamma_constraint=gamma_constraint,
renorm=renorm,
renorm_clipping=renorm_clipping,
renorm_momentum=renorm_momentum,
fused=fused,
trainable=trainable,
virtual_batch_size=virtual_batch_size,
adjustment=adjustment,
)
# Named as post_activation to not conflict with Layer self.activation.
self.post_activation = activations.get(post_activation)
self.is_quantized = is_quantized
if self.is_quantized:
# TODO(b/142132535): update when we move to new quantization scheme.
self.weight_quantizer = quantizers.LastValueQuantizer(
num_bits=8, per_axis=False, symmetric=False)
self.activation_quantizer = quantizers.MovingAverageQuantizer(
num_bits=8, per_axis=False, symmetric=False)
def __init__(
self,
# DepthwiseConv2D params
kernel_size,
strides=(1, 1),
padding='valid',
depth_multiplier=1,
data_format=None,
depthwise_initializer='glorot_uniform',
depthwise_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
depthwise_constraint=None,
bias_constraint=None,
name=None,
# BatchNormalization params
axis=-1,
momentum=0.99,
epsilon=1e-3,
center=True,
scale=True,
beta_initializer='zeros',
gamma_initializer='ones',
moving_mean_initializer='zeros',
moving_variance_initializer='ones',
beta_regularizer=None,
gamma_regularizer=None,
beta_constraint=None,
gamma_constraint=None,
renorm=False,
renorm_clipping=None,
renorm_momentum=0.99,
fused=None,
trainable=True,
virtual_batch_size=None,
adjustment=None,
# Post-batchnorm activation instance.
post_activation=None,
# quantization params
is_quantized=True,
**kwargs):
super(_DepthwiseConvBatchNorm2D, self).__init__(
kernel_size,
strides=strides,
padding=padding,
depth_multiplier=depth_multiplier,
data_format=data_format,
use_bias=False,
depthwise_initializer=depthwise_initializer,
depthwise_regularizer=depthwise_regularizer,
bias_regularizer=bias_regularizer,
activity_regularizer=activity_regularizer,
depthwise_constraint=depthwise_constraint,
bias_constraint=bias_constraint,
name=name,
**kwargs)
# TODO(b/187881826): conv_batchnorm should use v2 BatchNormalization layer
self.batchnorm = tf.compat.v1.layers.BatchNormalization(
axis=axis,
momentum=momentum,
epsilon=epsilon,
center=center,
scale=scale,
beta_initializer=beta_initializer,
gamma_initializer=gamma_initializer,
moving_mean_initializer=moving_mean_initializer,
moving_variance_initializer=moving_variance_initializer,
beta_regularizer=beta_regularizer,
gamma_regularizer=gamma_regularizer,
beta_constraint=beta_constraint,
gamma_constraint=gamma_constraint,
renorm=renorm,
renorm_clipping=renorm_clipping,
renorm_momentum=renorm_momentum,
fused=fused,
trainable=trainable,
virtual_batch_size=virtual_batch_size,
adjustment=adjustment,
)
self.post_activation = tf.keras.activations.get(post_activation)
self.is_quantized = is_quantized
if self.is_quantized:
self.weight_quantizer = default_8bit_quantizers.Default8BitConvWeightsQuantizer(
)
self.activation_quantizer = quantizers.MovingAverageQuantizer(
num_bits=8, per_axis=False, symmetric=False, narrow_range=False)
def testMovingAverageQuantizer(self):
quantizer = quantizers.MovingAverageQuantizer(**self.quant_params)
self._test_quantizer(quantizer)