def test_bias_update_to_dense(self):
"""
test bias correction on matmul layer
:return:
"""
tf.compat.v1.reset_default_graph()
inputs = tf.keras.Input(shape=(32, 32, 3,))
x = tf.keras.layers.Flatten()(inputs)
dense = tf.keras.layers.Dense(2, use_bias=False, activation=tf.nn.softmax, name="single_residual")(x)
# pylint: disable=no-member
_ = tf.nn.relu(dense)
init = tf.compat.v1.global_variables_initializer()
sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph())
sess.run(init)
dense_op = sess.graph.get_operation_by_name('single_residual/MatMul')
self.assertTrue(BiasUtils.is_bias_none(dense_op))
new_sess = BiasUtils.initialize_model_with_bias(sess, ['input_1'], ['Relu'])
dense_op = new_sess.graph.get_operation_by_name('single_residual/MatMul')
self.assertTrue(not BiasUtils.is_bias_none(dense_op))
new_sess.close()
def test_bias_add_with_conv(self):
"""
Test bias add on conv op
:return:
"""
tf.compat.v1.reset_default_graph()
inputs = tf.keras.Input(shape=(32, 32, 3,), name="inputs")
# create a conv without bias param
conv_op = tf.keras.layers.Conv2D(32, (3, 3), use_bias=False)(inputs)
bn_op = tf.keras.layers.BatchNormalization(fused=True)(conv_op)
# pylint: disable=no-member
_ = tf.nn.relu(bn_op)
init = tf.compat.v1.global_variables_initializer()
sess = tf.compat.v1.Session()
sess.run(init)
conv_op = sess.graph.get_operation_by_name('conv2d/Conv2D')
self.assertTrue(BiasUtils.is_bias_none(conv_op))
# new_sess = BiasUtils.initialize_model_with_bias(sess)
shape = BiasUtils.get_shape(conv_op)
numpy_data = np.random.rand(shape[0])
BiasUtils.update_bias_for_op(sess, conv_op, bias_as_numpy_array=numpy_data)
new_sess = save_and_load_graph('./temp_bn_fold', sess)
conv_op = new_sess.graph.get_operation_by_name('conv2d/Conv2D')
bias_as_numpy_data = BiasUtils.get_bias_as_numpy_data(new_sess, conv_op)
assert(not BiasUtils.is_bias_none(conv_op))
new_sess.close()
def scale_cls_set_with_conv_layers(model: tf.compat.v1.Session, cls_set: Tuple[tf.Operation, tf.Operation]) -> np.ndarray:
"""
API to invoke equalize layer params (update for weights and bias is in place)
:param model: active tf.compat.v1.Session
:param cls_set: Consecutive Conv layers Tuple whose weights and biases need to be equalized
:return: Scaling factor S_12 for each conv layer pair: numpy array
"""
with model.graph.as_default():
for module in cls_set:
if module.type not in ['Conv2D', 'DepthwiseConv2dNative']:
raise ValueError("Only conv layers are supported for cross layer equalization")
# Create structs for holding layer weights and bias parameters
prev_layer_params = libpymo.EqualizationParams()
curr_layer_params = libpymo.EqualizationParams()
# send as [Noc, Nic, kh, kw], TF format is [kh, kw, Nic, Noc]
prev_layer_params.weight = WeightTensorUtils.get_tensor_as_numpy_data(model, cls_set[0]). \
transpose((3, 2, 0, 1)).reshape(-1)
weight_shape = WeightTensorUtils.get_tensor_shape(cls_set[0])
prev_layer_params.weightShape = [weight_shape[3], weight_shape[2], weight_shape[0], weight_shape[1]]
prev_layer_params.isBiasNone = BiasUtils.is_bias_none(cls_set[0])
# send as [Noc, Nic, kh, kw], TF format is [kh, kw, Nic, Noc]
curr_layer_params.weight = WeightTensorUtils.get_tensor_as_numpy_data(model, cls_set[1]). \
transpose((3, 2, 0, 1)).reshape(-1)
weight_shape = WeightTensorUtils.get_tensor_shape(cls_set[1])
curr_layer_params.weightShape = [weight_shape[3], weight_shape[2], weight_shape[0], weight_shape[1]]
if not BiasUtils.is_bias_none(cls_set[0]):
prev_layer_params.bias = BiasUtils.get_bias_as_numpy_data(model, cls_set[0]).reshape(-1)
else:
prev_layer_params.isBiasNone = True
scaling_factor = libpymo.scaleLayerParams(prev_layer_params, curr_layer_params)
# convert received formats back to TF
# TF format is [kh, kw, Nic, Noc]
numpy_weight_reshaped = np.reshape(prev_layer_params.weight, prev_layer_params.weightShape). \
transpose((2, 3, 1, 0))
WeightTensorUtils.update_tensor_for_op(model, cls_set[0], numpy_weight_reshaped)
numpy_weight_reshaped = np.reshape(curr_layer_params.weight, curr_layer_params.weightShape). \
transpose((2, 3, 1, 0))
WeightTensorUtils.update_tensor_for_op(model, cls_set[1], numpy_weight_reshaped)
if not BiasUtils.is_bias_none(cls_set[0]):
numpy_bias_reshaped = np.reshape(prev_layer_params.bias, BiasUtils.get_shape(cls_set[0]))
BiasUtils.update_bias_for_op(model, cls_set[0], numpy_bias_reshaped)
return scaling_factor
def _get_conv_linear_params(model, layer_to_be_corrected):
"""
Extract weights and bias of given conv/linear layer
:param model: tf.compat.v1.Session type
:param layer_to_be_corrected: conv/linear layer as tf.Operation
:return: bias, weight and quantized weights as TensorParamBiasCorrection types
"""
bias_tensor = libpymo.TensorParamBiasCorrection()
# get weight tensor
weight_tensor, _ = get_weight_tensor_with_shape(
model, layer_to_be_corrected)
if weight_tensor is None:
logger.error('Weight tensor extraction failed for layer {%s}',
layer_to_be_corrected.name)
# get bias tensor, at this point we have initialized model layers to have bias add param.
assert not BiasUtils.is_bias_none(layer_to_be_corrected)
bias_tensor.data = BiasUtils.get_bias_as_numpy_data(
model, layer_to_be_corrected)
bias_tensor.shape = BiasUtils.get_shape(layer_to_be_corrected)
return bias_tensor, weight_tensor
def create_conv2d_dense_type_params(my_op: Op):
""" Create products for conv2d, dense, depthwise conv2d, and similar """
tf_op = my_op.get_module()
weight_op = WeightTensorUtils.get_read_op(tf_op)
create_and_connect_product('kernel', weight_op.outputs[0].shape, my_op, weight_op.outputs[0])
if not BiasUtils.is_bias_none(tf_op):
bias_op = BiasUtils.get_bias_read_op(tf_op)
create_and_connect_product('bias', bias_op.outputs[0].shape, my_op, bias_op.outputs[0])
def test_equalize_with_custom_model_no_bias(self):
"""
Test equalize with a custom model with conv without bias param
"""
tf.compat.v1.reset_default_graph()
sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph())
with sess.as_default():
inputs = tf.keras.Input(shape=(
32,
32,
3,
))
conv_op = tf.keras.layers.Conv2D(32, (3, 3),
use_bias=False)(inputs)
bn_op = tf.keras.layers.BatchNormalization(fused=True)(conv_op)
relu_1 = tf.nn.relu(bn_op)
conv2_op = tf.keras.layers.Conv2D(32, (3, 3),
use_bias=False)(relu_1)
bn_op_2 = tf.keras.layers.BatchNormalization(fused=True)(
conv2_op, training=False)
relu_2 = tf.nn.relu(bn_op_2)
init = tf.compat.v1.global_variables_initializer()
sess.run(init)
old_conv_op = sess.graph.get_operation_by_name('conv2d/Conv2D')
self.assertTrue(BiasUtils.is_bias_none(old_conv_op))
conv_op = sess.graph.get_operation_by_name('conv2d/Conv2D')
new_sess = equalize_model(sess, conv_op.inputs[0].op.name,
'Relu_1')
new_conv_op = new_sess.graph.get_operation_by_name('conv2d/Conv2D')
bias = BiasUtils.get_bias_as_numpy_data(new_sess, new_conv_op)
self.assertFalse(BiasUtils.is_bias_none(new_conv_op))
sess.close()
def test_bias_add_custom_model(self):
""" test update bias when no bias present """
tf.compat.v1.reset_default_graph()
tf.set_random_seed(0)
inputs = tf.keras.Input(shape=(
32,
32,
3,
))
conv_op = tf.keras.layers.Conv2D(32, (3, 3), use_bias=False)(inputs)
conv2_op = tf.keras.layers.Conv2D(32, (3, 3), use_bias=False)(inputs)
relu2 = tf.nn.relu(conv2_op)
add = tf.keras.layers.add([conv_op, relu2])
relu = tf.nn.relu(add)
init = tf.compat.v1.global_variables_initializer()
sess = tf.compat.v1.Session()
sess.run(init)
shape = WeightTensorUtils.get_tensor_shape(conv_op.op)
np.random.seed(0)
bias_data = np.random.rand(shape[3])
assert BiasUtils.is_bias_none(conv_op.op)
BiasUtils.update_bias_for_op(sess, conv_op.op, bias_data)
n_sess = aimet_tensorflow.utils.graph_saver.save_and_load_graph(
'./test_update', sess)
conv_op_updated = n_sess.graph.get_operation_by_name(conv_op.op.name)
assert not BiasUtils.is_bias_none(conv_op_updated)
updated_bias = BiasUtils.get_bias_as_numpy_data(
n_sess, conv_op_updated)
self.assertTrue(np.allclose(updated_bias, bias_data))
sess.close()
def _get_bias_tensor(sess: tf.compat.v1.Session,
conv: tf.Operation) -> libpymo.TensorParams():
"""
Get bias tensor in given conv op.
Packs bias in the format required for BN fold
(libpymo.TensorParams()).
:param sess: current session
:param conv: conv op
:return: return bias param in libpymo.TensorParams() format.
"""
# Bias tensor
bias_tensor = libpymo.TensorParams()
with sess.graph.as_default():
if not BiasUtils.is_bias_none(conv):
bias_tensor.shape = BiasUtils.get_shape(conv)
bias_tensor.data = BiasUtils.get_bias_as_numpy_data(sess, conv)
return bias_tensor
def test_equalize_fold_forward(self):
"""
Test equalize on a model with a forward bn fold
"""
tf.compat.v1.reset_default_graph()
inputs = tf.keras.Input(shape=(
32,
32,
3,
), name="inputs")
conv_op = tf.keras.layers.Conv2D(32, (3, 3))(inputs)
r_op = tf.nn.relu(conv_op)
bn_op = tf.keras.layers.BatchNormalization(fused=True)(r_op)
conv2_op = tf.keras.layers.Conv2D(32, (3, 3))(bn_op)
conv3_op = tf.keras.layers.Conv2D(32, (3, 3))(conv2_op)
_ = tf.nn.relu(conv3_op)
init = tf.compat.v1.global_variables_initializer()
sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph())
sess.run(init)
old_conv_op = sess.graph.get_operation_by_name('conv2d/Conv2D')
conv_bias_data_before_fold = BiasUtils.get_bias_as_numpy_data(
sess, old_conv_op)
conv_op = sess.graph.get_operation_by_name('conv2d/Conv2D')
new_sess = equalize_model(sess, conv_op.inputs[0].op.name, 'Relu_1')
new_conv_op = new_sess.graph.get_operation_by_name('conv2d/Conv2D')
self.assertFalse(BiasUtils.is_bias_none(new_conv_op))
conv_bias_data_after_fold = BiasUtils.get_bias_as_numpy_data(
new_sess, new_conv_op)
for i in range(len(conv_bias_data_before_fold)):
self.assertTrue(
conv_bias_data_before_fold[i] <= conv_bias_data_after_fold[i])
sess.close()
def _call_mo_correct_bias(corrected_model: tf.compat.v1.Session,
layer_name: str,
bias_correction: libpymo.BiasCorrection,
bias_shape: int):
"""
helper to perform bias correction using cpp backend
:param corrected_model: active tensorflow session with corrected model as tf.compat.v1.Session
:param layer_name: name of the layer to be bias corrected
:param bias_correction: bias correction inputs
:param bias_shape: shape of bias associated with the layer
:return: None, updates bias for the given layer
"""
bias_tensor = libpymo.TensorParamBiasCorrection()
layer_to_be_corrected = corrected_model.graph.get_operation_by_name(
layer_name)
with corrected_model.graph.as_default():
assert (layer_to_be_corrected.type
in ['Conv2D', 'DepthwiseConv2dNative', 'MatMul'])
if BiasUtils.is_bias_none(layer_to_be_corrected):
bias_tensor.data = np.zeros(bias_shape)
else:
# read bias from given op
bias_tensor.data = BiasUtils.get_bias_as_numpy_data(
corrected_model, layer_to_be_corrected)
# perform bias correction
bias_correction.correctBias(bias_tensor)
# this api updates bias or adds bias add to layer if not present
BiasUtils.update_bias_for_op(corrected_model,
layer_to_be_corrected,
np.array(bias_tensor.data))
def _fold_given_auto_selected_batch_norms(
sess: tf.compat.v1.Session,
layer_pairs: List[PairType]) -> tf.compat.v1.Session:
"""
Fold a given set of batch_norm layers into conv layers
:param sess: tf.compat.v1.Session
:param layer_pairs: pair of conv and bn layers
:return: new session with updated graph
"""
with sess.graph.as_default():
for pair in layer_pairs:
conv_linear, batchnorm, is_batch_norm_second = pair
assert conv_linear.type in [
'Conv2D', 'DepthwiseConv2dNative', 'MatMul'
]
# check flag
is_bias_valid = False
if not BiasUtils.is_bias_none(conv_linear):
is_bias_valid = True
bn_params = _get_bn_params(sess, batchnorm.op)
weight_tensor = _get_weight_tensor_transpose_reshape(
sess, conv_linear)
bias_tensor = _get_bias_tensor(sess, conv_linear)
bias = libpymo.fold(bn_params, weight_tensor, bias_tensor,
is_bias_valid, is_batch_norm_second)
# converting back to TF format [kh, kw, Nic, Noc] before updating weight tensor value
if conv_linear.type == 'DepthwiseConv2dNative':
# Depthwise conv layers in TF have outputs(Noc) set to 1.
# we send in format [Nic, Noc, kh, kw]
numpy_weight_reshaped = np.reshape(
weight_tensor.data, weight_tensor.shape).transpose(
(2, 3, 0, 1))
elif conv_linear.type == 'MatMul':
# o, i - convert to i , o
numpy_weight_reshaped = np.reshape(
weight_tensor.data,
[weight_tensor.shape[0], weight_tensor.shape[1]
]).transpose(1, 0)
else:
# conv2D case
# we sent in format [Noc, Nic, kh, kw]
numpy_weight_reshaped = np.reshape(
weight_tensor.data, weight_tensor.shape).transpose(
(2, 3, 1, 0))
WeightTensorUtils.update_tensor_for_op(sess, conv_linear,
numpy_weight_reshaped)
# remove bn op
BNUtils.skip_bn_op(sess, batchnorm.op, batchnorm.in_tensor,
batchnorm.out_tensor)
# update bias tensor, even in case there was no existing bias add op in given conv2D op.
bias_tensor_shape = [weight_tensor.shape[0]]
numpy_bias_reshaped = np.reshape(bias, bias_tensor_shape)
BiasUtils.update_bias_for_op(sess, conv_linear,
numpy_bias_reshaped)
# we edited the graph, so we should load and save for the metagraph associated with the session to be updated
after_bn_fold_sess = save_and_load_graph('./temp_bn_fold', sess)
return after_bn_fold_sess
def test_depthwise_custom(self):
""" test depthwise conv2d layer withput bias """
tf.compat.v1.reset_default_graph()
inputs = tf.keras.Input(shape=(
10,
10,
3,
))
x = tf.keras.layers.Conv2D(10, (1, 1))(inputs)
with tf.compat.v1.variable_scope("standalone_depthwise"):
x = tf.compat.v1.nn.depthwise_conv2d_native(
x,
tf.compat.v1.get_variable(
initializer=tf.random.truncated_normal(shape=(3, 3, 10,
1)),
name="depthwise_kernel"), [1, 1, 1, 1], 'VALID')
_ = tf.nn.relu(x)
init = tf.compat.v1.global_variables_initializer()
sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph())
sess.run(init)
op_list = sess.graph.get_operations()
depthwise_conv_op = sess.graph.get_operation_by_name(
'standalone_depthwise/DepthwiseConv2dNative')
input_op = sess.graph.get_operation_by_name('input_1')
output_op = sess.graph.get_operation_by_name('Relu')
input_op_names = ['input_1']
output_op_names = [output_op.name]
batch_size = 1
num_samples = 10
np.random.seed(0)
shape = input_op.outputs[0].shape
dataset = np.random.rand(10, 1, shape[1], shape[2], shape[3])
dataset = tf.convert_to_tensor(dataset)
dataset = tf.data.Dataset.from_tensor_slices(dataset)
quant_params = QuantParams(use_cuda=False)
bias_correction_params = BiasCorrectionParams(
batch_size=batch_size,
num_quant_samples=num_samples,
num_bias_correct_samples=num_samples,
input_op_names=input_op_names,
output_op_names=output_op_names)
assert (BiasUtils.is_bias_none(depthwise_conv_op))
new_sess = BiasCorrection.correct_bias(sess, bias_correction_params,
quant_params, dataset)
updated_conv_op = new_sess.graph.get_operation_by_name(
'standalone_depthwise/DepthwiseConv2dNative')
assert (not BiasUtils.is_bias_none(updated_conv_op))
sess.close()
new_sess.close()
def bias_correction_per_layer(
reference_model: tf.compat.v1.Session,
corrected_model: tf.compat.v1.Session,
bias_correct_params: BiasCorrectionParams,
layer_name_to_be_corrected: str, quant_params: QuantParams,
data_set: tf.data.Dataset) -> tf.compat.v1.Session:
"""
Helper function to perform empirical bias correction per layer.
:param reference_model: active tensorflow session for reference model
:param corrected_model: active tensorflow session for corrected model
:param bias_correct_params: bias correction params
:param layer_name_to_be_corrected: name of layer on which bias correction is to be performed
:param quant_params: Quantization specific params from user
:return: None, updates corrected model in-place.
"""
# Quantize model
quantize_model = BiasCorrection._get_quantized_model(
corrected_model, quant_params, bias_correct_params.input_op_names,
bias_correct_params.output_op_names,
bias_correct_params.num_quant_samples,
bias_correct_params.batch_size, data_set)
ref_layer = reference_model.graph.get_operation_by_name(
layer_name_to_be_corrected)
bias_correction = libpymo.BiasCorrection()
logger.info('Correcting layer %s', ref_layer.name)
n_batches_bias_correction = int(
np.ceil(bias_correct_params.num_bias_correct_samples /
bias_correct_params.batch_size))
reduced_dataset_iter = iter_first_x(data_set,
n_batches_bias_correction)
for batch_input in reduced_dataset_iter:
# reference model without corrected nodes
reference_output_batch = BiasCorrection._get_output_data(
reference_model, bias_correct_params.input_op_names,
ref_layer.name, batch_input)
quantized_model_output_batch = BiasCorrection._get_output_data(
quantize_model, bias_correct_params.input_op_names,
ref_layer.name, batch_input)
if ref_layer.type == 'MatMul':
extended_shape = np.concatenate(
(reference_output_batch.shape, np.array([1, 1])))
reference_output_batch = reference_output_batch.reshape(
extended_shape)
quantized_model_output_batch = quantized_model_output_batch.reshape(
extended_shape)
# we need to reshape from tensorflow shape NxHxWxC to NxCxHxW
bias_correction.storePreActivationOutput(
np.ascontiguousarray(reference_output_batch.transpose(0, 3, 1, 2)))
bias_correction.storeQuantizedPreActivationOutput(
np.ascontiguousarray(
quantized_model_output_batch.transpose(0, 3, 1, 2)))
bias_shape = None
# get shape for bias if the layer does not have bias
if BiasUtils.is_bias_none(ref_layer):
if ref_layer.type == 'MatMul':
bias_shape = reference_output_batch.shape[1]
elif ref_layer.type in ['Conv2D', 'DepthwiseConv2dNative']:
# for conv2d or depthwise conv2d
bias_shape = reference_output_batch.shape[3]
# bias is to be corrected in the corrected model graph
BiasCorrection._call_mo_correct_bias(corrected_model, ref_layer.name,
bias_correction, bias_shape)
logger.info('Completed empirical bias correction for layer %s',
ref_layer.name)
def test_bias_correction_model_tf_with_no_bias(self):
"""
Test bias correction for custom model
"""
tf.compat.v1.reset_default_graph()
inputs = tf.keras.Input(shape=(
32,
32,
3,
))
conv_op = tf.keras.layers.Conv2D(32, (3, 3), use_bias=False)(inputs)
relu_1 = tf.nn.relu(conv_op)
conv2_op = tf.keras.layers.Conv2D(32, (3, 3), use_bias=False)(relu_1)
relu_2 = tf.nn.relu(conv2_op)
conv3_op = tf.keras.layers.Conv2D(32, (3, 3), use_bias=False)(relu_2)
_ = tf.nn.relu(conv3_op)
init = tf.compat.v1.global_variables_initializer()
sess = tf.compat.v1.Session()
sess.run(init)
# updating random bias and weight for one conv
np.random.seed(0)
conv_op = sess.graph.get_operation_by_name('conv2d/Conv2D')
w_shape = WeightTensorUtils.get_tensor_shape(conv_op)
w_shape = WeightTensorUtils.get_tensor_shape(conv_op)
w_numpy_data = np.random.rand(w_shape[0], w_shape[1], w_shape[2],
w_shape[3])
# save and load the updated graph after high bias fold update
n_sess = save_and_load_graph('./test_update', sess)
conv_op = n_sess.graph.get_operation_by_name('conv2d/Conv2D')
input_op_name = conv_op.inputs[0].op.name
output_op = n_sess.graph.get_operation_by_name('Relu_2')
input_op_names = [input_op_name]
output_op_names = [output_op.name]
batch_size = 1
num_samples = 10
np.random.seed(0)
shape = conv_op.inputs[0].shape
dataset = np.random.rand(10, 1, shape[1], shape[2], shape[3])
dataset = tf.convert_to_tensor(dataset)
dataset = tf.data.Dataset.from_tensor_slices(dataset)
quant_params = QuantParams(quant_mode='tf', use_cuda=False)
bias_correction_params = BiasCorrectionParams(
batch_size=batch_size,
num_quant_samples=num_samples,
num_bias_correct_samples=num_samples,
input_op_names=input_op_names,
output_op_names=output_op_names)
conv_op = sess.graph.get_operation_by_name('conv2d_1/Conv2D')
assert (BiasUtils.is_bias_none(conv_op))
new_sess = BiasCorrection.correct_bias(
n_sess,
bias_correction_params,
quant_params,
dataset,
perform_only_empirical_bias_corr=False)
conv_op = new_sess.graph.get_operation_by_name('conv2d_1/Conv2D')
assert (not BiasUtils.is_bias_none(conv_op))
sess.close()
n_sess.close()
new_sess.close()
def scale_cls_set_with_depthwise_layers(model: tf.compat.v1.Session,
cls_set: Tuple[tf.Operation,
tf.Operation,
tf.Operation]) -> [np.ndarray, np.ndarray]:
"""
API to invoke equalize layer params for depth wise separable layers(update for weights and bias is in place)
:param model: active tf.compat.v1.Session
:param cls_set: Consecutive Conv layers whose weights and biases need to be equalized.
Second Conv layer is a depth-wise conv and third conv layer is point-wise conv
:return: Scaling factors S_12 and S_23 : numpy arrays
"""
# make sure you define the session and graph scope before making any graph updates.
with model.graph.as_default():
for module in cls_set:
if module.type not in ['Conv2D', 'DepthwiseConv2dNative']:
raise ValueError("Only conv layers are supported for cross layer equalization")
# Create structs for holding layer weights and bias parameters
prev_layer_params = libpymo.EqualizationParams()
curr_layer_params = libpymo.EqualizationParams()
next_layer_params = libpymo.EqualizationParams()
# send as [Noc, Nic, kh, kw], TF format is [kh, kw, Nic, Noc]
prev_layer_params.weight = WeightTensorUtils.get_tensor_as_numpy_data(model, cls_set[0]). \
transpose((3, 2, 0, 1)).reshape(-1)
weight_shape = WeightTensorUtils.get_tensor_shape(cls_set[0])
prev_layer_params.weightShape = [weight_shape[3], weight_shape[2], weight_shape[0], weight_shape[1]]
prev_layer_params.isBiasNone = BiasUtils.is_bias_none(cls_set[0])
# depthwise layer outputs is set to 1 in TF
# send as [Nic, Noc, kh, kw], TF format is [kh, kw, Nic, Noc]
curr_layer_params.weight = WeightTensorUtils.get_tensor_as_numpy_data(model, cls_set[1]). \
transpose((2, 3, 0, 1)).reshape(-1)
weight_shape = WeightTensorUtils.get_tensor_shape(cls_set[1])
# depthwise layer outputs is set to 1 in TF
# send as [Nic, Noc, kh, kw], TF format is [kh, kw, Nic, Noc]
curr_layer_params.weightShape = [weight_shape[2], weight_shape[3], weight_shape[0], weight_shape[1]]
curr_layer_params.isBiasNone = BiasUtils.is_bias_none(cls_set[1])
# send as [Noc, Nic, kh, kw] , TF format is [kh, kw, Nic, Noc]
next_layer_params.weight = WeightTensorUtils.get_tensor_as_numpy_data(model, cls_set[2]). \
transpose((3, 2, 0, 1)).reshape(-1)
weight_shape = WeightTensorUtils.get_tensor_shape(cls_set[2])
next_layer_params.weightShape = [weight_shape[3], weight_shape[2], weight_shape[0], weight_shape[1]]
if not BiasUtils.is_bias_none(cls_set[0]):
prev_layer_params.bias = BiasUtils.get_bias_as_numpy_data(model, cls_set[0]).reshape(-1)
else:
prev_layer_params.isBiasNone = True
if not BiasUtils.is_bias_none(cls_set[1]):
curr_layer_params.bias = BiasUtils.get_bias_as_numpy_data(model, cls_set[1]).reshape(-1)
else:
curr_layer_params.isBiasNone = True
scaling_params = libpymo.scaleDepthWiseSeparableLayer(prev_layer_params, curr_layer_params,
next_layer_params)
# convert received formats back to TF
# TF format is [kh, kw, Nic, Noc]
numpy_weight_reshaped_0 = np.reshape(prev_layer_params.weight, prev_layer_params.weightShape). \
transpose((2, 3, 1, 0))
WeightTensorUtils.update_tensor_for_op(model, cls_set[0], numpy_weight_reshaped_0)
# depthwise layer
numpy_weight_reshaped_1 = np.reshape(curr_layer_params.weight, curr_layer_params.weightShape). \
transpose((2, 3, 0, 1))
WeightTensorUtils.update_tensor_for_op(model, cls_set[1], numpy_weight_reshaped_1)
numpy_weight_reshaped_2 = np.reshape(next_layer_params.weight, next_layer_params.weightShape). \
transpose((2, 3, 1, 0))
WeightTensorUtils.update_tensor_for_op(model, cls_set[2], numpy_weight_reshaped_2)
if not BiasUtils.is_bias_none(cls_set[0]):
numpy_bias_reshaped = np.reshape(prev_layer_params.bias, BiasUtils.get_shape(cls_set[0]))
BiasUtils.update_bias_for_op(model, cls_set[0], numpy_bias_reshaped)
if not BiasUtils.is_bias_none(cls_set[1]):
numpy_bias_reshaped = np.reshape(curr_layer_params.bias, BiasUtils.get_shape(cls_set[1]))
BiasUtils.update_bias_for_op(model, cls_set[1], numpy_bias_reshaped)
return scaling_params.scalingMatrix12, scaling_params.scalingMatrix23
def bias_fold(sess: tf.compat.v1.Session, folded_pairs: List[Tuple[tf.Operation, tf.Operation]],
cls_set_info_list: List[ClsSetInfo]) -> tf.compat.v1.Session:
"""
Folds bias values greater than 3 * sigma to next layer's bias
:param sess: Current session
:param folded_pairs: Key: Conv/Linear layer Value: Corresponding folded BN layer
:param cls_set_info_list: List of info elements for each cls set
:return: updated session after graph updates from hbf
"""
with sess.graph.as_default():
# refresh the references saved during bn fold and cls.
cls_set_info_list, bn_layers = HighBiasFold._refresh_layer_set_info_before_hbf(sess, folded_pairs,
cls_set_info_list)
if not bn_layers:
logger.error('High Bias folding is not supported for models without BatchNorm Layers')
return sess
for cls_set_info in cls_set_info_list:
for cls_pair_info in cls_set_info.cls_pair_info_list:
# check if we have a corresponding bn layer
if cls_pair_info.layer1.name in bn_layers.keys():
# check if bias present in given conv2D(s)
if BiasUtils.is_bias_none(cls_pair_info.layer1) or BiasUtils.is_bias_none(cls_pair_info.layer2):
continue
prev_layer_params = libpymo.LayerParams()
curr_layer_params = libpymo.LayerParams()
scaling_parameter = cls_pair_info.scale_factor
prev_layer_bn_params =\
HighBiasFold.get_bn_params_for_bias_fold(sess,
bn_layers[cls_pair_info.layer1.name],
scaling_parameter)
prev_layer_params.activationIsRelu = cls_pair_info.relu_activation_between_layers
prev_layer_params.bias =\
BiasUtils.get_bias_as_numpy_data(sess, cls_pair_info.layer1).reshape(-1)
prev_bias_shape = BiasUtils.get_shape(cls_pair_info.layer1)
weight_shape = WeightTensorUtils.get_tensor_shape(cls_pair_info.layer1)
prev_layer_params.weightShape = [weight_shape[3], weight_shape[2], weight_shape[0],
weight_shape[1]]
curr_layer_params.bias =\
BiasUtils.get_bias_as_numpy_data(sess, cls_pair_info.layer2).reshape(-1)
curr_bias_shape = BiasUtils.get_shape(cls_pair_info.layer2)
weight_shape = WeightTensorUtils.get_tensor_shape(cls_pair_info.layer2)
# Handle depthwise layer case
# for a depthwise layer num outputs is set to 1 in TF
# send as [Nic, Noc, kh, kw], TF format is [kh, kw, Nic, Noc]
if cls_pair_info.layer2.type in ['DepthwiseConv2dNative']:
c_wt = WeightTensorUtils.get_tensor_as_numpy_data(
sess, cls_pair_info.layer2).transpose((2, 3, 0, 1))
curr_layer_params.weight = c_wt.reshape(-1)
curr_layer_params.weightShape = [weight_shape[2], weight_shape[3], weight_shape[0],
weight_shape[1]]
else:
# send as [Noc, Nic, kh, kw], TF format is [kh, kw, Nic, Noc]
c_wt = WeightTensorUtils.get_tensor_as_numpy_data(
sess, cls_pair_info.layer2).transpose((3, 2, 0, 1))
curr_layer_params.weight = c_wt.reshape(-1)
curr_layer_params.weightShape = [weight_shape[3], weight_shape[2], weight_shape[0],
weight_shape[1]]
libpymo.updateBias(prev_layer_params, curr_layer_params, prev_layer_bn_params)
BiasUtils.update_bias_for_op(sess, cls_pair_info.layer1, np.reshape(prev_layer_params.bias,
prev_bias_shape))
BiasUtils.update_bias_for_op(sess, cls_pair_info.layer2, np.reshape(curr_layer_params.bias,
curr_bias_shape))
else:
logger.info("skipping layer: {%s}", cls_pair_info.layer1.name)
# save and load the updated graph after high bias fold update
aftr_hbf_sess = save_and_load_graph('./temp_hbf', sess)
return aftr_hbf_sess
def test_bias_update_to_dense(self):
"""
test bias correction on matmul layer
:return:
"""
tf.compat.v1.reset_default_graph()
inputs = tf.keras.Input(shape=(
32,
32,
3,
))
x = tf.keras.layers.Flatten()(inputs)
dense = tf.keras.layers.Dense(2,
use_bias=False,
activation=tf.nn.softmax,
name="single_residual")(x)
_ = tf.nn.relu(dense)
init = tf.compat.v1.global_variables_initializer()
sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph())
sess.run(init)
op_list = sess.graph.get_operations()
input_op = sess.graph.get_operation_by_name('input_1')
output_op = sess.graph.get_operation_by_name('Relu')
input_op_names = ['input_1']
output_op_names = [output_op.name]
op_list = sess.graph.get_operations()
batch_size = 1
num_samples = 10
np.random.seed(0)
shape = input_op.outputs[0].shape
dataset = np.random.rand(10, 1, shape[1], shape[2], shape[3])
dataset = tf.convert_to_tensor(dataset)
dataset = tf.data.Dataset.from_tensor_slices(dataset)
quant_params = QuantParams(use_cuda=False)
bias_correction_params = BiasCorrectionParams(
batch_size=batch_size,
num_quant_samples=num_samples,
num_bias_correct_samples=num_samples,
input_op_names=input_op_names,
output_op_names=output_op_names)
dense_conv_op = sess.graph.get_operation_by_name(
'single_residual/MatMul')
assert (BiasUtils.is_bias_none(dense_conv_op))
new_sess = BiasCorrection.correct_bias(sess, bias_correction_params,
quant_params, dataset)
updated_dense_conv_op = new_sess.graph.get_operation_by_name(
'single_residual/MatMul')
bias = BiasUtils.get_bias_as_numpy_data(new_sess,
updated_dense_conv_op)
assert (not BiasUtils.is_bias_none(updated_dense_conv_op))
sess.close()
new_sess.close()