def apply(self, model, layer_quantize_map, quantize_registry, mode):
"""Implement vitis 8-bit transforms.
Currently this means the following.
1. Pull activations into layers, and apply fuse activations. (TODO)
2. Modify range in incoming layers for Concat. (TODO)
3. Fuse Conv2D/DepthwiseConv2D + BN into single layer.
Args:
model: Keras model to be quantized.
layer_quantize_map: Map with keys as layer names, and values as dicts
containing custom `QuantizeConfig`s which may have been passed with
layers.
quantize_registry: QuantizeRegistry object containing the quantize configs
for each layer.
mode: String object indicating the mode of the quantized model.
Returns:
(Transformed Keras model to better match TensorFlow Lite backend, updated
layer quantize map.)
"""
transforms = [
vitis_8bit_quantize_transforms.InputLayerQuantize(
quantize_registry.get_input_quantizer(), mode),
vitis_8bit_quantize_transforms.ConvActivationQuantize(),
vitis_8bit_quantize_transforms.AddActivationQuantize(),
]
return model_transformer.ModelTransformer(
model, transforms, set(layer_quantize_map.keys()),
layer_quantize_map).transform()
def apply(self, model, layer_quantize_map, remove_dropout, fold_conv_bn,
fold_bn, replace_relu6, include_cle, cle_steps):
"""Implement vitis 8-bit transforms.
All the transforms should not break the float model structure, and
the output of the transformed model should be consistant with the float
model.
"""
transforms = []
if remove_dropout:
transforms.append(vitis_optimize_transforms.RemoveDropout())
if fold_conv_bn:
transforms.append(vitis_optimize_transforms.Conv2DBatchNormFold())
if fold_bn:
transforms.append(vitis_optimize_transforms.BatchNormFold())
if replace_relu6:
transforms.append(vitis_optimize_transforms.ReplaceReLU6WithReLU())
transformed_model, layer_quantize_map = model_transformer.ModelTransformer(
model, transforms, None, layer_quantize_map).transform()
# Cross Layer Equalization
if include_cle:
cle_transforms = [
vitis_optimize_transforms.ConvConvCLE(),
vitis_optimize_transforms.ConvActConvCLE(),
]
progbar = keras.utils.Progbar(cle_steps)
print('[INFO] Start CrossLayerEqualization...')
for i in range(cle_steps):
progbar.update(i + 1)
tmp_model, tmp_layer_quantize_map = transformed_model, layer_quantize_map
transformed_model, layer_quantize_map = model_transformer.ModelTransformer(
tmp_model, cle_transforms, None,
tmp_layer_quantize_map).transform()
print('[INFO] CrossLayerEqualization Done.')
return transformed_model, layer_quantize_map
def _apply_availables(model, configs, available_transforms, candidate_layers,
layer_metadata):
transforms = []
for key in available_transforms:
if configs.get(key):
new_trans = available_transforms.get(key)
if isinstance(new_trans, list):
transforms.extend(new_trans)
else:
transforms.append(new_trans)
transformed_model, layer_metadata = model_transformer.ModelTransformer(
model, transforms, candidate_layers,
layer_metadata).recursive_transform()
return transformed_model, layer_metadata
def convert_quantize_strategy(model, conversion='tqt_to_pof2s'):
allowed_conversions = ['tqt_to_pof2s']
if not conversion in allowed_conversions:
logger.error(
'Invalid conversion {}, allowed conversions are: {}.'.format(
conversion, allowed_conversions))
if conversion == 'tqt_to_pof2s':
transforms = [
vitis_tqt_refine_transforms.ConvertTQTToPof2SQuantizeStrategy()
]
transformed_model, _ = model_transformer.ModelTransformer(
model, transforms, None, None).recursive_transform()
return transformed_model
return model
def apply(self, model, candidate_layers, layer_metadata):
"""Implement vitis 8-bit optimize transforms to make it more quantize-friendly.
All the transforms should not break the float model structure, and
the output of the transformed model should be consistant with the float
model.
"""
configs = self.get_configs()
available_transforms = collections.OrderedDict({
'remove_dropout':
vitis_optimize_transforms.RemoveDropout(),
'separate_conv_act':
vitis_optimize_transforms.SeparateConvAct(),
'convert_relu6_to_relu':
vitis_optimize_transforms.ConvertReLU6ToReLU(),
'convert_tf_op_to_keras':
vitis_optimize_transforms.ConvertTFOpToKeras(),
})
transformed_model, layer_metadata = _apply_availables(
model, configs, available_transforms, candidate_layers, layer_metadata)
# Train with bn is conflict with fold bn params
model = transformed_model
if configs['train_with_bn']:
transforms = [
vitis_optimize_transforms.FakeConvBNFold(),
]
transformed_model, _ = model_transformer.ModelTransformer(
model, transforms, None, None).recursive_transform()
else:
available_transforms = {
'fold_conv_bn': vitis_optimize_transforms.ConvBNFold(),
'convert_bn_to_dwconv': vitis_optimize_transforms.ConvertBNToDWConv(),
}
transformed_model, layer_metadata = _apply_availables(
model, configs, available_transforms, candidate_layers,
layer_metadata)
# Cross Layer Equalization
if configs['include_cle']:
cle_to_relu6 = configs['cle_to_relu6']
cle_balance_method = configs['cle_balance_method']
cle_weight_threshold = configs['cle_weight_threshold']
cle_transforms = [
vitis_equalization_transforms.ConvConvCLE(cle_to_relu6,
cle_balance_method,
cle_weight_threshold),
vitis_equalization_transforms.ConvActConvCLE(cle_to_relu6,
cle_balance_method,
cle_weight_threshold),
vitis_equalization_transforms.ConvReLUConvCLE(cle_to_relu6,
cle_balance_method,
cle_weight_threshold),
vitis_equalization_transforms.ConvReLUPadConvCLE(
cle_to_relu6, cle_balance_method, cle_weight_threshold)
]
cle_steps = configs['cle_steps']
progbar = keras.utils.Progbar(cle_steps)
logger.info('Start CrossLayerEqualization...')
for i in range(cle_steps):
progbar.update(i + 1)
tmp_model = transformed_model
transformed_model, layer_metadata = model_transformer.ModelTransformer(
tmp_model, cle_transforms, candidate_layers,
layer_metadata).recursive_transform()
logger.info('CrossLayerEqualization Done.')
if logger.debug_enabled():
model_utils.save_model(transformed_model, 'optimized_model.h5',
'./debug/')
return transformed_model, layer_metadata
def apply(self, model, candidate_layers, layer_metadata, quantize_registry,
mode):
"""Implement vitis 8-bit quantize transforms.
Args:
model: Keras model to be quantized.
quantize_registry: QuantizeRegistry object containing the quantize configs
for each layer.
mode: String object indicating the mode of the quantized model.
Returns:
(Quantized Keras model.)
"""
configs = self.get_configs()
available_pre_annotate_transforms = collections.OrderedDict({})
pre_annotated_model, layer_metadata = _apply_availables(
model, configs, available_pre_annotate_transforms, candidate_layers,
layer_metadata)
if logger.debug_enabled():
model_utils.save_model(pre_annotated_model, 'pre_annotated_model.h5',
'./debug/')
available_annotate_transforms = collections.OrderedDict({
'no_quant_in_conv_bn_act':
vitis_fs_quantize_transforms.NoQuantInConvBNAct(),
'no_quant_in_conv_act':
vitis_fs_quantize_transforms.NoQuantInConvAct(
pre_annotated_model, quantize_registry),
'no_quant_in_add_act':
vitis_fs_quantize_transforms.NoQuantInAddAct(
pre_annotated_model, quantize_registry),
})
annotated_model, layer_metadata = _apply_availables(
pre_annotated_model, configs, available_annotate_transforms,
candidate_layers, layer_metadata)
if logger.debug_enabled():
model_utils.save_model(annotated_model, 'annotated_model.h5', './debug/')
freeze_bn_delay = configs['freeze_bn_delay']
if freeze_bn_delay < 0:
freeze_bn_delay = None
quantize_transforms = [
vitis_fs_quantize_transforms.InputLayerQuantize(quantize_registry,
mode),
vitis_fs_quantize_transforms.ConvBNQuantize(quantize_registry, mode,
freeze_bn_delay),
vitis_fs_quantize_transforms.LayersQuantize(annotated_model,
quantize_registry, mode),
]
quantized_model, layer_metadata = model_transformer.ModelTransformer(
annotated_model, quantize_transforms, candidate_layers,
layer_metadata).recursive_transform()
input_quantize_transforms = [
vitis_fs_quantize_transforms.LayersInputQuantize(
quantized_model, quantize_registry, mode)
]
quantized_model, layer_metadata = model_transformer.ModelTransformer(
quantized_model, input_quantize_transforms, candidate_layers,
layer_metadata).recursive_transform()
return quantized_model, layer_metadata
def apply(self, quantized_model, candidate_layers, layer_metadata,
optimized_model, dataset, batch_size, steps, add_shape_info,
input_shape):
"""Implement vitis 8-bit float scale refine transforms.
Args:
qunantized_model: the quantized model to be refined.
optimized_model: the optimized float model used in fast finetune to generate fake label.
dataset: the dataset used in fast finetune.
batch_size: the batch size of dataset used in fast finetune.
steps: the steps of dataste used in fast finetune.
add_shape_info: bool, whether to add shape information to the refined model. Must be set True
for models with custom layers.
input_shape: the shape of the model inputs, if not set, the default shape in the model inputs
will be used.
Returns:
(Refined quantized model.)
"""
refined_model = quantized_model
configs = self.get_configs()
# Convert TQTQuantizer to Pof2SQuantizer
if configs['convert_to_pof2s_quantize_strategy']:
logger.info("Start Converting To Pof2S Quantize Strategy...")
transforms = [
vitis_tqt_refine_transforms.ConvertTQTToPof2SQuantizeStrategy(
)
]
refined_model, _ = model_transformer.ModelTransformer(
refined_model, transforms, None, None).recursive_transform()
logger.info("Converting To Pof2S Quantize Strategy Done.")
# Adjust quantize positions only when convert_to_pof2s_quantize_strategy is True
logger.info("Start Quantize Position Ajustment...")
adjust_vs = configs['adjust_dpu_sigmoid']
adjust_sc = configs['adjust_dpu_shift_cut']
adjust_sb = configs['adjust_dpu_shift_bias']
adjust_sr = configs['adjust_dpu_shift_read']
quantize_info = model_utils.get_quantize_info(refined_model)
adjusted_quantize_info = vitis_pof2s_refine_transforms.adjust_quantize_info(
refined_model, quantize_info, adjust_vs, adjust_sc, adjust_sb,
adjust_sr)
model_utils.set_quantize_info(refined_model,
adjusted_quantize_info)
logger.info("Quantize Position Ajustment Done.")
# Fast finetune
include_fast_ft = configs['include_fast_ft']
fast_ft_epochs = configs['fast_ft_epochs']
if include_fast_ft:
logger.info("Start Fast Finetuning...")
vitis_fast_finetune.fast_finetune(refined_model, optimized_model,
dataset, batch_size, steps,
fast_ft_epochs)
logger.info("Fast Finetuning Done.")
# # Bias correction
# include_bias_corr = configs['quantize_pipeline_config'][
# 'include_bias_corr']
# if include_bias_corr:
# logger.info("Start Bias Correction...")
# vitis_bias_correction.bias_correction(self._qcbev_model,
# self._optimized_model,
# calib_dataset, calib_batch_size,
# calib_steps)
# logger.info("Bias Correction Done.")
if add_shape_info:
logger.info("Start Getting Shape Information...")
shape_info = model_utils.get_shape(model=refined_model,
calib_dataset=dataset,
input_shape=input_shape)
if logger.debug_enabled():
model_utils.save_shape_info(shape_info, './debug/')
model_utils.save_model(refined_model,
'calibrated_model_add_shape.h5',
'./debug/')
logger.info("Getting Shape Information Done.")
return refined_model, layer_metadata
def apply(self, model, candidate_layers, layer_metadata, quantize_registry,
mode):
"""Implement vitis 8-bit quantize transforms.
Args:
model: Keras model to be quantized.
quantize_registry: QuantizeRegistry object containing the quantize configs
for each layer.
mode: String object indicating the mode of the quantized model.
Returns:
(Quantized Keras model.)
"""
configs = self.get_configs()
available_pre_annotate_transforms = collections.OrderedDict({
'convert_sigmoid_to_hard_sigmoid': [
vitis_pof2s_quantize_transforms.ConvertConvSigmoid(),
vitis_pof2s_quantize_transforms.ConvertActivationSigmoid(),
vitis_pof2s_quantize_transforms.ConvertConvSwish(),
vitis_pof2s_quantize_transforms.ConvertActivationSwish(),
],
'convert_hard_sigmoid_to_dpu_version': [
vitis_pof2s_quantize_transforms.ConvertHardSigmoidToDpuVersion(
),
],
'convert_average_pooling2d_to_dpu_version': [
vitis_pof2s_quantize_transforms.
ConvertAveragePooling2DToDpuVersion(),
vitis_pof2s_quantize_transforms.
ConvertGlobalAveragePooling2DToDpuVersion(),
],
'convert_leaky_relu_to_dpu_version': [
vitis_pof2s_quantize_transforms.ConvertLeakyReLUToDpuVersion(),
],
})
pre_annotated_model, layer_metadata = _apply_availables(
model, configs, available_pre_annotate_transforms,
candidate_layers, layer_metadata)
if logger.debug_enabled():
model_utils.save_model(pre_annotated_model,
'pre_annotated_model.h5', './debug/')
annotate_transforms = [
vitis_pof2s_quantize_transforms.AnnotateConvBNAct(),
vitis_pof2s_quantize_transforms.AnnotateConvAct(
pre_annotated_model, quantize_registry),
vitis_pof2s_quantize_transforms.AnnotateAddAct(
pre_annotated_model, quantize_registry),
vitis_pof2s_quantize_transforms.AnnotatePoolAct(
pre_annotated_model, quantize_registry),
]
annotated_model, layer_metadata = model_transformer.ModelTransformer(
pre_annotated_model, annotate_transforms, candidate_layers,
layer_metadata).recursive_transform()
if logger.debug_enabled():
model_utils.save_model(annotated_model, 'annotated_model.h5',
'./debug/')
quantize_transforms = [
vitis_pof2s_quantize_transforms.InputLayerQuantize(
quantize_registry, mode),
vitis_pof2s_quantize_transforms.ConvBNQuantize(
quantize_registry, mode, configs['freeze_bn_delay']),
vitis_pof2s_quantize_transforms.LayersQuantize(
annotated_model, quantize_registry, mode),
vitis_pof2s_quantize_transforms.LayersInputQuantize(
quantize_registry, mode),
vitis_pof2s_quantize_transforms.CustomLayerWrapper(
quantize_registry),
]
quantized_model, layer_metadata = model_transformer.ModelTransformer(
annotated_model, quantize_transforms, candidate_layers,
layer_metadata).recursive_transform()
return quantized_model, layer_metadata
def replace_hard_sigmoid(model):
transforms = [vitis_pof2s_quantize_transforms.ReplaceHardSigmoid()]
transformed_model, _ = model_transformer.ModelTransformer(
model, transforms, None, None).recursive_transform()
return transformed_model
def conv_bn_quantize_fold(model):
transforms = [vitis_pof2s_quantize_transforms.ConvBNQuantizeFold()]
transformed_model, _ = model_transformer.ModelTransformer(
model, transforms, None, None).recursive_transform()
return transformed_model
def real_conv_bn_fold(model):
transforms = [vitis_optimize_transforms.RealConvBNFold()]
transformed_model, _ = model_transformer.ModelTransformer(
model, transforms, None, None).recursive_transform()
return transformed_model
def separate_conv_act(model):
transforms = [vitis_optimize_transforms.SeparateConvAct()]
transformed_model, _ = model_transformer.ModelTransformer(
model, transforms, None, None).recursive_transform()
return transformed_model
def remove_layer(model, class_name, name='.*'):
"""Remove given layer from the model."""
transforms = [vitis_optimize_transforms.RemoveLayer(class_name, name)]
transformed_model, _ = model_transformer.ModelTransformer(
model, transforms, None, None).recursive_transform()
return transformed_model
