def quant_embedding(program, place, config=None, scope=None):
"""quantize lookup_table op parameters
Args:
program(paddle.static.Program): infer program
scope(paddle.static.Scope, optional): Scope records the mapping between variable names and variables, similar to brackets in programming languages. Usually users can use `paddle.static.global_scope() <https://www.paddlepaddle.org.cn/documentation/docs/zh/develop/api_cn/executor_cn/global_scope_cn.html>`_ . When ``None`` will use `paddle.static.global_scope() <https://www.paddlepaddle.org.cn/documentation/docs/zh/develop/api_cn/executor_cn/global_scope_cn.html>`_. Default : ``None``.
place(paddle.CPUPlace or paddle.CUDAPlace): This parameter represents the executor run on which device.
config(dict, optional): config to quantize. The keys are 'quantize_op_types'. For op in quantize_op_types, you can define 'quantize_type', \
'quantize_bits', 'dtype', 'threshold'. \
``quantize_type`` is quantize type, supported types are ['abs_max'], default is "abs_max".
``quantize_bits`` supported bits are [8] and default is 8.
``dtype`` is quantize dtype, supported dtype are ['int8'], default is 'int8'.
``threshold`` is threshold to clip tensor before quant. When threshold is not set, \
tensor will not be clipped.
Returns:
None
"""
config = config or {}
config = _merge_config(copy.deepcopy(_default_config), config)
scope = paddle.static.global_scope() if scope is None else scope
graph = IrGraph(core.Graph(program.desc), for_test=True)
quantize_params_map = {}
all_op = graph.all_op_nodes()
for op in all_op:
if op.inputs == [] and op.outputs == []:
continue
op_type = op.name()
if op_type in config['quantize_op_types']:
weight_name = op.input('W')[0]
if weight_name in quantize_params_map.values():
continue
embedding_node = graph._find_node_by_name(op.inputs,
op.input('W')[0])
for op_node in embedding_node.outputs:
if op_node.name() == 'fused_embedding_seq_pool':
_split_embedding_seq_pool(graph, op_node)
if config[op_type]['quantize_type'] == 'abs_max':
_quant_embedding_abs_max(graph, scope, place, config[op_type],
weight_name, embedding_node)
elif config[op_type]['quantize_type'] == 'log':
_quant_embedding_log(graph, scope, place, config[op_type],
weight_name, embedding_node)
quantize_params_map[weight_name] = _get_quant_var_name(weight_name)
for op in all_op:
if op.name() == 'fused_embedding_seq_pool':
graph.safe_remove_nodes(op)
return graph.to_program()
def create_quant_model(model,
params,
activation_quantize_type='moving_average_abs_max',
weight_quantize_type='channel_wise_abs_max',
save=False):
place = paddle.CUDAPlace(0)
scope = global_scope()
exe = paddle.static.Executor(place)
[inference_program, feed_target_names, fetch_targets
] = paddle.static.load_inference_model(path_prefix=None,
executor=exe,
model_filename=model,
params_filename=params)
graph = IrGraph(core.Graph(inference_program.desc), for_test=True)
out_scale_op_list = [
"conv2d",
"depthwise_conv2d",
"mul",
"matmul",
"relu",
"leaky_relu",
"relu6",
"sigmoid",
"tanh",
"prelu",
"swish",
"softmax",
"batch_norm",
"layer_norm",
"elementwise_add",
"pool2d",
"reshape2",
"transpose2",
"concat",
"elementwise_mul",
"scale",
"slice",
"hard_swish",
"hard_sigmoid",
"conv2d_transpose",
"gru",
"bilinear_interp",
"nearest_interp",
"trilinear_interp",
"flatten",
"flatten2",
"transpose",
"pad2d",
"reshape",
"layer_norm",
]
op_real_in_out_name = {
"conv2d": [["Input", "Filter"], ["Output"]],
"depthwise_conv2d": [["Input", "Filter"], ["Output"]],
"conv2d_transpose": [["Input", "Filter"], ["Output"]],
"mul": [["X", "Y"], ["Out"]],
"matmul": [["X", "Y"], ["Out"]],
"pool2d": [["X"], ["Out"]],
"elementwise_add": [["X", "Y"], ["Out"]],
"concat": [["X"], ["Out"]],
"softmax": [["X"], ["Out"]],
"argmax": [["X"], ["Out"]],
"transpose": [["X"], ["Out"]],
"equal": [["X", "Y"], ["Out"]],
"gather": [["X"], ["Out"]],
"greater_equal": [["X", "Y"], ["Out"]],
"greater_than": [["X", "Y"], ["Out"]],
"less_equal": [["X", "Y"], ["Out"]],
"less_than": [["X", "Y"], ["Out"]],
"mean": [["X"], ["Out"]],
"not_equal": [["X", "Y"], ["Out"]],
"reshape": [["X"], ["Out"]],
"reshape2": [["X"], ["Out"]],
"transpose2": [["X"], ["Out"]],
"bilinear_interp": [["X"], ["Out"]],
"nearest_interp": [["X"], ["Out"]],
"trilinear_interp": [["X"], ["Out"]],
"slice": [["Input"], ["Out"]],
"squeeze": [["X"], ["Out"]],
"elementwise_sub": [["X", "Y"], ["Out"]],
"relu": [["X"], ["Out"]],
"relu6": [["X"], ["Out"]],
"leaky_relu": [["X"], ["Out"]],
"prelu": [["X"], ["Out"]],
"tanh": [["X"], ["Out"]],
"swish": [["X"], ["Out"]],
"dropout": [["X"], ["Out"]],
"batch_norm": [["X"], ["Y"]],
"layer_norm": [["X"], ["Y"]],
"sigmoid": [["X"], ["Out"]],
"elementwise_mul": [["X", "Y"], ["Out"]],
"scale": [["X"], ["Out"]],
"hard_swish": [["X"], ["Out"]],
"hard_sigmoid": [["X"], ["Out"]],
"gru": [["Input", "Weight"], ["Hidden"]],
"lstm": [["Input", "Weight"], ["Hidden"]],
"pad2d": [["X"], ["Out"]],
"flatten": [["X"], ["Out"]],
"flatten2": [["X"], ["Out"]],
}
def _get_op_output_var_names(op):
""" """
assert isinstance(op, (IrNode, Operator)), \
"The input op should be IrNode or Operator."
var_names = []
op_name = op.name() if isinstance(op, IrNode) \
else op.type
if op_name not in op_real_in_out_name:
return []
name_list = op_real_in_out_name[op_name][1]
for name in name_list:
var_name = op.output(name)
if isinstance(var_name, list):
var_names.extend(var_name)
else:
var_names.append(var_name)
return var_names
transform_pass = QuantizationTransformPass(
scope=scope,
place=place,
activation_quantize_type=activation_quantize_type,
weight_quantize_type=weight_quantize_type)
transform_pass.apply(graph)
op_nodes = graph.all_op_nodes()
for op_node in op_nodes:
if op_node.name() in out_scale_op_list:
var_names = _get_op_output_var_names(op_node)
for var_name in var_names:
in_node = graph._find_node_by_name(op_node.outputs, var_name)
if in_node.dtype() not in \
[core.VarDesc.VarType.FP64, core.VarDesc.VarType.FP32]:
continue
op_node.op()._set_attr("out_threshold", 3.0)
# Freeze graph for inference, but the weight of fc/conv is still float type.
freeze_pass = QuantizationFreezePass(
scope=scope, place=place, weight_quantize_type=weight_quantize_type)
freeze_pass.apply(graph)
main_program = graph.to_program()
# modify fake_quantize_moving_average_abs_max(InScale) and fake_channel_wise_dequantize_max_abs(Scales)
op_nodes = graph.all_op_nodes()
for op_node in op_nodes:
if op_node.name() == 'fake_quantize_moving_average_abs_max':
var_name = op_node.input("InScale")[0]
tensor = scope.var(var_name).get_tensor()
tensor.set(np.array([1], dtype=np.float32), place)
elif op_node.name() == 'fake_channel_wise_dequantize_max_abs':
var_name = op_node.input("Scales")[0]
tensor = scope.var(var_name).get_tensor()
tensor.set(np.ones(tensor.shape(), dtype=np.float32), place)
if save:
fluid.io.save_inference_model('test_inference_model',
feed_target_names,
fetch_targets,
exe,
main_program=main_program)
feed_vars = [
main_program.global_block().var(name) for name in feed_target_names
]
serialized_program = paddle.static.serialize_program(feed_vars,
fetch_targets,
program=main_program)
serialized_params = paddle.static.serialize_persistables(
feed_vars, fetch_targets, executor=exe, program=main_program)
return serialized_program, serialized_params