def _eightbitize_input_to_node(self,
namespace_prefix,
original_input_name,
reshape_dims_name,
reduction_dims_name,
dtype=dtypes.quint8):
"""Takes one float input to an op, and converts it to quantized form."""
unique_input_name = helper.unique_node_name_from_input(
original_input_name)
if unique_input_name in self.quantized_node_dict:
quantized_tuple = self.quantized_node_dict[unique_input_name]
return quantized_tuple[0], quantized_tuple[1], quantized_tuple[2]
reshape_input_name = namespace_prefix + "_reshape_" + unique_input_name
min_input_name = namespace_prefix + "_min_" + unique_input_name
max_input_name = namespace_prefix + "_max_" + unique_input_name
quantize_input_name = namespace_prefix + "_quantize_" + unique_input_name
reshape_input_node = helper.create_node(
"Reshape", reshape_input_name,
[original_input_name, reshape_dims_name])
helper.set_attr_dtype(reshape_input_node, "T", dtypes.float32)
self.add_output_graph_node(reshape_input_node)
min_input_node = helper.create_node(
"Min", min_input_name, [reshape_input_name, reduction_dims_name])
helper.set_attr_dtype(min_input_node, "T", dtypes.float32)
helper.set_attr_dtype(min_input_node, "Tidx", dtypes.int32)
helper.set_attr_bool(min_input_node, "keep_dims", False)
self.add_output_graph_node(min_input_node)
max_input_node = helper.create_node(
"Max", max_input_name, [reshape_input_name, reduction_dims_name])
helper.set_attr_dtype(max_input_node, "T", dtypes.float32)
helper.set_attr_dtype(max_input_node, "Tidx", dtypes.int32)
helper.set_attr_bool(max_input_node, "keep_dims", False)
self.add_output_graph_node(max_input_node)
quantize_input_node = helper.create_node(
"QuantizeV2", quantize_input_name,
[original_input_name, min_input_name, max_input_name])
helper.set_attr_dtype(quantize_input_node, "T", dtype)
helper.set_attr_string(quantize_input_node, "mode", b"SCALED")
helper.set_attr_string(quantize_input_node, "round_mode",
b"HALF_TO_EVEN")
# if FLAGS.model_name in ["wide_deep_large_ds"]:
# set_attr_string(quantize_input_node, "mode", b"MIN_FIRST")
# else:
# set_attr_string(quantize_input_node, "mode",
# b"SCALED" if self.intel_cpu_eightbitize else b"MIN_FIRST")
# set_attr_string(quantize_input_node, "round_mode",
# b"HALF_TO_EVEN" if self.intel_cpu_eightbitize
# else b"HALF_AWAY_FROM_ZERO")
self.add_output_graph_node(quantize_input_node)
min_output_name = quantize_input_name + ":1"
max_output_name = quantize_input_name + ":2"
self.quantized_node_dict[unique_input_name] = (quantize_input_name,
min_output_name,
max_output_name)
return quantize_input_name, min_output_name, max_output_name
def add_dequantize_result_node(self,
quantized_output_name,
original_node_name,
min_tensor_index=1):
min_max_inputs = [
"%s:%s" % (quantized_output_name, min_tensor_index),
"%s:%s" % (quantized_output_name, (min_tensor_index + 1))
]
dequantize_name = original_node_name
dequantize_node = helper.create_node(
"Dequantize", dequantize_name,
[quantized_output_name, min_max_inputs[0], min_max_inputs[1]])
helper.set_attr_dtype(dequantize_node, "T", dtypes.quint8)
helper.set_attr_string(
dequantize_node, "mode",
b"SCALED" if self.intel_cpu_eightbitize else b"MIN_FIRST")
self.add_output_graph_node(dequantize_node)
def _intel_cpu_quantize_weight_eightbit(self,
parent,
input_node,
per_channel,
quantization_mode=b"SCALED"):
base_name = input_node.name + "_"
qint8_const_name = base_name + "qint8_const"
min_name = base_name + "min"
max_name = base_name + "max"
float_tensor = tensor_util.MakeNdarray(input_node.attr["value"].tensor)
epsilon = 1e-4 # Needs to be set empirically if accuracy is not satisfactory
if parent in ("Conv2D", "MatMul"):
if per_channel:
ranges = np.abs(float_tensor).max(axis=(0, 1, 2))
min_value = -ranges
max_value = ranges
# nudging min-max values outside epsilon radius around zero
ranges[ranges < epsilon] = epsilon
min_value[np.abs(min_value) < epsilon] = -epsilon
max_value[np.abs(max_value) < epsilon] = epsilon
qint8_tensor = (float_tensor * 127.0 / ranges).astype(np.int8)
else:
min_value = np.min(float_tensor.flatten())
max_value = np.max(float_tensor.flatten())
# Same processing of min-max as in quantize_weight_eightbit
# function.
if min_value > 0.0:
min_value = 0.0
if min_value == max_value:
if abs(min_value) < 0.000001:
max_value = min_value + 1.0
elif min_value > 0:
max_value = 2 * min_value
else:
max_value = min_value / 2.0
sess = session.Session()
with sess.as_default():
quantize_op = array_ops.quantize_v2(
float_tensor,
min_value,
max_value,
dtypes.qint8,
mode=quantization_mode,
round_mode="HALF_TO_EVEN")
qint8_tensor = quantize_op[0].eval()
# Updated min-max values should be passed to the next feeding node.
min_value = quantize_op[1].eval()
max_value = quantize_op[2].eval()
elif parent == "DepthwiseConv2dNative":
# get the max values based on dim 0 and 1 for depthwise conv
# since, the output channel will be dim 2 * dim 3
ranges = np.abs(float_tensor).max(axis=(0, 1))
ranges = ranges.flatten()
min_value = -ranges
max_value = ranges
# nudging min-max values outside epsilon radius around zero
ranges[ranges < epsilon] = epsilon
min_value[np.abs(min_value) < epsilon] = -epsilon
max_value[np.abs(max_value) < epsilon] = epsilon
# Since output channel will be 1 dim which is dim 2 * dim 3
# When divide by range, qint8_tensor needs to be 3 dim
# where, 3rd dim should be same dim of ranges
a, b, c, d = float_tensor.shape
qint8_tensor = (float_tensor.reshape(a, b, c * d) * 127.0 /
ranges).astype(np.int8)
# get the shape back to 4 dim
qint8_tensor = qint8_tensor.reshape(a, b, c, d)
shape = tensor_util.TensorShapeProtoToList(
input_node.attr["value"].tensor.tensor_shape)
qint8_const_node = helper.create_constant_node(qint8_const_name,
qint8_tensor,
dtypes.qint8,
shape=shape)
min_node = helper.create_constant_node(min_name, min_value,
dtypes.float32)
max_node = helper.create_constant_node(max_name, max_value,
dtypes.float32)
dequantize_node = helper.create_node(
"Dequantize", input_node.name,
[qint8_const_name, min_name, max_name])
helper.set_attr_dtype(dequantize_node, "T", dtypes.qint8)
helper.set_attr_string(dequantize_node, "mode", b"SCALED")
self.add_output_graph_node(qint8_const_node)
self.add_output_graph_node(min_node)
self.add_output_graph_node(max_node)
self.add_output_graph_node(dequantize_node)
