def load_filter_parameters(cls, G, params, input_tensors, output_tensors, opts, converted_to_conv=False):
if opts.get('load_tensors') or opts.get('load_quantization'):
params.weights = input_tensors[1].value
if converted_to_conv:
params.weights = params.weights.transpose(cls.TF_LITE_DW_FILTER_TRANSPOSE)
if params.has_bias:
params.biases = input_tensors[2].value
if opts.get('load_quantization'):
if input_tensors[0].qtype is None:
raise NoQuantizationError("quantization not present in tflite file")
weights_scales, biases_scales, w_mins, w_maxes = cls.fix_weights_and_biases(
params, input_tensors, opts)
biases_q = SymmetricMultBiasesQType(dtype=np.int32, scale=biases_scales.flatten())
weights_q = SymmetricMultQType(
dtype=np.int8, narrow_range=True, scale=weights_scales.flatten(), min_val=w_mins, max_val=w_maxes)
in_q = input_tensors[0].qtype
out_q = output_tensors[0].qtype
mulbiases_q = MultMulBiasScaleQType.from_filter(in_q, weights_q, out_q, params)
qrec = MultScalableFilterQuantizationRecord(in_qs=[in_q],
out_qs=[out_q],
mul_biases_q=mulbiases_q,
weights_q=weights_q,
biases_q=biases_q)
G.quantization[NodeId(params)] = qrec
def calculate_q(self, G, node, astats, in_qs, dtype, out_dtype=None):
del G
if out_dtype is None:
out_dtype = dtype
if isinstance(node, (PoolingParameters, OutputParameters)):
o_q = in_qs[0]
elif isinstance(node, SoftMaxParameters):
o_q = SymmetricMultQType(min_val=-1, max_val=1, dtype=np.int16, scale=2**(-15))
else:
o_q = SymmetricMultQType.from_min_max(min_val=astats['min'],
max_val=astats['max'],
dtype=out_dtype)
if isinstance(node, (MatrixAddParameters, MatrixSubParameters)):
qrec = MultAddQuantizationRecord(in_qs=in_qs, out_qs=[o_q])
elif isinstance(node, (MatrixBroadcastedLinearOpParameters, MatScaleFusionParameters, GlobalPoolParameters)):
qrec = MultQuantizationRecord(in_qs=in_qs, out_qs=[o_q])
elif isinstance(node, ConstantInputParameters):
qrec = MultConstantQuantizationRecord(out_qs=[o_q],
constants_are_quantized=False)
elif isinstance(node, (FcParameters, Conv2DParameters)):
weights_q = SymmetricMultQType.from_array(arr=node.weights,
quantized_dimension=self.get_quantized_dimension(node),
dtype=dtype, narrow_range=self._narrow_weights)
if node.has_bias:
biases_q = SymmetricMultBiasesQType(dtype=np.int32, scale=weights_q.scale * in_qs[0].scale)
else:
biases_q = SymmetricMultBiasesQType(dtype=np.int32, scale=np.array([1], dtype=np.int32))
mul_biases_q = MultMulBiasScaleQType.from_filter(in_qs[0], weights_q, o_q, node)
qrec = MultScalableFilterQuantizationRecord(in_qs=[in_qs[0]],
out_qs=[o_q],
weights_q=weights_q,
biases_q=biases_q,
mul_biases_q=mul_biases_q,
constants_are_quantized=False)
LOG.debug("filter %s qrec %s", node.name, qrec)
else:
qrec = MultQuantizationRecord(in_qs=in_qs, out_qs=[o_q])
return qrec
def calculate_q(self, G, node, astats, in_qs, dtype, out_dtype=None):
if out_dtype is None:
out_dtype = dtype
if isinstance(node, (PoolingParameters, OutputParameters, SplitParameters)):
o_q = in_qs[0]
elif isinstance(node, SoftMaxParameters):
o_q = SymmetricMultQType(min_val=-1, max_val=1, dtype=np.int16, scale=2**(-15))
else:
o_q = SymmetricMultQType.from_min_max(min_val=astats['range_out'][0]['min'],
max_val=astats['range_out'][0]['max'],
dtype=out_dtype)
if isinstance(node, (MatrixAddParameters, MatrixSubParameters)):
qrec = MultAddQuantizationRecord(in_qs=in_qs, out_qs=[o_q])
elif isinstance(node, ExpressionFusionParameters):
o_qs = [SymmetricMultQType.from_min_max(min_val=orange['min'],
max_val=orange['max'],
dtype=out_dtype)
for orange in astats['range_out']]
fusion_inputs = sorted([n for n in node.subgraph.inputs()
if isinstance(n, FusionInputParameters)],
key=lambda x: x.idx)
fusion_outputs = sorted([n for n in node.subgraph.outputs()
if isinstance(n, FusionOutputParameters)],
key=lambda x: x.idx)
node_scale_map = {fnode: in_qs[idx].scale
for idx, fnode in enumerate(fusion_inputs)}
for idx, fnode in enumerate(fusion_outputs):
node_scale_map[fnode] = o_qs[idx].scale
inp, outp, expr = node.decompose(node_scale_map=node_scale_map)
qrec = MultExpressionQuantizationRecord(in_qs=in_qs,
out_qs=o_qs,
inputs=inp,
output_exprs=outp,
intermediate_exprs=expr)
elif isinstance(node, (MatrixBroadcastedLinearOpParameters, MatScaleFusionParameters, GlobalPoolParameters)):
qrec = MultQuantizationRecord(in_qs=in_qs, out_qs=[o_q])
elif isinstance(node, SplitParameters):
qrec = MultQuantizationRecord(in_qs=in_qs, out_qs=[o_q]*node.num_splits)
elif isinstance(node, ConstantInputParameters):
if node.value_quantization:
qrec = MultConstantQuantizationRecord(out_qs=[node.value_quantization],
constants_are_quantized=True)
else:
qrec = MultConstantQuantizationRecord(out_qs=[o_q],
constants_are_quantized=False)
elif isinstance(node, (FcParameters, Conv2DParameters)):
weights_q = SymmetricMultQType.from_array(arr=node.weights,
quantized_dimension=self.get_quantized_dimension(
node),
dtype=dtype, narrow_range=self._narrow_weights)
if node.has_bias:
biases_q = SymmetricMultBiasesQType(
dtype=np.int32, scale=weights_q.scale * in_qs[0].scale)
else:
biases_q = SymmetricMultBiasesQType(
dtype=np.int32, scale=np.array([1], dtype=np.int32))
mul_biases_q = MultMulBiasScaleQType.from_filter(in_qs[0], weights_q, o_q, node)
qrec = MultScalableFilterQuantizationRecord(in_qs=[in_qs[0]],
out_qs=[o_q],
weights_q=weights_q,
biases_q=biases_q,
mul_biases_q=mul_biases_q,
constants_are_quantized=False)
LOG.debug("filter %s qrec %s", node.name, qrec)
elif isinstance(node, RNNParameters):
input_nodes = {RNNParameters.INPUT_NAMES[edge.to_idx]: edge.from_node
for edge in G.in_edges(node.name)
if isinstance(edge.from_node, ConstantInputParameters)}
names = {val: idx for idx, val in enumerate(RNNParameters.INPUT_NAMES)}
# quantization_mode: extended, autotiler
# state_width: 16bit or 8bit
opts = self.get_options(node)
if opts['mode'] == "extended":
in_w_scale = in_qs[names['i_2_i_w']].scale * in_qs[0].scale
state_w_scale = in_qs[names['r_2_i_w']].scale
i_2_a_q = MultMulBiasScaleQType(scale=in_w_scale/state_w_scale)
s_2_s_q = MultMulBiasScaleQType(scale=state_w_scale)
s_2_o_q = MultMulBiasScaleQType(scale=1/o_q.scale)
self.rescale_constant(input_nodes['i_b'], state_w_scale, dtype=np.int32)
qrec = MultScalableRnnQuantizationRecord(
in_qs=in_qs,
out_qs=[o_q],
i_2_a_q=i_2_a_q,
s_2_s_q=s_2_s_q,
s_2_o_q=s_2_o_q
)
elif opts['mode'] == 'autotiler':
in_and_state_scale = np.maximum(in_qs[0].scale, o_q.scale)
in_and_state_w_scale = np.maximum(
in_qs[names['i_2_i_w']].scale, in_qs[names['r_2_i_w']].scale)
in_qs[0].scale = in_and_state_scale
o_q.scale = in_and_state_scale
self.rescale_constant(input_nodes['i_state'], in_and_state_scale)
self.rescale_constant(input_nodes['i_2_i_w'], in_and_state_w_scale)
self.rescale_constant(input_nodes['r_2_i_w'], in_and_state_w_scale)
state_w_scale = in_and_state_scale * in_and_state_w_scale
self.rescale_constant(input_nodes['i_b'], state_w_scale, dtype=np.int32)
s_2_s_q = MultMulBiasScaleQType(scale=state_w_scale/in_and_state_scale)
qrec = MultScalableRnnQuantizationRecord(
in_qs=in_qs,
out_qs=[o_q],
s_2_s_q=s_2_s_q,
)
elif isinstance(node, LSTMParameters):
input_nodes = {LSTMParameters.INPUT_NAMES[edge.to_idx]: edge.from_node
for edge in G.in_edges(node.name)
if isinstance(edge.from_node, ConstantInputParameters)}
names = {val: idx for idx, val in enumerate(LSTMParameters.INPUT_NAMES)}
if node.cell_clip:
cell_max = node.cell_clip
else:
cell_max = max(abs(astats['range_cell'][var]) for var in ['min', 'max'])
cell_int_bits = calc_bits(cell_max)
in_qs[names['c_state']].recalculate_scale(-cell_max,
cell_max)
LOG.debug("cell bits %d max %d cell range %d",
cell_int_bits,
cell_max,
in_qs[names['c_state']].range)
# worst case is (internal_q * 3) + 2 = 32 (1 for 1 and 1 for sign) i.e. 10
# but also (internal_q * 2) + cell_bits = 32
int_q = min((32-cell_int_bits)//2, 10)
# in and out and state are all in the same scale
in_and_out_scale = np.maximum(in_qs[0].scale, o_q.scale)
in_and_state_scale = np.maximum(in_and_out_scale, in_qs[names['i_state']].scale)
in_qs[0].scale = in_and_state_scale
o_q.scale = in_and_state_scale
self.rescale_constant(input_nodes['i_state'], in_and_state_scale)
scale_pairs = {chan: ('i_2_%s_w' % chan, 'r_2_%s_w' % chan)
for chan in ['i', 'o', 'c', 'f']}
scales = {k: np.maximum(in_qs[names[namei]].scale, in_qs[names[namer]].scale)
for k, (namei, namer) in scale_pairs.items()}
for k, (namei, namer) in scale_pairs.items():
self.rescale_constant(input_nodes[namei], scales[k])
self.rescale_constant(input_nodes[namer], scales[k])
int_scale = pow(2, -int_q)
int2_scale = pow(2, -(int_q*2))
int3_scale = pow(2, -(int_q*3))
# compute scales for perceptrons
pscales = {k: scales[k] * in_and_state_scale for k in ['i', 'o', 'c', 'f']}
scale_qtypes = {"r_2_%s_q" % k: MultMulBiasScaleQType(
scale=pscale/int_scale) for k, pscale in pscales.items()}
scale_qtypes['cell_in_q'] = MultMulBiasScaleQType(
scale=in_qs[names['c_state']].scale/int_scale)
# TODO - Check cell clip here
scale_qtypes['cell_out_q'] = MultMulBiasScaleQType(
scale=int2_scale/in_qs[names['c_state']].scale)
scale_qtypes['state_out_q'] = MultMulBiasScaleQType(scale=int3_scale/in_and_state_scale)
# set internal scale
scale_qtypes['i_qtype'] = QType(q=int_q, bits=32, signed=True)
# set biases to output of perceptron
for k in ['i', 'o', 'c', 'f']:
self.rescale_constant(input_nodes["%s_b" % k], pscales[k], dtype=np.int32)
qrec = MultScalableLstmQuantizationRecord(
in_qs=in_qs,
out_qs=[o_q],
**scale_qtypes,
)
else:
qrec = MultQuantizationRecord(in_qs=in_qs, out_qs=[o_q])
return qrec