def bindings_generator(cls, gen, node, qrec, in_eparams, out_eparams,
cname) -> bool:
if isinstance(node,
(ActivationFusionBase, BroadcastableActivationFusion)):
cnodes = node.contained_nodes()
quants = [
gen.G.quantization[NodeId(node, fnode)] for fnode in cnodes
]
if isinstance(cnodes[0], MatrixAddParameters):
qrec = quants[0]
out_q = quants[1]
else:
return False
else:
out_q = qrec
set_add_in_scale(qrec)
scaled_idx = qrec.cache['scaled_idx']
not_scaled_idx = 0 if scaled_idx else 1
gen.bindings.append(
CommentBindingList("Node {} in1q {} in2q {} outq {}", cname,
qrec.in_qs[scaled_idx],
qrec.in_qs[not_scaled_idx], out_q.out_qs[0]))
gen.bindings.append(
NodeBindingList(cname, GNodeArgEdge(in_eparams[scaled_idx]),
GNodeArgEdge(in_eparams[not_scaled_idx]),
GNodeArgEdge(out_eparams[0], "GNA_OUT"),
GNodeArgNode(node, 'infos')))
return True
def set_matadd_bindings(gen,
node,
step_idx,
in_eparams,
out_eparams,
cname,
qrec,
out_q=None):
del step_idx
if out_q is None:
out_q = qrec
set_add_in_scale(qrec)
scaled_idx = qrec.cache['scaled_idx']
not_scaled_idx = 0 if scaled_idx else 1
gen.bindings.append(
CommentBindingList("Node {} in1q {} in2q {} outq {}", cname,
qrec.in_qs[scaled_idx], qrec.in_qs[not_scaled_idx],
out_q.out_qs[0]))
if isinstance(node, PaddedAddFusionParameters):
gen.bindings.append(
NodeBindingList(cname, GNodeArgEdge(in_eparams[scaled_idx]),
GNodeArgEdge(in_eparams[not_scaled_idx]),
GNodeArgEdge(out_eparams[0], "GNA_OUT"),
GNodeArgNode(node, 'infos'),
GNodeArgNode(node.contained_nodes()[0], 'infos')))
else:
gen.bindings.append(
NodeBindingList(cname, GNodeArgEdge(in_eparams[scaled_idx]),
GNodeArgEdge(in_eparams[not_scaled_idx]),
GNodeArgEdge(out_eparams[0], "GNA_OUT"),
GNodeArgNode(node, 'infos')))
def execute(cls, params,
in_tensors,
qrec: QRec,
**kwargs):
in_tensors = qrec.prepare_inputs(params, in_tensors, ktype="symmetric")
if isinstance(params, Broadcastable) and params.is_broadcasted:
in_tensors = params.broadcast_inputs(in_tensors)
func = PIECEWISE_OPS[params.__class__]
op = func['op']
if func['is_mult']:
compute_in_out_scale(qrec, in_idx=(0, 1), out_idx=0)
scale_mul_biases_q = qrec.cache['scale_mul_biases_q']
i1 = in_tensors[0].astype(np.int32)
i2 = in_tensors[1].astype(np.int32)
out_tensor = scale_mul_biases_q.apply_scales(op(i1, i2, np.int32))
else:
# larger scale should be scaled
set_add_in_scale(qrec)
scale_mul_biases_q = qrec.cache['scale_mul_biases_q']
if qrec.cache['scaled_idx']:
i1 = in_tensors[0].astype(np.int32)
i2 = qrec.cache['scale_in_mul_biases_q'].apply_scales(in_tensors[1])
else:
i1 = qrec.cache['scale_in_mul_biases_q'].apply_scales(in_tensors[0])
i2 = in_tensors[1].astype(np.int32)
out_tensor = scale_mul_biases_q.apply_scales(op(i1, i2, None))
return qrec.get_outputs(params, [qrec.out_qs[0].clip(out_tensor)], ktype="symmetric")
def bindings_generator(cls, gen, node, qrec, in_eparams, out_eparams, cname) -> bool:
step_idx = node.step_idx
cnodes = node.contained_nodes()
quants = [gen.G.quantization[NodeId(node, fnode)] for fnode in cnodes]
add_node = [node for node in cnodes if isinstance(
node, MatrixAddParameters)]
if add_node:
quants = [gen.G.quantization[NodeId(
node, fnode)] for fnode in cnodes]
set_add_in_scale(quants[1])
scaled_idx = quants[1].cache['scaled_idx']
not_scaled_idx = 0 if scaled_idx else 1
gen.bindings.append(
CommentBindingList("Node {} in1q {} in2q {} outq {}", cname,
quants[1].in_qs[scaled_idx], quants[1].in_qs[not_scaled_idx], quants[-1].out_qs[0])
)
gen.bindings.append(
NodeBindingList(cname, GNodeArgEdge(in_eparams[scaled_idx]),
GNodeArgEdge(in_eparams[not_scaled_idx]),
GNodeArgEdge(out_eparams[0], "GNA_OUT"),
GNodeArgNode(node, 'infos'),
GNodeArgNode(node.contained_nodes()[0], 'infos')
))
return True
def globals_generator(cls, gen, node, qrec, pnode, fnode) -> bool:
if isinstance(pnode, PaddedAddFusionParameters):
return False
if isinstance(pnode,
(ActivationFusionBase, BroadcastableActivationFusion)):
cnodes = pnode.contained_nodes()
quants = [
gen.G.quantization[NodeId(pnode, cnode)] for cnode in cnodes
]
if isinstance(fnode, MatrixAddParameters):
set_add_in_scale(quants[0])
if not quants[0].in_qs[0].signed:
add_quant_bias_value1 = np.bitwise_and(
quants[0].cache['add_bias_offset'], 0xFF)
add_quant_bias_value2 = np.bitwise_and(
quants[0].cache['add_bias_offset'], 0xFF00) >> 8
else:
add_quant_bias_value1 = add_quant_bias_value2 = None
act_infos(
gen,
pnode,
pnode,
cnodes[1],
quants[1],
extra1=quants[0].cache['scale_in_mul_biases_q'].qbiases[0],
extra2=quants[0].cache['scale_in_mul_biases_q'].qnorms[0],
extra3=quants[0].cache['scale_mul_biases_q'].qbiases[0],
extra4=quants[0].cache['scale_mul_biases_q'].qnorms[0],
extra5=add_quant_bias_value1,
extra6=add_quant_bias_value2)
return True
else:
return False
set_add_in_scale(qrec)
if not qrec.in_qs[0].signed:
add_quant_bias_value1 = np.bitwise_and(
qrec.cache['add_bias_offset'], 0xFF)
add_quant_bias_value2 = np.bitwise_and(
qrec.cache['add_bias_offset'], 0xFF00) >> 8
else:
add_quant_bias_value1 = add_quant_bias_value2 = None
act_infos(gen,
pnode,
pnode,
None,
None,
extra1=qrec.cache['scale_in_mul_biases_q'].qbiases[0],
extra2=qrec.cache['scale_in_mul_biases_q'].qnorms[0],
extra3=qrec.cache['scale_mul_biases_q'].qbiases[0],
extra4=qrec.cache['scale_mul_biases_q'].qnorms[0],
extra5=add_quant_bias_value1,
extra6=add_quant_bias_value2)
return True
def set_matadd_bindings(gen, node, step_idx, in_eparams, out_eparams, cname, qrec, out_q=None):
del step_idx
if out_q is None:
out_q = qrec
set_add_in_scale(qrec)
scaled_idx = qrec.cache['scaled_idx']
not_scaled_idx = 0 if scaled_idx else 1
gen.bindings.append(
CommentBindingList("Node {} in1q {} in2q {} outq {}", cname,
qrec.in_qs[scaled_idx], qrec.in_qs[not_scaled_idx], out_q.out_qs[0])
)
gen.bindings.append(
NodeBindingList(cname, GNodeArgEdge(in_eparams[scaled_idx]),
GNodeArgEdge(in_eparams[not_scaled_idx]),
GNodeArgEdge(out_eparams[0], "GNA_OUT")
))
def globals_generator(cls, gen, node, qrec, pnode, fnode) -> bool:
cnodes = node.contained_nodes()
quants = [gen.G.quantization[NodeId(node, fnode)] for fnode in cnodes]
for qrec in quants:
compute_in_out_scale(qrec)
act_node = [cnode for cnode in cnodes if isinstance(
cnode, ActivationParameters)]
act_node = act_node[0] if act_node else None
act_qrec = quants[-1] if act_node else None
set_add_in_scale(quants[1])
act_infos(gen, pnode, pnode, act_node, act_qrec,
extra1=quants[1].cache['scale_in_mul_biases_q'].qbiases[0],
extra2=quants[1].cache['scale_in_mul_biases_q'].qnorms[0],
extra3=quants[1].cache['scale_mul_biases_q'].qbiases[0],
extra4=quants[1].cache['scale_mul_biases_q'].qnorms[0])
act_infos(gen, pnode, cnodes[0], act_node, act_qrec, extra_name="Pad",
extra1=quants[1].cache['scale_mul_biases_q'].qbiases[0],
extra2=quants[1].cache['scale_mul_biases_q'].qnorms[0])
return True