def bindings_generator(cls, gen, node, qrec, in_eparams, out_eparams, cname) -> bool: if isinstance(node, (ActivationFusionBase, BroadcastableActivationFusion)): cnodes = node.contained_nodes() quants = [ gen.G.quantization[NodeId(node, fnode)] for fnode in cnodes ] if isinstance(cnodes[0], MatrixAddParameters): qrec = quants[0] out_q = quants[1] else: return False else: out_q = qrec set_add_in_scale(qrec) scaled_idx = qrec.cache['scaled_idx'] not_scaled_idx = 0 if scaled_idx else 1 gen.bindings.append( CommentBindingList("Node {} in1q {} in2q {} outq {}", cname, qrec.in_qs[scaled_idx], qrec.in_qs[not_scaled_idx], out_q.out_qs[0])) gen.bindings.append( NodeBindingList(cname, GNodeArgEdge(in_eparams[scaled_idx]), GNodeArgEdge(in_eparams[not_scaled_idx]), GNodeArgEdge(out_eparams[0], "GNA_OUT"), GNodeArgNode(node, 'infos'))) return True
def set_matadd_bindings(gen, node, step_idx, in_eparams, out_eparams, cname, qrec, out_q=None): del step_idx if out_q is None: out_q = qrec set_add_in_scale(qrec) scaled_idx = qrec.cache['scaled_idx'] not_scaled_idx = 0 if scaled_idx else 1 gen.bindings.append( CommentBindingList("Node {} in1q {} in2q {} outq {}", cname, qrec.in_qs[scaled_idx], qrec.in_qs[not_scaled_idx], out_q.out_qs[0])) if isinstance(node, PaddedAddFusionParameters): gen.bindings.append( NodeBindingList(cname, GNodeArgEdge(in_eparams[scaled_idx]), GNodeArgEdge(in_eparams[not_scaled_idx]), GNodeArgEdge(out_eparams[0], "GNA_OUT"), GNodeArgNode(node, 'infos'), GNodeArgNode(node.contained_nodes()[0], 'infos'))) else: gen.bindings.append( NodeBindingList(cname, GNodeArgEdge(in_eparams[scaled_idx]), GNodeArgEdge(in_eparams[not_scaled_idx]), GNodeArgEdge(out_eparams[0], "GNA_OUT"), GNodeArgNode(node, 'infos')))
def execute(cls, params, in_tensors, qrec: QRec, **kwargs): in_tensors = qrec.prepare_inputs(params, in_tensors, ktype="symmetric") if isinstance(params, Broadcastable) and params.is_broadcasted: in_tensors = params.broadcast_inputs(in_tensors) func = PIECEWISE_OPS[params.__class__] op = func['op'] if func['is_mult']: compute_in_out_scale(qrec, in_idx=(0, 1), out_idx=0) scale_mul_biases_q = qrec.cache['scale_mul_biases_q'] i1 = in_tensors[0].astype(np.int32) i2 = in_tensors[1].astype(np.int32) out_tensor = scale_mul_biases_q.apply_scales(op(i1, i2, np.int32)) else: # larger scale should be scaled set_add_in_scale(qrec) scale_mul_biases_q = qrec.cache['scale_mul_biases_q'] if qrec.cache['scaled_idx']: i1 = in_tensors[0].astype(np.int32) i2 = qrec.cache['scale_in_mul_biases_q'].apply_scales(in_tensors[1]) else: i1 = qrec.cache['scale_in_mul_biases_q'].apply_scales(in_tensors[0]) i2 = in_tensors[1].astype(np.int32) out_tensor = scale_mul_biases_q.apply_scales(op(i1, i2, None)) return qrec.get_outputs(params, [qrec.out_qs[0].clip(out_tensor)], ktype="symmetric")
def bindings_generator(cls, gen, node, qrec, in_eparams, out_eparams, cname) -> bool: step_idx = node.step_idx cnodes = node.contained_nodes() quants = [gen.G.quantization[NodeId(node, fnode)] for fnode in cnodes] add_node = [node for node in cnodes if isinstance( node, MatrixAddParameters)] if add_node: quants = [gen.G.quantization[NodeId( node, fnode)] for fnode in cnodes] set_add_in_scale(quants[1]) scaled_idx = quants[1].cache['scaled_idx'] not_scaled_idx = 0 if scaled_idx else 1 gen.bindings.append( CommentBindingList("Node {} in1q {} in2q {} outq {}", cname, quants[1].in_qs[scaled_idx], quants[1].in_qs[not_scaled_idx], quants[-1].out_qs[0]) ) gen.bindings.append( NodeBindingList(cname, GNodeArgEdge(in_eparams[scaled_idx]), GNodeArgEdge(in_eparams[not_scaled_idx]), GNodeArgEdge(out_eparams[0], "GNA_OUT"), GNodeArgNode(node, 'infos'), GNodeArgNode(node.contained_nodes()[0], 'infos') )) return True
def globals_generator(cls, gen, node, qrec, pnode, fnode) -> bool: if isinstance(pnode, PaddedAddFusionParameters): return False if isinstance(pnode, (ActivationFusionBase, BroadcastableActivationFusion)): cnodes = pnode.contained_nodes() quants = [ gen.G.quantization[NodeId(pnode, cnode)] for cnode in cnodes ] if isinstance(fnode, MatrixAddParameters): set_add_in_scale(quants[0]) if not quants[0].in_qs[0].signed: add_quant_bias_value1 = np.bitwise_and( quants[0].cache['add_bias_offset'], 0xFF) add_quant_bias_value2 = np.bitwise_and( quants[0].cache['add_bias_offset'], 0xFF00) >> 8 else: add_quant_bias_value1 = add_quant_bias_value2 = None act_infos( gen, pnode, pnode, cnodes[1], quants[1], extra1=quants[0].cache['scale_in_mul_biases_q'].qbiases[0], extra2=quants[0].cache['scale_in_mul_biases_q'].qnorms[0], extra3=quants[0].cache['scale_mul_biases_q'].qbiases[0], extra4=quants[0].cache['scale_mul_biases_q'].qnorms[0], extra5=add_quant_bias_value1, extra6=add_quant_bias_value2) return True else: return False set_add_in_scale(qrec) if not qrec.in_qs[0].signed: add_quant_bias_value1 = np.bitwise_and( qrec.cache['add_bias_offset'], 0xFF) add_quant_bias_value2 = np.bitwise_and( qrec.cache['add_bias_offset'], 0xFF00) >> 8 else: add_quant_bias_value1 = add_quant_bias_value2 = None act_infos(gen, pnode, pnode, None, None, extra1=qrec.cache['scale_in_mul_biases_q'].qbiases[0], extra2=qrec.cache['scale_in_mul_biases_q'].qnorms[0], extra3=qrec.cache['scale_mul_biases_q'].qbiases[0], extra4=qrec.cache['scale_mul_biases_q'].qnorms[0], extra5=add_quant_bias_value1, extra6=add_quant_bias_value2) return True
def set_matadd_bindings(gen, node, step_idx, in_eparams, out_eparams, cname, qrec, out_q=None): del step_idx if out_q is None: out_q = qrec set_add_in_scale(qrec) scaled_idx = qrec.cache['scaled_idx'] not_scaled_idx = 0 if scaled_idx else 1 gen.bindings.append( CommentBindingList("Node {} in1q {} in2q {} outq {}", cname, qrec.in_qs[scaled_idx], qrec.in_qs[not_scaled_idx], out_q.out_qs[0]) ) gen.bindings.append( NodeBindingList(cname, GNodeArgEdge(in_eparams[scaled_idx]), GNodeArgEdge(in_eparams[not_scaled_idx]), GNodeArgEdge(out_eparams[0], "GNA_OUT") ))
def globals_generator(cls, gen, node, qrec, pnode, fnode) -> bool: cnodes = node.contained_nodes() quants = [gen.G.quantization[NodeId(node, fnode)] for fnode in cnodes] for qrec in quants: compute_in_out_scale(qrec) act_node = [cnode for cnode in cnodes if isinstance( cnode, ActivationParameters)] act_node = act_node[0] if act_node else None act_qrec = quants[-1] if act_node else None set_add_in_scale(quants[1]) act_infos(gen, pnode, pnode, act_node, act_qrec, extra1=quants[1].cache['scale_in_mul_biases_q'].qbiases[0], extra2=quants[1].cache['scale_in_mul_biases_q'].qnorms[0], extra3=quants[1].cache['scale_mul_biases_q'].qbiases[0], extra4=quants[1].cache['scale_mul_biases_q'].qnorms[0]) act_infos(gen, pnode, cnodes[0], act_node, act_qrec, extra_name="Pad", extra1=quants[1].cache['scale_mul_biases_q'].qbiases[0], extra2=quants[1].cache['scale_mul_biases_q'].qnorms[0]) return True