def set_fc_bindings(gen,
step_idx,
in_eparams,
out_eparams,
cname,
params,
linear_q,
out_q=None):
if out_q is None:
out_q = linear_q
gen.bindings.append(
CommentBindingList("Node {} inq {} weightsq {} outq {}", params.name,
linear_q.in_qs[0].q, linear_q.in_qs[1].q,
out_q.out_qs[0].q))
gen.bindings.append(
NodeBindingList(cname, GNodeArgEdge(in_eparams[0]),
GNodeArgEdge(in_eparams[1]),
GNodeArgEdge(in_eparams[2]),
GNodeArgEdge(out_eparams[0], "GNA_OUT")))
def set_mat_vect_mul_bindings(gen,
node,
step_idx,
in_eparams,
out_eparams,
cname,
qrec,
out_q=None):
del step_idx
if out_q is None:
out_q = qrec
gen.bindings.append(
CommentBindingList("Node {} in1q {} in2q {} outq {}", cname,
qrec.in_qs[0], qrec.in_qs[1], out_q.out_qs[0]))
gen.bindings.append(
NodeBindingList(cname, GNodeArgEdge(in_eparams[0]),
GNodeArgEdge(in_eparams[1]),
GNodeArgEdge(out_eparams[0], "GNA_OUT"),
GNodeArgNode(node, 'infos')))
def set_conv_bindings(gen,
step_idx,
in_eparams,
out_eparams,
cname,
conv_params,
conv_q,
out_q=None):
del step_idx
if out_q is None:
out_q = conv_q
gen.bindings.append(
CommentBindingList("Node {} inq {} weightsq {} outq {} biasesq {}",
cname, conv_q.in_qs[0], conv_q.in_qs[1],
out_q.out_qs[0], conv_q.in_qs[2]))
gen.bindings.append(
NodeBindingList(cname, GNodeArgEdge(in_eparams[0]),
GNodeArgEdge(in_eparams[1]),
GNodeArgEdge(in_eparams[2]),
GNodeArgEdge(out_eparams[0], "GNA_OUT")))
def set_matadd_bindings(gen, node, step_idx, in_eparams, out_eparams, cname, qrec, out_q=None):
del step_idx
if out_q is None:
out_q = qrec
scaled_idx = qrec.scaled_idx
not_scaled_idx = 0 if scaled_idx else 1
gen.bindings.append(
CommentBindingList("Node {} in1q {} in2q {} outq {}", cname,
qrec.in_qs[scaled_idx], qrec.in_qs[not_scaled_idx], out_q.out_qs[0])
)
if isinstance(node, PaddedAddFusionParameters):
gen.bindings.append(
NodeBindingList(cname, GNodeArgEdge(in_eparams[scaled_idx]),
GNodeArgEdge(in_eparams[not_scaled_idx]),
GNodeArgEdge(out_eparams[0], "GNA_OUT"),
GNodeArgNode(node, 'infos'),
GNodeArgNode(node.contained_nodes()[0], 'infos')
))
else:
gen.bindings.append(
NodeBindingList(cname, GNodeArgEdge(in_eparams[scaled_idx]),
GNodeArgEdge(in_eparams[not_scaled_idx]),
GNodeArgEdge(out_eparams[0], "GNA_OUT"),
GNodeArgNode(node, 'infos')
))
def bindings_generator(cls, gen, node, qrec, in_eparams, out_eparams,
cname) -> bool:
step_idx = node.step_idx
cnodes = node.contained_nodes()
quants = [gen.G.quantization[NodeId(node, fnode)] for fnode in cnodes]
add_node = [
node for node in cnodes if isinstance(node, MatrixAddParameters)
]
if add_node:
quants = [
gen.G.quantization[NodeId(node, fnode)] for fnode in cnodes
]
gen.bindings.append(
CommentBindingList("Node {} in1q {} in2q {} outq {}", cname,
quants[1].in_qs[0], quants[1].in_qs[1],
quants[-1].out_qs[0]))
gen.bindings.append(
NodeBindingList(cname, GNodeArgEdge(in_eparams[0]),
GNodeArgEdge(in_eparams[1]),
GNodeArgEdge(out_eparams[0], "GNA_OUT")))
return True
def set_matadd_bindings(gen,
node,
step_idx,
in_eparams,
out_eparams,
cname,
qrec,
out_q=None):
del step_idx
if out_q is None:
out_q = qrec
scaled_idx = qrec.scaled_idx
not_scaled_idx = 0 if scaled_idx else 1
gen.bindings.append(
CommentBindingList("Node {} in1q {} in2q {} outq {}", cname,
qrec.in_qs[scaled_idx], qrec.in_qs[not_scaled_idx],
out_q.out_qs[0]))
gen.bindings.append(
NodeBindingList(cname, GNodeArgEdge(in_eparams[scaled_idx]),
GNodeArgEdge(in_eparams[not_scaled_idx]),
GNodeArgEdge(out_eparams[0], "GNA_OUT"),
GNodeArgNode(node, 'infos')))
def set_ssd_postprocess_bindings(gen,
in_eparams,
out_eparams,
cname,
node,
node_q,
out_q=None):
if out_q is None:
out_q = node_q
gen.bindings.append(
CommentBindingList(
"Node {} offsetsq {} scoresq {} anchorsq {} outboxesq {}",
node.name, str(node_q.in_qs[0]), str(node_q.in_qs[1]),
str(node_q.in_qs[2]), str(out_q.out_qs[0])))
gen.bindings.append(
NodeBindingList(cname, GNodeArgEdge(in_eparams[0]),
GNodeArgEdge(in_eparams[1]),
GNodeArgEdge(in_eparams[2]),
GNodeArgEdge(out_eparams[0], "GNA_OUT"),
GNodeArgNode(node, SSD_SCALES),
GNodeArgNode(node,
SSD_NORMS), GNodeArgNode(node, INFOS)))
def bindings_generator(cls, gen, node, qrec, in_eparams, out_eparams,
cname) -> bool:
if isinstance(node, (ActivationFusion, BroadcastableActivationFusion)):
cnodes = node.contained_nodes()
quants = [
gen.G.quantization[NodeId(node, fnode)] for fnode in cnodes
]
if isinstance(cnodes[0], MatrixAddParameters):
qrec = quants[0]
out_q = quants[1]
else:
return False
else:
out_q = qrec
gen.bindings.append(
CommentBindingList("Node {} in1q {} in2q {} outq {}", cname,
qrec.in_qs[0], qrec.in_qs[1], out_q.out_qs[0]))
gen.bindings.append(
NodeBindingList(cname, GNodeArgEdge(in_eparams[0]),
GNodeArgEdge(in_eparams[1]),
GNodeArgEdge(out_eparams[0], "GNA_OUT")))
return True
def set_fc_bindings(gen,
step_idx,
in_eparams,
out_eparams,
cname,
params,
linear_q,
out_q=None):
del step_idx
if out_q is None:
out_q = linear_q
gen.bindings.append(
CommentBindingList("Node {} inq {} weightsq {} outq {}", params.name,
linear_q.in_qs[0], linear_q.weights_q,
out_q.out_qs[0]))
gen.bindings.append(
NodeBindingList(cname, GNodeArgEdge(in_eparams[0]),
GNodeArgNode(params, WEIGHTS),
GNodeArgNode(params, BIASES),
GNodeArgEdge(out_eparams[0], "GNA_OUT"),
GNodeArgNode(params, MULSCALE),
GNodeArgNode(params, MULSHIFT),
GNodeArgNode(params, INFOS)))
def set_conv_bindings(gen, step_idx, in_eparams, out_eparams, cname, params, conv_q,
out_q=None):
if out_q is None:
out_q = conv_q
gen.bindings.append(
CommentBindingList("Node {} inq {} weightsq {} outq {} biasesq {}", cname,
conv_q.in_qs[0].q, conv_q.in_qs[1].q, out_q.out_qs[0].q, conv_q.in_qs[2].q)
)
if params.has_mul_bias:
gen.bindings.append(
NodeBindingList(cname, GNodeArgEdge(in_eparams[0]), GNodeArgEdge(in_eparams[1]),
GNodeArgEdge(in_eparams[2]), GNodeArgNode(params, MULSCALE),
GNodeArgEdge(out_eparams[0], "GNA_OUT")))
else:
gen.bindings.append(
NodeBindingList(cname, GNodeArgEdge(in_eparams[0]), GNodeArgEdge(in_eparams[1]),
GNodeArgEdge(in_eparams[2]),
GNodeArgEdge(out_eparams[0], "GNA_OUT")))
def bindings_generator(cls, gen, node, qrec, in_eparams, out_eparams,
cname) -> bool:
gen.bindings.append(CommentBindingList("Node {}", node.name))
gen.bindings.append(
NodeBindingList(cname, GNodeArgEdge(in_eparams[0]),
GNodeArgEdge(in_eparams[1]),
GNodeArgEdge(in_eparams[2]),
GNodeArgEdge(out_eparams[0], "GNA_OUT"),
GNodeArgEdge(out_eparams[1], "GNA_OUT"),
GNodeArgEdge(out_eparams[2], "GNA_OUT")))
return True
def bindings_generator(cls, gen, node, qrec, in_eparams, out_eparams,
cname) -> bool:
gen.bindings.append(
CommentBindingList(
f"Node {cname} input_q {qrec.in_qs[0].q} window_q {qrec.in_qs[1].q} fft_twiddles_q {qrec.in_qs[2].q} rfft_twiddles {qrec.in_qs[4].q} -> out_q {qrec.out_qs[0]}"
))
binding_list = [
GNodeArgEdge(in_eparams[0]),
GNodeArgEdge(out_eparams[0], "GNA_OUT"),
GNodeArgEdge(in_eparams[2]),
GNodeArgEdge(in_eparams[4]),
GNodeArgEdge(in_eparams[3]),
GNodeArgEdge(in_eparams[5]),
GNodeArgEdge(in_eparams[6])
]
if node.win_fn is not None:
binding_list.insert(5, GNodeArgEdge(in_eparams[1]))
if not qrec.out_qs[0].dtype in [np.float16, np.float32, bfloat16]:
binding_list.append(Imm(node.quant_norm))
if node.n_dct:
binding_list.append(GNodeArgEdge(in_eparams[7]))
gen.bindings.append(NodeBindingList(cname, *binding_list))
return True
def bindings_generator(cls, gen, node, qrec, in_eparams, out_eparams,
cname) -> bool:
names = node.get_name_indexes()
gen.bindings.append(
CommentBindingList("Node {} inq {} weightsq {} outq {} biasesq {}",
cname, qrec.in_qs[0],
qrec.in_qs[names['r_2_i_w']], qrec.out_qs[0],
qrec.in_qs[names['i_b']]))
num_seq = num_sequences(node)
if num_seq > 1:
gen.locals.append(
LocalArgInfo("uint8", f"S{node.step_idx}_StateInternal01"))
if num_seq > 2:
gen.locals.append(
LocalArgInfo("uint8", f"S{node.step_idx}_StateInternal02"))
i_state_eparams = in_eparams[names['i_state']]
reset_name = i_state_eparams.creating_node.reset_name if node.rnn_states_as_inputs else "Reset"
gen.bindings.append(
NodeBindingList(
cname, GNodeArgEdge(i_state_eparams, direction="GNA_INOUT"),
GNodeArgEdge(f"S{node.step_idx}_StateInternal01",
alias=i_state_eparams,
direction="GNA_INOUT")
if num_seq > 1 else NoArg(),
GNodeArgEdge(f"S{node.step_idx}_StateInternal02",
alias=f"S{node.step_idx}_StateInternal01",
direction="GNA_INOUT")
if num_seq > 2 else NoArg(), GNodeArgEdge(in_eparams[0]),
GNodeArgNode(node, "scalenorm"),
GNodeArgEdge(in_eparams[names['r_2_i_w']]),
GNodeArgEdge(in_eparams[names['i_2_i_w']])
if not node.rnn_same_inout_scale else NoArg(),
GNodeArgEdge(in_eparams[names['i_b']]),
GNodeArgEdge(out_eparams[0], direction="GNA_OUT"),
GNodeArgNode(node, INFOS), GArgName(reset_name)))
return True
def set_matscale_bindings(gen, in_eparams, out_eparams, cname, params, node_q):
if params.fusion_type == "vec_scalar":
gen.bindings.append(
CommentBindingList("Node {} inq1 {} inq2 {} inq3 {} outq {}",
params.name, node_q.in_qs[0].q,
node_q.in_qs[1].q, node_q.in_qs[2].q,
node_q.out_qs[0].q))
gen.bindings.append(
NodeBindingList(cname, GNodeArgEdge(in_eparams[0]),
GNodeArgEdge(in_eparams[1]),
GNodeArgEdge(in_eparams[2]),
GNodeArgEdge(out_eparams[0], "GNA_OUT")))
else:
gen.bindings.append(
CommentBindingList("Node {} inq1 {} inq2 {} outq {}", params.name,
node_q.in_qs[0].q, node_q.in_qs[1].q,
node_q.out_qs[0].q))
gen.bindings.append(
NodeBindingList(cname, GNodeArgEdge(in_eparams[0]),
GNodeArgEdge(in_eparams[1]),
GNodeArgEdge(out_eparams[0], "GNA_OUT")))
def bindings_generator(cls, gen, node, qrec, in_eparams, out_eparams,
cname) -> bool:
gen.bindings.append(
CommentBindingList(
f"Node {cname} input_q {qrec.in_qs[0].q} window_q {qrec.in_qs[1].q} fft_twiddles_q {qrec.in_qs[2].q} rfft_twiddles {qrec.in_qs[4].q} -> out_q {qrec.out_qs[0]}"
))
binding_list = [
GNodeArgEdge(in_eparams[0]),
GNodeArgEdge(out_eparams[0], "GNA_OUT"),
GNodeArgEdge(in_eparams[2]),
GNodeArgEdge(in_eparams[4]),
GNodeArgEdge(in_eparams[3])
]
if node.win_fn is not None:
binding_list.append(GNodeArgEdge(in_eparams[1]))
gen.bindings.append(NodeBindingList(cname, *binding_list))
return True
def bindings_generator(cls, gen, node, qrec, in_eparams, out_eparams,
cname) -> bool:
names = node.get_name_indexes()
gen.bindings.append(
CommentBindingList("Node {} inq {} outq {}", cname, qrec.in_qs[0],
qrec.out_qs[0]))
num_seq = num_sequences(node)
step_idx = node.step_idx
in_ctype = "char" if qrec.in_qs[0].dtype_bits == 8 else "short"
if num_seq > 1:
gen.locals.append(
LocalArgInfo(f"signed {in_ctype}",
f"S{step_idx}_CellInternal01"))
gen.locals.append(
LocalArgInfo(f"unsigned {in_ctype}",
f"S{step_idx}_StateInternal01"))
if num_seq > 2:
gen.locals.append(
LocalArgInfo(f"signed {in_ctype}",
f"S{step_idx}_CellInternal02"))
gen.locals.append(
LocalArgInfo(f"unsigned {in_ctype}",
f"S{step_idx}_StateInternal02"))
i_state_eparams = in_eparams[names['i_state']]
c_state_eparams = in_eparams[names['c_state']]
reset_name = i_state_eparams.creating_node.reset_name if node.rnn_states_as_inputs else "Reset"
bindings = [
GNodeArgEdge(c_state_eparams, direction="GNA_INOUT"),
GNodeArgEdge(i_state_eparams, direction="GNA_INOUT"),
GNodeArgEdge("S%s_CellInternal01" % step_idx,
alias=c_state_eparams,
direction="GNA_INOUT") if num_seq > 1 else NoArg(),
GNodeArgEdge("S%s_StateInternal01" % step_idx,
alias=i_state_eparams,
direction="GNA_INOUT") if num_seq > 1 else NoArg(),
GNodeArgEdge("S%s_CellInternal02" % step_idx,
alias="S%s_CellInternal01" % step_idx,
direction="GNA_INOUT") if num_seq > 2 else NoArg(),
GNodeArgEdge("S%s_StateInternal02" % step_idx,
alias="S%s_CellInternal01" % step_idx,
direction="GNA_INOUT") if num_seq > 2 else NoArg(),
GNodeArgEdge(in_eparams[0]),
GNodeArgNode(node, "scalenorm")
]
for gate in ['f', 'i', 'c', 'o']:
for inp_t in ['r', 'i']:
bindings.append(
GNodeArgEdge(in_eparams[names[f'{inp_t}_2_{gate}_w']]))
bindings.append(GNodeArgEdge(in_eparams[names[f'{gate}_b']]))
bindings.extend([
GNodeArgEdge(out_eparams[0], direction="GNA_OUT"),
GNodeArgNode(node, INFOS),
GArgName(reset_name)
])
gen.bindings.append(NodeBindingList(cname, *bindings))
return True
def set_gru_bindings(gen, step_idx, in_eparams, out_eparams, cname, rnn_params,
rnn_q):
names = rnn_params.get_name_indexes()
gen.bindings.append(
CommentBindingList("Node {} inq {} outq {}", cname, rnn_q.in_qs[0],
rnn_q.out_qs[0]))
num_seq = num_sequences(rnn_params)
if num_seq > 1:
gen.locals.append(
LocalArgInfo("int8", "S%s_StateInternal01" % step_idx))
if num_seq > 2:
gen.locals.append(
LocalArgInfo("int8", "S%s_StateInternal02" % step_idx))
i_state_eparams = in_eparams[names['h_state']]
reset_name = i_state_eparams.creating_node.reset_name if not rnn_params.rnn_states_as_inputs else "Reset"
gen.bindings.append(
NodeBindingList(
cname, GNodeArgEdge(i_state_eparams, direction="GNA_INOUT"),
GNodeArgEdge("S%s_StateInternal01" % step_idx,
alias=i_state_eparams,
direction="GNA_INOUT") if num_seq > 1 else NoArg(),
GNodeArgEdge("S%s_StateInternal02" % step_idx,
alias="S%s_CellInternal01" % step_idx,
direction="GNA_INOUT") if num_seq > 2 else NoArg(),
GNodeArgEdge(in_eparams[0]),
GNodeArgEdge(in_eparams[names['r_2_r_w']]),
GNodeArgEdge(in_eparams[names['r_b']]),
GNodeArgEdge(in_eparams[names['r_2_z_w']]),
GNodeArgEdge(in_eparams[names['z_b']]),
GNodeArgEdge(in_eparams[names['r_2_h_w']]),
GNodeArgEdge(in_eparams[names['w_h_b']]),
GNodeArgEdge(in_eparams[names['r_h_b']]),
GNodeArgEdge(out_eparams[0], direction="GNA_OUT"),
GNodeArgNode(rnn_params, INFOS), GArgName(reset_name)))
def set_lstm_bindings(gen, step_idx, in_eparams, out_eparams, cname,
rnn_params, rnn_q):
names = rnn_params.get_name_indexes()
gen.bindings.append(
CommentBindingList("Node {} inq {} outq {}", cname, rnn_q.in_qs[0],
rnn_q.out_qs[0]))
c_state_eparams = in_eparams[names['c_state']]
i_state_eparams = in_eparams[names['i_state']]
num_seq = num_sequences(rnn_params)
if num_seq > 1:
gen.locals.append(LocalArgInfo("int8",
"S%s_CellInternal01" % step_idx))
gen.locals.append(
LocalArgInfo("int8", "S%s_StateInternal01" % step_idx))
if num_seq > 2:
gen.locals.append(LocalArgInfo("int8",
"S%s_CellInternal02" % step_idx))
gen.locals.append(
LocalArgInfo("int8", "S%s_StateInternal02" % step_idx))
gen.bindings.append(
NodeBindingList(
cname, GNodeArgEdge(c_state_eparams, direction="GNA_INOUT"),
GNodeArgEdge(i_state_eparams, direction="GNA_INOUT"),
GNodeArgEdge("S%s_CellInternal01" % step_idx,
alias=c_state_eparams,
direction="GNA_INOUT") if num_seq > 1 else NoArg(),
GNodeArgEdge("S%s_StateInternal01" % step_idx,
alias=i_state_eparams,
direction="GNA_INOUT") if num_seq > 1 else NoArg(),
GNodeArgEdge("S%s_CellInternal02" % step_idx,
alias="S%s_CellInternal01" % step_idx,
direction="GNA_INOUT") if num_seq > 2 else NoArg(),
GNodeArgEdge("S%s_StateInternal02" % step_idx,
alias="S%s_CellInternal01" % step_idx,
direction="GNA_INOUT") if num_seq > 2 else NoArg(),
GNodeArgEdge(in_eparams[0]),
GNodeArgEdge(in_eparams[names['r_2_f_w']]),
GNodeArgEdge(in_eparams[names['f_b']]),
GNodeArgEdge(in_eparams[names['r_2_i_w']]),
GNodeArgEdge(in_eparams[names['i_b']]),
GNodeArgEdge(in_eparams[names['r_2_c_w']]),
GNodeArgEdge(in_eparams[names['c_b']]),
GNodeArgEdge(in_eparams[names['r_2_o_w']]),
GNodeArgEdge(in_eparams[names['o_b']]),
GNodeArgEdge(out_eparams[0], direction="GNA_OUT"),
GNodeArgNode(rnn_params, INFOS),
GArgName(i_state_eparams.creating_node.reset_name)))
def set_in_out_bindings(gen,
in_eparams,
out_eparams,
cname,
node,
node_q,
out_q=None):
if out_q is None:
out_q = node_q
gen.bindings.append(
CommentBindingList("Node {} inq {} outq {}", node.name,
str(node_q.in_qs[0]), str(out_q.out_qs[0])))
if len(node.in_dims) == 3:
if len(node_q.in_qs[0].scale) > 1:
gen.bindings.append(
NodeBindingList(cname, GNodeArgEdge(in_eparams[1]),
GNodeArgEdge(in_eparams[0]),
GNodeArgEdge(in_eparams[2]),
GNodeArgEdge(out_eparams[0], "GNA_OUT"),
GNodeArgNode(node, MULSCALE),
GNodeArgNode(node, MULSHIFT),
GNodeArgNode(node, INFOS)))
else:
gen.bindings.append(
NodeBindingList(cname, GNodeArgEdge(in_eparams[1]),
GNodeArgEdge(in_eparams[0]),
GNodeArgEdge(in_eparams[2]),
GNodeArgEdge(out_eparams[0], "GNA_OUT"),
GNodeArgNode(node, INFOS)))
else:
gen.bindings.append(
NodeBindingList(cname, GNodeArgEdge(in_eparams[1]),
GNodeArgEdge(in_eparams[0]),
GNodeArgEdge(out_eparams[0], "GNA_OUT"),
GNodeArgNode(node, INFOS)))