def apply(self, model):
streamline_transformations = [
ConvertSubToAdd(),
ConvertDivToMul(),
BatchNormToAffine(),
ConvertSignToThres(),
MoveMulPastMaxPool(),
MoveScalarLinearPastInvariants(),
AbsorbSignBiasIntoMultiThreshold(),
MoveAddPastMul(),
MoveScalarAddPastMatMul(),
MoveAddPastConv(),
MoveScalarMulPastMatMul(),
MoveScalarMulPastConv(),
MoveAddPastMul(),
CollapseRepeatedAdd(),
CollapseRepeatedMul(),
MoveMulPastMaxPool(),
AbsorbAddIntoMultiThreshold(),
FactorOutMulSignMagnitude(),
AbsorbMulIntoMultiThreshold(),
Absorb1BitMulIntoMatMul(),
Absorb1BitMulIntoConv(),
RoundAndClipThresholds(),
]
for trn in streamline_transformations:
model = model.transform(trn)
model = model.transform(RemoveIdentityOps())
model = model.transform(GiveUniqueNodeNames())
model = model.transform(GiveReadableTensorNames())
model = model.transform(InferDataTypes())
return (model, False)
def test_end2end_mobilenet_streamline():
model = load_test_checkpoint_or_skip(build_dir +
"/end2end_mobilenet_tidy.onnx")
model = model.transform(Streamline())
additional_streamline_transformations = [
DoubleToSingleFloat(),
reorder.MoveMulPastDWConv(),
absorb.AbsorbMulIntoMultiThreshold(),
ChangeDataLayoutQuantAvgPool2d(),
InferDataLayouts(),
reorder.MoveTransposePastScalarMul(),
absorb.AbsorbTransposeIntoFlatten(),
reorder.MoveFlattenPastAffine(),
reorder.MoveFlattenPastTopK(),
reorder.MoveScalarMulPastMatMul(),
CollapseRepeatedMul(),
RemoveIdentityOps(),
RoundAndClipThresholds(),
]
for trn in additional_streamline_transformations:
model = model.transform(trn)
model = model.transform(GiveUniqueNodeNames())
model = model.transform(GiveReadableTensorNames())
model = model.transform(InferDataTypes())
model.save(build_dir + "/end2end_mobilenet_streamlined.onnx")
assert (len(model.get_nodes_by_op_type("Add")) == 1
) # only final quantized bias Add op remains
assert len(model.get_nodes_by_op_type("Mul")) == 0 # no Mul ops remain
def step_resnet50_streamline_linear(model: ModelWrapper,
cfg: DataflowBuildConfig):
streamline_transformations = [
AbsorbScalarMulAddIntoTopK(
), # before MoveAddPastMul to avoid int->float
ConvertSubToAdd(),
ConvertDivToMul(),
RemoveIdentityOps(),
CollapseRepeatedMul(),
BatchNormToAffine(),
ConvertSignToThres(),
MoveAddPastMul(),
MoveScalarAddPastMatMul(),
MoveAddPastConv(),
MoveScalarMulPastMatMul(),
MoveScalarMulPastConv(),
MoveScalarLinearPastInvariants(),
MoveAddPastMul(),
CollapseRepeatedAdd(),
CollapseRepeatedMul(),
AbsorbAddIntoMultiThreshold(),
FactorOutMulSignMagnitude(),
MoveMaxPoolPastMultiThreshold(),
AbsorbMulIntoMultiThreshold(),
Absorb1BitMulIntoMatMul(),
Absorb1BitMulIntoConv(),
RoundAndClipThresholds(),
]
for trn in streamline_transformations:
model = model.transform(trn)
model = model.transform(GiveUniqueNodeNames())
return model
def test_remove_identity_ops(op, as_first_node, approx):
# set up onnx model
inp = helper.make_tensor_value_info("inp", TensorProto.FLOAT, [1, 4, 1, 1])
mul = helper.make_tensor_value_info("mul", TensorProto.FLOAT, [])
shape = helper.make_tensor_value_info("shape", TensorProto.FLOAT, [2])
div = helper.make_tensor_value_info("div", TensorProto.FLOAT, [])
matmul = helper.make_tensor_value_info("matmul", TensorProto.FLOAT, [4, 2])
outp = helper.make_tensor_value_info("outp", TensorProto.FLOAT, [1, 2])
mul_node = helper.make_node("Mul", ["inp", "mul"], ["mul_out"])
reshape_node = helper.make_node("Reshape", ["mul_out", "shape"], ["reshape_out"])
div_node = helper.make_node("Div", ["reshape_out", "div"], ["div_out"])
matmul_node = helper.make_node("MatMul", ["div_out", "matmul"], ["outp"])
graph = helper.make_graph(
nodes=[mul_node, reshape_node, div_node, matmul_node],
name="identity-graph",
inputs=[inp],
outputs=[outp],
value_info=[mul, shape, div, matmul],
)
model = helper.make_model(graph, producer_name="mulpastconv-model")
model = ModelWrapper(model)
inp_values = gen_finn_dt_tensor(DataType["INT2"], [1, 4, 1, 1])
mul_values = np.random.uniform(low=0.1, high=0.99, size=(1)).astype(np.float32)
shape_values = np.asarray([1, -1], dtype=np.int64)
div_values = np.random.uniform(low=0.1, high=0.99, size=(1)).astype(np.float32)
matmul_values = gen_finn_dt_tensor(DataType["INT2"], [4, 2])
model.set_initializer("mul", mul_values)
model.set_initializer("shape", shape_values)
model.set_initializer("div", div_values)
model.set_initializer("matmul", matmul_values)
insert_identity_op(model, op, as_first_node, approx)
model = model.transform(InferShapes())
model = model.transform(InferDataTypes())
idict = {"inp": inp_values}
odict = oxe.execute_onnx(model, idict)
out_before = odict["outp"]
num_of_nodes_before = len(model.graph.node)
model = model.transform(RemoveIdentityOps())
num_of_nodes_after = len(model.graph.node)
assert num_of_nodes_before - 1 == num_of_nodes_after
odict = oxe.execute_onnx(model, idict)
out_after = odict["outp"]
assert np.isclose(out_before, out_after, atol=1e-3).all()
def apply(self, model):
# Extract the bias from Conv node
model = model.transform(ExtractBiasFromConv())
# Gemm operations are not supported by FINN, so we convert them to MatMul
model = model.transform(GemmToMatMul())
model = model.transform(FoldTransposeIntoQuantInit())
# Make sure the datatypes exist, these are required for folding the weights
model = model.transform(InferDataTypes())
# Fold weights
model = model.transform(FoldQuantWeights())
# Convert activations
model = model.transform(
ConvertQuantActToMultiThreshold(
filter_function=self._filter_function, ))
# Recompute datatypes
model = model.transform(InferDataTypes())
# Convert AvgPool -> Mul -> Trunc structure to QuantAvgPool2d
model = model.transform(AvgPoolAndTruncToQuantAvgPool())
# Remove empty padding if it exists
model = model.transform(RemoveIdentityOps())
return model, False
def step_mobilenet_streamline(model: ModelWrapper, cfg: DataflowBuildConfig):
model = model.transform(Streamline())
additional_streamline_transformations = [
DoubleToSingleFloat(),
reorder.MoveMulPastDWConv(),
absorb.AbsorbMulIntoMultiThreshold(),
ChangeDataLayoutQuantAvgPool2d(),
InferDataLayouts(),
reorder.MoveTransposePastScalarMul(),
absorb.AbsorbTransposeIntoFlatten(),
reorder.MoveFlattenPastAffine(),
reorder.MoveFlattenPastTopK(),
reorder.MoveScalarMulPastMatMul(),
CollapseRepeatedMul(),
RemoveIdentityOps(),
RoundAndClipThresholds(),
]
for trn in additional_streamline_transformations:
model = model.transform(trn)
model = model.transform(GiveUniqueNodeNames())
model = model.transform(GiveReadableTensorNames())
model = model.transform(InferDataTypes())
return model
def apply(self, model):
graph = model.graph
node_ind = 0
for n in graph.node:
node_ind += 1
if n.op_type == "Gemm":
# Check for correct ONNX version
model_onnx_version = model.model.opset_import[0].version
if model_onnx_version != 9:
warnings.warn(
f"The GemmToMatMul transformation only offers explicit support "
f"for version 9 of the Gemm node, but the ONNX version of the "
f"supplied model is {model_onnx_version}. "
f"Thus the transformation may fail or "
f"return incomplete results.")
running_node_index = node_ind
# Transpose A?
transA = get_by_name(n.attribute, "transA")
if transA is not None and transA.i:
# Insert transpose node
shape = model.get_tensor_shape(n.input[0])
if shape is not None:
shape = tuple(reversed(shape))
inp_trans_out = helper.make_tensor_value_info(
model.make_new_valueinfo_name(),
TensorProto.FLOAT,
shape,
)
graph.value_info.append(inp_trans_out)
inp_trans_node = helper.make_node("Transpose",
[n.input[0]],
[inp_trans_out.name])
graph.node.insert(running_node_index, inp_trans_node)
running_node_index += 1
dt = model.get_tensor_datatype(n.input[0])
if dt != DataType["FLOAT32"]:
model.set_tensor_datatype(inp_trans_out.name, dt)
n.input[0] = inp_trans_out.name
# Transpose B?
transB = get_by_name(n.attribute, "transB")
if transB is not None and transB.i:
# Insert transpose node
shape = model.get_tensor_shape(n.input[1])
if shape is not None:
shape = tuple(reversed(shape))
inp_trans_out = helper.make_tensor_value_info(
model.make_new_valueinfo_name(),
TensorProto.FLOAT,
shape,
)
graph.value_info.append(inp_trans_out)
inp_trans_node = helper.make_node("Transpose",
[n.input[1]],
[inp_trans_out.name])
graph.node.insert(running_node_index, inp_trans_node)
running_node_index += 1
# Copy over the datatype
dt = model.get_tensor_datatype(n.input[1])
if dt != DataType["FLOAT32"]:
model.set_tensor_datatype(inp_trans_out.name, dt)
n.input[1] = inp_trans_out.name
# Insert MatMul: A * B
matMul_node = helper.make_node("MatMul",
[n.input[0], n.input[1]],
[n.output[0]])
graph.node.insert(running_node_index, matMul_node)
matMul_node = graph.node[running_node_index]
running_node_index += 1
# Insert Mul: (A*B) * alpha
alpha = get_by_name(n.attribute, "alpha")
if alpha is None:
alpha = np.array(1.0)
else:
alpha = np.array(alpha.f)
mul_tensor = helper.make_tensor_value_info(
model.make_new_valueinfo_name(),
TensorProto.FLOAT,
None,
)
graph.value_info.append(mul_tensor)
model.set_initializer(mul_tensor.name, alpha)
A_shape = model.get_tensor_shape(n.input[0])
B_shape = model.get_tensor_shape(n.input[1])
if A_shape is not None and B_shape is not None:
shape = [A_shape[0], B_shape[1]]
else:
shape = None
act_mul_tensor = helper.make_tensor_value_info(
model.make_new_valueinfo_name(),
TensorProto.FLOAT,
shape,
)
graph.value_info.append(act_mul_tensor)
mul_node = helper.make_node(
"Mul",
[act_mul_tensor.name, mul_tensor.name],
[n.output[0]],
)
graph.node.insert(running_node_index, mul_node)
mul_node_main_branch = graph.node[running_node_index]
running_node_index += 1
matMul_node.output[0] = act_mul_tensor.name
# Other branch: Insert Mul: beta * C
beta = get_by_name(n.attribute, "beta")
if beta is None:
beta = np.array(1.0)
else:
beta = np.array(beta.f)
mul_tensor = helper.make_tensor_value_info(
model.make_new_valueinfo_name(),
TensorProto.FLOAT,
None,
)
graph.value_info.append(mul_tensor)
model.set_initializer(mul_tensor.name, beta)
C_shape = model.get_tensor_shape(n.input[2])
act_mul_tensor = helper.make_tensor_value_info(
model.make_new_valueinfo_name(),
TensorProto.FLOAT,
C_shape,
)
graph.value_info.append(act_mul_tensor)
mul_node = helper.make_node(
"Mul",
[n.input[2], mul_tensor.name],
[act_mul_tensor.name],
)
graph.node.insert(running_node_index, mul_node)
running_node_index += 1
dt = model.get_tensor_datatype(n.input[2])
if dt != DataType["FLOAT32"]:
model.set_tensor_datatype(act_mul_tensor.name, dt)
n.input[2] = act_mul_tensor.name
# Insert Add: ((A*B) * alpha) + (beta * C)
shape = model.get_tensor_shape(mul_node_main_branch.input[0])
act_add_tensor = helper.make_tensor_value_info(
model.make_new_valueinfo_name(),
TensorProto.FLOAT,
shape,
)
graph.value_info.append(act_add_tensor)
mul_node_main_branch.output[0] = act_add_tensor.name
add_node = helper.make_node(
"Add",
[act_add_tensor.name, n.input[2]],
[n.output[0]],
)
graph.node.insert(running_node_index, add_node)
running_node_index += 1
# Delete Gemm node
graph.node.remove(n)
# Remove potential unity multiplications from alpha and beta attributes
model = model.transform(RemoveIdentityOps())
return model, True
return model, False