def apply(self, model):
streamline_transformations = [
ConvertSubToAdd(),
ConvertDivToMul(),
BatchNormToAffine(),
ConvertSignToThres(),
MoveMulPastMaxPool(),
MoveScalarLinearPastInvariants(),
AbsorbSignBiasIntoMultiThreshold(),
MoveAddPastMul(),
MoveScalarAddPastMatMul(),
MoveAddPastConv(),
MoveScalarMulPastMatMul(),
MoveScalarMulPastConv(),
MoveAddPastMul(),
CollapseRepeatedAdd(),
CollapseRepeatedMul(),
MoveMulPastMaxPool(),
AbsorbAddIntoMultiThreshold(),
FactorOutMulSignMagnitude(),
AbsorbMulIntoMultiThreshold(),
Absorb1BitMulIntoMatMul(),
Absorb1BitMulIntoConv(),
RoundAndClipThresholds(),
]
for trn in streamline_transformations:
model = model.transform(trn)
model = model.transform(RemoveIdentityOps())
model = model.transform(GiveUniqueNodeNames())
model = model.transform(GiveReadableTensorNames())
model = model.transform(InferDataTypes())
return (model, False)
def test_end2end_mobilenet_streamline():
model = load_test_checkpoint_or_skip(build_dir +
"/end2end_mobilenet_tidy.onnx")
model = model.transform(Streamline())
additional_streamline_transformations = [
DoubleToSingleFloat(),
reorder.MoveMulPastDWConv(),
absorb.AbsorbMulIntoMultiThreshold(),
ChangeDataLayoutQuantAvgPool2d(),
InferDataLayouts(),
reorder.MoveTransposePastScalarMul(),
absorb.AbsorbTransposeIntoFlatten(),
reorder.MoveFlattenPastAffine(),
reorder.MoveFlattenPastTopK(),
reorder.MoveScalarMulPastMatMul(),
CollapseRepeatedMul(),
RemoveIdentityOps(),
RoundAndClipThresholds(),
]
for trn in additional_streamline_transformations:
model = model.transform(trn)
model = model.transform(GiveUniqueNodeNames())
model = model.transform(GiveReadableTensorNames())
model = model.transform(InferDataTypes())
model.save(build_dir + "/end2end_mobilenet_streamlined.onnx")
assert (len(model.get_nodes_by_op_type("Add")) == 1
) # only final quantized bias Add op remains
assert len(model.get_nodes_by_op_type("Mul")) == 0 # no Mul ops remain
def step_resnet50_streamline_linear(model: ModelWrapper,
cfg: DataflowBuildConfig):
streamline_transformations = [
AbsorbScalarMulAddIntoTopK(
), # before MoveAddPastMul to avoid int->float
ConvertSubToAdd(),
ConvertDivToMul(),
RemoveIdentityOps(),
CollapseRepeatedMul(),
BatchNormToAffine(),
ConvertSignToThres(),
MoveAddPastMul(),
MoveScalarAddPastMatMul(),
MoveAddPastConv(),
MoveScalarMulPastMatMul(),
MoveScalarMulPastConv(),
MoveScalarLinearPastInvariants(),
MoveAddPastMul(),
CollapseRepeatedAdd(),
CollapseRepeatedMul(),
AbsorbAddIntoMultiThreshold(),
FactorOutMulSignMagnitude(),
MoveMaxPoolPastMultiThreshold(),
AbsorbMulIntoMultiThreshold(),
Absorb1BitMulIntoMatMul(),
Absorb1BitMulIntoConv(),
RoundAndClipThresholds(),
]
for trn in streamline_transformations:
model = model.transform(trn)
model = model.transform(GiveUniqueNodeNames())
return model
def test_end2end_tfc_w1a1_convert_to_hls_layers():
model = ModelWrapper(build_dir + "/end2end_tfc_w1a1_streamlined.onnx")
model = model.transform(ConvertBipolarMatMulToXnorPopcount())
model = model.transform(absorb.AbsorbAddIntoMultiThreshold())
model = model.transform(absorb.AbsorbMulIntoMultiThreshold())
model = model.transform(RoundAndClipThresholds())
model = model.transform(to_hls.InferBinaryStreamingFCLayer())
model.save(build_dir + "/end2end_tfc_w1a1_hls_layers.onnx")
def step_mobilenet_lower_convs(model: ModelWrapper, cfg: DataflowBuildConfig):
model = model.transform(LowerConvsToMatMul())
model = model.transform(absorb.AbsorbTransposeIntoMultiThreshold())
model = model.transform(GiveUniqueNodeNames())
model = model.transform(GiveReadableTensorNames())
model = model.transform(InferDataTypes())
model = model.transform(RoundAndClipThresholds())
model = model.transform(InferDataLayouts())
return model
def test_end2end_mobilenet_lowering():
model = load_test_checkpoint_or_skip(build_dir +
"/end2end_mobilenet_streamlined.onnx")
model = model.transform(LowerConvsToMatMul())
model = model.transform(absorb.AbsorbTransposeIntoMultiThreshold())
model = model.transform(GiveUniqueNodeNames())
model = model.transform(GiveReadableTensorNames())
model = model.transform(InferDataTypes())
model = model.transform(RoundAndClipThresholds())
model.save(build_dir + "/end2end_mobilenet_lowered.onnx")
def streamline(model, binary=True):
log("Streamline transformations launched")
model = model.transform(MoveScalarLinearPastInvariants())
model = model.transform(Streamline())
# Absorb add and mul in thresholds
model = model.transform(absorb.AbsorbAddIntoMultiThreshold())
model = model.transform(absorb.AbsorbMulIntoMultiThreshold())
# Absorb add-mul in top-k
model = model.transform(absorb.AbsorbScalarMulAddIntoTopK())
model = model.transform(RoundAndClipThresholds())
# Tidy-up
model = model.transform(InferDataLayouts())
model = model.transform(RemoveUnusedTensors())
log("Streamline transformations completed")
save(model, "3_streamlined")
return model
def step_mobilenet_streamline(model: ModelWrapper, cfg: DataflowBuildConfig):
model = model.transform(Streamline())
additional_streamline_transformations = [
DoubleToSingleFloat(),
reorder.MoveMulPastDWConv(),
absorb.AbsorbMulIntoMultiThreshold(),
ChangeDataLayoutQuantAvgPool2d(),
InferDataLayouts(),
reorder.MoveTransposePastScalarMul(),
absorb.AbsorbTransposeIntoFlatten(),
reorder.MoveFlattenPastAffine(),
reorder.MoveFlattenPastTopK(),
reorder.MoveScalarMulPastMatMul(),
CollapseRepeatedMul(),
RemoveIdentityOps(),
RoundAndClipThresholds(),
]
for trn in additional_streamline_transformations:
model = model.transform(trn)
model = model.transform(GiveUniqueNodeNames())
model = model.transform(GiveReadableTensorNames())
model = model.transform(InferDataTypes())
return model
def test_convert_to_hls_layers_tfc_w1a1():
tfc = get_test_model_trained("TFC", 1, 1)
bo.export_finn_onnx(tfc, (1, 1, 28, 28), export_onnx_path)
model = ModelWrapper(export_onnx_path)
model = model.transform(InferShapes())
model = model.transform(FoldConstants())
model = model.transform(GiveUniqueNodeNames())
model = model.transform(GiveReadableTensorNames())
model = model.transform(Streamline())
model = model.transform(ConvertBipolarMatMulToXnorPopcount())
model = model.transform(absorb.AbsorbAddIntoMultiThreshold())
model = model.transform(absorb.AbsorbMulIntoMultiThreshold())
model = model.transform(RoundAndClipThresholds())
model = model.transform(to_hls.InferBinaryStreamingFCLayer())
fc0 = model.graph.node[2]
assert fc0.op_type == "StreamingFCLayer_Batch"
assert model.get_tensor_shape(fc0.input[0]) == [1, 784]
assert model.get_tensor_shape(fc0.input[1]) == [784, 64]
assert model.get_tensor_shape(fc0.input[2]) == [64, 1]
fc1 = model.graph.node[3]
assert fc1.op_type == "StreamingFCLayer_Batch"
assert model.get_tensor_shape(fc1.input[0]) == [1, 64]
assert model.get_tensor_shape(fc1.input[1]) == [64, 64]
assert model.get_tensor_shape(fc1.input[2]) == [64, 1]
fc2 = model.graph.node[4]
assert fc2.op_type == "StreamingFCLayer_Batch"
assert model.get_tensor_shape(fc2.input[0]) == [1, 64]
assert model.get_tensor_shape(fc2.input[1]) == [64, 64]
assert model.get_tensor_shape(fc2.input[2]) == [64, 1]
fc3 = model.graph.node[5]
assert fc3.op_type == "StreamingFCLayer_Batch"
assert model.get_tensor_shape(fc3.input[0]) == [1, 64]
assert model.get_tensor_shape(fc3.input[1]) == [64, 10]
fc0w = getCustomOp(fc0)
fc0w.set_nodeattr("SIMD", 784)
fc0w.set_nodeattr("PE", 16)
fc1w = getCustomOp(fc1)
fc1w.set_nodeattr("SIMD", 16)
fc1w.set_nodeattr("PE", 16)
fc2w = getCustomOp(fc2)
fc2w.set_nodeattr("SIMD", 16)
fc2w.set_nodeattr("PE", 16)
fc3w = getCustomOp(fc3)
fc3w.set_nodeattr("SIMD", 16)
fc3w.set_nodeattr("PE", 10)
model = model.transform(PrepareCppSim())
model = model.transform(CompileCppSim())
model = model.transform(SetExecMode("cppsim"))
raw_i = get_data("finn", "data/onnx/mnist-conv/test_data_set_0/input_0.pb")
input_tensor = onnx.load_tensor_from_string(raw_i)
# run using FINN-based execution
input_dict = {"global_in": nph.to_array(input_tensor)}
output_dict = oxe.execute_onnx(model, input_dict)
produced = output_dict[list(output_dict.keys())[0]]
# run using PyTorch/Brevitas
input_tensor = torch.from_numpy(nph.to_array(input_tensor)).float()
assert input_tensor.shape == (1, 1, 28, 28)
# do forward pass in PyTorch/Brevitas
expected = tfc.forward(input_tensor).detach().numpy()
assert np.isclose(produced, expected, atol=1e-3).all()
os.remove(export_onnx_path)