def step_resnet50_streamline_linear(model: ModelWrapper,
cfg: DataflowBuildConfig):
streamline_transformations = [
AbsorbScalarMulAddIntoTopK(
), # before MoveAddPastMul to avoid int->float
ConvertSubToAdd(),
ConvertDivToMul(),
RemoveIdentityOps(),
CollapseRepeatedMul(),
BatchNormToAffine(),
ConvertSignToThres(),
MoveAddPastMul(),
MoveScalarAddPastMatMul(),
MoveAddPastConv(),
MoveScalarMulPastMatMul(),
MoveScalarMulPastConv(),
MoveScalarLinearPastInvariants(),
MoveAddPastMul(),
CollapseRepeatedAdd(),
CollapseRepeatedMul(),
AbsorbAddIntoMultiThreshold(),
FactorOutMulSignMagnitude(),
MoveMaxPoolPastMultiThreshold(),
AbsorbMulIntoMultiThreshold(),
Absorb1BitMulIntoMatMul(),
Absorb1BitMulIntoConv(),
RoundAndClipThresholds(),
]
for trn in streamline_transformations:
model = model.transform(trn)
model = model.transform(GiveUniqueNodeNames())
return model
def apply(self, model):
streamline_transformations = [
ConvertSubToAdd(),
ConvertDivToMul(),
BatchNormToAffine(),
ConvertSignToThres(),
MoveMulPastMaxPool(),
MoveScalarLinearPastInvariants(),
AbsorbSignBiasIntoMultiThreshold(),
MoveAddPastMul(),
MoveScalarAddPastMatMul(),
MoveAddPastConv(),
MoveScalarMulPastMatMul(),
MoveScalarMulPastConv(),
MoveAddPastMul(),
CollapseRepeatedAdd(),
CollapseRepeatedMul(),
MoveMulPastMaxPool(),
AbsorbAddIntoMultiThreshold(),
FactorOutMulSignMagnitude(),
AbsorbMulIntoMultiThreshold(),
Absorb1BitMulIntoMatMul(),
Absorb1BitMulIntoConv(),
RoundAndClipThresholds(),
]
for trn in streamline_transformations:
model = model.transform(trn)
model = model.transform(RemoveIdentityOps())
model = model.transform(GiveUniqueNodeNames())
model = model.transform(GiveReadableTensorNames())
model = model.transform(InferDataTypes())
return (model, False)
def step_streamline(model: ModelWrapper, cfg: DataflowBuildConfig):
"""Run streamlining on given model. Streamlining involves moving floating point
scale/shift parameters around, collapsing adjacent ones into a single parameter,
then absorbing the scale/shift into the following `MultiThreshold` node.
Streamlining requires careful topology design and cannot be applied to all
topologies.
"""
model = model.transform(MoveScalarLinearPastInvariants())
model = model.transform(Streamline())
need_lowering = len(model.get_nodes_by_op_type("Conv")) > 0
if need_lowering:
model = model.transform(LowerConvsToMatMul())
model = model.transform(MakeMaxPoolNHWC())
model = model.transform(absorb.AbsorbTransposeIntoMultiThreshold())
model = model.transform(ConvertBipolarMatMulToXnorPopcount())
model = model.transform(Streamline())
# absorb final add-mul nodes into TopK
model = model.transform(absorb.AbsorbScalarMulAddIntoTopK())
model = model.transform(InferDataLayouts())
model = model.transform(RemoveUnusedTensors())
if VerificationStepType.STREAMLINED_PYTHON in cfg._resolve_verification_steps(
):
verify_step(model, cfg, "streamlined_python", need_parent=False)
return model
def streamline(model, binary=True):
log("Streamline transformations launched")
model = model.transform(MoveScalarLinearPastInvariants())
model = model.transform(Streamline())
# Absorb add and mul in thresholds
model = model.transform(absorb.AbsorbAddIntoMultiThreshold())
model = model.transform(absorb.AbsorbMulIntoMultiThreshold())
# Absorb add-mul in top-k
model = model.transform(absorb.AbsorbScalarMulAddIntoTopK())
model = model.transform(RoundAndClipThresholds())
# Tidy-up
model = model.transform(InferDataLayouts())
model = model.transform(RemoveUnusedTensors())
log("Streamline transformations completed")
save(model, "3_streamlined")
return model
def test_streamline(self, topology, wbits, abits):
prev_chkpt_name = get_checkpoint_name(topology, wbits, abits, "pre_post")
model = load_test_checkpoint_or_skip(prev_chkpt_name)
# move past any reshapes to be able to streamline input scaling
model = model.transform(MoveScalarLinearPastInvariants())
model = model.transform(Streamline())
if "fc" not in topology:
model = model.transform(LowerConvsToMatMul())
model = model.transform(MakeMaxPoolNHWC())
model = model.transform(absorb.AbsorbTransposeIntoMultiThreshold())
model = model.transform(ConvertBipolarMatMulToXnorPopcount())
model = model.transform(Streamline())
# absorb final add-mul nodes into TopK
model = model.transform(absorb.AbsorbScalarMulAddIntoTopK())
model = model.transform(InferDataLayouts())
model = model.transform(RemoveUnusedTensors())
model.save(get_checkpoint_name(topology, wbits, abits, "streamline"))
def test_convert_to_hls_conv_fc_transition(conv_config, depthwise,
use_reshape):
np.random.seed(0)
idt = DataType["UINT4"]
odt = DataType["UINT4"]
conv_weight_dt = DataType["INT4"]
fc_weight_dt = DataType["INT4"]
input_shape, kernel_shape, stride, pad = conv_config
kernel_size_h, kernel_size_w = kernel_shape
input_size_h, input_size_w = input_shape
stride_h, stride_w = stride
pad_h, pad_w = pad
in_chn = 4
fc_filters = 16
if depthwise is True:
group = out_chn = in_chn
conv_param_shape = [out_chn, 1, kernel_size_h, kernel_size_w]
else:
group = 1
out_chn = 8
conv_param_shape = [out_chn, in_chn, kernel_size_h, kernel_size_w]
output_size_h = compute_conv_output_dim(input_size_h, kernel_size_h,
stride_h, 2 * pad_h)
output_size_w = compute_conv_output_dim(input_size_w, kernel_size_w,
stride_w, 2 * pad_w)
input_shape = [1, in_chn, input_size_h, input_size_w]
fc_param_shape = [out_chn * output_size_h * output_size_w, fc_filters]
output_shape = [1, fc_filters]
conv_config = {}
conv_config["dilations"] = [1, 1]
conv_config["group"] = group
conv_config["kernel_shape"] = [kernel_size_h, kernel_size_w]
conv_config["pads"] = [pad_h, pad_w, pad_h, pad_w]
conv_config["strides"] = [stride_h, stride_w]
global_in = helper.make_tensor_value_info("global_in", TensorProto.FLOAT,
input_shape)
global_out = helper.make_tensor_value_info("global_out", TensorProto.FLOAT,
output_shape)
value_info = [
helper.make_tensor_value_info("conv_param", TensorProto.FLOAT,
conv_param_shape),
helper.make_tensor_value_info("thres1_param", TensorProto.FLOAT,
(out_chn, 15)),
helper.make_tensor_value_info("matmul_param", TensorProto.FLOAT,
fc_param_shape),
helper.make_tensor_value_info("thres2_param", TensorProto.FLOAT,
(fc_filters, 15)),
helper.make_tensor_value_info("reshape_shape", TensorProto.INT64, []),
]
if use_reshape:
flatten_node = helper.make_node("Reshape",
["thres1_out", "reshape_shape"],
["flatten_out"])
else:
flatten_node = helper.make_node("Flatten", ["thres1_out"],
["flatten_out"],
axis=1)
modelproto = helper.make_model(
helper.make_graph(
name="test",
inputs=[global_in],
outputs=[global_out],
value_info=value_info,
nodes=[
helper.make_node("Conv", ["global_in", "conv_param"],
["conv_out"], **conv_config),
helper.make_node(
"MultiThreshold",
["conv_out", "thres1_param"],
["thres1_out"],
domain="finn.custom_op.general",
out_dtype="UINT4",
),
flatten_node,
helper.make_node("MatMul", ["flatten_out", "matmul_param"],
["matmul_out"]),
helper.make_node(
"MultiThreshold",
["matmul_out", "thres2_param"],
["global_out"],
domain="finn.custom_op.general",
out_dtype="UINT4",
),
],
))
model = ModelWrapper(modelproto)
model.set_tensor_datatype("global_in", idt)
model.set_tensor_layout("global_in", DataLayout.NCHW)
model.set_tensor_datatype("global_out", odt)
model.set_tensor_datatype("conv_param", conv_weight_dt)
model.set_tensor_datatype("matmul_param", fc_weight_dt)
model.set_tensor_datatype("thres1_param", DataType["INT32"])
model.set_tensor_datatype("thres2_param", DataType["INT32"])
model.set_initializer("conv_param",
gen_finn_dt_tensor(conv_weight_dt, conv_param_shape))
model.set_initializer("thres1_param",
get_multithreshold_rand_params(out_chn, 15, seed=0))
model.set_initializer(
"thres2_param", get_multithreshold_rand_params(fc_filters, 15, seed=0))
model.set_initializer("matmul_param",
gen_finn_dt_tensor(fc_weight_dt, fc_param_shape))
model.set_initializer("reshape_shape", np.array([1, -1]))
model = model.transform(InferShapes())
model = model.transform(InferDataTypes())
model = model.transform(InferDataLayouts())
# streamlining
new_model = model.transform(MoveScalarLinearPastInvariants())
new_model = new_model.transform(Streamline())
new_model = new_model.transform(LowerConvsToMatMul())
new_model = new_model.transform(absorb.AbsorbTransposeIntoMultiThreshold())
new_model = new_model.transform(Streamline())
new_model = new_model.transform(InferDataLayouts())
new_model = new_model.transform(RemoveUnusedTensors())
# convert_to_hls
if depthwise is True:
new_model = new_model.transform(to_hls.InferVVAU())
new_model = new_model.transform(to_hls.InferQuantizedStreamingFCLayer())
new_model = new_model.transform(to_hls.InferThresholdingLayer())
new_model = new_model.transform(to_hls.InferConvInpGen())
new_model = new_model.transform(to_hls.InferStreamingMaxPool())
new_model = new_model.transform(RemoveCNVtoFCFlatten())
new_model = new_model.transform(absorb.AbsorbConsecutiveTransposes())
new_model = new_model.transform(GiveUniqueNodeNames())
new_model = new_model.transform(InferDataLayouts())
# prepare cppsim
new_model = new_model.transform(PrepareCppSim())
new_model = new_model.transform(CompileCppSim())
new_model = new_model.transform(SetExecMode("cppsim"))
# check for correct execution
x = gen_finn_dt_tensor(idt, input_shape)
inp_dict = {model.graph.input[0].name: x}
assert oxe.compare_execution(model, new_model, inp_dict)
num_transpose = len(new_model.get_nodes_by_op_type("Transpose"))
num_flatten = len(new_model.get_nodes_by_op_type("Flatten"))
num_reshape = len(new_model.get_nodes_by_op_type("Reshape"))
# check if transpose->flatten was removed
assert num_transpose == 1 and num_flatten == 0 and num_reshape == 0
def test_convert_to_hls_layers_synthetic(ch, ifmdim, idt):
model = make_model(ch, ifmdim)
model.save(export_onnx_path)
model = ModelWrapper(export_onnx_path)
model = model.transform(InferShapes())
model = model.transform(FoldConstants())
model = model.transform(GiveUniqueNodeNames())
model = model.transform(GiveReadableTensorNames())
model = model.transform(InferDataLayouts())
# model.save("golden.onnx")
# generate test vectors of correct shape
if ifmdim == -1:
input_tensor_shape = (1, ch)
else:
input_tensor_shape = (1, ch, ifmdim, ifmdim)
x = gen_finn_dt_tensor(idt, input_tensor_shape)
# generate expected value from streamlined net
input_dict = {model.graph.input[0].name: x}
output_dict = oxe.execute_onnx(model, input_dict, True)
produced_sum = output_dict[model.graph.output[0].name]
chw_mul = model.get_initializer(model.graph.node[-1].input[1])
chw_mul = 1
expected_sum = chw_mul * np.sum(2 * (2 * x + 15.0),
axis=(2, 3)) / (ifmdim * ifmdim)
assert (produced_sum.flatten() == expected_sum.flatten()).all()
model = model.transform(InferDataLayouts())
# convert to hls
model.set_tensor_datatype(model.graph.input[0].name, idt)
# extra streamlining
model = model.transform(MoveScalarLinearPastInvariants())
model = model.transform(MoveAddPastMul())
model = model.transform(CollapseRepeatedMul())
model = model.transform(CollapseRepeatedAdd())
# insert top-k node, which should absorb linear ops before it
model = model.transform(InferShapes())
model = model.transform(InferDataLayouts())
model = model.transform(InferDataTypes())
model = model.transform(to_hls.InferChannelwiseLinearLayer())
model = model.transform(to_hls.InferAddStreamsLayer())
model = model.transform(to_hls.InferGlobalAccPoolLayer())
model = model.transform(MoveScalarLinearPastInvariants())
model = model.transform(InsertTopK())
model = model.transform(AbsorbScalarMulAddIntoTopK())
model = model.transform(InferDataTypes())
model = model.transform(to_hls.InferLabelSelectLayer())
model = model.transform(AbsorbConsecutiveTransposes())
model = model.transform(InferDataTypes())
model = model.transform(to_hls.InferLabelSelectLayer())
model = model.transform(to_hls.InferDuplicateStreamsLayer())
model = model.transform(SortGraph())
# model.save("golden_hls.onnx")
# check topology status
finn_nodes = model.get_finn_nodes()
assert len(finn_nodes) == 9
add_nodes = model.get_nodes_by_op_type("AddStreams_Batch")
assert len(add_nodes) == 1
pool_nodes = model.get_nodes_by_op_type("GlobalAccPool_Batch")
assert len(pool_nodes) == 1
label_nodes = model.get_nodes_by_op_type("LabelSelect_Batch")
assert len(label_nodes) == 1
channelwise_nodes = model.get_nodes_by_op_type("ChannelwiseOp_Batch")
assert len(channelwise_nodes) == 5
dup_nodes = model.get_nodes_by_op_type("DuplicateStreams_Batch")
assert len(dup_nodes) == 1
model = model.transform(PrepareCppSim())
model = model.transform(CompileCppSim())
model = model.transform(SetExecMode("cppsim"))
output_dict = oxe.execute_onnx(model, input_dict, True)
produced_topk_hls = output_dict[model.graph.output[0].name]
topk_input = output_dict[model.graph.node[-1].input[0]]
assert soft_verify_topk(topk_input, produced_topk_hls, 5)
os.remove(export_onnx_path)