def test_end2end_mobilenet_convert_to_hls_layers():
model = load_test_checkpoint_or_skip(build_dir + "/end2end_mobilenet_lowered.onnx")
model = model.transform(to_hls.InferPool_Batch())
model = model.transform(to_hls.InferConvInpGen())
model = model.transform(to_hls.InferVVAU())
model = model.transform(to_hls.InferQuantizedStreamingFCLayer(mem_mode))
model = model.transform(to_hls.InferChannelwiseLinearLayer())
model = model.transform(to_hls.InferLabelSelectLayer())
model = model.transform(InferShapes())
model = model.transform(GiveUniqueNodeNames())
model = model.transform(GiveReadableTensorNames())
model.save(build_dir + "/end2end_mobilenet_hls_layers.onnx")
def step_mobilenet_convert_to_hls_layers(model: ModelWrapper,
cfg: DataflowBuildConfig):
mem_mode = cfg.default_mem_mode.value
model = model.transform(to_hls.InferPool_Batch())
model = model.transform(to_hls.InferConvInpGen())
model = model.transform(to_hls.InferVVAU())
model = model.transform(to_hls.InferQuantizedStreamingFCLayer(mem_mode))
model = model.transform(to_hls.InferChannelwiseLinearLayer())
model = model.transform(to_hls.InferLabelSelectLayer())
model = model.transform(InferShapes())
model = model.transform(GiveUniqueNodeNames())
model = model.transform(GiveReadableTensorNames())
return model
def test_convert_to_hls_conv_layer(conv_config, depthwise, exec_mode):
kernel_size, stride, pad = conv_config
np.random.seed(0)
idt = DataType.UINT4
in_feature_dim = 7
in_chn = 16
if depthwise is True:
group = out_chn = in_chn
conv_param_shape = [out_chn, 1, kernel_size, kernel_size]
else:
group = 1
out_chn = 20
conv_param_shape = [out_chn, in_chn, kernel_size, kernel_size]
out_feature_dim = compute_conv_output_dim(in_feature_dim, kernel_size, stride, pad)
input_shape = [1, in_chn, in_feature_dim, in_feature_dim]
output_shape = [1, out_chn, out_feature_dim, out_feature_dim]
conv_weight_dt = DataType.UINT4
conv_config = {}
conv_config["dilations"] = [1, 1]
conv_config["group"] = group
conv_config["kernel_shape"] = [kernel_size, kernel_size]
conv_config["pads"] = [pad, pad, pad, pad]
conv_config["strides"] = [stride, stride]
top_in = helper.make_tensor_value_info("top_in", TensorProto.FLOAT, input_shape)
top_out = helper.make_tensor_value_info("top_out", TensorProto.FLOAT, output_shape)
value_info = [
helper.make_tensor_value_info("p1", TensorProto.FLOAT, conv_param_shape)
]
modelproto = helper.make_model(
helper.make_graph(
name="conv_test",
inputs=[top_in],
outputs=[top_out],
value_info=value_info,
nodes=[
helper.make_node("Conv", ["top_in", "p1"], ["top_out"], **conv_config)
],
)
)
model = ModelWrapper(modelproto)
model.set_tensor_datatype("top_in", idt)
model.set_tensor_datatype("top_out", idt)
model.set_tensor_datatype("p1", conv_weight_dt)
model.set_initializer("p1", gen_finn_dt_tensor(conv_weight_dt, conv_param_shape))
model = model.transform(InferShapes())
model = model.transform(InferDataTypes())
new_model = model.transform(LowerConvsToMatMul())
new_model = new_model.transform(to_hls.InferConvInpGen())
if depthwise is True:
new_model = new_model.transform(to_hls.InferVVAU())
else:
new_model = new_model.transform(to_hls.InferQuantizedStreamingFCLayer())
fc_node = new_model.get_nodes_by_op_type("StreamingFCLayer_Batch")[0]
fc_inst = getCustomOp(fc_node)
mw = fc_inst.get_nodeattr("MW")
mh = fc_inst.get_nodeattr("MH")
pe_cands = list(filter(lambda x: mh % x == 0, range(2, mh + 1)))
simd_cands = list(filter(lambda x: mw % x == 0, range(2, mw + 1)))
fc_inst.set_nodeattr("PE", pe_cands[0])
fc_inst.set_nodeattr("SIMD", simd_cands[0])
new_model = new_model.transform(GiveUniqueNodeNames())
new_model = new_model.transform(InferShapes())
new_model = new_model.transform(InferDataTypes())
if exec_mode == "cppsim":
new_model = new_model.transform(PrepareCppSim())
new_model = new_model.transform(CompileCppSim())
new_model = new_model.transform(SetExecMode("cppsim"))
elif exec_mode == "rtlsim":
new_model = new_model.transform(SetExecMode("rtlsim"))
new_model = new_model.transform(GiveUniqueNodeNames())
new_model = new_model.transform(PrepareIP("xc7z020clg400-1", 5))
new_model = new_model.transform(HLSSynthIP())
new_model = new_model.transform(PrepareRTLSim())
else:
raise Exception("Unknown exec_mode")
x = gen_finn_dt_tensor(idt, input_shape)
inp_dict = {model.graph.input[0].name: x}
assert oxe.compare_execution(model, new_model, inp_dict)
if kernel_size == 1 and stride > 1 and pad == 0:
assert new_model.graph.node[1].op_type == "DownSampler"
if exec_mode == "rtlsim":
node = new_model.get_nodes_by_op_type("DownSampler")[0]
inst = getCustomOp(node)
cycles_rtlsim = inst.get_nodeattr("cycles_rtlsim")
exp_cycles_dict = new_model.analysis(exp_cycles_per_layer)
exp_cycles = exp_cycles_dict[node.name]
assert np.isclose(exp_cycles, cycles_rtlsim, atol=11)
assert exp_cycles != 0
if pad == 1:
padding_node = new_model.get_nodes_by_op_type("FMPadding_Batch")[0]
padding_inst = getCustomOp(padding_node)
assert padding_inst.get_nodeattr("SIMD") == in_chn
if depthwise is True and exec_mode == "rtlsim":
node = new_model.get_nodes_by_op_type("Vector_Vector_Activate_Batch")[0]
inst = getCustomOp(node)
cycles_rtlsim = inst.get_nodeattr("cycles_rtlsim")
exp_cycles_dict = new_model.analysis(exp_cycles_per_layer)
exp_cycles = exp_cycles_dict[node.name]
assert np.isclose(exp_cycles, cycles_rtlsim, atol=11)
assert exp_cycles != 0
def test_convert_to_hls_conv_fc_transition(conv_config, depthwise,
use_reshape):
np.random.seed(0)
idt = DataType["UINT4"]
odt = DataType["UINT4"]
conv_weight_dt = DataType["INT4"]
fc_weight_dt = DataType["INT4"]
input_shape, kernel_shape, stride, pad = conv_config
kernel_size_h, kernel_size_w = kernel_shape
input_size_h, input_size_w = input_shape
stride_h, stride_w = stride
pad_h, pad_w = pad
in_chn = 4
fc_filters = 16
if depthwise is True:
group = out_chn = in_chn
conv_param_shape = [out_chn, 1, kernel_size_h, kernel_size_w]
else:
group = 1
out_chn = 8
conv_param_shape = [out_chn, in_chn, kernel_size_h, kernel_size_w]
output_size_h = compute_conv_output_dim(input_size_h, kernel_size_h,
stride_h, 2 * pad_h)
output_size_w = compute_conv_output_dim(input_size_w, kernel_size_w,
stride_w, 2 * pad_w)
input_shape = [1, in_chn, input_size_h, input_size_w]
fc_param_shape = [out_chn * output_size_h * output_size_w, fc_filters]
output_shape = [1, fc_filters]
conv_config = {}
conv_config["dilations"] = [1, 1]
conv_config["group"] = group
conv_config["kernel_shape"] = [kernel_size_h, kernel_size_w]
conv_config["pads"] = [pad_h, pad_w, pad_h, pad_w]
conv_config["strides"] = [stride_h, stride_w]
global_in = helper.make_tensor_value_info("global_in", TensorProto.FLOAT,
input_shape)
global_out = helper.make_tensor_value_info("global_out", TensorProto.FLOAT,
output_shape)
value_info = [
helper.make_tensor_value_info("conv_param", TensorProto.FLOAT,
conv_param_shape),
helper.make_tensor_value_info("thres1_param", TensorProto.FLOAT,
(out_chn, 15)),
helper.make_tensor_value_info("matmul_param", TensorProto.FLOAT,
fc_param_shape),
helper.make_tensor_value_info("thres2_param", TensorProto.FLOAT,
(fc_filters, 15)),
helper.make_tensor_value_info("reshape_shape", TensorProto.INT64, []),
]
if use_reshape:
flatten_node = helper.make_node("Reshape",
["thres1_out", "reshape_shape"],
["flatten_out"])
else:
flatten_node = helper.make_node("Flatten", ["thres1_out"],
["flatten_out"],
axis=1)
modelproto = helper.make_model(
helper.make_graph(
name="test",
inputs=[global_in],
outputs=[global_out],
value_info=value_info,
nodes=[
helper.make_node("Conv", ["global_in", "conv_param"],
["conv_out"], **conv_config),
helper.make_node(
"MultiThreshold",
["conv_out", "thres1_param"],
["thres1_out"],
domain="finn.custom_op.general",
out_dtype="UINT4",
),
flatten_node,
helper.make_node("MatMul", ["flatten_out", "matmul_param"],
["matmul_out"]),
helper.make_node(
"MultiThreshold",
["matmul_out", "thres2_param"],
["global_out"],
domain="finn.custom_op.general",
out_dtype="UINT4",
),
],
))
model = ModelWrapper(modelproto)
model.set_tensor_datatype("global_in", idt)
model.set_tensor_layout("global_in", DataLayout.NCHW)
model.set_tensor_datatype("global_out", odt)
model.set_tensor_datatype("conv_param", conv_weight_dt)
model.set_tensor_datatype("matmul_param", fc_weight_dt)
model.set_tensor_datatype("thres1_param", DataType["INT32"])
model.set_tensor_datatype("thres2_param", DataType["INT32"])
model.set_initializer("conv_param",
gen_finn_dt_tensor(conv_weight_dt, conv_param_shape))
model.set_initializer("thres1_param",
get_multithreshold_rand_params(out_chn, 15, seed=0))
model.set_initializer(
"thres2_param", get_multithreshold_rand_params(fc_filters, 15, seed=0))
model.set_initializer("matmul_param",
gen_finn_dt_tensor(fc_weight_dt, fc_param_shape))
model.set_initializer("reshape_shape", np.array([1, -1]))
model = model.transform(InferShapes())
model = model.transform(InferDataTypes())
model = model.transform(InferDataLayouts())
# streamlining
new_model = model.transform(MoveScalarLinearPastInvariants())
new_model = new_model.transform(Streamline())
new_model = new_model.transform(LowerConvsToMatMul())
new_model = new_model.transform(absorb.AbsorbTransposeIntoMultiThreshold())
new_model = new_model.transform(Streamline())
new_model = new_model.transform(InferDataLayouts())
new_model = new_model.transform(RemoveUnusedTensors())
# convert_to_hls
if depthwise is True:
new_model = new_model.transform(to_hls.InferVVAU())
new_model = new_model.transform(to_hls.InferQuantizedStreamingFCLayer())
new_model = new_model.transform(to_hls.InferThresholdingLayer())
new_model = new_model.transform(to_hls.InferConvInpGen())
new_model = new_model.transform(to_hls.InferStreamingMaxPool())
new_model = new_model.transform(RemoveCNVtoFCFlatten())
new_model = new_model.transform(absorb.AbsorbConsecutiveTransposes())
new_model = new_model.transform(GiveUniqueNodeNames())
new_model = new_model.transform(InferDataLayouts())
# prepare cppsim
new_model = new_model.transform(PrepareCppSim())
new_model = new_model.transform(CompileCppSim())
new_model = new_model.transform(SetExecMode("cppsim"))
# check for correct execution
x = gen_finn_dt_tensor(idt, input_shape)
inp_dict = {model.graph.input[0].name: x}
assert oxe.compare_execution(model, new_model, inp_dict)
num_transpose = len(new_model.get_nodes_by_op_type("Transpose"))
num_flatten = len(new_model.get_nodes_by_op_type("Flatten"))
num_reshape = len(new_model.get_nodes_by_op_type("Reshape"))
# check if transpose->flatten was removed
assert num_transpose == 1 and num_flatten == 0 and num_reshape == 0