def test_end2end_ext_weights_build():
model_file = get_checkpoint_name("download")
load_test_checkpoint_or_skip(model_file)
build_env = get_build_env(build_kind, target_clk_ns)
folding_config_file = pk.resource_filename(
"finn.qnn-data", "test_ext_weights/tfc-w1a1-extw.json")
output_dir = make_build_dir("test_end2end_ext_weights_build")
cfg = build.DataflowBuildConfig(
output_dir=output_dir,
folding_config_file=folding_config_file,
synth_clk_period_ns=target_clk_ns,
board=build_env["board"],
shell_flow_type=build_cfg.ShellFlowType.VIVADO_ZYNQ,
generate_outputs=[
build_cfg.DataflowOutputType.ESTIMATE_REPORTS,
build_cfg.DataflowOutputType.BITFILE,
build_cfg.DataflowOutputType.PYNQ_DRIVER,
build_cfg.DataflowOutputType.DEPLOYMENT_PACKAGE,
],
)
build.build_dataflow_cfg(model_file, cfg)
assert os.path.isfile(output_dir + "/deploy/bitfile/finn-accel.bit")
assert os.path.isfile(output_dir + "/deploy/bitfile/finn-accel.hwh")
assert os.path.isfile(output_dir + "/deploy/driver/driver.py")
assert os.path.isfile(output_dir +
"/deploy/driver/runtime_weights/idma0.npy")
if os.path.isdir(get_checkpoint_name("build")):
shutil.rmtree(get_checkpoint_name("build"))
shutil.copytree(output_dir + "/deploy", get_checkpoint_name("build"))
def test_throughput_hw(self, topology, wbits, abits, kind):
prev_chkpt_name = get_checkpoint_name(topology, wbits, abits, "deploy_" + kind)
end2end_example = "%s_w%da%d_%s" % (topology, wbits, abits, kind)
model = load_test_checkpoint_or_skip(prev_chkpt_name) # NOQA
cfg = get_build_env(kind, target_clk_ns)
if cfg["ip"] == "":
pytest.skip("PYNQ board IP address not specified")
ret = dict()
# try a range of batch sizes, some may fail due to insufficient DMA
# buffers
bsize_range_in = [8 ** i for i in range(5)]
bsize_range = []
for bsize in bsize_range_in:
res = throughput_test_remote(model, bsize)
if res is not None:
ret[bsize] = res
bsize_range.append(bsize)
else:
# assume we reached largest possible N
break
y = [ret[key]["runtime[ms]"] for key in bsize_range]
lrret = linregress(bsize_range, y)
ret_str = ""
ret_str += "\n" + "%s Throughput Test Results" % end2end_example
ret_str += "\n" + "-----------------------------"
ret_str += "\n" + "From linear regression:"
ret_str += "\n" + "Invocation overhead: %f ms" % lrret.intercept
ret_str += "\n" + "Time per sample: %f ms" % lrret.slope
ret_str += "\n" + "Raw data:"
ret_str += "\n" + "{:<8} {:<16} {:<16} {:<16} {:<16} {:<16}".format(
"N", "runtime[ms]", "fclk[mhz]", "fps", "DRAM rd[Mb/s]", "DRAM wr[Mb/s]"
)
for k in bsize_range:
v = ret[k]
ret_str += "\n" + "{:<8} {:<16} {:<16} {:<16} {:<16} {:<16}".format(
k,
np.round(v["runtime[ms]"], 4),
v["fclk[mhz]"],
np.round(v["throughput[images/s]"], 2),
np.round(v["DRAM_in_bandwidth[Mb/s]"], 2),
np.round(v["DRAM_out_bandwidth[Mb/s]"], 2),
)
ret_str += "\n" + "-----------------------------"
warnings.warn(ret_str)
largest_bsize = bsize_range[-1]
update_dashboard_data(
topology, wbits, abits, "fclk[mhz]", ret[largest_bsize]["fclk[mhz]"]
)
update_dashboard_data(
topology,
wbits,
abits,
"throughput[images/s]",
ret[largest_bsize]["throughput[images/s]"],
)
def test_ipgen(self, topology, wbits, abits, kind):
if kind == "alveo" and ("VITIS_PATH" not in os.environ):
pytest.skip("VITIS_PATH not set")
prev_chkpt_name = get_checkpoint_name(topology, wbits, abits, "fold")
model = load_test_checkpoint_or_skip(prev_chkpt_name)
test_fpga_part = get_build_env(kind, target_clk_ns)["part"]
model = model.transform(GiveUniqueNodeNames())
model = model.transform(PrepareIP(test_fpga_part, target_clk_ns))
model = model.transform(HLSSynthIP())
model.save(get_checkpoint_name(topology, wbits, abits, "ipgen_" + kind))
def test_end2end_cybsec_mlp_run_on_hw():
build_env = get_build_env(build_kind, target_clk_ns)
assets_dir = pk.resource_filename("finn.qnn-data", "cybsec-mlp/")
deploy_dir = get_checkpoint_name("build")
if not os.path.isdir(deploy_dir):
pytest.skip(deploy_dir +
" not found from previous test step, skipping")
driver_dir = deploy_dir + "/driver"
assert os.path.isdir(driver_dir)
# put all assets into driver dir
shutil.copy(assets_dir + "/validate-unsw-nb15.py", driver_dir)
# put a copy of binarized dataset into driver dir
dataset_url = (
"https://zenodo.org/record/4519767/files/unsw_nb15_binarized.npz?download=1"
)
dataset_local = driver_dir + "/unsw_nb15_binarized.npz"
if not os.path.isfile(dataset_local):
wget.download(dataset_url, out=dataset_local)
assert os.path.isfile(dataset_local)
# create a shell script for running validation: 10 batches x 10 imgs
with open(driver_dir + "/validate.sh", "w") as f:
f.write("""#!/bin/bash
cd %s/driver
echo %s | sudo -S python3.6 validate-unsw-nb15.py --batchsize=10 --limit_batches=10
""" % (
build_env["target_dir"] + "/end2end_cybsecmlp_build",
build_env["password"],
))
# set up rsync command
remote_target = "%s@%s:%s" % (
build_env["username"],
build_env["ip"],
build_env["target_dir"],
)
rsync_res = subprocess.run(["rsync", "-avz", deploy_dir, remote_target])
assert rsync_res.returncode == 0
remote_verif_cmd = [
"ssh",
"%s@%s" % (build_env["username"], build_env["ip"]),
"sh",
build_env["target_dir"] +
"/end2end_cybsecmlp_build/driver/validate.sh",
]
verif_res = subprocess.run(
remote_verif_cmd,
stdout=subprocess.PIPE,
universal_newlines=True,
input=build_env["password"],
)
assert verif_res.returncode == 0
log_output = verif_res.stdout.split("\n")
assert log_output[-3] == "batch 10 / 10 : total OK 93 NOK 7"
assert log_output[-2] == "Final accuracy: 93.000000"
def test_build(self, topology, wbits, abits, kind):
if kind == "alveo" and ("VITIS_PATH" not in os.environ):
pytest.skip("VITIS_PATH not set")
prev_chkpt_name = get_checkpoint_name(topology, wbits, abits, "ipgen_" + kind)
model = load_test_checkpoint_or_skip(prev_chkpt_name)
cfg = get_build_env(kind, target_clk_ns)
model = model.transform(cfg["build_fxn"])
model = model.transform(AnnotateResources("synth"))
synth_dct = eval(model.get_metadata_prop("res_total_top_synth"))
for (k, v) in synth_dct.items():
update_dashboard_data(topology, wbits, abits, k, v)
update_dashboard_data(topology, wbits, abits, "board", cfg["board"])
model.save(get_checkpoint_name(topology, wbits, abits, "build_" + kind))
def test_end2end_ext_weights_dataset():
# make sure we have local copies of mnist dataset files
subprocess.check_output(["mkdir", "-p", mnist_local])
for f in mnist_files:
if not os.path.isfile(mnist_local + "/" + f):
wget.download(mnist_url + "/" + f, out=mnist_local + "/" + f)
assert os.path.isfile(mnist_local + "/" + f)
# rsync to board
build_env = get_build_env(build_kind, target_clk_ns)
mnist_target = "%s@%s:%s" % (build_env["username"], build_env["ip"],
"/tmp/")
rsync_dataset_cmd = ["rsync", "-rv", mnist_local + "/", mnist_target]
subprocess.check_output(rsync_dataset_cmd)
def test_set_fifo_depths(self, topology, wbits, abits, kind):
prev_chkpt_name = get_checkpoint_name(topology, wbits, abits, "ipgen_" + kind)
model = load_test_checkpoint_or_skip(prev_chkpt_name)
test_fpga_part = get_build_env(kind, target_clk_ns)["part"]
model = model.transform(InsertAndSetFIFODepths(test_fpga_part, target_clk_ns))
fifo_layers = model.get_nodes_by_op_type("StreamingFIFO")
assert len(fifo_layers) > 0
hls_layers = model.get_finn_nodes()
for node in hls_layers:
if node.op_type != "StreamingFIFO":
op_inst = getCustomOp(node)
assert op_inst.get_nodeattr("inFIFODepth") == 0
assert op_inst.get_nodeattr("outFIFODepth") == 0
model.save(get_checkpoint_name(topology, wbits, abits, "fifodepth_" + kind))
def test_deploy(self, topology, wbits, abits, kind):
prev_chkpt_name = get_checkpoint_name(topology, wbits, abits, "build_" + kind)
model = load_test_checkpoint_or_skip(prev_chkpt_name)
cfg = get_build_env(kind, target_clk_ns)
if cfg["ip"] == "":
pytest.skip("PYNQ board IP address not specified")
model = model.transform(
DeployToPYNQ(
cfg["ip"],
cfg["port"],
cfg["username"],
cfg["password"],
cfg["target_dir"],
)
)
# save the model to be able to link it to the parent
model.save(get_checkpoint_name(topology, wbits, abits, "deploy_" + kind))
def test_end2end_cybsec_mlp_build(QONNX_export):
model_file = get_checkpoint_name("export", QONNX_export)
load_test_checkpoint_or_skip(model_file)
build_env = get_build_env(build_kind, target_clk_ns)
output_dir = make_build_dir(
f"test_end2end_cybsec_mlp_build_QONNX-{QONNX_export}")
cfg = build.DataflowBuildConfig(
output_dir=output_dir,
target_fps=1000000,
synth_clk_period_ns=target_clk_ns,
board=build_env["board"],
shell_flow_type=build_cfg.ShellFlowType.VIVADO_ZYNQ,
generate_outputs=[
build_cfg.DataflowOutputType.ESTIMATE_REPORTS,
build_cfg.DataflowOutputType.BITFILE,
build_cfg.DataflowOutputType.PYNQ_DRIVER,
build_cfg.DataflowOutputType.DEPLOYMENT_PACKAGE,
],
)
build.build_dataflow_cfg(model_file, cfg)
# check the generated files
assert os.path.isfile(output_dir + "/time_per_step.json")
assert os.path.isfile(output_dir + "/final_hw_config.json")
assert os.path.isfile(output_dir + "/driver/driver.py")
est_cycles_report = output_dir + "/report/estimate_layer_cycles.json"
assert os.path.isfile(est_cycles_report)
est_res_report = output_dir + "/report/estimate_layer_resources.json"
assert os.path.isfile(est_res_report)
assert os.path.isfile(output_dir +
"/report/estimate_network_performance.json")
assert os.path.isfile(output_dir + "/bitfile/finn-accel.bit")
assert os.path.isfile(output_dir + "/bitfile/finn-accel.hwh")
assert os.path.isfile(output_dir + "/report/post_synth_resources.xml")
assert os.path.isfile(output_dir + "/report/post_route_timing.rpt")
# examine the report contents
with open(est_cycles_report, "r") as f:
est_cycles_dict = json.load(f)
assert est_cycles_dict["StreamingFCLayer_Batch_0"] == 80
assert est_cycles_dict["StreamingFCLayer_Batch_1"] == 64
with open(est_res_report, "r") as f:
est_res_dict = json.load(f)
assert est_res_dict["total"]["LUT"] == 11360.0
assert est_res_dict["total"]["BRAM_18K"] == 36.0
shutil.copytree(output_dir + "/deploy",
get_checkpoint_name("build", QONNX_export))
def test_end2end_ext_weights_run_on_hw():
build_env = get_build_env(build_kind, target_clk_ns)
deploy_dir = get_checkpoint_name("build")
if not os.path.isdir(deploy_dir):
pytest.skip(deploy_dir +
" not found from previous test step, skipping")
driver_dir = deploy_dir + "/driver"
assert os.path.isdir(driver_dir)
# create a shell script for running validation: 10 batches x 10 imgs
with open(driver_dir + "/validate.sh", "w") as f:
f.write("""#!/bin/bash
cd %s/driver
echo %s | sudo -S python3.6 validate.py --dataset mnist --bitfile %s
""" % (
build_env["target_dir"] + "/end2end_ext_weights_build",
build_env["password"],
"../bitfile/finn-accel.bit",
))
# set up rsync command
remote_target = "%s@%s:%s" % (
build_env["username"],
build_env["ip"],
build_env["target_dir"],
)
rsync_res = subprocess.run(["rsync", "-avz", deploy_dir, remote_target])
assert rsync_res.returncode == 0
remote_verif_cmd = [
"ssh",
"%s@%s" % (build_env["username"], build_env["ip"]),
"sh",
build_env["target_dir"] +
"/end2end_ext_weights_build/driver/validate.sh",
]
verif_res = subprocess.run(
remote_verif_cmd,
stdout=subprocess.PIPE,
universal_newlines=True,
input=build_env["password"],
)
assert verif_res.returncode == 0
log_output = verif_res.stdout.split("\n")
assert log_output[-3] == "batch 100 / 100 : total OK 9296 NOK 704"
assert log_output[-2] == "Final accuracy: 92.960000"
def test_ipstitch_rtlsim(self, topology, wbits, abits, kind):
prev_chkpt_name = get_checkpoint_name(
topology, wbits, abits, "fifodepth_" + kind
)
model = load_test_checkpoint_or_skip(prev_chkpt_name)
test_fpga_part = get_build_env(kind, target_clk_ns)["part"]
model = model.transform(InsertDWC())
model = model.transform(GiveUniqueNodeNames())
model = model.transform(AnnotateCycles())
perf = model.analysis(dataflow_performance)
latency = perf["critical_path_cycles"]
# rtlsim only supports impl_style=rtl for StreamingFIFO, ensure that
for fifo_layer in model.get_nodes_by_op_type("StreamingFIFO"):
getCustomOp(fifo_layer).set_nodeattr("impl_style", "rtl")
model = model.transform(PrepareIP(test_fpga_part, target_clk_ns))
model = model.transform(HLSSynthIP())
model = model.transform(CreateStitchedIP(test_fpga_part, target_clk_ns))
model = model.transform(PrepareRTLSim())
model.set_metadata_prop("exec_mode", "rtlsim")
os.environ["LIVENESS_THRESHOLD"] = str(int(latency * 1.1))
if rtlsim_trace:
model.set_metadata_prop(
"rtlsim_trace", "%s_w%da%d.vcd" % (topology, wbits, abits)
)
os.environ["RTLSIM_TRACE_DEPTH"] = "3"
rtlsim_chkpt = get_checkpoint_name(
topology, wbits, abits, "ipstitch_rtlsim_" + kind
)
model.save(rtlsim_chkpt)
parent_chkpt = get_checkpoint_name(topology, wbits, abits, "dataflow_parent")
(input_tensor_npy, output_tensor_npy) = get_golden_io_pair(
topology, wbits, abits, return_topk=1
)
y = execute_parent(parent_chkpt, rtlsim_chkpt, input_tensor_npy)
model = ModelWrapper(rtlsim_chkpt)
perf["cycles_rtlsim"] = model.get_metadata_prop("cycles_rtlsim")
# warnings.warn("Estimated & rtlsim performance: " + str(perf))
# for (k, v) in perf.items():
# update_dashboard_data(topology, wbits, abits, k, v)
update_dashboard_data(
topology, wbits, abits, "cycles_rtlsim", perf["cycles_rtlsim"]
)
assert np.isclose(y, output_tensor_npy).all()
def test_end2end_access_board():
build_env = get_build_env("zynq", 5)
if build_env["ip"] == "":
pytest.skip("PYNQ board IP address not specified")
remote_cmd_base = [
"ssh",
"-o",
"PreferredAuthentications=publickey",
"-o",
"PasswordAuthentication=no",
"%s@%s" % (build_env["username"], build_env["ip"]),
]
test_text = "BoardIsAccessible"
touch_cmd = remote_cmd_base + ["echo %s" % test_text]
verif_res = subprocess.run(touch_cmd,
stdout=subprocess.PIPE,
universal_newlines=True)
assert verif_res.returncode == 0
assert verif_res.stdout.split("\n")[0] == test_text
def test_run_on_hw(self, topology, wbits, abits, kind):
prev_chkpt_name = get_checkpoint_name(topology, wbits, abits, "deploy_" + kind)
model = load_test_checkpoint_or_skip(prev_chkpt_name) # NOQA
cfg = get_build_env(kind, target_clk_ns)
if cfg["ip"] == "":
pytest.skip("PYNQ board IP address not specified")
(input_tensor_npy, output_tensor_npy) = get_golden_io_pair(
topology, wbits, abits, return_topk=1
)
parent_model = load_test_checkpoint_or_skip(
get_checkpoint_name(topology, wbits, abits, "dataflow_parent")
)
iname = parent_model.graph.input[0].name
oname = parent_model.graph.output[0].name
sdp_node = parent_model.get_nodes_by_op_type("StreamingDataflowPartition")[0]
sdp_node = getCustomOp(sdp_node)
sdp_node.set_nodeattr("model", prev_chkpt_name)
ret = execute_onnx(parent_model, {iname: input_tensor_npy}, True)
y = ret[oname]
assert np.isclose(y, output_tensor_npy).all()
