def verify_step(model: ModelWrapper, cfg: DataflowBuildConfig, step_name: str, need_parent: bool): print("Running verification for " + step_name) verify_out_dir = cfg.output_dir + "/verification_output" intermediate_models_dir = cfg.output_dir + "/intermediate_models" os.makedirs(verify_out_dir, exist_ok=True) (in_npy, exp_out_npy) = cfg._resolve_verification_io_pair() if need_parent: assert (cfg.save_intermediate_models ), "Enable save_intermediate_models for verification" parent_model_fn = intermediate_models_dir + "/dataflow_parent.onnx" child_model_fn = intermediate_models_dir + "/verify_%s.onnx" % step_name model.save(child_model_fn) out_npy = execute_parent(parent_model_fn, child_model_fn, in_npy) else: inp_tensor_name = model.graph.input[0].name out_tensor_name = model.graph.output[0].name inp_dict = {inp_tensor_name: in_npy} out_dict = execute_onnx(model, inp_dict) out_npy = out_dict[out_tensor_name] res = np.isclose(exp_out_npy, out_npy, atol=1e-3).all() res_to_str = {True: "SUCCESS", False: "FAIL"} res_str = res_to_str[res] verification_output_fn = verify_out_dir + "/verify_%s_%s.npy" % (step_name, res_str) np.save(verification_output_fn, out_npy) print("Verification for %s : %s" % (step_name, res_str))
def test_cppsim(self, topology, wbits, abits): prev_chkpt_name = get_checkpoint_name(topology, wbits, abits, "fold") model = load_test_checkpoint_or_skip(prev_chkpt_name) model = model.transform(PrepareCppSim()) model = model.transform(CompileCppSim()) model = model.transform(SetExecMode("cppsim")) cppsim_chkpt = get_checkpoint_name(topology, wbits, abits, "cppsim") model.save(cppsim_chkpt) parent_chkpt = get_checkpoint_name(topology, wbits, abits, "dataflow_parent") (input_tensor_npy, output_tensor_npy) = get_golden_io_pair( topology, wbits, abits, return_topk=1 ) y = execute_parent(parent_chkpt, cppsim_chkpt, input_tensor_npy) assert np.isclose(y, output_tensor_npy).all()
def test_ipstitch_rtlsim(self, topology, wbits, abits, kind): prev_chkpt_name = get_checkpoint_name( topology, wbits, abits, "fifodepth_" + kind ) model = load_test_checkpoint_or_skip(prev_chkpt_name) test_fpga_part = get_build_env(kind, target_clk_ns)["part"] model = model.transform(InsertDWC()) model = model.transform(GiveUniqueNodeNames()) model = model.transform(AnnotateCycles()) perf = model.analysis(dataflow_performance) latency = perf["critical_path_cycles"] # rtlsim only supports impl_style=rtl for StreamingFIFO, ensure that for fifo_layer in model.get_nodes_by_op_type("StreamingFIFO"): getCustomOp(fifo_layer).set_nodeattr("impl_style", "rtl") model = model.transform(PrepareIP(test_fpga_part, target_clk_ns)) model = model.transform(HLSSynthIP()) model = model.transform(CreateStitchedIP(test_fpga_part, target_clk_ns)) model = model.transform(PrepareRTLSim()) model.set_metadata_prop("exec_mode", "rtlsim") os.environ["LIVENESS_THRESHOLD"] = str(int(latency * 1.1)) if rtlsim_trace: model.set_metadata_prop( "rtlsim_trace", "%s_w%da%d.vcd" % (topology, wbits, abits) ) os.environ["RTLSIM_TRACE_DEPTH"] = "3" rtlsim_chkpt = get_checkpoint_name( topology, wbits, abits, "ipstitch_rtlsim_" + kind ) model.save(rtlsim_chkpt) parent_chkpt = get_checkpoint_name(topology, wbits, abits, "dataflow_parent") (input_tensor_npy, output_tensor_npy) = get_golden_io_pair( topology, wbits, abits, return_topk=1 ) y = execute_parent(parent_chkpt, rtlsim_chkpt, input_tensor_npy) model = ModelWrapper(rtlsim_chkpt) perf["cycles_rtlsim"] = model.get_metadata_prop("cycles_rtlsim") # warnings.warn("Estimated & rtlsim performance: " + str(perf)) # for (k, v) in perf.items(): # update_dashboard_data(topology, wbits, abits, k, v) update_dashboard_data( topology, wbits, abits, "cycles_rtlsim", perf["cycles_rtlsim"] ) assert np.isclose(y, output_tensor_npy).all()
def measure_top1_accuracy(model_chkpt, dataset, parent_chkpt=None): if dataset == "cifar10": trainx, trainy, testx, testy, valx, valy = cifar.load_cifar_data( "/workspace/finn/dataset", download=True, one_hot=False ) elif dataset == "mnist": trainx, trainy, testx, testy, valx, valy = mnist.load_mnist_data( "/workspace/finn/dataset", download=True, one_hot=False ) else: raise Exception("Unrecognized dataset") # move from dataset_loader layout to ONNX layout: NHWC -> NCHW testx = testx.transpose(0, 3, 1, 2) model = ModelWrapper(model_chkpt) iname = model.graph.input[0].name oname = model.graph.output[0].name if parent_chkpt is None: ishape = model.get_tensor_shape(iname) else: parent_model = ModelWrapper(parent_chkpt) parent_iname = parent_model.graph.input[0].name ishape = parent_model.get_tensor_shape(parent_iname) ok = 0 nok = 0 n_batches = testx.shape[0] for i in range(n_batches): tdata = testx[i].reshape(ishape).astype(np.float32) exp = testy[i].item() if parent_chkpt is not None: y = execute_parent(parent_chkpt, model_chkpt, tdata) else: y = execute_onnx(model, {iname: tdata}, False)[oname] ret = y.item() if ret == exp: ok += 1 else: nok += 1 if i % 10 == 0: print("%d : OK %d NOK %d " % (i, ok, nok)) acc_top1 = ok * 100.0 / (ok + nok) warnings.warn("Final OK %d NOK %d top-1 %f" % (ok, nok, acc_top1)) return acc_top1