def test_brevitas_debug(): finn_onnx = "test_brevitas_debug.onnx" fc = get_test_model_trained("TFC", 2, 2) dbg_hook = bo.enable_debug(fc) bo.export_finn_onnx(fc, (1, 1, 28, 28), finn_onnx) model = ModelWrapper(finn_onnx) model = model.transform(InferShapes()) model = model.transform(FoldConstants()) model = model.transform(RemoveStaticGraphInputs()) assert len(model.graph.input) == 1 assert len(model.graph.output) == 1 # load one of the test vectors raw_i = get_data("finn", "data/onnx/mnist-conv/test_data_set_0/input_0.pb") input_tensor = onnx.load_tensor_from_string(raw_i) # run using FINN-based execution input_dict = {"0": nph.to_array(input_tensor)} output_dict = oxe.execute_onnx(model, input_dict, return_full_exec_context=True) produced = output_dict[model.graph.output[0].name] # run using PyTorch/Brevitas input_tensor = torch.from_numpy(nph.to_array(input_tensor)).float() assert input_tensor.shape == (1, 1, 28, 28) # do forward pass in PyTorch/Brevitas expected = fc.forward(input_tensor).detach().numpy() assert np.isclose(produced, expected, atol=1e-3).all() # check all tensors at debug markers names_brevitas = set(dbg_hook.values.keys()) names_finn = set(output_dict.keys()) names_common = names_brevitas.intersection(names_finn) assert len(names_common) == 16 for dbg_name in names_common: tensor_pytorch = dbg_hook.values[dbg_name].detach().numpy() tensor_finn = output_dict[dbg_name] assert np.isclose(tensor_finn, tensor_pytorch, atol=1e-5).all() os.remove(finn_onnx)
def test_debug_finn_onnx_export(): model, cfg = model_with_cfg(REF_MODEL, pretrained=False) debug_hook = enable_debug(model) input_tensor = torch.randn(1, 3, 32, 32) export_finn_onnx(model, input_shape=input_tensor.shape, export_path='debug.onnx') model(input_tensor) assert debug_hook.values
def test_brevitas_compare_exported_mobilenet(): if "IMAGENET_VAL_PATH" not in os.environ.keys(): pytest.skip("Can't do validation without IMAGENET_VAL_PATH") n_images = 10 debug_mode = False export_onnx_path = make_build_dir("test_brevitas_mobilenet-v1_") # export preprocessing preproc_onnx = export_onnx_path + "/quant_mobilenet_v1_4b_preproc.onnx" preproc = NormalizePreProc(mean, std, ch) bo.export_finn_onnx(preproc, (1, 3, 224, 224), preproc_onnx) preproc_model = ModelWrapper(preproc_onnx) preproc_model = preproc_model.transform(InferShapes()) preproc_model = preproc_model.transform(GiveUniqueNodeNames()) preproc_model = preproc_model.transform(GiveUniqueParameterTensors()) preproc_model = preproc_model.transform(GiveReadableTensorNames()) # export the actual MobileNet-v1 finn_onnx = export_onnx_path + "/quant_mobilenet_v1_4b.onnx" mobilenet = get_test_model_trained("mobilenet", 4, 4) if debug_mode: dbg_hook = bo.enable_debug(mobilenet) bo.export_finn_onnx(mobilenet, (1, 3, 224, 224), finn_onnx) model = ModelWrapper(finn_onnx) model = model.transform(InferShapes()) model = model.transform(FoldConstants()) model = model.transform(RemoveStaticGraphInputs()) model = model.transform(InsertTopK()) # get initializer from Mul that will be absorbed into topk a0 = model.get_initializer(model.get_nodes_by_op_type("Mul")[-1].input[1]) model = model.transform(absorb.AbsorbScalarMulAddIntoTopK()) model = model.transform(InferShapes()) model = model.transform(InferDataTypes()) model = model.transform(InferDataLayouts()) model = model.transform(GiveUniqueNodeNames()) model = model.transform(GiveUniqueParameterTensors()) model = model.transform(GiveReadableTensorNames()) model.save(export_onnx_path + "/quant_mobilenet_v1_4b_wo_preproc.onnx") # create merged preprocessing + MobileNet-v1 model model = model.transform(MergeONNXModels(preproc_model)) model.save(export_onnx_path + "/quant_mobilenet_v1_4b.onnx") with open( export_onnx_path + "/mobilenet_validation.csv", "w", newline="" ) as csvfile: writer = csv.writer(csvfile) writer.writerow( [ "goldenID", "brevitasTop5", "brevitasTop5[%]", "finnTop5", "finnTop5[%]", "top5equal", "top5%equal", ] ) csvfile.flush() workload = imagenet_util.get_val_images(n_images, interleave_classes=True) all_inds_ok = True all_probs_ok = True for (img_path, target_id) in workload: img_np = imagenet_util.load_resize_crop(img_path) img_torch = torch.from_numpy(img_np).float() # do forward pass in PyTorch/Brevitas input_tensor = preproc.forward(img_torch) expected = mobilenet.forward(input_tensor).detach().numpy() expected_topk = expected.flatten() expected_top5 = np.argsort(expected_topk)[-5:] expected_top5 = np.flip(expected_top5) expected_top5_prob = [] for index in expected_top5: expected_top5_prob.append(expected_topk[index]) idict = {model.graph.input[0].name: img_np} odict = oxe.execute_onnx(model, idict, return_full_exec_context=True) produced = odict[model.graph.output[0].name] produced_prob = odict["TopK_0_out0"] * a0 inds_ok = (produced.flatten() == expected_top5).all() probs_ok = np.isclose(produced_prob.flatten(), expected_top5_prob).all() all_inds_ok = all_inds_ok and inds_ok all_probs_ok = all_probs_ok and probs_ok writer.writerow( [ str(target_id), str(expected_top5), str(expected_top5_prob), str(produced.flatten()), str(produced_prob.flatten()), str(inds_ok), str(probs_ok), ] ) csvfile.flush() if ((not inds_ok) or (not probs_ok)) and debug_mode: print("Results differ for %s" % img_path) # check all tensors at debug markers names_brevitas = set(dbg_hook.values.keys()) names_finn = set(odict.keys()) names_common = names_brevitas.intersection(names_finn) for dbg_name in names_common: if not np.isclose( dbg_hook.values[dbg_name].detach().numpy(), odict[dbg_name], atol=1e-3, ).all(): print("Tensor %s differs between Brevitas and FINN" % dbg_name) assert all_inds_ok and all_probs_ok
def test_brevitas_debug(QONNX_export, QONNX_FINN_conversion): if (not QONNX_export) and QONNX_FINN_conversion: pytest.skip( "This test configuration is not valid and is thus skipped.") finn_onnx = "test_brevitas_debug.onnx" fc = get_test_model_trained("TFC", 2, 2) ishape = (1, 1, 28, 28) if QONNX_export: dbg_hook = bo.enable_debug(fc, proxy_level=True) BrevitasONNXManager.export(fc, ishape, finn_onnx) # DebugMarkers have the brevitas.onnx domain, so that needs adjusting model = ModelWrapper(finn_onnx) dbg_nodes = model.get_nodes_by_op_type("DebugMarker") for dbg_node in dbg_nodes: dbg_node.domain = "finn.custom_op.general" model.save(finn_onnx) qonnx_cleanup(finn_onnx, out_file=finn_onnx) if QONNX_FINN_conversion: model = ModelWrapper(finn_onnx) model = model.transform(ConvertQONNXtoFINN()) model.save(finn_onnx) else: dbg_hook = bo.enable_debug(fc) bo.export_finn_onnx(fc, ishape, finn_onnx) model = ModelWrapper(finn_onnx) # DebugMarkers have the brevitas.onnx domain, so that needs adjusting # ToDo: We should probably have transformation pass, which does this # domain conversion for us? dbg_nodes = model.get_nodes_by_op_type("DebugMarker") for dbg_node in dbg_nodes: dbg_node.domain = "finn.custom_op.general" model = model.transform(InferShapes()) model = model.transform(FoldConstants()) model = model.transform(RemoveStaticGraphInputs()) model.save(finn_onnx) model = ModelWrapper(finn_onnx) assert len(model.graph.input) == 1 assert len(model.graph.output) == 1 # load one of the test vectors raw_i = get_data("finn.data", "onnx/mnist-conv/test_data_set_0/input_0.pb") input_tensor = onnx.load_tensor_from_string(raw_i) # run using FINN-based execution input_dict = {model.graph.input[0].name: nph.to_array(input_tensor)} output_dict = oxe.execute_onnx(model, input_dict, return_full_exec_context=True) produced = output_dict[model.graph.output[0].name] # run using PyTorch/Brevitas input_tensor = torch.from_numpy(nph.to_array(input_tensor)).float() assert input_tensor.shape == (1, 1, 28, 28) # do forward pass in PyTorch/Brevitas expected = fc.forward(input_tensor).detach().numpy() assert np.isclose(produced, expected, atol=1e-3).all() # check all tensors at debug markers names_brevitas = set(dbg_hook.values.keys()) names_finn = set(output_dict.keys()) names_common = names_brevitas.intersection(names_finn) # The different exports return debug markers in different numbers and places print(len(names_common)) if QONNX_export and not QONNX_FINN_conversion: assert len(names_common) == 12 elif QONNX_export and QONNX_FINN_conversion: assert len(names_common) == 8 else: assert len(names_common) == 16 for dbg_name in names_common: if QONNX_export: tensor_pytorch = dbg_hook.values[dbg_name].value.detach().numpy() else: tensor_pytorch = dbg_hook.values[dbg_name].detach().numpy() tensor_finn = output_dict[dbg_name] assert np.isclose(tensor_finn, tensor_pytorch, atol=1e-5).all() os.remove(finn_onnx)