def verify_ell_output_in_vision_model(ell_map, cntk_model, testing_info): _logger = logger.get() _logger.info("Verification of model output starting") try: cntk_input_tensor = np.random.random((cntk_model.arguments[0].shape)).astype(np.float32) * 255 ell_input_tensor = memory_shapes.get_tensor_in_ell_order(cntk_input_tensor, "channel_row_column") ell_input_tensor = ell_input_tensor.ravel().astype(np.float32) # Get output from CNTK model cntk_output = get_output_from_cntk_model(cntk_model, cntk_input_tensor, testing_info) # Get computed ELL result _logger.info("Getting computed ELL results") result_from_compute = np.array(ell_map.Compute(ell_input_tensor)) # Get compiled ELL result _logger.info("Getting compiled ELL results") compiler_options = ell.model.MapCompilerOptions() compiler_options.useBlas = True compiled_ell_map = ell_map.Compile("host", "model", "predict", compilerOptions=compiler_options) result_from_compiled = np.array(compiled_ell_map.Compute(ell_input_tensor)) # Verify the computed result against the cntk result np.testing.assert_array_almost_equal( cntk_output, result_from_compute, decimal=4, err_msg=('results for computed ELL model do not match CNTK output!')) _logger.info("Verified computed result against CNTK") # Verify the compiled result against the cntk result np.testing.assert_array_almost_equal( cntk_output, result_from_compiled, decimal=4, err_msg=( 'results for compiled ELL model do not match CNTK output!')) _logger.info("Verified compiled result against CNTK") # Verify the compiled result agrees with the computed result np.testing.assert_array_almost_equal( result_from_compute, result_from_compiled, decimal=4, err_msg=( 'results for computed ELL model do not match results from compiled ELL model!')) _logger.info("Verified compiled result against computed result") # Get timing info total_time = 0 num_frames = 50 _logger.info("Sending {} frames through model...".format(num_frames)) for i in range(num_frames): cntk_input_tensor = np.random.random((cntk_model.arguments[0].shape)).astype(np.float32) * 255 ell_input_tensor = memory_shapes.get_tensor_in_ell_order(cntk_input_tensor, "channel_row_column") ell_input_tensor = ell_input_tensor.ravel().astype(np.float32) start = time.time() result_from_compiled = np.array(compiled_ell_map.Compute(ell_input_tensor, dtype=np.float32)) end = time.time() total_time += end - start total_time /= num_frames _logger.info("Average speed: {:.0f}ms/frame".format(total_time * 1000)) except BaseException as exception: _logger.error("Verification of model output failed") raise exception _logger.info("Verification of model output complete")
def verify_ell_output_in_vision_model(ell_map, torch_model, onnx_nodes, testing_info): _logger.info("Verification of model output starting") try: torch_model.eval() ell_input_shape = ell_map.GetInputShape() ell_shape = (ell_input_shape.rows, ell_input_shape.columns, ell_input_shape.channels) pytorch_input_shape = (ell_shape[2], ell_shape[0], ell_shape[1]) torch_input_tensor_ = np.random.random( (1, pytorch_input_shape[0], pytorch_input_shape[1], pytorch_input_shape[2])) * 255 torch_input_tensor = np.float32(torch_input_tensor_) order = "channel" if len(torch_input_tensor.shape) == 4: order = "filter_channel_row_column" elif len(torch_input_tensor.shape) == 3: order = "channel_row_column" print("Checking shapes: torch shape =", torch_input_tensor.shape) print("Order", order) ell_input_tensor = memory_shapes.get_tensor_in_ell_order( torch_input_tensor, order).ravel().astype(np.float32) ell_input_tensor = np.float32(ell_input_tensor) _logger.info("Get manual test output") flatten_shape = int(torch_input_tensor.shape[1]) * int( torch_input_tensor.shape[2]) * int(torch_input_tensor.shape[3]) torch_input_tensor_ = torch.from_numpy(torch_input_tensor) test_tensor = torch_input_tensor_.view(-1, flatten_shape) # manual_compute_result = fc_manual_test(onnx_nodes, test_tensor) # Get output from PyTorch model _logger.info("Getting PyTorch results") with torch.no_grad(): torch_output_ = torch_model.forward( torch.from_numpy(torch_input_tensor).float()) order = "channel" if len(torch_output_.shape) == 4: order = "filter_channel_row_column" elif len(torch_output_.shape) == 3: order = "channel_row_column" torch_output = memory_shapes.get_tensor_in_ell_order( torch_output_.numpy(), order) print("Check shapes: Ell input shape =", ell_input_tensor.shape) # Get computed ELL result _logger.info("Getting computed ELL results") result_from_compute = np.array( ell_map.Compute(ell_input_tensor, dtype=np.float32)) # result_from_compute = result_from_compute.reshape((1, 1, 10)) # Get compiled ELL result _logger.info("Getting compiled ELL results") compiler_options = ell.model.MapCompilerOptions() compiler_options.useBlas = True compiled_ell_map = ell_map.Compile("host", "model", "predict", compilerOptions=compiler_options, dtype=np.float32) result_from_compiled = np.array( compiled_ell_map.Compute(ell_input_tensor, dtype=np.float32)) print("Check shapes: torch shape = ", torch_output.shape, "ell shape = ", result_from_compute.shape) # Verify the computed result against the pytorch result np.testing.assert_array_almost_equal( torch_output[-1], result_from_compute[-1], decimal=4, err_msg=( 'results for computed ELL model do not match ONNX output!')) _logger.info("Verified computed result against ONNX") # Manual test # torch_output_ = torch_output_.reshape((1,10)) # np.testing.assert_array_almost_equal( # torch_output[-1], manual_compute_result, decimal=4, err_msg=( # 'results for manual computation do not match ELL output!')) # print("torch_output_: ", torch_output_[-1, :]) # print("manual_compute_result:", manual_compute_result[-1, :]) # _logger.info("Verified manual result against ONNX") # Verify the compiled result against the onnx result # torch_output = torch_output.reshape((10,)) # np.testing.assert_array_almost_equal( # torch_output[0,-1], result_from_compiled , decimal=4, err_msg=( # 'results for compiled ELL model do not match ONNX output!')) # _logger.info("Verified compiled result against ONNX") # Verify the compiled result agrees with the computed result # np.testing.assert_array_almost_equal( # result_from_compute, result_from_compiled, decimal=4, err_msg=( # 'results for computed ELL model do not match results from compiled ELL model!')) # _logger.info("Verified compiled result against computed result") # Get timing info # total_time = 0 # num_frames = 50 # _logger.info("Sending {} frames through model...".format(num_frames)) # for i in range(num_frames): # torch_input_tensor = np.random.random((torch_model.arguments[0].shape)).astype(np.float32) * 255 # ell_input_tensor = memory_shapes.get_tensor_in_ell_order(torch_input_tensor, "channel_row_column").ravel().astype(np.float32) # start = time.time() # result_from_compiled = np.array(compiled_ell_map.Compute(ell_input_tensor, dtype=np.float32)) # end = time.time() # total_time += end - start # total_time /= num_frames # _logger.info("Average speed: {:.0f}ms/frame".format(total_time * 1000)) except BaseException as exception: _logger.error("Verification of model output failed") raise exception _logger.info("Verification of model output complete")
model.forward(tensor) # print("Torch model", model) _logger.info("Exporting model to ONNX") torch_out = torch.onnx._export(model, tensor, "model.onnx", export_params=True, verbose=True) onnx_model = onnx.load("model.onnx") ell_map, onnx_nodes = onnx_converter("model.onnx") test_input = torch.Tensor([[[[1, 1], [1, 1]]]]) ell_input_tensor = memory_shapes.get_tensor_in_ell_order( test_input.numpy(), "filter_channel_row_column").ravel().astype(np.float32) ell_out = np.array(ell_map.Compute(ell_input_tensor, dtype=np.float32)) ell_out = ell_out.reshape((1, 2)) model.eval flatten_shape = int(test_input.size(1)) * int(test_input.size(2)) * int( test_input.size(3)) test_tensor = test_input.view(-1, flatten_shape) torch_out = model.forward(test_tensor) # np.testing.assert_almost_equal(torch_out.data.cpu().numpy(), ell_out, decimal=3, err_msg=( # 'results for compiled ELL model do not match ONNX output!')) # Manual test print("Manual test: \n") torch_output_tensor = model.forward(test_tensor)
def verify_compiled_ell_nodes_in_vision_model(modelFile, cntk_model, model_cntk_nodes, ordered_importer_nodes, step_interval_msec=None, lag_threshold_msec=None, plot_model=False, verify_model={"audio": False, "vision": False}): _logger = logger.get() # For convenient lookup, map from the cntk intermediate node to the # importer node cntk_nodes_map = {} for cntk_node in model_cntk_nodes: cntk_nodes_map[cntk_node.uid] = cntk_node cntk_input_tensor = np.random.random((cntk_model.arguments[0].shape)).astype(np.float32) * 255 ell_input_tensor = cntk_input_tensor if len(cntk_model.arguments[0].shape) == 1: ell_input_tensor = cntk_input_tensor.reshape((1, 1, cntk_model.arguments[0].shape[0])) ell_input_tensor = memory_shapes.get_tensor_in_ell_order(ell_input_tensor, "channel_row_column") ell_input_tensor = ell_input_tensor.ravel().astype(np.float32) cntk_nodes = [cntk_nodes_map[ordered_importer_nodes[0].id]] for i in range(1, len(ordered_importer_nodes)): cntk_node = cntk_nodes_map[ordered_importer_nodes[i].id] cntk_nodes.append(cntk_node) try: _logger.info("---- Testing compiled ELL nodes using prefix of {} CNTK nodes ----".format(i + 1)) _logger.info("Last CNTK node in chain: {}".format(cntk_nodes[i].uid)) # Create an ImporterModel from the CNTK nodes importer_model = import_nodes(cntk_nodes) if len(importer_model.nodes) > 0: # Use the common importer engine to drive conversion of the # ImporterModel to ELL layers importer_engine = common.importer.ImporterEngine(step_interval_msec=step_interval_msec, lag_threshold_msec=lag_threshold_msec) ell_map = importer_engine.convert_nodes(importer_model) prefix_ordered_importer_nodes, _ = importer_engine.get_importer_node_to_ell_mapping() if prefix_ordered_importer_nodes[-1].id != ordered_importer_nodes[i].id: _logger.info("Skipping..., can't test output of node {} yet".format(ordered_importer_nodes[i].id)) continue else: _logger.info("Skipping...") continue # Feed input to the ELL model _logger.info("Getting computed ELL results") ell_map.Compute(ell_input_tensor) model_clone = None if cntk_node.op_name != "UserFunction": model_clone = cntk_node.clone(CloneMethod.clone) if not model_clone: print("Skipping...") continue if verify_model["audio"]: _logger.info("Verification of audio models is not supported at this time, skipping verification") elif verify_model["vision"]: testing_info = {"apply_softmax": False} try: # Get output from CNTK model cntk_output = get_output_from_cntk_model(model_clone, cntk_input_tensor, testing_info) # Get compiled ELL result _logger.info("Getting compiled ELL results") compiler_options = ell.model.MapCompilerOptions() compiler_options.useBlas = True compiled_ell_map = ell_map.Compile("host", "model", "predict", compilerOptions=compiler_options) result_from_compiled = np.array(compiled_ell_map.Compute(ell_input_tensor)) output_shape = cntk_output.shape if (len(output_shape) == 3): if cntk_output.size == result_from_compiled.size: result_from_compiled = result_from_compiled.reshape(output_shape) else: padding = ordered_importer_nodes[i].output_padding["size"] output_shape_with_padding = (output_shape[0] + 2 * padding, output_shape[1] + 2 * padding, output_shape[2]) result_from_compiled = result_from_compiled.reshape(output_shape_with_padding) # Remove padding and look at active region only result_from_compiled = result_from_compiled[padding:output_shape[0] + padding, padding:output_shape[1] + padding, :] # Put the ELL results into same order as CNTK # if prefix_ordered_importer_nodes[-1].output_shapes[0][1] == "channel_row_column": print(result_from_compiled.shape, cntk_output.shape) # result_from_compiled = memory_shapes.get_tensor_in_ell_order(result_from_compiled, "xyz") # print(result_from_compiled) # print(cntk_output) # Compare results. Some layers have large numbers (e.g > 500.734) and some small numbers # (e.g. 0.0038453). To make the comparison more resilient and meaningful for large numbers, # normalize before comparing, since comparison is being done on significant digits. max = cntk_output.max() if max > 100: cntk_output = cntk_output / max result_from_compiled = result_from_compiled / max # Verify the compiled result against the cntk result np.testing.assert_array_almost_equal( cntk_output, result_from_compiled, decimal=4, err_msg=( 'results for compiled ELL nodes do not match CNTK output!')) _logger.info("---- passed ----") _logger.info("") except BaseException as exception: _logger.error("Error occurred verifying compiled ELL nodes of imported model") basename, ext = os.path.splitext(modelFile) ell_map.Save(basename + ".ell.compiled_node_verification_failed") raise exception except BaseException as exception: _logger.error("Error occurred attempting to convert cntk layers to ELL model: " + str(exception)) raise exception
def verify_ell_nodes_in_vision_model(ell_map, cntk_model, cntk_nodes, ordered_importer_nodes, node_mapping, testing_info): _logger = logger.get() _logger.info("\n\nVerification of model nodes starting") cntk_node_results = None try: # Get input to the CNTK model cntk_input_tensor = np.random.random((cntk_model.arguments[0].shape)).astype(np.float32) * 255 ell_input_tensor = cntk_input_tensor if len(cntk_model.arguments[0].shape) == 1: ell_input_tensor = cntk_input_tensor.reshape((1, 1, cntk_model.arguments[0].shape[0])) ell_input_tensor = memory_shapes.get_tensor_in_ell_order(ell_input_tensor, "channel_row_column") ell_input_tensor = ell_input_tensor.ravel().astype(np.float32) # For convenient lookup, map from the cntk intermediate node to the # importer node cntk_nodes_map = {} for cntk_node in cntk_nodes: cntk_nodes_map[cntk_node.uid] = cntk_node # Feed input to the ELL model _logger.info("Getting computed ELL results") ell_map.Compute(ell_input_tensor) # Walk list of importer nodes for importer_node in ordered_importer_nodes: if importer_node.operation_type in ["Input", "Passthrough", "Reshape", "Skip", "Softmax"]: if importer_node.operation_type == "Softmax": testing_info["apply_softmax"] = True continue _logger.info("Looking at node: {}".format(importer_node)) # Get the CNTK output values cntk_node = cntk_nodes_map[importer_node.id] try: if cntk_node.op_name != "UserFunction": clone = cntk_node.clone(CloneMethod.clone) except BaseException: _logger.info("Couldn't clone {}, skipping".format(cntk_node.uid)) continue # Get output from CNTK model _logger.info("Getting CNTK results") if (len(clone.arguments) > 1): arg1_output = np.zeros(clone.arguments[1].shape).astype(np.float32) cntk_node_results = clone.eval({clone.arguments[0]: [cntk_input_tensor], clone.arguments[1]: arg1_output}) else: cntk_node_results = clone.eval({clone.arguments[0]: [cntk_input_tensor]}) # Reorder cntk node output cntk_node_results = get_node_output_in_ell_order(cntk_node_results) # Get the results from the last ELL node for that group ell_node = node_mapping[importer_node.id][-1] ell_node_output_port = ell_node.GetOutputPort("output") ell_node_results = np.zeros((ell_node_output_port.Size(),), dtype=np.float32) for i in range(ell_node_output_port.Size()): ell_node_results[i] = ell_node_output_port.GetDoubleOutput(i) output_shape = cntk_node_results.shape if (len(output_shape) == 3): padding = importer_node.output_padding["size"] output_shape_with_padding = (output_shape[0] + 2 * padding, output_shape[1] + 2 * padding, output_shape[2]) ell_node_results = ell_node_results.reshape(output_shape_with_padding) # Remove padding and look at active region only ell_node_results = ell_node_results[padding:output_shape[0] + padding, padding:output_shape[1] + padding, :] # Compare results. Some layers have large numbers (e.g > 500.734) and some small numbers # (e.g. 0.0038453). To make the comparison more resilient and meaningful for large numbers, # normalize before comparing, since comparison is being done on significant digits. max = cntk_node_results.max() if max > 100: cntk_node_results = cntk_node_results / max ell_node_results = ell_node_results / max np.testing.assert_allclose( cntk_node_results, ell_node_results, rtol=1e-04, atol=1e-04, err_msg=( 'results for compiled ELL model do not match CNTK output!')) _logger.info("Output for {} verified\n".format(importer_node.id)) except BaseException as exception: _logger.error("Verification of model output failed") # if cntk_node_results is not None: # print_comparison(cntk_node_results, ell_node_results) raise exception _logger.info("Verification of model nodes complete\n")
def verify_ell_output_in_vision_model(ell_map, cntk_model, testing_info): _logger.info("Verification of model output starting") try: cntk_input_tensor = np.random.random((cntk_model.arguments[0].shape)).astype(np.float32) * 255 ell_input_tensor = memory_shapes.get_tensor_in_ell_order(cntk_input_tensor, "channel_row_column").ravel().astype(np.float32) # Get output from CNTK model _logger.info("Getting CNTK results") if (len(cntk_model.arguments) > 1): arg1_output = np.zeros(cntk_model.arguments[1].shape).astype(np.float32) cntk_output = cntk_model.eval({cntk_model.arguments[0]:[cntk_input_tensor], cntk_model.arguments[1]:arg1_output}) else: cntk_output = cntk_model.eval({cntk_model.arguments[0]:[cntk_input_tensor]}) size = 0 if isinstance(cntk_output,dict): for key in cntk_model.outputs: shape = key.shape if len(shape) > 0: s = np.max(shape) if (s > size): size = s cntk_output = cntk_output[key][0] else: cntk_output = cntk_output[0] # Check whether softmax needs to be applied or not. if testing_info["apply_softmax"]: cntk_output = softmax(cntk_output).eval() # Get computed ELL result _logger.info("Getting computed ELL results") result_from_compute = np.array(ell_map.Compute(ell_input_tensor, dtype=np.float32)) # Get compiled ELL result _logger.info("Getting compiled ELL results") compiler_options = ell.model.MapCompilerOptions() compiler_options.useBlas = True compiled_ell_map = ell_map.Compile("host", "model", "predict", compilerOptions=compiler_options, dtype=np.float32) result_from_compiled = np.array(compiled_ell_map.Compute(ell_input_tensor, dtype=np.float32)) # Verify the computed result against the cntk result np.testing.assert_array_almost_equal( cntk_output, result_from_compute, decimal=4, err_msg=( 'results for computed ELL model do not match CNTK output!')) _logger.info("Verified computed result against CNTK") # Verify the compiled result against the cntk result np.testing.assert_array_almost_equal( cntk_output, result_from_compiled, decimal=4, err_msg=( 'results for compiled ELL model do not match CNTK output!')) _logger.info("Verified compiled result against CNTK") # Verify the compiled result agrees with the computed result np.testing.assert_array_almost_equal( result_from_compute, result_from_compiled, decimal=4, err_msg=( 'results for computed ELL model do not match results from compiled ELL model!')) _logger.info("Verified compiled result against computed result") # Get timing info total_time = 0 num_frames = 50 _logger.info("Sending {} frames through model...".format(num_frames)) for i in range(num_frames): cntk_input_tensor = np.random.random((cntk_model.arguments[0].shape)).astype(np.float32) * 255 ell_input_tensor = memory_shapes.get_tensor_in_ell_order(cntk_input_tensor, "channel_row_column").ravel().astype(np.float32) start = time.time() result_from_compiled = np.array(compiled_ell_map.Compute(ell_input_tensor, dtype=np.float32)) end = time.time() total_time += end - start total_time /= num_frames _logger.info("Average speed: {:.0f}ms/frame".format(total_time * 1000)) except BaseException as exception: _logger.error("Verification of model output failed") raise exception _logger.info("Verification of model output complete")
def verify_ell_model(self, onnx_model_path, verify_compiled=True, arg_max_only=False): """ Test each operation in the onnx graph by creating a custom pytorch layer for each node then run forward with the onnx node weight on both ell and pytorch node. If verify_compiled is True then also test compiled ELL model. """ self._logger.info("Model verification started") try: # install debug hooks into torch model so we capture output of every layer. info = LayerInfo(self.torch_model) # get the pytorch model output self.torch_model.eval() model_name = os.path.basename(onnx_model_path) model_name = os.path.splitext(model_name)[0] ell_map, ordered_importer_nodes = onnx_to_ell.convert_onnx_to_ell( onnx_model_path) ell_map.Save(model_name + ".ell") # Get compiled ELL result if verify_compiled: self._logger.info("Getting compiled ELL results") compiler_options = ell.model.MapCompilerOptions() compiler_options.useBlas = True compiled_ell_map = ell_map.Compile( "host", "model", "predict", compilerOptions=compiler_options, dtype=np.float32) input_index = 0 for test_input in self.input_tensors: # get torch model output info.clear() torch_out = self.torch_model.forward( test_input).data.numpy().ravel() test_input = test_input.detach().cpu().numpy( ) # convert to numpy order = self.get_order(test_input.shape) ell_input_tensor = memory_shapes.get_tensor_in_ell_order( test_input, order) ell_flat_input = ell_input_tensor.ravel().astype(np.float32) if verify_compiled: ell_out_compiled = np.array( compiled_ell_map.Compute(ell_flat_input, dtype=np.float32)) # must come after the compiled Compute so that map contains valid outputs for layer by layer test ell_out = np.array( ell_map.Compute(ell_input_tensor, dtype=np.float32)) ell_nodes = get_nodes(ell_map) if not arg_max_only: # Compare the layers of the torch model with the coorresponding layers of the ELL model for key in info.layers.keys(): if input_index == 0: self._logger.info( "----- Comparing Layer {} output -----".format( key)) torch_output = info.layers[key]["output"].detach( ).numpy().ravel() node = get_matching_node_output(ell_nodes, key) if node is not None: port = node.GetOutputPort("output") shape = tuple(port.GetMemoryLayout().size) extent = tuple(port.GetMemoryLayout().extent) offset = tuple(port.GetMemoryLayout().offset) # padding = tuple(port.GetMemoryLayout().padding) # output shape includes padding ell_output = np.array( port.GetDoubleOutput()).astype(np.float32) # now to compare ell (row,col,channel) with torch (channel,row,col) we have to reorder ell_output = get_active_region( ell_output, shape, extent, offset) ell_output = np.moveaxis(ell_output, 2, 0).ravel() # close = np.allclose(torch_output, ell_output, atol=1e-3) np.testing.assert_almost_equal( torch_output, ell_output, decimal=3, err_msg= ('results for ELL layer {} do not match torch output for row {}' .format(node.GetRuntimeTypeName(), input_index))) # compare whole model output but only the argmax of it, # because sometimes model has Softmax but ELL does not. torch_prediction = np.argmax(torch_out) ell_prediction = np.argmax(ell_out) if verify_compiled: compiled_prediction = np.argmax(ell_out_compiled) msg = "argmax of ELL result {}, ELL compiled result {} and argmax of torch output {} on row {}" msg = msg.format(ell_prediction, compiled_prediction, torch_prediction, input_index) else: msg = "argmax of ELL result {} and argmax of torch output {} on row {}".format( ell_prediction, torch_prediction, input_index) self._logger.info(msg) np.testing.assert_equal(torch_prediction, ell_prediction, msg) if verify_compiled: np.testing.assert_equal(torch_prediction, compiled_prediction, msg) input_index += 1 except BaseException as exception: self._logger.error("Verification of model output failed: " + str(exception)) raise self._logger.info("Verification of model output complete")
def verify_ell_model(onnx_model): """ Test each operation in the onnx graph by creating a custom pytorch layer for each node then run forward with the onnx node weight on both ell and pytorch node """ _logger.info("Model verification started") try: ell_map, onnx_nodes = onnx_converter(onnx_model) ell_map.Save("model.ell") # get model input shape ell_input_shape = ell_map.GetInputShape() ell_shape = (ell_input_shape.rows, ell_input_shape.columns, ell_input_shape.channels) model_input_shape = (ell_shape[2], ell_shape[0], ell_shape[1]) test_input = torch.randn( (1, model_input_shape[0], model_input_shape[1], model_input_shape[2])) * 255 order = "channel" if len(test_input.shape) == 4: order = "filter_channel_row_column" elif len(test_input.shape) == 3: order = "channel_row_column" elif len(test_input.shape) == 2: order = "row_column" ell_input_tensor = memory_shapes.get_tensor_in_ell_order( test_input.numpy(), order).ravel().astype(np.float32) ell_out = np.array(ell_map.Compute(ell_input_tensor, dtype=np.float32)) _logger.info("############ ell_output:", ell_out) model.eval() torch_out = model.forward(test_input) _logger.info("############ torch_output:", torch_out) ell_out = ell_out.reshape(torch_out.size()) # np.testing.assert_almost_equal(torch_out.data.cpu().numpy(), ell_out, decimal=3, err_msg=( # 'results for compiled ELL model do not match ONNX output!')) # Manual test _logger.info("Manual test: \n") manual_compute_result = test_graph(onnx_nodes, test_input) np.testing.assert_array_almost_equal( manual_compute_result.detach().numpy(), ell_out, decimal=4, err_msg=( 'results for compute ELL model do not match ONNX output!')) _logger.info("Verified compute result against ONNX") _logger.info("########### Compute test passed ########### ") _logger.info("ell_output:", ell_out) _logger.info("torch_output", torch_out) _logger.info("onnx_output", manual_compute_result) _logger.info("###################### ") # Get compiled ELL result _logger.info("Getting compiled ELL results") compiler_options = ell.model.MapCompilerOptions() compiler_options.useBlas = True compiled_ell_map = ell_map.Compile("host", "model", "predict", compilerOptions=compiler_options, dtype=np.float32) result_from_compiled = np.array( compiled_ell_map.Compute(ell_input_tensor, dtype=np.float32)) ell_out_compiled = result_from_compiled.reshape(torch_out.size()) # Verify the computed result against the onnx result np.testing.assert_array_almost_equal( torch_out, ell_out_compiled, decimal=4, err_msg=( 'results for compiled ELL model do not match ONNX output!')) _logger.info("Verified compiled result against ONNX") except BaseException as exception: _logger.error("Verification of model output failed") raise exception _logger.info("Verification of model output complete")