def verify_ell_output_in_vision_model(ell_map, cntk_model, testing_info):
_logger = logger.get()
_logger.info("Verification of model output starting")
try:
cntk_input_tensor = np.random.random((cntk_model.arguments[0].shape)).astype(np.float32) * 255
ell_input_tensor = memory_shapes.get_tensor_in_ell_order(cntk_input_tensor, "channel_row_column")
ell_input_tensor = ell_input_tensor.ravel().astype(np.float32)
# Get output from CNTK model
cntk_output = get_output_from_cntk_model(cntk_model, cntk_input_tensor, testing_info)
# Get computed ELL result
_logger.info("Getting computed ELL results")
result_from_compute = np.array(ell_map.Compute(ell_input_tensor))
# Get compiled ELL result
_logger.info("Getting compiled ELL results")
compiler_options = ell.model.MapCompilerOptions()
compiler_options.useBlas = True
compiled_ell_map = ell_map.Compile("host", "model", "predict", compilerOptions=compiler_options)
result_from_compiled = np.array(compiled_ell_map.Compute(ell_input_tensor))
# Verify the computed result against the cntk result
np.testing.assert_array_almost_equal(
cntk_output, result_from_compute, decimal=4,
err_msg=('results for computed ELL model do not match CNTK output!'))
_logger.info("Verified computed result against CNTK")
# Verify the compiled result against the cntk result
np.testing.assert_array_almost_equal(
cntk_output, result_from_compiled, decimal=4, err_msg=(
'results for compiled ELL model do not match CNTK output!'))
_logger.info("Verified compiled result against CNTK")
# Verify the compiled result agrees with the computed result
np.testing.assert_array_almost_equal(
result_from_compute, result_from_compiled, decimal=4, err_msg=(
'results for computed ELL model do not match results from compiled ELL model!'))
_logger.info("Verified compiled result against computed result")
# Get timing info
total_time = 0
num_frames = 50
_logger.info("Sending {} frames through model...".format(num_frames))
for i in range(num_frames):
cntk_input_tensor = np.random.random((cntk_model.arguments[0].shape)).astype(np.float32) * 255
ell_input_tensor = memory_shapes.get_tensor_in_ell_order(cntk_input_tensor, "channel_row_column")
ell_input_tensor = ell_input_tensor.ravel().astype(np.float32)
start = time.time()
result_from_compiled = np.array(compiled_ell_map.Compute(ell_input_tensor, dtype=np.float32))
end = time.time()
total_time += end - start
total_time /= num_frames
_logger.info("Average speed: {:.0f}ms/frame".format(total_time * 1000))
except BaseException as exception:
_logger.error("Verification of model output failed")
raise exception
_logger.info("Verification of model output complete")
def verify_ell_output_in_vision_model(ell_map, torch_model, onnx_nodes,
testing_info):
_logger.info("Verification of model output starting")
try:
torch_model.eval()
ell_input_shape = ell_map.GetInputShape()
ell_shape = (ell_input_shape.rows, ell_input_shape.columns,
ell_input_shape.channels)
pytorch_input_shape = (ell_shape[2], ell_shape[0], ell_shape[1])
torch_input_tensor_ = np.random.random(
(1, pytorch_input_shape[0], pytorch_input_shape[1],
pytorch_input_shape[2])) * 255
torch_input_tensor = np.float32(torch_input_tensor_)
order = "channel"
if len(torch_input_tensor.shape) == 4:
order = "filter_channel_row_column"
elif len(torch_input_tensor.shape) == 3:
order = "channel_row_column"
print("Checking shapes: torch shape =", torch_input_tensor.shape)
print("Order", order)
ell_input_tensor = memory_shapes.get_tensor_in_ell_order(
torch_input_tensor, order).ravel().astype(np.float32)
ell_input_tensor = np.float32(ell_input_tensor)
_logger.info("Get manual test output")
flatten_shape = int(torch_input_tensor.shape[1]) * int(
torch_input_tensor.shape[2]) * int(torch_input_tensor.shape[3])
torch_input_tensor_ = torch.from_numpy(torch_input_tensor)
test_tensor = torch_input_tensor_.view(-1, flatten_shape)
# manual_compute_result = fc_manual_test(onnx_nodes, test_tensor)
# Get output from PyTorch model
_logger.info("Getting PyTorch results")
with torch.no_grad():
torch_output_ = torch_model.forward(
torch.from_numpy(torch_input_tensor).float())
order = "channel"
if len(torch_output_.shape) == 4:
order = "filter_channel_row_column"
elif len(torch_output_.shape) == 3:
order = "channel_row_column"
torch_output = memory_shapes.get_tensor_in_ell_order(
torch_output_.numpy(), order)
print("Check shapes: Ell input shape =", ell_input_tensor.shape)
# Get computed ELL result
_logger.info("Getting computed ELL results")
result_from_compute = np.array(
ell_map.Compute(ell_input_tensor, dtype=np.float32))
# result_from_compute = result_from_compute.reshape((1, 1, 10))
# Get compiled ELL result
_logger.info("Getting compiled ELL results")
compiler_options = ell.model.MapCompilerOptions()
compiler_options.useBlas = True
compiled_ell_map = ell_map.Compile("host",
"model",
"predict",
compilerOptions=compiler_options,
dtype=np.float32)
result_from_compiled = np.array(
compiled_ell_map.Compute(ell_input_tensor, dtype=np.float32))
print("Check shapes: torch shape = ", torch_output.shape,
"ell shape = ", result_from_compute.shape)
# Verify the computed result against the pytorch result
np.testing.assert_array_almost_equal(
torch_output[-1],
result_from_compute[-1],
decimal=4,
err_msg=(
'results for computed ELL model do not match ONNX output!'))
_logger.info("Verified computed result against ONNX")
# Manual test
# torch_output_ = torch_output_.reshape((1,10))
# np.testing.assert_array_almost_equal(
# torch_output[-1], manual_compute_result, decimal=4, err_msg=(
# 'results for manual computation do not match ELL output!'))
# print("torch_output_: ", torch_output_[-1, :])
# print("manual_compute_result:", manual_compute_result[-1, :])
# _logger.info("Verified manual result against ONNX")
# Verify the compiled result against the onnx result
# torch_output = torch_output.reshape((10,))
# np.testing.assert_array_almost_equal(
# torch_output[0,-1], result_from_compiled , decimal=4, err_msg=(
# 'results for compiled ELL model do not match ONNX output!'))
# _logger.info("Verified compiled result against ONNX")
# Verify the compiled result agrees with the computed result
# np.testing.assert_array_almost_equal(
# result_from_compute, result_from_compiled, decimal=4, err_msg=(
# 'results for computed ELL model do not match results from compiled ELL model!'))
# _logger.info("Verified compiled result against computed result")
# Get timing info
# total_time = 0
# num_frames = 50
# _logger.info("Sending {} frames through model...".format(num_frames))
# for i in range(num_frames):
# torch_input_tensor = np.random.random((torch_model.arguments[0].shape)).astype(np.float32) * 255
# ell_input_tensor = memory_shapes.get_tensor_in_ell_order(torch_input_tensor, "channel_row_column").ravel().astype(np.float32)
# start = time.time()
# result_from_compiled = np.array(compiled_ell_map.Compute(ell_input_tensor, dtype=np.float32))
# end = time.time()
# total_time += end - start
# total_time /= num_frames
# _logger.info("Average speed: {:.0f}ms/frame".format(total_time * 1000))
except BaseException as exception:
_logger.error("Verification of model output failed")
raise exception
_logger.info("Verification of model output complete")
model.forward(tensor)
# print("Torch model", model)
_logger.info("Exporting model to ONNX")
torch_out = torch.onnx._export(model,
tensor,
"model.onnx",
export_params=True,
verbose=True)
onnx_model = onnx.load("model.onnx")
ell_map, onnx_nodes = onnx_converter("model.onnx")
test_input = torch.Tensor([[[[1, 1], [1, 1]]]])
ell_input_tensor = memory_shapes.get_tensor_in_ell_order(
test_input.numpy(), "filter_channel_row_column").ravel().astype(np.float32)
ell_out = np.array(ell_map.Compute(ell_input_tensor, dtype=np.float32))
ell_out = ell_out.reshape((1, 2))
model.eval
flatten_shape = int(test_input.size(1)) * int(test_input.size(2)) * int(
test_input.size(3))
test_tensor = test_input.view(-1, flatten_shape)
torch_out = model.forward(test_tensor)
# np.testing.assert_almost_equal(torch_out.data.cpu().numpy(), ell_out, decimal=3, err_msg=(
# 'results for compiled ELL model do not match ONNX output!'))
# Manual test
print("Manual test: \n")
torch_output_tensor = model.forward(test_tensor)
def verify_compiled_ell_nodes_in_vision_model(modelFile, cntk_model, model_cntk_nodes, ordered_importer_nodes,
step_interval_msec=None, lag_threshold_msec=None, plot_model=False,
verify_model={"audio": False, "vision": False}):
_logger = logger.get()
# For convenient lookup, map from the cntk intermediate node to the
# importer node
cntk_nodes_map = {}
for cntk_node in model_cntk_nodes:
cntk_nodes_map[cntk_node.uid] = cntk_node
cntk_input_tensor = np.random.random((cntk_model.arguments[0].shape)).astype(np.float32) * 255
ell_input_tensor = cntk_input_tensor
if len(cntk_model.arguments[0].shape) == 1:
ell_input_tensor = cntk_input_tensor.reshape((1, 1, cntk_model.arguments[0].shape[0]))
ell_input_tensor = memory_shapes.get_tensor_in_ell_order(ell_input_tensor, "channel_row_column")
ell_input_tensor = ell_input_tensor.ravel().astype(np.float32)
cntk_nodes = [cntk_nodes_map[ordered_importer_nodes[0].id]]
for i in range(1, len(ordered_importer_nodes)):
cntk_node = cntk_nodes_map[ordered_importer_nodes[i].id]
cntk_nodes.append(cntk_node)
try:
_logger.info("---- Testing compiled ELL nodes using prefix of {} CNTK nodes ----".format(i + 1))
_logger.info("Last CNTK node in chain: {}".format(cntk_nodes[i].uid))
# Create an ImporterModel from the CNTK nodes
importer_model = import_nodes(cntk_nodes)
if len(importer_model.nodes) > 0:
# Use the common importer engine to drive conversion of the
# ImporterModel to ELL layers
importer_engine = common.importer.ImporterEngine(step_interval_msec=step_interval_msec,
lag_threshold_msec=lag_threshold_msec)
ell_map = importer_engine.convert_nodes(importer_model)
prefix_ordered_importer_nodes, _ = importer_engine.get_importer_node_to_ell_mapping()
if prefix_ordered_importer_nodes[-1].id != ordered_importer_nodes[i].id:
_logger.info("Skipping..., can't test output of node {} yet".format(ordered_importer_nodes[i].id))
continue
else:
_logger.info("Skipping...")
continue
# Feed input to the ELL model
_logger.info("Getting computed ELL results")
ell_map.Compute(ell_input_tensor)
model_clone = None
if cntk_node.op_name != "UserFunction":
model_clone = cntk_node.clone(CloneMethod.clone)
if not model_clone:
print("Skipping...")
continue
if verify_model["audio"]:
_logger.info("Verification of audio models is not supported at this time, skipping verification")
elif verify_model["vision"]:
testing_info = {"apply_softmax": False}
try:
# Get output from CNTK model
cntk_output = get_output_from_cntk_model(model_clone, cntk_input_tensor, testing_info)
# Get compiled ELL result
_logger.info("Getting compiled ELL results")
compiler_options = ell.model.MapCompilerOptions()
compiler_options.useBlas = True
compiled_ell_map = ell_map.Compile("host", "model", "predict", compilerOptions=compiler_options)
result_from_compiled = np.array(compiled_ell_map.Compute(ell_input_tensor))
output_shape = cntk_output.shape
if (len(output_shape) == 3):
if cntk_output.size == result_from_compiled.size:
result_from_compiled = result_from_compiled.reshape(output_shape)
else:
padding = ordered_importer_nodes[i].output_padding["size"]
output_shape_with_padding = (output_shape[0] + 2 * padding,
output_shape[1] + 2 * padding,
output_shape[2])
result_from_compiled = result_from_compiled.reshape(output_shape_with_padding)
# Remove padding and look at active region only
result_from_compiled = result_from_compiled[padding:output_shape[0] + padding,
padding:output_shape[1] + padding, :]
# Put the ELL results into same order as CNTK
# if prefix_ordered_importer_nodes[-1].output_shapes[0][1] == "channel_row_column":
print(result_from_compiled.shape, cntk_output.shape)
# result_from_compiled = memory_shapes.get_tensor_in_ell_order(result_from_compiled, "xyz")
# print(result_from_compiled)
# print(cntk_output)
# Compare results. Some layers have large numbers (e.g > 500.734) and some small numbers
# (e.g. 0.0038453). To make the comparison more resilient and meaningful for large numbers,
# normalize before comparing, since comparison is being done on significant digits.
max = cntk_output.max()
if max > 100:
cntk_output = cntk_output / max
result_from_compiled = result_from_compiled / max
# Verify the compiled result against the cntk result
np.testing.assert_array_almost_equal(
cntk_output, result_from_compiled, decimal=4, err_msg=(
'results for compiled ELL nodes do not match CNTK output!'))
_logger.info("---- passed ----")
_logger.info("")
except BaseException as exception:
_logger.error("Error occurred verifying compiled ELL nodes of imported model")
basename, ext = os.path.splitext(modelFile)
ell_map.Save(basename + ".ell.compiled_node_verification_failed")
raise exception
except BaseException as exception:
_logger.error("Error occurred attempting to convert cntk layers to ELL model: " + str(exception))
raise exception
def verify_ell_nodes_in_vision_model(ell_map, cntk_model, cntk_nodes, ordered_importer_nodes, node_mapping,
testing_info):
_logger = logger.get()
_logger.info("\n\nVerification of model nodes starting")
cntk_node_results = None
try:
# Get input to the CNTK model
cntk_input_tensor = np.random.random((cntk_model.arguments[0].shape)).astype(np.float32) * 255
ell_input_tensor = cntk_input_tensor
if len(cntk_model.arguments[0].shape) == 1:
ell_input_tensor = cntk_input_tensor.reshape((1, 1, cntk_model.arguments[0].shape[0]))
ell_input_tensor = memory_shapes.get_tensor_in_ell_order(ell_input_tensor, "channel_row_column")
ell_input_tensor = ell_input_tensor.ravel().astype(np.float32)
# For convenient lookup, map from the cntk intermediate node to the
# importer node
cntk_nodes_map = {}
for cntk_node in cntk_nodes:
cntk_nodes_map[cntk_node.uid] = cntk_node
# Feed input to the ELL model
_logger.info("Getting computed ELL results")
ell_map.Compute(ell_input_tensor)
# Walk list of importer nodes
for importer_node in ordered_importer_nodes:
if importer_node.operation_type in ["Input", "Passthrough", "Reshape", "Skip", "Softmax"]:
if importer_node.operation_type == "Softmax":
testing_info["apply_softmax"] = True
continue
_logger.info("Looking at node: {}".format(importer_node))
# Get the CNTK output values
cntk_node = cntk_nodes_map[importer_node.id]
try:
if cntk_node.op_name != "UserFunction":
clone = cntk_node.clone(CloneMethod.clone)
except BaseException:
_logger.info("Couldn't clone {}, skipping".format(cntk_node.uid))
continue
# Get output from CNTK model
_logger.info("Getting CNTK results")
if (len(clone.arguments) > 1):
arg1_output = np.zeros(clone.arguments[1].shape).astype(np.float32)
cntk_node_results = clone.eval({clone.arguments[0]: [cntk_input_tensor],
clone.arguments[1]: arg1_output})
else:
cntk_node_results = clone.eval({clone.arguments[0]: [cntk_input_tensor]})
# Reorder cntk node output
cntk_node_results = get_node_output_in_ell_order(cntk_node_results)
# Get the results from the last ELL node for that group
ell_node = node_mapping[importer_node.id][-1]
ell_node_output_port = ell_node.GetOutputPort("output")
ell_node_results = np.zeros((ell_node_output_port.Size(),), dtype=np.float32)
for i in range(ell_node_output_port.Size()):
ell_node_results[i] = ell_node_output_port.GetDoubleOutput(i)
output_shape = cntk_node_results.shape
if (len(output_shape) == 3):
padding = importer_node.output_padding["size"]
output_shape_with_padding = (output_shape[0] + 2 * padding,
output_shape[1] + 2 * padding,
output_shape[2])
ell_node_results = ell_node_results.reshape(output_shape_with_padding)
# Remove padding and look at active region only
ell_node_results = ell_node_results[padding:output_shape[0] + padding,
padding:output_shape[1] + padding, :]
# Compare results. Some layers have large numbers (e.g > 500.734) and some small numbers
# (e.g. 0.0038453). To make the comparison more resilient and meaningful for large numbers,
# normalize before comparing, since comparison is being done on significant digits.
max = cntk_node_results.max()
if max > 100:
cntk_node_results = cntk_node_results / max
ell_node_results = ell_node_results / max
np.testing.assert_allclose(
cntk_node_results, ell_node_results, rtol=1e-04, atol=1e-04, err_msg=(
'results for compiled ELL model do not match CNTK output!'))
_logger.info("Output for {} verified\n".format(importer_node.id))
except BaseException as exception:
_logger.error("Verification of model output failed")
# if cntk_node_results is not None:
# print_comparison(cntk_node_results, ell_node_results)
raise exception
_logger.info("Verification of model nodes complete\n")
def verify_ell_output_in_vision_model(ell_map, cntk_model, testing_info):
_logger.info("Verification of model output starting")
try:
cntk_input_tensor = np.random.random((cntk_model.arguments[0].shape)).astype(np.float32) * 255
ell_input_tensor = memory_shapes.get_tensor_in_ell_order(cntk_input_tensor, "channel_row_column").ravel().astype(np.float32)
# Get output from CNTK model
_logger.info("Getting CNTK results")
if (len(cntk_model.arguments) > 1):
arg1_output = np.zeros(cntk_model.arguments[1].shape).astype(np.float32)
cntk_output = cntk_model.eval({cntk_model.arguments[0]:[cntk_input_tensor], cntk_model.arguments[1]:arg1_output})
else:
cntk_output = cntk_model.eval({cntk_model.arguments[0]:[cntk_input_tensor]})
size = 0
if isinstance(cntk_output,dict):
for key in cntk_model.outputs:
shape = key.shape
if len(shape) > 0:
s = np.max(shape)
if (s > size):
size = s
cntk_output = cntk_output[key][0]
else:
cntk_output = cntk_output[0]
# Check whether softmax needs to be applied or not.
if testing_info["apply_softmax"]:
cntk_output = softmax(cntk_output).eval()
# Get computed ELL result
_logger.info("Getting computed ELL results")
result_from_compute = np.array(ell_map.Compute(ell_input_tensor, dtype=np.float32))
# Get compiled ELL result
_logger.info("Getting compiled ELL results")
compiler_options = ell.model.MapCompilerOptions()
compiler_options.useBlas = True
compiled_ell_map = ell_map.Compile("host", "model", "predict", compilerOptions=compiler_options, dtype=np.float32)
result_from_compiled = np.array(compiled_ell_map.Compute(ell_input_tensor, dtype=np.float32))
# Verify the computed result against the cntk result
np.testing.assert_array_almost_equal(
cntk_output, result_from_compute, decimal=4, err_msg=(
'results for computed ELL model do not match CNTK output!'))
_logger.info("Verified computed result against CNTK")
# Verify the compiled result against the cntk result
np.testing.assert_array_almost_equal(
cntk_output, result_from_compiled, decimal=4, err_msg=(
'results for compiled ELL model do not match CNTK output!'))
_logger.info("Verified compiled result against CNTK")
# Verify the compiled result agrees with the computed result
np.testing.assert_array_almost_equal(
result_from_compute, result_from_compiled, decimal=4, err_msg=(
'results for computed ELL model do not match results from compiled ELL model!'))
_logger.info("Verified compiled result against computed result")
# Get timing info
total_time = 0
num_frames = 50
_logger.info("Sending {} frames through model...".format(num_frames))
for i in range(num_frames):
cntk_input_tensor = np.random.random((cntk_model.arguments[0].shape)).astype(np.float32) * 255
ell_input_tensor = memory_shapes.get_tensor_in_ell_order(cntk_input_tensor, "channel_row_column").ravel().astype(np.float32)
start = time.time()
result_from_compiled = np.array(compiled_ell_map.Compute(ell_input_tensor, dtype=np.float32))
end = time.time()
total_time += end - start
total_time /= num_frames
_logger.info("Average speed: {:.0f}ms/frame".format(total_time * 1000))
except BaseException as exception:
_logger.error("Verification of model output failed")
raise exception
_logger.info("Verification of model output complete")
def verify_ell_model(self,
onnx_model_path,
verify_compiled=True,
arg_max_only=False):
"""
Test each operation in the onnx graph by creating a custom pytorch layer for each node then
run forward with the onnx node weight on both ell and pytorch node. If verify_compiled is
True then also test compiled ELL model.
"""
self._logger.info("Model verification started")
try:
# install debug hooks into torch model so we capture output of every layer.
info = LayerInfo(self.torch_model)
# get the pytorch model output
self.torch_model.eval()
model_name = os.path.basename(onnx_model_path)
model_name = os.path.splitext(model_name)[0]
ell_map, ordered_importer_nodes = onnx_to_ell.convert_onnx_to_ell(
onnx_model_path)
ell_map.Save(model_name + ".ell")
# Get compiled ELL result
if verify_compiled:
self._logger.info("Getting compiled ELL results")
compiler_options = ell.model.MapCompilerOptions()
compiler_options.useBlas = True
compiled_ell_map = ell_map.Compile(
"host",
"model",
"predict",
compilerOptions=compiler_options,
dtype=np.float32)
input_index = 0
for test_input in self.input_tensors:
# get torch model output
info.clear()
torch_out = self.torch_model.forward(
test_input).data.numpy().ravel()
test_input = test_input.detach().cpu().numpy(
) # convert to numpy
order = self.get_order(test_input.shape)
ell_input_tensor = memory_shapes.get_tensor_in_ell_order(
test_input, order)
ell_flat_input = ell_input_tensor.ravel().astype(np.float32)
if verify_compiled:
ell_out_compiled = np.array(
compiled_ell_map.Compute(ell_flat_input,
dtype=np.float32))
# must come after the compiled Compute so that map contains valid outputs for layer by layer test
ell_out = np.array(
ell_map.Compute(ell_input_tensor, dtype=np.float32))
ell_nodes = get_nodes(ell_map)
if not arg_max_only:
# Compare the layers of the torch model with the coorresponding layers of the ELL model
for key in info.layers.keys():
if input_index == 0:
self._logger.info(
"----- Comparing Layer {} output -----".format(
key))
torch_output = info.layers[key]["output"].detach(
).numpy().ravel()
node = get_matching_node_output(ell_nodes, key)
if node is not None:
port = node.GetOutputPort("output")
shape = tuple(port.GetMemoryLayout().size)
extent = tuple(port.GetMemoryLayout().extent)
offset = tuple(port.GetMemoryLayout().offset)
# padding = tuple(port.GetMemoryLayout().padding) # output shape includes padding
ell_output = np.array(
port.GetDoubleOutput()).astype(np.float32)
# now to compare ell (row,col,channel) with torch (channel,row,col) we have to reorder
ell_output = get_active_region(
ell_output, shape, extent, offset)
ell_output = np.moveaxis(ell_output, 2, 0).ravel()
# close = np.allclose(torch_output, ell_output, atol=1e-3)
np.testing.assert_almost_equal(
torch_output,
ell_output,
decimal=3,
err_msg=
('results for ELL layer {} do not match torch output for row {}'
.format(node.GetRuntimeTypeName(),
input_index)))
# compare whole model output but only the argmax of it,
# because sometimes model has Softmax but ELL does not.
torch_prediction = np.argmax(torch_out)
ell_prediction = np.argmax(ell_out)
if verify_compiled:
compiled_prediction = np.argmax(ell_out_compiled)
msg = "argmax of ELL result {}, ELL compiled result {} and argmax of torch output {} on row {}"
msg = msg.format(ell_prediction, compiled_prediction,
torch_prediction, input_index)
else:
msg = "argmax of ELL result {} and argmax of torch output {} on row {}".format(
ell_prediction, torch_prediction, input_index)
self._logger.info(msg)
np.testing.assert_equal(torch_prediction, ell_prediction, msg)
if verify_compiled:
np.testing.assert_equal(torch_prediction,
compiled_prediction, msg)
input_index += 1
except BaseException as exception:
self._logger.error("Verification of model output failed: " +
str(exception))
raise
self._logger.info("Verification of model output complete")
def verify_ell_model(onnx_model):
"""
Test each operation in the onnx graph by creating a
custom pytorch layer for each node then run forward
with the onnx node weight on both ell and pytorch node
"""
_logger.info("Model verification started")
try:
ell_map, onnx_nodes = onnx_converter(onnx_model)
ell_map.Save("model.ell")
# get model input shape
ell_input_shape = ell_map.GetInputShape()
ell_shape = (ell_input_shape.rows, ell_input_shape.columns,
ell_input_shape.channels)
model_input_shape = (ell_shape[2], ell_shape[0], ell_shape[1])
test_input = torch.randn(
(1, model_input_shape[0], model_input_shape[1],
model_input_shape[2])) * 255
order = "channel"
if len(test_input.shape) == 4:
order = "filter_channel_row_column"
elif len(test_input.shape) == 3:
order = "channel_row_column"
elif len(test_input.shape) == 2:
order = "row_column"
ell_input_tensor = memory_shapes.get_tensor_in_ell_order(
test_input.numpy(), order).ravel().astype(np.float32)
ell_out = np.array(ell_map.Compute(ell_input_tensor, dtype=np.float32))
_logger.info("############ ell_output:", ell_out)
model.eval()
torch_out = model.forward(test_input)
_logger.info("############ torch_output:", torch_out)
ell_out = ell_out.reshape(torch_out.size())
# np.testing.assert_almost_equal(torch_out.data.cpu().numpy(), ell_out, decimal=3, err_msg=(
# 'results for compiled ELL model do not match ONNX output!'))
# Manual test
_logger.info("Manual test: \n")
manual_compute_result = test_graph(onnx_nodes, test_input)
np.testing.assert_array_almost_equal(
manual_compute_result.detach().numpy(),
ell_out,
decimal=4,
err_msg=(
'results for compute ELL model do not match ONNX output!'))
_logger.info("Verified compute result against ONNX")
_logger.info("########### Compute test passed ########### ")
_logger.info("ell_output:", ell_out)
_logger.info("torch_output", torch_out)
_logger.info("onnx_output", manual_compute_result)
_logger.info("###################### ")
# Get compiled ELL result
_logger.info("Getting compiled ELL results")
compiler_options = ell.model.MapCompilerOptions()
compiler_options.useBlas = True
compiled_ell_map = ell_map.Compile("host",
"model",
"predict",
compilerOptions=compiler_options,
dtype=np.float32)
result_from_compiled = np.array(
compiled_ell_map.Compute(ell_input_tensor, dtype=np.float32))
ell_out_compiled = result_from_compiled.reshape(torch_out.size())
# Verify the computed result against the onnx result
np.testing.assert_array_almost_equal(
torch_out,
ell_out_compiled,
decimal=4,
err_msg=(
'results for compiled ELL model do not match ONNX output!'))
_logger.info("Verified compiled result against ONNX")
except BaseException as exception:
_logger.error("Verification of model output failed")
raise exception
_logger.info("Verification of model output complete")
