def test_cast(self):
X = np.random.randn(1, 2, 3).astype(np.float32)
for to_type in ['INT8', caffe2_pb2.TensorProto.INT8,
'DOUBLE', caffe2_pb2.TensorProto.DOUBLE]:
predict_net = caffe2_pb2.NetDef()
predict_net.name = 'test-cast-net'
predict_net.external_input[:] = ['X']
predict_net.external_output[:] = ['Y']
predict_net.op.extend([
core.CreateOperator(
'Cast',
inputs=['X'],
outputs=['Y'],
to=to_type,
),
])
ws, c2_outputs = c2_native_run_net(
init_net=None,
predict_net=predict_net,
inputs=[X])
onnx_model = c2_onnx.caffe2_net_to_onnx_model(
predict_net=predict_net,
value_info={
'X': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[X.dtype], X.shape)
})
onnx_outputs = c2.run_model(onnx_model, inputs=[X])
self.assertSameOutputs(c2_outputs, onnx_outputs)
def test_fc(self):
X_fake = np.zeros((3, 1, 3, 1, 7), dtype=np.float32)
X = np.random.randn(5, 2, 3, 1, 7).astype(np.float32)
W = np.random.randn(11, 21).astype(np.float32)
B = np.random.randn(11).astype(np.float32)
predict_net = caffe2_pb2.NetDef()
predict_net.name = 'test-fc-net'
predict_net.external_input[:] = ['X', 'W', 'B']
predict_net.external_output[:] = ['Y']
predict_net.op.extend([
core.CreateOperator(
'FC',
inputs=['X', 'W', 'B'],
outputs=['Y'],
axis=2,
),
])
ws, c2_outputs = c2_native_run_net(
init_net=None,
predict_net=predict_net,
inputs=[X, W, B])
onnx_model = c2_onnx.caffe2_net_to_onnx_model(
predict_net=predict_net,
value_info={
'X': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[X.dtype], X_fake.shape),
'W': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[W.dtype], W.shape),
'B': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[B.dtype], B.shape),
})
onnx_outputs = c2.run_model(onnx_model, inputs=[X, W, B])
self.assertSameOutputs(c2_outputs, onnx_outputs)
def test_qconv_model(self):
class ConvModel(torch.nn.Module):
def __init__(self):
super(ConvModel, self).__init__()
self.qconfig = torch.quantization.default_qconfig
self.fc1 = torch.quantization.QuantWrapper(
torch.nn.Conv2d(3, 5, 2, bias=True).to(dtype=torch.float))
def forward(self, x):
x = self.fc1(x)
return x
torch.backends.quantized.engine = "qnnpack"
qconfig = torch.quantization.default_qconfig
model = ConvModel()
model.qconfig = qconfig
model = torch.quantization.prepare(model)
model = torch.quantization.convert(model)
x_numpy = np.random.rand(1, 3, 6, 6).astype(np.float32)
x = torch.from_numpy(x_numpy).to(dtype=torch.float)
outputs = model(x)
input_names = ["x"]
onnx_model = self.export_to_onnx(model, x, input_names)
y = np.expand_dims(x_numpy, axis=0)
caffe_res = c2.run_model(onnx_model, dict(zip(input_names, y)))[0]
output_diff = np.absolute(np.squeeze(outputs.numpy()) - caffe_res)
max_diff = np.amax(output_diff)
# Permute pytorch output to NHWC
# This check had to be changed to account for changes in
# qnnpack's requant logic.
np.testing.assert_(max_diff <= 1,
"Maximum absolute difference must be less than 1")
def test_elementwiselinear(self):
X = np.random.randn(4, 2, 5, 7, 3).astype(np.float32)
W = np.random.randn(21).astype(np.float32)
B = np.random.randn(21).astype(np.float32)
predict_net = caffe2_pb2.NetDef()
predict_net.name = 'test-elementwiselinear-net'
predict_net.external_input[:] = ['X', 'W', 'B']
predict_net.external_output[:] = ['Y']
predict_net.op.extend([
core.CreateOperator(
'ElementwiseLinear',
inputs=['X', 'W', 'B'],
outputs=['Y'],
axis=3,
),
])
ws, c2_outputs = c2_native_run_net(
init_net=None,
predict_net=predict_net,
inputs=[X, W, B])
onnx_model = c2_onnx.caffe2_net_to_onnx_model(
predict_net=predict_net,
value_info={
'X': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[X.dtype], X.shape),
'W': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[W.dtype], W.shape),
'B': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[B.dtype], B.shape),
})
onnx_outputs = c2.run_model(onnx_model, inputs=[X, W, B])
self.assertSameOutputs(c2_outputs, onnx_outputs)
def test_qconv_model(self):
class ConvModel(torch.nn.Module):
def __init__(self):
super(ConvModel, self).__init__()
self.qconfig = torch.quantization.default_qconfig
self.fc1 = torch.quantization.QuantWrapper(
torch.nn.Conv2d(3, 5, 2, bias=True).to(dtype=torch.float))
def forward(self, x):
x = self.fc1(x)
return x
torch.backends.quantized.engine = "qnnpack"
qconfig = torch.quantization.default_qconfig
model = ConvModel()
model.qconfig = qconfig
model = torch.quantization.prepare(model)
model = torch.quantization.convert(model)
x_numpy = np.random.rand(1, 3, 6, 6).astype(np.float32)
x = torch.from_numpy(x_numpy).to(dtype=torch.float)
outputs = model(x)
input_names = ["x"]
onnx_model = self.export_to_onnx(model, x, input_names)
y = np.expand_dims(x_numpy, axis=0)
caffe_res = c2.run_model(onnx_model, dict(zip(input_names, y)))[0]
# Permute pytorch output to NHWC
np.testing.assert_almost_equal(outputs.numpy(), caffe_res, decimal=3)
def _test_onnx_importer(self,
model_name,
data_input_index,
opset_version=onnx.defs.onnx_opset_version()):
model_dir = _download_onnx_model(model_name, opset_version)
model_def = onnx.load(os.path.join(model_dir, 'model.onnx'))
input_blob_dims = [
int(x.dim_value) for x in
model_def.graph.input[data_input_index].type.tensor_type.shape.dim
]
op_inputs = [x.name for x in model_def.graph.input]
op_outputs = [x.name for x in model_def.graph.output]
print("{}".format(op_inputs))
data = np.random.randn(*input_blob_dims).astype(np.float32)
Y_c2 = c2.run_model(model_def, {op_inputs[data_input_index]: data})
op = convert_onnx_model_to_trt_op(model_def, verbosity=3)
device_option = core.DeviceOption(caffe2_pb2.CUDA, 0)
op.device_option.CopyFrom(device_option)
Y_trt = None
ws = Workspace()
with core.DeviceScope(device_option):
ws.FeedBlob(op_inputs[data_input_index], data)
ws.RunOperatorsOnce([op])
output_values = [ws.FetchBlob(name) for name in op_outputs]
Y_trt = namedtupledict('Outputs', op_outputs)(*output_values)
np.testing.assert_allclose(Y_c2, Y_trt, rtol=1e-3)
def test_slice(self):
X = np.random.randn(1, 2, 3).astype(np.float32)
starts = np.array([0, 1, 0], dtype=np.int32)
ends = np.array([-1, 2, 3], dtype=np.int32)
predict_net = caffe2_pb2.NetDef()
predict_net.name = 'test-slice-net'
predict_net.external_input[:] = ['X']
predict_net.external_output[:] = ['Y']
predict_net.op.extend([
core.CreateOperator(
'Slice',
inputs=['X'],
outputs=['Y'],
starts=starts,
ends=ends,
),
])
ws, (Y,) = c2_native_run_net(
init_net=None,
predict_net=predict_net,
inputs=[X])
onnx_model = c2_onnx.caffe2_net_to_onnx_model(
predict_net=predict_net,
value_info={
'X': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[X.dtype], X.shape)
})
Y, = c2.run_model(onnx_model, inputs=[X])
np.testing.assert_almost_equal(Y, X[:, 1:2, :])
def test_qlinear_model(self):
class LinearModel(torch.nn.Module):
def __init__(self):
super(LinearModel, self).__init__()
self.qconfig = torch.quantization.default_qconfig
self.fc1 = torch.quantization.QuantWrapper(
torch.nn.Linear(5, 10).to(dtype=torch.float))
def forward(self, x):
x = self.fc1(x)
return x
torch.backends.quantized.engine = "qnnpack"
qconfig = torch.quantization.default_qconfig
model = LinearModel()
model.qconfig = qconfig
model = torch.quantization.prepare(model)
model = torch.quantization.convert(model)
x_numpy = np.random.rand(1, 2, 5).astype(np.float32)
x = torch.from_numpy(x_numpy).to(dtype=torch.float)
outputs = model(x)
input_names = ["x"]
onnx_model = self.export_to_onnx(model, x, input_names)
caffe_res = c2.run_model(onnx_model, dict(zip(input_names,
x_numpy)))[0]
np.testing.assert_almost_equal(np.squeeze(outputs.numpy()),
caffe_res,
decimal=3)
def test_upsample(self):
X = np.random.randn(1, 1, 2, 2).astype(np.float32)
width_scale = 2.0
height_scale = 2.0
predict_net = caffe2_pb2.NetDef()
predict_net.name = 'test-upsample-net'
predict_net.external_input[:] = ['X']
predict_net.external_output[:] = ['Y']
predict_net.op.extend([
core.CreateOperator(
'ResizeNearest',
inputs=['X'],
outputs=['Y'],
width_scale=width_scale,
height_scale=height_scale,
),
])
ws, c2_outputs = c2_native_run_net(init_net=None,
predict_net=predict_net,
inputs=[X])
onnx_model = c2_onnx.caffe2_net_to_onnx_model(
predict_net=predict_net,
value_info={
'X': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[X.dtype], X.shape)
})
onnx_outputs = c2.run_model(onnx_model, inputs=[X])
self.assertSameOutputs(c2_outputs, onnx_outputs)
def generic_test(self, model, sample_inputs, input_names=None, decimal=3, relaxed_check=False):
torch.backends.quantized.engine = "qnnpack"
pt_inputs = tuple(torch.from_numpy(x) for x in sample_inputs)
model.qconfig = torch.ao.quantization.get_default_qconfig("qnnpack")
q_model = torch.ao.quantization.prepare(model, inplace=False)
q_model = torch.ao.quantization.convert(q_model, inplace=False)
traced_model = torch.jit.trace(q_model, pt_inputs)
buf = io.BytesIO()
torch.jit.save(traced_model, buf)
buf.seek(0)
q_model = torch.jit.load(buf)
q_model.eval()
output = q_model(*pt_inputs)
f = io.BytesIO()
torch.onnx.export(q_model, pt_inputs, f, input_names=input_names,
operator_export_type=torch.onnx.OperatorExportTypes.ONNX_ATEN_FALLBACK)
f.seek(0)
onnx_model = onnx.load(f)
caffe_res = c2.run_model(onnx_model, dict(zip(input_names, sample_inputs)))[0]
# Due to change in requantization logic for certain ops such conv, linear
# in pytorch's integration of qnnpack, numerics may have a mismatc with C2.
# This mismatch should not be off my more than 1.
# This flag helps us override default behavior under certain circumstances.
if relaxed_check:
output_diff = np.absolute(np.squeeze(output.detach().numpy()) - caffe_res)
max_diff = np.amax(output_diff)
# This check had to be changed to account for changes in
# qnnpack's requant logic.
np.testing.assert_(max_diff <= 1, "Maximum absolute difference must be less than 1")
else:
np.testing.assert_almost_equal(output.detach().numpy(), caffe_res, decimal=decimal)
def test_concat(self):
I0 = np.random.randn(20, 4).astype(np.float32)
I1 = np.random.randn(20, 4).astype(np.float32)
for i in range(2):
predict_net = caffe2_pb2.NetDef()
predict_net.name = 'test-concat-net'
predict_net.external_input[:] = ['I0', 'I1']
predict_net.external_output[:] = ['Y', 'output_dim']
predict_net.op.extend([
core.CreateOperator(
'Concat',
inputs=['I0', 'I1'],
outputs=['Y', 'output_dim'],
axis=1,
add_axis=(1 if i == 0 else 0),
),
])
ws, c2_outputs = c2_native_run_net(init_net=None,
predict_net=predict_net,
inputs=[I0, I1])
onnx_model = c2_onnx.caffe2_net_to_onnx_model(
predict_net=predict_net,
value_info={
'I0':
(onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[I0.dtype], I0.shape),
'I1':
(onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[I1.dtype], I1.shape),
})
onnx_outputs = c2.run_model(onnx_model, inputs=[I0, I1])
self.assertSameOutputs(c2_outputs, onnx_outputs)
def test_slice(self):
X = np.random.randn(1, 2, 3).astype(np.float32)
starts = np.array([0, 1, 0], dtype=np.int32)
ends = np.array([-1, 2, 3], dtype=np.int32)
predict_net = caffe2_pb2.NetDef()
predict_net.name = 'test-slice-net'
predict_net.external_input[:] = ['X']
predict_net.external_output[:] = ['Y']
predict_net.op.extend([
core.CreateOperator(
'Slice',
inputs=['X'],
outputs=['Y'],
starts=starts,
ends=ends,
),
])
ws, c2_outputs = c2_native_run_net(init_net=None,
predict_net=predict_net,
inputs=[X])
onnx_model = c2_onnx.caffe2_net_to_onnx_model(
predict_net=predict_net,
value_info={
'X': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[X.dtype], X.shape)
})
onnx_outputs = c2.run_model(onnx_model, inputs=[X])
self.assertSameOutputs(c2_outputs, onnx_outputs)
def export_model(filename, verbose=False):
dummy_input = torch.linspace(0,
2,
steps=seq_len * output_dim,
device=device).reshape(seq_len, output_dim)
if not isinstance(model, LSTMModel):
dummy_input = dummy_input.flatten()
torch.onnx.export(model, dummy_input, filename, verbose=verbose)
model.eval()
result = model(dummy_input).cpu().detach().numpy()
# check the model
dummy_input = dummy_input.cpu().detach().numpy()
import onnx
import caffe2.python.onnx.backend as caffe_backend
onnx_model = onnx.load(filename)
onnx.checker.check_model(onnx_model)
result2 = caffe_backend.run_model(onnx_model, [dummy_input])[0]
if not np.isclose(result, result2).all():
raise Exception("model is not consistent: {} {}".format(
result, result2))
onnx_model = onnx.utils.polish_model(onnx_model)
onnx.save(onnx_model, filename.replace(".onnx", "_opt.onnx"))
def test_mergedim(self):
X = np.random.randn(2, 3, 1, 5).astype(np.float32)
predict_net = caffe2_pb2.NetDef()
predict_net.name = 'test-mergedim-net'
predict_net.external_input[:] = ['X']
predict_net.external_output[:] = ['Y']
predict_net.op.extend([
core.CreateOperator(
'MergeDim',
inputs=['X'],
outputs=['Y'],
),
])
ws, c2_outputs = c2_native_run_net(
init_net=None,
predict_net=predict_net,
inputs=[X])
onnx_model = c2_onnx.caffe2_net_to_onnx_model(
predict_net=predict_net,
value_info={
'X': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[X.dtype], X.shape),
})
onnx_outputs = c2.run_model(onnx_model, inputs=[X])
self.assertSameOutputs(c2_outputs, onnx_outputs)
def test_cast(self):
X = np.random.randn(1, 2, 3).astype(np.float32)
for to_type in ['INT8', caffe2_pb2.TensorProto.INT8,
'DOUBLE', caffe2_pb2.TensorProto.DOUBLE]:
predict_net = caffe2_pb2.NetDef()
predict_net.name = 'test-cast-net'
predict_net.external_input[:] = ['X']
predict_net.external_output[:] = ['Y']
predict_net.op.extend([
core.CreateOperator(
'Cast',
inputs=['X'],
outputs=['Y'],
to=to_type,
),
])
ws, c2_outputs = c2_native_run_net(
init_net=None,
predict_net=predict_net,
inputs=[X])
onnx_model = c2_onnx.caffe2_net_to_onnx_model(
predict_net=predict_net,
value_info={
'X': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[X.dtype], X.shape)
})
onnx_outputs = c2.run_model(onnx_model, inputs=[X])
self.assertSameOutputs(c2_outputs, onnx_outputs)
def generic_test(self, model, sample_inputs, input_names=None, permute=False):
pt_inputs = tuple(torch.from_numpy(x) for x in sample_inputs)
model.qconfig = torch.quantization.default_qconfig
q_model = torch.quantization.prepare(model, inplace=False)
q_model = torch.quantization.convert(q_model, inplace=False)
if permute:
# Permute input to caffe2 to be NHWC layout
X_nhwc = np.ascontiguousarray(sample_inputs[0].transpose([0, 2, 3, 1]))
pytorch_res = q_model(*pt_inputs)
f = io.BytesIO()
torch.onnx.export(q_model, pt_inputs, f, input_names=input_names, operator_export_type=torch.onnx.OperatorExportTypes.ONNX_ATEN_FALLBACK)
f.seek(0)
onnx_model = onnx.load(f)
if permute:
caffe_res = c2.run_model(onnx_model, dict(zip(input_names, (X_nhwc,))))[0]
np.testing.assert_almost_equal(pytorch_res.permute(0, 2, 3, 1).numpy(), caffe_res, decimal=3)
else:
caffe_res = c2.run_model(onnx_model, dict(zip(input_names, sample_inputs)))[0]
np.testing.assert_almost_equal(pytorch_res.numpy(), caffe_res, decimal=3)
def processs_model(model):
onnx.checker.check_model(model)
input_shape = get_info_tensor_shape(model.graph.input[0])
dummy_input = np.linspace(
0, 1,
num=np.prod(input_shape)).reshape(*input_shape).astype(np.float32)
result1 = caffe_backend.run_model(model, [dummy_input])[0]
model = graph_pass(model, initializers_to_constants)
model = graph_pass(model, shape_to_constant)
model = graph_pass(model, constant_ops)
model = graph_pass(model, lstm_final_hidden)
result2 = caffe_backend.run_model(model, [dummy_input])[0]
if not np.isclose(result1, result2).all():
raise Exception("model is not consistent: {} {}".format(
result1, result2))
print('Export Completed Successfully')
return model
def test_reducemean(self):
X = np.random.randn(4, 6, 10, 5, 3).astype(np.float32)
predict_net = caffe2_pb2.NetDef()
predict_net.name = 'test-reducemean-net'
predict_net.external_input[:] = ['X']
predict_net.external_output[:] = [
'reduce_front_mean',
'reduce_back_mean',
'reduce_mean_0',
'reduce_mean_1',
]
predict_net.op.extend([
core.CreateOperator(
'ReduceFrontMean',
inputs=['X'],
outputs=['reduce_front_mean'],
num_reduce_dim=2,
),
core.CreateOperator(
'ReduceBackMean',
inputs=['X'],
outputs=['reduce_back_mean'],
num_reduce_dim=2,
),
core.CreateOperator(
'ReduceMean',
inputs=['X'],
outputs=['reduce_mean_0'],
axes=[1, 3],
keepdims=0,
),
core.CreateOperator(
'ReduceMean',
inputs=['X'],
outputs=['reduce_mean_1'],
axes=[1, 3],
keepdims=1,
),
])
ws, c2_outputs = c2_native_run_net(
init_net=None,
predict_net=predict_net,
inputs=[X])
onnx_model = c2_onnx.caffe2_net_to_onnx_model(
predict_net=predict_net,
value_info={
'X': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[X.dtype], X.shape)
})
onnx_outputs = c2.run_model(onnx_model, inputs=[X])
self.assertSameOutputs(c2_outputs, onnx_outputs)
def _evaluate_caffe2(onnx_model, inputs):
from caffe2.python.onnx.backend import run_model
if not isinstance(inputs, list):
inputs = [inputs]
outputs = run_model(onnx_model, inputs)
adjusted_outputs = dict()
for output in onnx_model.graph.output:
adjusted_outputs[output.name] = outputs[output.name]
return adjusted_outputs[onnx_model.graph.output[0].name]
def main():
warnings.filterwarnings('ignore')
parser = argparse.ArgumentParser(description="Tool to compare output of onnx model to \
sample data obtained from the onnx model-zoo.")
parser.add_argument(
"--input",
type=Text,
required=True,
help="Path to onnx model."
)
args = parser.parse_args()
input_path = args.input
test_data_dir = os.path.join(os.path.dirname(os.path.abspath(input_path)), "test_data_set_0")
print(input_path)
print(test_data_dir)
model = onnx.load(input_path)
# Load inputs
inputs = []
inputs_num = len(glob.glob(os.path.join(test_data_dir, 'input_*.pb')))
for i in range(inputs_num):
input_file = os.path.join(test_data_dir, 'input_{}.pb'.format(i))
tensor = onnx.TensorProto()
with open(input_file, 'rb') as f:
tensor.ParseFromString(f.read())
inputs.append(numpy_helper.to_array(tensor))
# Load reference outputs
ref_outputs = []
ref_outputs_num = len(glob.glob(os.path.join(test_data_dir, 'output_*.pb')))
for i in range(ref_outputs_num):
output_file = os.path.join(test_data_dir, 'output_{}.pb'.format(i))
tensor = onnx.TensorProto()
with open(output_file, 'rb') as f:
tensor.ParseFromString(f.read())
ref_outputs.append(numpy_helper.to_array(tensor))
# Run the model on the backend
outputs = list(backend.run_model(model, inputs, device='CPU'))
# Compare the results with reference outputs.
for ref_o, o in zip(ref_outputs, outputs):
np.testing.assert_almost_equal(ref_o, o, decimal=4)
print("Reference output matches computed output of network.")
def generic_test(self, model, sample_inputs, input_names=None):
torch.backends.quantized.engine = "qnnpack"
pt_inputs = tuple(torch.from_numpy(x) for x in sample_inputs)
model.qconfig = torch.quantization.default_qconfig
q_model = torch.quantization.prepare(model, inplace=False)
q_model = torch.quantization.convert(q_model, inplace=False)
pytorch_res = q_model(*pt_inputs)
f = io.BytesIO()
torch.onnx.export(q_model, pt_inputs, f, input_names=input_names, operator_export_type=torch.onnx.OperatorExportTypes.ONNX_ATEN_FALLBACK)
f.seek(0)
onnx_model = onnx.load(f)
caffe_res = c2.run_model(onnx_model, dict(zip(input_names, sample_inputs)))[0]
np.testing.assert_almost_equal(pytorch_res.numpy(), caffe_res, decimal=3)
def test():
# input = np.random.randn(1, 5, 5, 3).astype(np.float32)
input = np.random.randn(1, 784).astype(np.float32)
# model = onnx.load('pb/onnx_reshape.pb')
model = onnx.load('pb/onnx_gh_mnist_nchw.pb')
outputs_caffe2 = backend.run_model(model, [input])
print(outputs_caffe2)
outputs_tf = run_model(model, [input])
# m = nn.AvgPool2d(3, stride=2, padding=[1, 1], count_include_pad=False)
print(outputs_tf)
print(np.allclose(outputs_caffe2, outputs_tf, rtol=1e-3))
def test_resnet50(self):
input_blob_dims = (1, 3, 224, 224)
model_dir = _download_onnx_model('resnet50')
model_def = onnx.load(os.path.join(model_dir, 'model.onnx'))
op_inputs = [x.name for x in model_def.graph.input]
op_outputs = [x.name for x in model_def.graph.output]
n, c, h, w = input_blob_dims
data = np.random.randn(n, c, h, w).astype(np.float32)
Y_c2 = c2.run_model(model_def, {op_inputs[0]: data})
op = convert_onnx_model_to_trt_op(model_def)
device_option = core.DeviceOption(caffe2_pb2.CUDA, 0)
op.device_option.CopyFrom(device_option)
Y_trt = None
ws = Workspace()
with core.DeviceScope(device_option):
ws.FeedBlob(op_inputs[0], data)
ws.RunOperatorsOnce([op])
output_values = [ws.FetchBlob(name) for name in op_outputs]
Y_trt = namedtupledict('Outputs', op_outputs)(*output_values)
np.testing.assert_allclose(Y_c2, Y_trt, rtol=1e-3)
def _test_onnx_importer(self, model_name, data_input_index = 0):
model_dir = _download_onnx_model(model_name)
model_def = onnx.load(os.path.join(model_dir, 'model.onnx'))
input_blob_dims = [int(x.dim_value) for x in model_def.graph.input[data_input_index].type.tensor_type.shape.dim]
op_inputs = [x.name for x in model_def.graph.input]
op_outputs = [x.name for x in model_def.graph.output]
print("{}".format(op_inputs))
data = np.random.randn(*input_blob_dims).astype(np.float32)
Y_c2 = c2.run_model(model_def, {op_inputs[data_input_index]: data})
op = convert_onnx_model_to_trt_op(model_def, verbosity=3)
device_option = core.DeviceOption(caffe2_pb2.CUDA, 0)
op.device_option.CopyFrom(device_option)
Y_trt = None
ws = Workspace()
with core.DeviceScope(device_option):
ws.FeedBlob(op_inputs[data_input_index], data)
ws.RunOperatorsOnce([op])
output_values = [ws.FetchBlob(name) for name in op_outputs]
Y_trt = namedtupledict('Outputs', op_outputs)(*output_values)
np.testing.assert_allclose(Y_c2, Y_trt, rtol=1e-3)
def export_model(filename, verbose=False):
dummy_input = np.linspace(0, 2, num=seq_len*output_dim, dtype=np.float32)\
.reshape(1, seq_len, output_dim)
dummy_input = prepare_shape(dummy_input)
dummy_input = torch.from_numpy(dummy_input).to(device)
torch.onnx.export(model, dummy_input, filename, verbose=verbose)
model.eval()
result = model(dummy_input).cpu().detach().numpy()
# check the model
dummy_input = dummy_input.cpu().detach().numpy()
import onnx
import caffe2.python.onnx.backend as caffe_backend
onnx_model = onnx.load(filename)
result2 = caffe_backend.run_model(onnx_model, [dummy_input])[0]
if not np.isclose(result, result2).all():
raise Exception("model is not consistent: {} {}".format(result, result2))
onnx_model = onnx.utils.polish_model(onnx_model)
onnx.save(onnx_model, filename.replace(".onnx", "_opt.onnx"))
def run_onnx_model_in_caffe2(orig_model, name='my_model.onnx'):
import numpy as np
import math, onnx
from caffe2.python.onnx.backend import run_model
from caffe2.python import core, workspace
model_file = onnx.load(name)
onnx.checker.check_model(model_file)
print(onnx.helper.printable_graph(model_file.graph))
is_close = True
for row in range(n_fragments):
for col in range(n_targets):
target = fragment_target_scores[row][col]
# TODO
# check_model seems fine.
# printable_graph shows reasonable model.
# but, the line below fails with:
# RuntimeError: Inferred shape and existing shape differ in rank: (0) vs (1)
prediction = run_model(model_file,
np.array([row, col]).astype(np.float32))
if not math.isclose(prediction, target, rel_tol=1e-4):
is_close = False
print('~~~~~~~~', prediction, target)
return is_close
def test_convert_end2end(self):
predict_net_f = tempfile.NamedTemporaryFile()
init_net_f = tempfile.NamedTemporaryFile()
onnx_model_f = tempfile.NamedTemporaryFile()
x = 'X'
w = 'W'
b = 'b'
y = 'Y'
predict_net = caffe2_pb2.NetDef()
predict_net.name = 'test-convert-end2end'
predict_net.external_input[:] = [x, w, b]
predict_net.external_output[:] = [y]
predict_net.op.extend([
core.CreateOperator(
'FC',
inputs=[x, w, b],
outputs=[y],
axis=2,
),
])
predict_net_f.write(predict_net.SerializeToString())
predict_net_f.flush()
init_net = caffe2_pb2.NetDef()
init_net.name = 'test-convert-end2end-init'
init_net.external_output[:] = [w, b]
x_val = np.random.randn(1, 3, 2).astype(np.float32)
w_val = np.random.randn(4, 2).astype(np.float32)
b_val = np.random.randn(4).astype(np.float32)
init_net.op.extend([
core.CreateOperator(
'GivenTensorFill',
[],
[w],
values=w_val,
shape=w_val.shape,
),
core.CreateOperator(
'GivenTensorFill',
[],
[b],
values=b_val,
shape=b_val.shape,
),
])
init_net_f.write(init_net.SerializeToString())
init_net_f.flush()
y_val = np.matmul(x_val, w_val.transpose()) + b_val
for _ in range(5):
self._run_command(
caffe2_to_onnx,
[
predict_net_f.name,
'--caffe2-init-net',
init_net_f.name,
'--output',
onnx_model_f.name,
'--value-info',
json.dumps({
x: (TensorProto.FLOAT, (1, 3, 2)),
}),
],
catch_exceptions=False,
)
onnx_model_f.seek(0)
onnx_model = ModelProto()
onnx_model.ParseFromString(onnx_model_f.read())
np.testing.assert_almost_equal(
c2.run_model(onnx_model,
{onnx_model.graph.input[0].name: x_val}), [y_val])
self._run_command(onnx_to_caffe2, [
onnx_model_f.name,
'--output',
predict_net_f.name,
'--init-net-output',
init_net_f.name,
])
predict_net_f.seek(0)
predict_net = caffe2_pb2.NetDef()
predict_net.ParseFromString(predict_net_f.read())
init_net_f.seek(0)
init_net = caffe2_pb2.NetDef()
init_net.ParseFromString(init_net_f.read())
x = predict_net.external_input[0]
np.testing.assert_almost_equal(
c2_native_run_net(init_net=init_net,
predict_net=predict_net,
inputs={x: x_val})[1], [y_val])
def main():
parser = argparse.ArgumentParser(
description="This tool creates appropriate sample input and output data for a given neural network. "
"This is done by running the given network (onnx model) on matching data "
"(supported file types: {}). As backend caffe2 will be used.".format(supported_file_types)
)
parser.add_argument(
"--model",
type=Text,
required=True,
help="Path to the onnx model."
)
parser.add_argument(
"--file",
type=Text,
required=False,
help="Path to the data the network should process. If not provided random values will be used."
)
parser.add_argument(
"--range",
type=int,
nargs=2,
required=False,
default=[0, 1],
help="Range for random input values. Default is [0,1)"
)
parser.add_argument(
"--shape",
type=int,
nargs='+',
required=True,
help="Required input shape. Format \"--shape 1 1 1 16000\" for (1, 1, 1, 16000) with "
"(batch_size, num_input_ch, height, width)."
)
parser.add_argument(
"--seed",
type=int,
nargs=1,
required=False,
default=None,
help="Seed to give to np.random(). Default is None."
)
args = parser.parse_args()
if args.file:
input_file = args.file
input_file_path = os.path.dirname(input_file)
input_file_name = os.path.basename(input_file)
# Check file type
file_type = magic.from_file(input_file, mime=True)
if file_type not in supported_file_types:
print("Unsupported file type: {}. At the moment this tool only supports: {}".format(file_type,
supported_file_types))
exit(1)
else:
print("No input file specified. Generating random input.")
file_type = 'random'
input_file = "random"
input_file_name = input_file
r = args.range
onnx_model = args.model
onnx_file_path = os.path.dirname(onnx_model)
input_shape = tuple(args.shape)
model = onnx.load(onnx_model)
onnx.checker.check_model(model)
packed_input = list(bytes())
magic_input = bytes("FCI\n", "ascii")
packed_output = list(bytes())
magic_output = bytes("FCO\n", "ascii")
if input_shape[0] != 1:
print("ERROR: Batch sizes != 1 not supported at the moment.")
exit(1)
else:
print("Input shape from parameters is: {}".format(input_shape))
# Process input data and write it to our custom binary format.
if file_type == 'audio/x-wav':
input_data = read(input_file)
input_data = np.array(input_data[1], dtype=float)
input_data = input_data/max(input_data)
input_data = input_data.astype(np.float32)
packed_input.append(struct.pack('{}s'.format(len(magic_input)), magic_input))
packed_input.append(struct.pack('i', input_shape[0])) # Number of batches
packed_input.append(struct.pack('i', input_shape[1])) # Number of channels
packed_input.append(struct.pack('i', 1)) # Channel height
packed_input.append(struct.pack('i', input_shape[2])) # Channel width
packed_input.append(struct.pack('f' * len(input_data), *input_data)) # Data
tupac = bytes("end\n", "ascii")
packed_input.append(struct.pack('{}s'.format(len(tupac)), tupac))
in_path = "{}_{}_input.data".format(os.path.splitext(input_file_name)[0],
os.path.splitext(os.path.basename(onnx_model))[0])
in_path = os.path.join(onnx_file_path, in_path)
print("Saving input to {}".format(in_path))
with open(in_path, "wb") as f:
for packed_struct in packed_input:
f.write(packed_struct)
input_data = input_data.reshape(input_shape)
elif file_type == 'image/png':
input_data = None
print("ERROR: File type {} currently not supported.".format(file_type))
exit(1)
elif file_type == 'image/jpeg':
input_data = imageio.imread(input_file)
input_data = np.array(input_data, dtype=float)
input_data = input_data.astype(np.float32)
packed_input.append(struct.pack('{}s'.format(len(magic_input)), magic_input))
packed_input.append(struct.pack('i', input_shape[0])) # Number of batches
packed_input.append(struct.pack('i', input_shape[1])) # Number of channels
packed_input.append(struct.pack('i', input_shape[2])) # Channel height
packed_input.append(struct.pack('i', input_shape[3])) # Channel width
input_data = input_data.reshape(input_shape)
for channel in input_data[0]:
for row in channel:
packed_input.append(struct.pack('f' * len(row), *row)) # Data
tupac = bytes("end\n", "ascii")
packed_input.append(struct.pack('{}s'.format(len(tupac)), tupac))
in_path = "{}_{}_input.data".format(os.path.splitext(input_file_name)[0],
os.path.splitext(os.path.basename(onnx_model))[0])
in_path = os.path.join(onnx_file_path, in_path)
print("Saving input to {}".format(in_path))
with open(in_path, "wb") as f:
for packed_struct in packed_input:
f.write(packed_struct)
elif file_type == 'image/x-portable-greymap':
input_data = imageio.imread(input_file)
input_data = np.array(input_data, dtype=float)
input_data = input_data / 255.0
input_data = input_data.astype(np.float32)
packed_input.append(struct.pack('{}s'.format(len(magic_input)), magic_input))
packed_input.append(struct.pack('i', input_shape[0])) # Number of batches
packed_input.append(struct.pack('i', input_shape[1])) # Number of channels
packed_input.append(struct.pack('i', input_shape[2])) # Channel height
packed_input.append(struct.pack('i', input_shape[3])) # Channel width
for row in input_data:
packed_input.append(struct.pack('f' * len(row), *row)) # Data
tupac = bytes("end\n", "ascii")
packed_input.append(struct.pack('{}s'.format(len(tupac)), tupac))
in_path = "{}_{}_input.data".format(os.path.splitext(input_file_name)[0],
os.path.splitext(os.path.basename(onnx_model))[0])
in_path = os.path.join(onnx_file_path, in_path)
print("Saving input to {}".format(in_path))
with open(in_path, "wb") as f:
for packed_struct in packed_input:
f.write(packed_struct)
input_data = input_data.reshape(input_shape)
elif file_type == 'random':
np.random.seed(args.seed)
input_data = np.random.uniform(r[0], r[1], input_shape)
input_data = input_data.astype(np.float32)
if len(input_shape) == 4:
packed_input.append(struct.pack('{}s'.format(len(magic_input)), magic_input))
packed_input.append(struct.pack('i', input_shape[0])) # Number of batches
packed_input.append(struct.pack('i', input_shape[1])) # Number of channels
packed_input.append(struct.pack('i', input_shape[2])) # Channel height
packed_input.append(struct.pack('i', input_shape[3])) # Channel width
for channel in input_data[0]:
for row in channel:
packed_input.append(struct.pack('f' * len(row), *row)) # Data
tupac = bytes("end\n", "ascii")
packed_input.append(struct.pack('{}s'.format(len(tupac)), tupac))
elif len(input_shape) == 3:
packed_input.append(struct.pack('{}s'.format(len(magic_input)), magic_input))
packed_input.append(struct.pack('i', input_shape[0])) # Number of batches
packed_input.append(struct.pack('i', input_shape[1])) # Number of channels
packed_input.append(struct.pack('i', 1)) # Channel height
packed_input.append(struct.pack('i', input_shape[2])) # Channel width
for channel in input_data[0]:
packed_input.append(struct.pack('f' * len(channel), *channel)) # Data
tupac = bytes("end\n", "ascii")
packed_input.append(struct.pack('{}s'.format(len(tupac)), tupac))
elif len(input_shape) == 2:
packed_input.append(struct.pack('{}s'.format(len(magic_input)), magic_input))
packed_input.append(struct.pack('i', input_shape[0])) # Number of batches
packed_input.append(struct.pack('i', 1)) # Number of channels
packed_input.append(struct.pack('i', 1)) # Channel height
packed_input.append(struct.pack('i', input_shape[1])) # Channel width
packed_input.append(struct.pack('f' * len(input_data[0]), *input_data[0])) # Data
tupac = bytes("end\n", "ascii")
packed_input.append(struct.pack('{}s'.format(len(tupac)), tupac))
else:
print("ERROR: Unsupported input shape length: {}".format(input_shape))
exit(1)
in_path = "{}_{}_input.data".format(os.path.splitext(input_file_name)[0],
os.path.splitext(os.path.basename(onnx_model))[0])
in_path = os.path.join(onnx_file_path, in_path)
print("Saving input to {}".format(in_path))
with open(in_path, "wb") as f:
for packed_struct in packed_input:
f.write(packed_struct)
else:
input_data = None
print("ERROR: Something went wrong during input data processing...")
exit(1)
# Run inference on input data.
print("Running inference...")
outputs = backend.run_model(model, input_data, device='CPU')
output_shape = outputs[0].shape
print("Shape of output data: {}".format(output_shape))
# Write outputs into our custom binary format.
packed_output.append(struct.pack('{}s'.format(len(magic_output)), magic_output))
packed_output.append(struct.pack('i', output_shape[0])) # Number of batches
packed_output.append(struct.pack('i', output_shape[1])) # Number of outputs
for output in outputs[0]:
packed_output.append(struct.pack('f'*len(output), *output)) # Data
tupac = bytes("end\n", "ascii")
packed_output.append(struct.pack('{}s'.format(len(tupac)), tupac))
out_path = "{}_{}_output.data".format(os.path.splitext(input_file_name)[0],
os.path.splitext(os.path.basename(onnx_model))[0])
out_path = os.path.join(onnx_file_path, out_path)
print("Saving output to {}".format(out_path))
with open(out_path, "wb") as f:
for packed_struct in packed_output:
f.write(packed_struct)
node_ops = [
op_type_to_creator[node.op_type](graph, node) for node in graph.node
if node.op_type
]
def forward(input):
dict = {graph.input[0].name: input, "": None}
for op in node_ops:
op(dict)
return dict[graph.output[0].name]
return forward
if __name__ == '__main__':
import sys
model = onnx.load(sys.argv[1])
input_shape = get_info_tensor_shape(model.graph.input[0])
dummy_input = np.linspace(
0, 1,
num=np.prod(input_shape)).reshape(*input_shape).astype(np.float32)
f = create_graph_forward(model.graph)
result1 = caffe_backend.run_model(model, [dummy_input])[0]
result2 = f(dummy_input)
if not np.isclose(result1, result2).all():
raise Exception("model is not consistent: {} {}".format(
result1, result2))
print('Forward Test Passed %s' % result2)
def test_convert_end2end(self):
predict_net_f = tempfile.NamedTemporaryFile()
init_net_f = tempfile.NamedTemporaryFile()
onnx_model_f = tempfile.NamedTemporaryFile()
x = 'X'
w = 'W'
b = 'b'
y = 'Y'
predict_net = caffe2_pb2.NetDef()
predict_net.name = 'test-convert-end2end'
predict_net.external_input[:] = [x, w, b]
predict_net.external_output[:] = [y]
predict_net.op.extend([
core.CreateOperator(
'FC',
inputs=[x, w, b],
outputs=[y],
axis=2,
),
])
predict_net_f.write(predict_net.SerializeToString())
predict_net_f.flush()
init_net = caffe2_pb2.NetDef()
init_net.name = 'test-convert-end2end-init'
init_net.external_output[:] = [w, b]
x_val = np.random.randn(1, 3, 2).astype(np.float32)
w_val = np.random.randn(4, 2).astype(np.float32)
b_val = np.random.randn(4).astype(np.float32)
init_net.op.extend([
core.CreateOperator(
'GivenTensorFill',
[],
[w],
values=w_val,
shape=w_val.shape,
),
core.CreateOperator(
'GivenTensorFill',
[],
[b],
values=b_val,
shape=b_val.shape,
),
])
init_net_f.write(init_net.SerializeToString())
init_net_f.flush()
y_val = np.matmul(x_val, w_val.transpose()) + b_val
for _ in range(5):
self._run_command(
caffe2_to_onnx, [
predict_net_f.name,
'--caffe2-init-net', init_net_f.name,
'--output', onnx_model_f.name,
'--value-info',
json.dumps({
x: (TensorProto.FLOAT, (1, 3, 2)),
}),
],
catch_exceptions=False,
)
onnx_model_f.seek(0)
onnx_model = ModelProto()
onnx_model.ParseFromString(onnx_model_f.read())
np.testing.assert_almost_equal(
c2.run_model(
onnx_model, {onnx_model.graph.input[0].name: x_val}),
[y_val])
self._run_command(
onnx_to_caffe2, [
onnx_model_f.name,
'--output', predict_net_f.name,
'--init-net-output', init_net_f.name,
])
predict_net_f.seek(0)
predict_net = caffe2_pb2.NetDef()
predict_net.ParseFromString(predict_net_f.read())
init_net_f.seek(0)
init_net = caffe2_pb2.NetDef()
init_net.ParseFromString(init_net_f.read())
x = predict_net.external_input[0]
np.testing.assert_almost_equal(c2_native_run_net(init_net=init_net,
predict_net=predict_net,
inputs={x: x_val})[1],
[y_val])
import os
import caffe2.python.onnx.backend as caffe2_bk
import numpy as np
import tensorflow as tf
import onnx
import onnx_tf.backend as tf_bk
with open(os.path.join('pb/onnx_gh_mnist_nchw.pb'), "rb") as f:
onnx_model = onnx.load(f)
onnx.checker.check_model(onnx_model)
# print(onnx_model)
img = np.random.randn(50, 784).astype(np.float32)
img = (img - np.min(img)) / (np.max(img) - np.min(img))
output_c2 = caffe2_bk.run_model(onnx_model, [img])
# output_tf = tf_bk.run_model(onnx_model, [img])
#
print(output_c2)
# print(output_tf)
# print(np.allclose(output_c2, output_tf))
#!/usr/bin/env python3 import onnx import onnxruntime from onnx import optimizer import numpy as np import onnx.helper import caffe2.python.onnx.backend as c2 model_path = "test_maxpool.onnx" print(model_path) input_data = np.random.rand(1, 1, 64, 64).astype(np.float32) model = onnx.load(model_path) input_names = ["x"] x_c2 = input_data #x_c2 = np.transpose(input_data, [0, 2, 3, 1]) y = np.expand_dims(x_c2, axis=0) caffe_res = c2.run_model(model, dict(zip(input_names, y)))[0] print(caffe_res)
