def test_cast(self): X = np.random.randn(1, 2, 3).astype(np.float32) for to_type in ['INT8', caffe2_pb2.TensorProto.INT8, 'DOUBLE', caffe2_pb2.TensorProto.DOUBLE]: predict_net = caffe2_pb2.NetDef() predict_net.name = 'test-cast-net' predict_net.external_input[:] = ['X'] predict_net.external_output[:] = ['Y'] predict_net.op.extend([ core.CreateOperator( 'Cast', inputs=['X'], outputs=['Y'], to=to_type, ), ]) ws, c2_outputs = c2_native_run_net( init_net=None, predict_net=predict_net, inputs=[X]) onnx_model = c2_onnx.caffe2_net_to_onnx_model( predict_net=predict_net, value_info={ 'X': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[X.dtype], X.shape) }) onnx_outputs = c2.run_model(onnx_model, inputs=[X]) self.assertSameOutputs(c2_outputs, onnx_outputs)
def test_fc(self): X_fake = np.zeros((3, 1, 3, 1, 7), dtype=np.float32) X = np.random.randn(5, 2, 3, 1, 7).astype(np.float32) W = np.random.randn(11, 21).astype(np.float32) B = np.random.randn(11).astype(np.float32) predict_net = caffe2_pb2.NetDef() predict_net.name = 'test-fc-net' predict_net.external_input[:] = ['X', 'W', 'B'] predict_net.external_output[:] = ['Y'] predict_net.op.extend([ core.CreateOperator( 'FC', inputs=['X', 'W', 'B'], outputs=['Y'], axis=2, ), ]) ws, c2_outputs = c2_native_run_net( init_net=None, predict_net=predict_net, inputs=[X, W, B]) onnx_model = c2_onnx.caffe2_net_to_onnx_model( predict_net=predict_net, value_info={ 'X': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[X.dtype], X_fake.shape), 'W': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[W.dtype], W.shape), 'B': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[B.dtype], B.shape), }) onnx_outputs = c2.run_model(onnx_model, inputs=[X, W, B]) self.assertSameOutputs(c2_outputs, onnx_outputs)
def test_qconv_model(self): class ConvModel(torch.nn.Module): def __init__(self): super(ConvModel, self).__init__() self.qconfig = torch.quantization.default_qconfig self.fc1 = torch.quantization.QuantWrapper( torch.nn.Conv2d(3, 5, 2, bias=True).to(dtype=torch.float)) def forward(self, x): x = self.fc1(x) return x torch.backends.quantized.engine = "qnnpack" qconfig = torch.quantization.default_qconfig model = ConvModel() model.qconfig = qconfig model = torch.quantization.prepare(model) model = torch.quantization.convert(model) x_numpy = np.random.rand(1, 3, 6, 6).astype(np.float32) x = torch.from_numpy(x_numpy).to(dtype=torch.float) outputs = model(x) input_names = ["x"] onnx_model = self.export_to_onnx(model, x, input_names) y = np.expand_dims(x_numpy, axis=0) caffe_res = c2.run_model(onnx_model, dict(zip(input_names, y)))[0] output_diff = np.absolute(np.squeeze(outputs.numpy()) - caffe_res) max_diff = np.amax(output_diff) # Permute pytorch output to NHWC # This check had to be changed to account for changes in # qnnpack's requant logic. np.testing.assert_(max_diff <= 1, "Maximum absolute difference must be less than 1")
def test_elementwiselinear(self): X = np.random.randn(4, 2, 5, 7, 3).astype(np.float32) W = np.random.randn(21).astype(np.float32) B = np.random.randn(21).astype(np.float32) predict_net = caffe2_pb2.NetDef() predict_net.name = 'test-elementwiselinear-net' predict_net.external_input[:] = ['X', 'W', 'B'] predict_net.external_output[:] = ['Y'] predict_net.op.extend([ core.CreateOperator( 'ElementwiseLinear', inputs=['X', 'W', 'B'], outputs=['Y'], axis=3, ), ]) ws, c2_outputs = c2_native_run_net( init_net=None, predict_net=predict_net, inputs=[X, W, B]) onnx_model = c2_onnx.caffe2_net_to_onnx_model( predict_net=predict_net, value_info={ 'X': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[X.dtype], X.shape), 'W': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[W.dtype], W.shape), 'B': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[B.dtype], B.shape), }) onnx_outputs = c2.run_model(onnx_model, inputs=[X, W, B]) self.assertSameOutputs(c2_outputs, onnx_outputs)
def test_qconv_model(self): class ConvModel(torch.nn.Module): def __init__(self): super(ConvModel, self).__init__() self.qconfig = torch.quantization.default_qconfig self.fc1 = torch.quantization.QuantWrapper( torch.nn.Conv2d(3, 5, 2, bias=True).to(dtype=torch.float)) def forward(self, x): x = self.fc1(x) return x torch.backends.quantized.engine = "qnnpack" qconfig = torch.quantization.default_qconfig model = ConvModel() model.qconfig = qconfig model = torch.quantization.prepare(model) model = torch.quantization.convert(model) x_numpy = np.random.rand(1, 3, 6, 6).astype(np.float32) x = torch.from_numpy(x_numpy).to(dtype=torch.float) outputs = model(x) input_names = ["x"] onnx_model = self.export_to_onnx(model, x, input_names) y = np.expand_dims(x_numpy, axis=0) caffe_res = c2.run_model(onnx_model, dict(zip(input_names, y)))[0] # Permute pytorch output to NHWC np.testing.assert_almost_equal(outputs.numpy(), caffe_res, decimal=3)
def _test_onnx_importer(self, model_name, data_input_index, opset_version=onnx.defs.onnx_opset_version()): model_dir = _download_onnx_model(model_name, opset_version) model_def = onnx.load(os.path.join(model_dir, 'model.onnx')) input_blob_dims = [ int(x.dim_value) for x in model_def.graph.input[data_input_index].type.tensor_type.shape.dim ] op_inputs = [x.name for x in model_def.graph.input] op_outputs = [x.name for x in model_def.graph.output] print("{}".format(op_inputs)) data = np.random.randn(*input_blob_dims).astype(np.float32) Y_c2 = c2.run_model(model_def, {op_inputs[data_input_index]: data}) op = convert_onnx_model_to_trt_op(model_def, verbosity=3) device_option = core.DeviceOption(caffe2_pb2.CUDA, 0) op.device_option.CopyFrom(device_option) Y_trt = None ws = Workspace() with core.DeviceScope(device_option): ws.FeedBlob(op_inputs[data_input_index], data) ws.RunOperatorsOnce([op]) output_values = [ws.FetchBlob(name) for name in op_outputs] Y_trt = namedtupledict('Outputs', op_outputs)(*output_values) np.testing.assert_allclose(Y_c2, Y_trt, rtol=1e-3)
def test_slice(self): X = np.random.randn(1, 2, 3).astype(np.float32) starts = np.array([0, 1, 0], dtype=np.int32) ends = np.array([-1, 2, 3], dtype=np.int32) predict_net = caffe2_pb2.NetDef() predict_net.name = 'test-slice-net' predict_net.external_input[:] = ['X'] predict_net.external_output[:] = ['Y'] predict_net.op.extend([ core.CreateOperator( 'Slice', inputs=['X'], outputs=['Y'], starts=starts, ends=ends, ), ]) ws, (Y,) = c2_native_run_net( init_net=None, predict_net=predict_net, inputs=[X]) onnx_model = c2_onnx.caffe2_net_to_onnx_model( predict_net=predict_net, value_info={ 'X': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[X.dtype], X.shape) }) Y, = c2.run_model(onnx_model, inputs=[X]) np.testing.assert_almost_equal(Y, X[:, 1:2, :])
def test_qlinear_model(self): class LinearModel(torch.nn.Module): def __init__(self): super(LinearModel, self).__init__() self.qconfig = torch.quantization.default_qconfig self.fc1 = torch.quantization.QuantWrapper( torch.nn.Linear(5, 10).to(dtype=torch.float)) def forward(self, x): x = self.fc1(x) return x torch.backends.quantized.engine = "qnnpack" qconfig = torch.quantization.default_qconfig model = LinearModel() model.qconfig = qconfig model = torch.quantization.prepare(model) model = torch.quantization.convert(model) x_numpy = np.random.rand(1, 2, 5).astype(np.float32) x = torch.from_numpy(x_numpy).to(dtype=torch.float) outputs = model(x) input_names = ["x"] onnx_model = self.export_to_onnx(model, x, input_names) caffe_res = c2.run_model(onnx_model, dict(zip(input_names, x_numpy)))[0] np.testing.assert_almost_equal(np.squeeze(outputs.numpy()), caffe_res, decimal=3)
def test_upsample(self): X = np.random.randn(1, 1, 2, 2).astype(np.float32) width_scale = 2.0 height_scale = 2.0 predict_net = caffe2_pb2.NetDef() predict_net.name = 'test-upsample-net' predict_net.external_input[:] = ['X'] predict_net.external_output[:] = ['Y'] predict_net.op.extend([ core.CreateOperator( 'ResizeNearest', inputs=['X'], outputs=['Y'], width_scale=width_scale, height_scale=height_scale, ), ]) ws, c2_outputs = c2_native_run_net(init_net=None, predict_net=predict_net, inputs=[X]) onnx_model = c2_onnx.caffe2_net_to_onnx_model( predict_net=predict_net, value_info={ 'X': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[X.dtype], X.shape) }) onnx_outputs = c2.run_model(onnx_model, inputs=[X]) self.assertSameOutputs(c2_outputs, onnx_outputs)
def generic_test(self, model, sample_inputs, input_names=None, decimal=3, relaxed_check=False): torch.backends.quantized.engine = "qnnpack" pt_inputs = tuple(torch.from_numpy(x) for x in sample_inputs) model.qconfig = torch.ao.quantization.get_default_qconfig("qnnpack") q_model = torch.ao.quantization.prepare(model, inplace=False) q_model = torch.ao.quantization.convert(q_model, inplace=False) traced_model = torch.jit.trace(q_model, pt_inputs) buf = io.BytesIO() torch.jit.save(traced_model, buf) buf.seek(0) q_model = torch.jit.load(buf) q_model.eval() output = q_model(*pt_inputs) f = io.BytesIO() torch.onnx.export(q_model, pt_inputs, f, input_names=input_names, operator_export_type=torch.onnx.OperatorExportTypes.ONNX_ATEN_FALLBACK) f.seek(0) onnx_model = onnx.load(f) caffe_res = c2.run_model(onnx_model, dict(zip(input_names, sample_inputs)))[0] # Due to change in requantization logic for certain ops such conv, linear # in pytorch's integration of qnnpack, numerics may have a mismatc with C2. # This mismatch should not be off my more than 1. # This flag helps us override default behavior under certain circumstances. if relaxed_check: output_diff = np.absolute(np.squeeze(output.detach().numpy()) - caffe_res) max_diff = np.amax(output_diff) # This check had to be changed to account for changes in # qnnpack's requant logic. np.testing.assert_(max_diff <= 1, "Maximum absolute difference must be less than 1") else: np.testing.assert_almost_equal(output.detach().numpy(), caffe_res, decimal=decimal)
def test_concat(self): I0 = np.random.randn(20, 4).astype(np.float32) I1 = np.random.randn(20, 4).astype(np.float32) for i in range(2): predict_net = caffe2_pb2.NetDef() predict_net.name = 'test-concat-net' predict_net.external_input[:] = ['I0', 'I1'] predict_net.external_output[:] = ['Y', 'output_dim'] predict_net.op.extend([ core.CreateOperator( 'Concat', inputs=['I0', 'I1'], outputs=['Y', 'output_dim'], axis=1, add_axis=(1 if i == 0 else 0), ), ]) ws, c2_outputs = c2_native_run_net(init_net=None, predict_net=predict_net, inputs=[I0, I1]) onnx_model = c2_onnx.caffe2_net_to_onnx_model( predict_net=predict_net, value_info={ 'I0': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[I0.dtype], I0.shape), 'I1': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[I1.dtype], I1.shape), }) onnx_outputs = c2.run_model(onnx_model, inputs=[I0, I1]) self.assertSameOutputs(c2_outputs, onnx_outputs)
def test_slice(self): X = np.random.randn(1, 2, 3).astype(np.float32) starts = np.array([0, 1, 0], dtype=np.int32) ends = np.array([-1, 2, 3], dtype=np.int32) predict_net = caffe2_pb2.NetDef() predict_net.name = 'test-slice-net' predict_net.external_input[:] = ['X'] predict_net.external_output[:] = ['Y'] predict_net.op.extend([ core.CreateOperator( 'Slice', inputs=['X'], outputs=['Y'], starts=starts, ends=ends, ), ]) ws, c2_outputs = c2_native_run_net(init_net=None, predict_net=predict_net, inputs=[X]) onnx_model = c2_onnx.caffe2_net_to_onnx_model( predict_net=predict_net, value_info={ 'X': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[X.dtype], X.shape) }) onnx_outputs = c2.run_model(onnx_model, inputs=[X]) self.assertSameOutputs(c2_outputs, onnx_outputs)
def export_model(filename, verbose=False): dummy_input = torch.linspace(0, 2, steps=seq_len * output_dim, device=device).reshape(seq_len, output_dim) if not isinstance(model, LSTMModel): dummy_input = dummy_input.flatten() torch.onnx.export(model, dummy_input, filename, verbose=verbose) model.eval() result = model(dummy_input).cpu().detach().numpy() # check the model dummy_input = dummy_input.cpu().detach().numpy() import onnx import caffe2.python.onnx.backend as caffe_backend onnx_model = onnx.load(filename) onnx.checker.check_model(onnx_model) result2 = caffe_backend.run_model(onnx_model, [dummy_input])[0] if not np.isclose(result, result2).all(): raise Exception("model is not consistent: {} {}".format( result, result2)) onnx_model = onnx.utils.polish_model(onnx_model) onnx.save(onnx_model, filename.replace(".onnx", "_opt.onnx"))
def test_mergedim(self): X = np.random.randn(2, 3, 1, 5).astype(np.float32) predict_net = caffe2_pb2.NetDef() predict_net.name = 'test-mergedim-net' predict_net.external_input[:] = ['X'] predict_net.external_output[:] = ['Y'] predict_net.op.extend([ core.CreateOperator( 'MergeDim', inputs=['X'], outputs=['Y'], ), ]) ws, c2_outputs = c2_native_run_net( init_net=None, predict_net=predict_net, inputs=[X]) onnx_model = c2_onnx.caffe2_net_to_onnx_model( predict_net=predict_net, value_info={ 'X': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[X.dtype], X.shape), }) onnx_outputs = c2.run_model(onnx_model, inputs=[X]) self.assertSameOutputs(c2_outputs, onnx_outputs)
def test_cast(self): X = np.random.randn(1, 2, 3).astype(np.float32) for to_type in ['INT8', caffe2_pb2.TensorProto.INT8, 'DOUBLE', caffe2_pb2.TensorProto.DOUBLE]: predict_net = caffe2_pb2.NetDef() predict_net.name = 'test-cast-net' predict_net.external_input[:] = ['X'] predict_net.external_output[:] = ['Y'] predict_net.op.extend([ core.CreateOperator( 'Cast', inputs=['X'], outputs=['Y'], to=to_type, ), ]) ws, c2_outputs = c2_native_run_net( init_net=None, predict_net=predict_net, inputs=[X]) onnx_model = c2_onnx.caffe2_net_to_onnx_model( predict_net=predict_net, value_info={ 'X': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[X.dtype], X.shape) }) onnx_outputs = c2.run_model(onnx_model, inputs=[X]) self.assertSameOutputs(c2_outputs, onnx_outputs)
def generic_test(self, model, sample_inputs, input_names=None, permute=False): pt_inputs = tuple(torch.from_numpy(x) for x in sample_inputs) model.qconfig = torch.quantization.default_qconfig q_model = torch.quantization.prepare(model, inplace=False) q_model = torch.quantization.convert(q_model, inplace=False) if permute: # Permute input to caffe2 to be NHWC layout X_nhwc = np.ascontiguousarray(sample_inputs[0].transpose([0, 2, 3, 1])) pytorch_res = q_model(*pt_inputs) f = io.BytesIO() torch.onnx.export(q_model, pt_inputs, f, input_names=input_names, operator_export_type=torch.onnx.OperatorExportTypes.ONNX_ATEN_FALLBACK) f.seek(0) onnx_model = onnx.load(f) if permute: caffe_res = c2.run_model(onnx_model, dict(zip(input_names, (X_nhwc,))))[0] np.testing.assert_almost_equal(pytorch_res.permute(0, 2, 3, 1).numpy(), caffe_res, decimal=3) else: caffe_res = c2.run_model(onnx_model, dict(zip(input_names, sample_inputs)))[0] np.testing.assert_almost_equal(pytorch_res.numpy(), caffe_res, decimal=3)
def processs_model(model): onnx.checker.check_model(model) input_shape = get_info_tensor_shape(model.graph.input[0]) dummy_input = np.linspace( 0, 1, num=np.prod(input_shape)).reshape(*input_shape).astype(np.float32) result1 = caffe_backend.run_model(model, [dummy_input])[0] model = graph_pass(model, initializers_to_constants) model = graph_pass(model, shape_to_constant) model = graph_pass(model, constant_ops) model = graph_pass(model, lstm_final_hidden) result2 = caffe_backend.run_model(model, [dummy_input])[0] if not np.isclose(result1, result2).all(): raise Exception("model is not consistent: {} {}".format( result1, result2)) print('Export Completed Successfully') return model
def test_reducemean(self): X = np.random.randn(4, 6, 10, 5, 3).astype(np.float32) predict_net = caffe2_pb2.NetDef() predict_net.name = 'test-reducemean-net' predict_net.external_input[:] = ['X'] predict_net.external_output[:] = [ 'reduce_front_mean', 'reduce_back_mean', 'reduce_mean_0', 'reduce_mean_1', ] predict_net.op.extend([ core.CreateOperator( 'ReduceFrontMean', inputs=['X'], outputs=['reduce_front_mean'], num_reduce_dim=2, ), core.CreateOperator( 'ReduceBackMean', inputs=['X'], outputs=['reduce_back_mean'], num_reduce_dim=2, ), core.CreateOperator( 'ReduceMean', inputs=['X'], outputs=['reduce_mean_0'], axes=[1, 3], keepdims=0, ), core.CreateOperator( 'ReduceMean', inputs=['X'], outputs=['reduce_mean_1'], axes=[1, 3], keepdims=1, ), ]) ws, c2_outputs = c2_native_run_net( init_net=None, predict_net=predict_net, inputs=[X]) onnx_model = c2_onnx.caffe2_net_to_onnx_model( predict_net=predict_net, value_info={ 'X': (onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[X.dtype], X.shape) }) onnx_outputs = c2.run_model(onnx_model, inputs=[X]) self.assertSameOutputs(c2_outputs, onnx_outputs)
def _evaluate_caffe2(onnx_model, inputs): from caffe2.python.onnx.backend import run_model if not isinstance(inputs, list): inputs = [inputs] outputs = run_model(onnx_model, inputs) adjusted_outputs = dict() for output in onnx_model.graph.output: adjusted_outputs[output.name] = outputs[output.name] return adjusted_outputs[onnx_model.graph.output[0].name]
def main(): warnings.filterwarnings('ignore') parser = argparse.ArgumentParser(description="Tool to compare output of onnx model to \ sample data obtained from the onnx model-zoo.") parser.add_argument( "--input", type=Text, required=True, help="Path to onnx model." ) args = parser.parse_args() input_path = args.input test_data_dir = os.path.join(os.path.dirname(os.path.abspath(input_path)), "test_data_set_0") print(input_path) print(test_data_dir) model = onnx.load(input_path) # Load inputs inputs = [] inputs_num = len(glob.glob(os.path.join(test_data_dir, 'input_*.pb'))) for i in range(inputs_num): input_file = os.path.join(test_data_dir, 'input_{}.pb'.format(i)) tensor = onnx.TensorProto() with open(input_file, 'rb') as f: tensor.ParseFromString(f.read()) inputs.append(numpy_helper.to_array(tensor)) # Load reference outputs ref_outputs = [] ref_outputs_num = len(glob.glob(os.path.join(test_data_dir, 'output_*.pb'))) for i in range(ref_outputs_num): output_file = os.path.join(test_data_dir, 'output_{}.pb'.format(i)) tensor = onnx.TensorProto() with open(output_file, 'rb') as f: tensor.ParseFromString(f.read()) ref_outputs.append(numpy_helper.to_array(tensor)) # Run the model on the backend outputs = list(backend.run_model(model, inputs, device='CPU')) # Compare the results with reference outputs. for ref_o, o in zip(ref_outputs, outputs): np.testing.assert_almost_equal(ref_o, o, decimal=4) print("Reference output matches computed output of network.")
def generic_test(self, model, sample_inputs, input_names=None): torch.backends.quantized.engine = "qnnpack" pt_inputs = tuple(torch.from_numpy(x) for x in sample_inputs) model.qconfig = torch.quantization.default_qconfig q_model = torch.quantization.prepare(model, inplace=False) q_model = torch.quantization.convert(q_model, inplace=False) pytorch_res = q_model(*pt_inputs) f = io.BytesIO() torch.onnx.export(q_model, pt_inputs, f, input_names=input_names, operator_export_type=torch.onnx.OperatorExportTypes.ONNX_ATEN_FALLBACK) f.seek(0) onnx_model = onnx.load(f) caffe_res = c2.run_model(onnx_model, dict(zip(input_names, sample_inputs)))[0] np.testing.assert_almost_equal(pytorch_res.numpy(), caffe_res, decimal=3)
def test(): # input = np.random.randn(1, 5, 5, 3).astype(np.float32) input = np.random.randn(1, 784).astype(np.float32) # model = onnx.load('pb/onnx_reshape.pb') model = onnx.load('pb/onnx_gh_mnist_nchw.pb') outputs_caffe2 = backend.run_model(model, [input]) print(outputs_caffe2) outputs_tf = run_model(model, [input]) # m = nn.AvgPool2d(3, stride=2, padding=[1, 1], count_include_pad=False) print(outputs_tf) print(np.allclose(outputs_caffe2, outputs_tf, rtol=1e-3))
def test_resnet50(self): input_blob_dims = (1, 3, 224, 224) model_dir = _download_onnx_model('resnet50') model_def = onnx.load(os.path.join(model_dir, 'model.onnx')) op_inputs = [x.name for x in model_def.graph.input] op_outputs = [x.name for x in model_def.graph.output] n, c, h, w = input_blob_dims data = np.random.randn(n, c, h, w).astype(np.float32) Y_c2 = c2.run_model(model_def, {op_inputs[0]: data}) op = convert_onnx_model_to_trt_op(model_def) device_option = core.DeviceOption(caffe2_pb2.CUDA, 0) op.device_option.CopyFrom(device_option) Y_trt = None ws = Workspace() with core.DeviceScope(device_option): ws.FeedBlob(op_inputs[0], data) ws.RunOperatorsOnce([op]) output_values = [ws.FetchBlob(name) for name in op_outputs] Y_trt = namedtupledict('Outputs', op_outputs)(*output_values) np.testing.assert_allclose(Y_c2, Y_trt, rtol=1e-3)
def _test_onnx_importer(self, model_name, data_input_index = 0): model_dir = _download_onnx_model(model_name) model_def = onnx.load(os.path.join(model_dir, 'model.onnx')) input_blob_dims = [int(x.dim_value) for x in model_def.graph.input[data_input_index].type.tensor_type.shape.dim] op_inputs = [x.name for x in model_def.graph.input] op_outputs = [x.name for x in model_def.graph.output] print("{}".format(op_inputs)) data = np.random.randn(*input_blob_dims).astype(np.float32) Y_c2 = c2.run_model(model_def, {op_inputs[data_input_index]: data}) op = convert_onnx_model_to_trt_op(model_def, verbosity=3) device_option = core.DeviceOption(caffe2_pb2.CUDA, 0) op.device_option.CopyFrom(device_option) Y_trt = None ws = Workspace() with core.DeviceScope(device_option): ws.FeedBlob(op_inputs[data_input_index], data) ws.RunOperatorsOnce([op]) output_values = [ws.FetchBlob(name) for name in op_outputs] Y_trt = namedtupledict('Outputs', op_outputs)(*output_values) np.testing.assert_allclose(Y_c2, Y_trt, rtol=1e-3)
def export_model(filename, verbose=False): dummy_input = np.linspace(0, 2, num=seq_len*output_dim, dtype=np.float32)\ .reshape(1, seq_len, output_dim) dummy_input = prepare_shape(dummy_input) dummy_input = torch.from_numpy(dummy_input).to(device) torch.onnx.export(model, dummy_input, filename, verbose=verbose) model.eval() result = model(dummy_input).cpu().detach().numpy() # check the model dummy_input = dummy_input.cpu().detach().numpy() import onnx import caffe2.python.onnx.backend as caffe_backend onnx_model = onnx.load(filename) result2 = caffe_backend.run_model(onnx_model, [dummy_input])[0] if not np.isclose(result, result2).all(): raise Exception("model is not consistent: {} {}".format(result, result2)) onnx_model = onnx.utils.polish_model(onnx_model) onnx.save(onnx_model, filename.replace(".onnx", "_opt.onnx"))
def run_onnx_model_in_caffe2(orig_model, name='my_model.onnx'): import numpy as np import math, onnx from caffe2.python.onnx.backend import run_model from caffe2.python import core, workspace model_file = onnx.load(name) onnx.checker.check_model(model_file) print(onnx.helper.printable_graph(model_file.graph)) is_close = True for row in range(n_fragments): for col in range(n_targets): target = fragment_target_scores[row][col] # TODO # check_model seems fine. # printable_graph shows reasonable model. # but, the line below fails with: # RuntimeError: Inferred shape and existing shape differ in rank: (0) vs (1) prediction = run_model(model_file, np.array([row, col]).astype(np.float32)) if not math.isclose(prediction, target, rel_tol=1e-4): is_close = False print('~~~~~~~~', prediction, target) return is_close
def test_convert_end2end(self): predict_net_f = tempfile.NamedTemporaryFile() init_net_f = tempfile.NamedTemporaryFile() onnx_model_f = tempfile.NamedTemporaryFile() x = 'X' w = 'W' b = 'b' y = 'Y' predict_net = caffe2_pb2.NetDef() predict_net.name = 'test-convert-end2end' predict_net.external_input[:] = [x, w, b] predict_net.external_output[:] = [y] predict_net.op.extend([ core.CreateOperator( 'FC', inputs=[x, w, b], outputs=[y], axis=2, ), ]) predict_net_f.write(predict_net.SerializeToString()) predict_net_f.flush() init_net = caffe2_pb2.NetDef() init_net.name = 'test-convert-end2end-init' init_net.external_output[:] = [w, b] x_val = np.random.randn(1, 3, 2).astype(np.float32) w_val = np.random.randn(4, 2).astype(np.float32) b_val = np.random.randn(4).astype(np.float32) init_net.op.extend([ core.CreateOperator( 'GivenTensorFill', [], [w], values=w_val, shape=w_val.shape, ), core.CreateOperator( 'GivenTensorFill', [], [b], values=b_val, shape=b_val.shape, ), ]) init_net_f.write(init_net.SerializeToString()) init_net_f.flush() y_val = np.matmul(x_val, w_val.transpose()) + b_val for _ in range(5): self._run_command( caffe2_to_onnx, [ predict_net_f.name, '--caffe2-init-net', init_net_f.name, '--output', onnx_model_f.name, '--value-info', json.dumps({ x: (TensorProto.FLOAT, (1, 3, 2)), }), ], catch_exceptions=False, ) onnx_model_f.seek(0) onnx_model = ModelProto() onnx_model.ParseFromString(onnx_model_f.read()) np.testing.assert_almost_equal( c2.run_model(onnx_model, {onnx_model.graph.input[0].name: x_val}), [y_val]) self._run_command(onnx_to_caffe2, [ onnx_model_f.name, '--output', predict_net_f.name, '--init-net-output', init_net_f.name, ]) predict_net_f.seek(0) predict_net = caffe2_pb2.NetDef() predict_net.ParseFromString(predict_net_f.read()) init_net_f.seek(0) init_net = caffe2_pb2.NetDef() init_net.ParseFromString(init_net_f.read()) x = predict_net.external_input[0] np.testing.assert_almost_equal( c2_native_run_net(init_net=init_net, predict_net=predict_net, inputs={x: x_val})[1], [y_val])
def main(): parser = argparse.ArgumentParser( description="This tool creates appropriate sample input and output data for a given neural network. " "This is done by running the given network (onnx model) on matching data " "(supported file types: {}). As backend caffe2 will be used.".format(supported_file_types) ) parser.add_argument( "--model", type=Text, required=True, help="Path to the onnx model." ) parser.add_argument( "--file", type=Text, required=False, help="Path to the data the network should process. If not provided random values will be used." ) parser.add_argument( "--range", type=int, nargs=2, required=False, default=[0, 1], help="Range for random input values. Default is [0,1)" ) parser.add_argument( "--shape", type=int, nargs='+', required=True, help="Required input shape. Format \"--shape 1 1 1 16000\" for (1, 1, 1, 16000) with " "(batch_size, num_input_ch, height, width)." ) parser.add_argument( "--seed", type=int, nargs=1, required=False, default=None, help="Seed to give to np.random(). Default is None." ) args = parser.parse_args() if args.file: input_file = args.file input_file_path = os.path.dirname(input_file) input_file_name = os.path.basename(input_file) # Check file type file_type = magic.from_file(input_file, mime=True) if file_type not in supported_file_types: print("Unsupported file type: {}. At the moment this tool only supports: {}".format(file_type, supported_file_types)) exit(1) else: print("No input file specified. Generating random input.") file_type = 'random' input_file = "random" input_file_name = input_file r = args.range onnx_model = args.model onnx_file_path = os.path.dirname(onnx_model) input_shape = tuple(args.shape) model = onnx.load(onnx_model) onnx.checker.check_model(model) packed_input = list(bytes()) magic_input = bytes("FCI\n", "ascii") packed_output = list(bytes()) magic_output = bytes("FCO\n", "ascii") if input_shape[0] != 1: print("ERROR: Batch sizes != 1 not supported at the moment.") exit(1) else: print("Input shape from parameters is: {}".format(input_shape)) # Process input data and write it to our custom binary format. if file_type == 'audio/x-wav': input_data = read(input_file) input_data = np.array(input_data[1], dtype=float) input_data = input_data/max(input_data) input_data = input_data.astype(np.float32) packed_input.append(struct.pack('{}s'.format(len(magic_input)), magic_input)) packed_input.append(struct.pack('i', input_shape[0])) # Number of batches packed_input.append(struct.pack('i', input_shape[1])) # Number of channels packed_input.append(struct.pack('i', 1)) # Channel height packed_input.append(struct.pack('i', input_shape[2])) # Channel width packed_input.append(struct.pack('f' * len(input_data), *input_data)) # Data tupac = bytes("end\n", "ascii") packed_input.append(struct.pack('{}s'.format(len(tupac)), tupac)) in_path = "{}_{}_input.data".format(os.path.splitext(input_file_name)[0], os.path.splitext(os.path.basename(onnx_model))[0]) in_path = os.path.join(onnx_file_path, in_path) print("Saving input to {}".format(in_path)) with open(in_path, "wb") as f: for packed_struct in packed_input: f.write(packed_struct) input_data = input_data.reshape(input_shape) elif file_type == 'image/png': input_data = None print("ERROR: File type {} currently not supported.".format(file_type)) exit(1) elif file_type == 'image/jpeg': input_data = imageio.imread(input_file) input_data = np.array(input_data, dtype=float) input_data = input_data.astype(np.float32) packed_input.append(struct.pack('{}s'.format(len(magic_input)), magic_input)) packed_input.append(struct.pack('i', input_shape[0])) # Number of batches packed_input.append(struct.pack('i', input_shape[1])) # Number of channels packed_input.append(struct.pack('i', input_shape[2])) # Channel height packed_input.append(struct.pack('i', input_shape[3])) # Channel width input_data = input_data.reshape(input_shape) for channel in input_data[0]: for row in channel: packed_input.append(struct.pack('f' * len(row), *row)) # Data tupac = bytes("end\n", "ascii") packed_input.append(struct.pack('{}s'.format(len(tupac)), tupac)) in_path = "{}_{}_input.data".format(os.path.splitext(input_file_name)[0], os.path.splitext(os.path.basename(onnx_model))[0]) in_path = os.path.join(onnx_file_path, in_path) print("Saving input to {}".format(in_path)) with open(in_path, "wb") as f: for packed_struct in packed_input: f.write(packed_struct) elif file_type == 'image/x-portable-greymap': input_data = imageio.imread(input_file) input_data = np.array(input_data, dtype=float) input_data = input_data / 255.0 input_data = input_data.astype(np.float32) packed_input.append(struct.pack('{}s'.format(len(magic_input)), magic_input)) packed_input.append(struct.pack('i', input_shape[0])) # Number of batches packed_input.append(struct.pack('i', input_shape[1])) # Number of channels packed_input.append(struct.pack('i', input_shape[2])) # Channel height packed_input.append(struct.pack('i', input_shape[3])) # Channel width for row in input_data: packed_input.append(struct.pack('f' * len(row), *row)) # Data tupac = bytes("end\n", "ascii") packed_input.append(struct.pack('{}s'.format(len(tupac)), tupac)) in_path = "{}_{}_input.data".format(os.path.splitext(input_file_name)[0], os.path.splitext(os.path.basename(onnx_model))[0]) in_path = os.path.join(onnx_file_path, in_path) print("Saving input to {}".format(in_path)) with open(in_path, "wb") as f: for packed_struct in packed_input: f.write(packed_struct) input_data = input_data.reshape(input_shape) elif file_type == 'random': np.random.seed(args.seed) input_data = np.random.uniform(r[0], r[1], input_shape) input_data = input_data.astype(np.float32) if len(input_shape) == 4: packed_input.append(struct.pack('{}s'.format(len(magic_input)), magic_input)) packed_input.append(struct.pack('i', input_shape[0])) # Number of batches packed_input.append(struct.pack('i', input_shape[1])) # Number of channels packed_input.append(struct.pack('i', input_shape[2])) # Channel height packed_input.append(struct.pack('i', input_shape[3])) # Channel width for channel in input_data[0]: for row in channel: packed_input.append(struct.pack('f' * len(row), *row)) # Data tupac = bytes("end\n", "ascii") packed_input.append(struct.pack('{}s'.format(len(tupac)), tupac)) elif len(input_shape) == 3: packed_input.append(struct.pack('{}s'.format(len(magic_input)), magic_input)) packed_input.append(struct.pack('i', input_shape[0])) # Number of batches packed_input.append(struct.pack('i', input_shape[1])) # Number of channels packed_input.append(struct.pack('i', 1)) # Channel height packed_input.append(struct.pack('i', input_shape[2])) # Channel width for channel in input_data[0]: packed_input.append(struct.pack('f' * len(channel), *channel)) # Data tupac = bytes("end\n", "ascii") packed_input.append(struct.pack('{}s'.format(len(tupac)), tupac)) elif len(input_shape) == 2: packed_input.append(struct.pack('{}s'.format(len(magic_input)), magic_input)) packed_input.append(struct.pack('i', input_shape[0])) # Number of batches packed_input.append(struct.pack('i', 1)) # Number of channels packed_input.append(struct.pack('i', 1)) # Channel height packed_input.append(struct.pack('i', input_shape[1])) # Channel width packed_input.append(struct.pack('f' * len(input_data[0]), *input_data[0])) # Data tupac = bytes("end\n", "ascii") packed_input.append(struct.pack('{}s'.format(len(tupac)), tupac)) else: print("ERROR: Unsupported input shape length: {}".format(input_shape)) exit(1) in_path = "{}_{}_input.data".format(os.path.splitext(input_file_name)[0], os.path.splitext(os.path.basename(onnx_model))[0]) in_path = os.path.join(onnx_file_path, in_path) print("Saving input to {}".format(in_path)) with open(in_path, "wb") as f: for packed_struct in packed_input: f.write(packed_struct) else: input_data = None print("ERROR: Something went wrong during input data processing...") exit(1) # Run inference on input data. print("Running inference...") outputs = backend.run_model(model, input_data, device='CPU') output_shape = outputs[0].shape print("Shape of output data: {}".format(output_shape)) # Write outputs into our custom binary format. packed_output.append(struct.pack('{}s'.format(len(magic_output)), magic_output)) packed_output.append(struct.pack('i', output_shape[0])) # Number of batches packed_output.append(struct.pack('i', output_shape[1])) # Number of outputs for output in outputs[0]: packed_output.append(struct.pack('f'*len(output), *output)) # Data tupac = bytes("end\n", "ascii") packed_output.append(struct.pack('{}s'.format(len(tupac)), tupac)) out_path = "{}_{}_output.data".format(os.path.splitext(input_file_name)[0], os.path.splitext(os.path.basename(onnx_model))[0]) out_path = os.path.join(onnx_file_path, out_path) print("Saving output to {}".format(out_path)) with open(out_path, "wb") as f: for packed_struct in packed_output: f.write(packed_struct)
node_ops = [ op_type_to_creator[node.op_type](graph, node) for node in graph.node if node.op_type ] def forward(input): dict = {graph.input[0].name: input, "": None} for op in node_ops: op(dict) return dict[graph.output[0].name] return forward if __name__ == '__main__': import sys model = onnx.load(sys.argv[1]) input_shape = get_info_tensor_shape(model.graph.input[0]) dummy_input = np.linspace( 0, 1, num=np.prod(input_shape)).reshape(*input_shape).astype(np.float32) f = create_graph_forward(model.graph) result1 = caffe_backend.run_model(model, [dummy_input])[0] result2 = f(dummy_input) if not np.isclose(result1, result2).all(): raise Exception("model is not consistent: {} {}".format( result1, result2)) print('Forward Test Passed %s' % result2)
def test_convert_end2end(self): predict_net_f = tempfile.NamedTemporaryFile() init_net_f = tempfile.NamedTemporaryFile() onnx_model_f = tempfile.NamedTemporaryFile() x = 'X' w = 'W' b = 'b' y = 'Y' predict_net = caffe2_pb2.NetDef() predict_net.name = 'test-convert-end2end' predict_net.external_input[:] = [x, w, b] predict_net.external_output[:] = [y] predict_net.op.extend([ core.CreateOperator( 'FC', inputs=[x, w, b], outputs=[y], axis=2, ), ]) predict_net_f.write(predict_net.SerializeToString()) predict_net_f.flush() init_net = caffe2_pb2.NetDef() init_net.name = 'test-convert-end2end-init' init_net.external_output[:] = [w, b] x_val = np.random.randn(1, 3, 2).astype(np.float32) w_val = np.random.randn(4, 2).astype(np.float32) b_val = np.random.randn(4).astype(np.float32) init_net.op.extend([ core.CreateOperator( 'GivenTensorFill', [], [w], values=w_val, shape=w_val.shape, ), core.CreateOperator( 'GivenTensorFill', [], [b], values=b_val, shape=b_val.shape, ), ]) init_net_f.write(init_net.SerializeToString()) init_net_f.flush() y_val = np.matmul(x_val, w_val.transpose()) + b_val for _ in range(5): self._run_command( caffe2_to_onnx, [ predict_net_f.name, '--caffe2-init-net', init_net_f.name, '--output', onnx_model_f.name, '--value-info', json.dumps({ x: (TensorProto.FLOAT, (1, 3, 2)), }), ], catch_exceptions=False, ) onnx_model_f.seek(0) onnx_model = ModelProto() onnx_model.ParseFromString(onnx_model_f.read()) np.testing.assert_almost_equal( c2.run_model( onnx_model, {onnx_model.graph.input[0].name: x_val}), [y_val]) self._run_command( onnx_to_caffe2, [ onnx_model_f.name, '--output', predict_net_f.name, '--init-net-output', init_net_f.name, ]) predict_net_f.seek(0) predict_net = caffe2_pb2.NetDef() predict_net.ParseFromString(predict_net_f.read()) init_net_f.seek(0) init_net = caffe2_pb2.NetDef() init_net.ParseFromString(init_net_f.read()) x = predict_net.external_input[0] np.testing.assert_almost_equal(c2_native_run_net(init_net=init_net, predict_net=predict_net, inputs={x: x_val})[1], [y_val])
import os import caffe2.python.onnx.backend as caffe2_bk import numpy as np import tensorflow as tf import onnx import onnx_tf.backend as tf_bk with open(os.path.join('pb/onnx_gh_mnist_nchw.pb'), "rb") as f: onnx_model = onnx.load(f) onnx.checker.check_model(onnx_model) # print(onnx_model) img = np.random.randn(50, 784).astype(np.float32) img = (img - np.min(img)) / (np.max(img) - np.min(img)) output_c2 = caffe2_bk.run_model(onnx_model, [img]) # output_tf = tf_bk.run_model(onnx_model, [img]) # print(output_c2) # print(output_tf) # print(np.allclose(output_c2, output_tf))
#!/usr/bin/env python3 import onnx import onnxruntime from onnx import optimizer import numpy as np import onnx.helper import caffe2.python.onnx.backend as c2 model_path = "test_maxpool.onnx" print(model_path) input_data = np.random.rand(1, 1, 64, 64).astype(np.float32) model = onnx.load(model_path) input_names = ["x"] x_c2 = input_data #x_c2 = np.transpose(input_data, [0, 2, 3, 1]) y = np.expand_dims(x_c2, axis=0) caffe_res = c2.run_model(model, dict(zip(input_names, y)))[0] print(caffe_res)