def _test_torch_model_single_io(torch_model,
torch_input_shape,
coreml_input_shape,
minimum_ios_deployment_target='12',
decimal=4,
opset_version=9):
# run torch model
torch_input = torch.rand(*torch_input_shape)
torch_out_raw = torch_model(torch_input)
if isinstance(torch_out_raw, tuple):
torch_out = torch_out_raw[0].detach().numpy()
else:
torch_out = torch_out_raw.detach().numpy()
# convert to onnx model
model_dir = tempfile.mkdtemp()
if DEBUG:
model_dir = '/tmp'
onnx_file = os.path.join(model_dir, 'torch_model.onnx')
torch.onnx.export(torch_model,
torch_input,
onnx_file,
opset_version=opset_version)
onnx_model = onnx.load(onnx_file)
# convert to coreml and run
coreml_model = convert(
onnx_model,
minimum_ios_deployment_target=minimum_ios_deployment_target)
output_name = [o.name for o in onnx_model.graph.output][0]
initializer_names = {t.name for t in onnx_model.graph.initializer}
input_name = [
i.name for i in onnx_model.graph.input
if i.name not in initializer_names
][0]
input_numpy = torch_input.detach().numpy()
if SupportedVersion.is_nd_array_supported(minimum_ios_deployment_target):
input_dict = {input_name: input_numpy} # type: ignore
else:
input_dict = {
input_name: np.reshape(input_numpy, coreml_input_shape)
} # type: ignore
coreml_out = coreml_model.predict(input_dict, useCPUOnly=True)[output_name]
if DEBUG:
coreml_model.save(model_dir + '/torch_model.mlmodel')
print('coreml_out')
print(np.squeeze(coreml_out))
print('torch_out')
print(np.squeeze(torch_out))
print('coreml out shape ', coreml_out.shape)
print('torch out shape: ', torch_out.shape)
# compare
_assert_outputs([torch_out], [coreml_out], decimal=decimal) # type: ignore
# delete onnx model
if not DEBUG:
if os.path.exists(model_dir):
shutil.rmtree(model_dir)
class CoreMLTestingBackend(
CoreMLBackendND if SupportedVersion.is_nd_array_supported(TARGET_IOS)
else CoreMLBackend):
@classmethod
def run_node(
cls,
node, # type: onnx.NodeProto
inputs, # type: Sequence[numpy.ndarray[Any]]
device='CPU', # type: Text
outputs_info=None, # type: Optional[Sequence[Tuple[numpy.dtype, Tuple[int, ...]]]]
):
# type: (...) -> onnx.namedtupledict
'''
CoreML requires full model for prediction, not just single layer.
Also input/output shapes are required to build CoreML spec for model.
As a temporary decision we use caffe2 backend for shape inference
task to build the appropriate ONNX model and convert it to
CoreML model.
'''
super(CoreMLTestingBackend, cls).run_node(node, inputs, device)
assert outputs_info is not None, "CoreML needs output shapes"
graph_inputs = []
for i in range(len(inputs)):
input_ = inputs[i]
value_info = onnx.helper.make_tensor_value_info(
name=node.input[i],
elem_type=onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[input_.dtype],
shape=input_.shape)
graph_inputs.append(value_info)
graph_outputs = []
for i in range(len(outputs_info)):
output_info = outputs_info[i]
value_info = onnx.helper.make_tensor_value_info(
name=node.output[i],
elem_type=onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[output_info[0]],
shape=output_info[1])
graph_outputs.append(value_info)
graph = onnx.helper.make_graph(
nodes=[node],
name='dummy',
inputs=graph_inputs,
outputs=graph_outputs,
)
model = onnx.helper.make_model(graph)
return cls.prepare(model).run(inputs)
def _coreml_forward_model(model, # type: ModelProto
input_dict, # type: Dict[Text, np._ArrayLike[Any]]
output_names, # type: Sequence[Text]
target_ios='12'
):
# type: (...) -> np.ndarray[Any]
if not SupportedVersion.is_nd_array_supported(target_ios):
for k, arr in input_dict.items():
if len(arr.shape) == 4:
input_dict[k] = arr[0]
for k,v in input_dict.items():
if len(v.shape) == 2 and v.shape[0] == 1:
input_dict[k] = v.flatten()
coreml_out = model.predict(input_dict, useCPUOnly=True)
return np.array([coreml_out[name] for name in output_names])
def exclude_test_cases(backend_test):
# Test cases to be disabled till coreml 2
unsupported_tests_till_coreml2 = [
# Failing due to tolerance (in particular due to the way outputs are compared,
'test_log_example_cpu',
'test_operator_add_broadcast_cpu',
'test_operator_add_size1_right_broadcast_cpu',
'test_operator_add_size1_singleton_broadcast_cpu',
'test_operator_addconstant_cpu',
'test_sign_cpu',
'test_sign_model_cpu',
# Dynamic ONNX ops not supported in CoreML
'test_reshape_extended_dims_cpu',
'test_reshape_negative_dim_cpu',
'test_reshape_one_dim_cpu',
'test_reshape_reduced_dims_cpu',
'test_reshape_reordered_dims_cpu',
'test_gemm_broadcast_cpu',
'test_gemm_nobroadcast_cpu',
'test_upsample_nearest_cpu',
# Failure due to axis alignment between ONNX and CoreML
'test_add_bcast_cpu', # (3,4,5, shaped tensor + (5,, shaped tensor
'test_div_bcast_cpu', # (3,4,5, shaped tensor + (5,, shaped tensor
'test_mul_bcast_cpu', # (3,4,5, shaped tensor + (5,, shaped tensor
'test_sub_bcast_cpu', # (3,4,5, shaped tensor + (5,, shaped tensor
'test_operator_index_cpu', # input: [1,1]: cannot slice along batch axis
'test_concat_2d_axis_0_cpu', # cannot slice along batch axis
'test_argmax_default_axis_example_cpu', # batch axis
'test_argmin_default_axis_example_cpu', # batch axis
'test_AvgPool1d_cpu', # 3-D tensor unsqueezed to 4D with axis = 3 and then pool
'test_AvgPool1d_stride_cpu', # same as above
# Possibly Fixable by the converter
'test_slice_start_out_of_bounds_cpu', # starts and ends exceed input size
# "Supported ops, but Unsupported parameters by CoreML"
'test_logsoftmax_axis_1_cpu', # this one converts but fails at runtime: must give error during conversion
'test_logsoftmax_default_axis_cpu', # this one converts but fails at runtime: must give error during conversion
'test_softmax_axis_1_cpu', # this one converts but fails at runtime: must give error during conversion
'test_softmax_default_axis_cpu', # this one converts but fails at runtime: must give error during conversion
'test_logsoftmax_axis_0_cpu',
'test_logsoftmax_axis_2_cpu',
'test_softmax_axis_0_cpu',
'test_softmax_axis_2_cpu',
'test_log_softmax_dim3_cpu',
'test_log_softmax_lastdim_cpu',
'test_softmax_functional_dim3_cpu',
'test_PReLU_1d_multiparam_cpu', # this one converts but fails at runtime: must give error during conversion
'test_mvn_cpu', # not sure why there is numerical mismatch
'test_flatten_axis2_cpu', # 2,3,4,5 --> 6,20
'test_flatten_axis3_cpu', # 2,3,4,5 --> 24,5
# Unsupported ops by CoreML
'test_constant_cpu',
'test_hardmax_axis_0_cpu',
'test_hardmax_axis_1_cpu',
'test_hardmax_axis_2_cpu',
'test_hardmax_default_axis_cpu',
'test_hardmax_example_cpu',
'test_hardmax_one_hot_cpu',
'test_matmul_2d_cpu',
'test_matmul_3d_cpu',
'test_matmul_4d_cpu',
'test_AvgPool3d_cpu',
'test_AvgPool3d_stride_cpu',
'test_BatchNorm3d_eval_cpu',
'test_Conv3d_cpu',
'test_Conv3d_dilated_cpu',
'test_Conv3d_dilated_strided_cpu',
'test_Conv3d_groups_cpu',
'test_Conv3d_no_bias_cpu',
'test_Conv3d_stride_cpu',
'test_Conv3d_stride_padding_cpu',
'test_MaxPool3d_cpu',
'test_MaxPool3d_stride_cpu',
'test_MaxPool3d_stride_padding_cpu',
'test_PReLU_3d_cpu',
'test_PReLU_3d_multiparam_cpu',
'test_AvgPool3d_stride1_pad0_gpu_input_cpu',
'test_BatchNorm3d_momentum_eval_cpu',
'test_BatchNorm1d_3d_input_eval_cpu',
'test_averagepool_3d_default_cpu',
'test_maxpool_3d_default_cpu',
'test_split_variable_parts_1d_cpu',
'test_split_variable_parts_2d_cpu',
'test_split_variable_parts_default_axis_cpu',
'test_cast_DOUBLE_to_FLOAT_cpu',
'test_cast_FLOAT_to_DOUBLE_cpu',
'test_shape_cpu',
'test_shape_example_cpu',
'test_size_cpu',
'test_size_example_cpu',
'test_top_k_cpu',
'test_PoissonNLLLLoss_no_reduce_cpu',
'test_operator_mm_cpu',
'test_operator_addmm_cpu',
'test_tile_cpu',
'test_tile_precomputed_cpu',
'test_acos_cpu',
'test_acos_example_cpu',
'test_asin_cpu',
'test_asin_example_cpu',
'test_atan_cpu',
'test_atan_example_cpu',
'test_cos_cpu',
'test_cos_example_cpu',
'test_sin_cpu',
'test_sin_example_cpu',
'test_tan_cpu',
'test_tan_example_cpu',
'test_dropout_cpu',
'test_dropout_default_cpu',
'test_dropout_random_cpu',
'test_gru_seq_length_cpu',
'test_identity_cpu',
'test_asin_example_cpu',
'test_reduce_log_sum_exp_default_axes_keepdims_example_cpu',
'test_reduce_log_sum_exp_default_axes_keepdims_random_cpu',
'test_reduce_log_sum_exp_do_not_keepdims_example_cpu',
'test_reduce_log_sum_exp_do_not_keepdims_random_cpu',
'test_reduce_log_sum_exp_keepdims_example_cpu',
'test_reduce_log_sum_exp_keepdims_random_cpu',
'test_rnn_seq_length_cpu',
'test_operator_repeat_cpu',
'test_operator_repeat_dim_overflow_cpu',
'test_thresholdedrelu_example_cpu', #different convention for CoreML
'test_expand_shape_model1_cpu',
'test_expand_shape_model2_cpu',
'test_expand_shape_model3_cpu',
'test_expand_shape_model4_cpu',
'test_expand_dim_changed_cpu',
'test_expand_dim_unchanged_cpu',
'test_operator_chunk_cpu', # unequal splits
'test_operator_permute2_cpu', # rank 6 input
'test_maxpool_with_argmax_2d_precomputed_pads_cpu',
'test_maxpool_with_argmax_2d_precomputed_strides_cpu',
'test_gather_1_cpu',
'test_gather_0_cpu',
'test_Embedding_cpu', # gather op
'test_Embedding_sparse_cpu', # gather op
'test_pow_bcast_scalar_cpu',
'test_pow_bcast_array_cpu',
'test_pow_cpu',
'test_pow_example_cpu',
'test_operator_pow_cpu',
'test_shrink_cpu',
'test_acosh_cpu',
'test_asinh_cpu',
'test_atanh_cpu',
'test_erf_cpu',
'test_isnan_cpu',
'test_where_example_cpu',
'test_acosh_example_cpu',
'test_asinh_example_cpu',
'test_atanh_example_cpu',
'test_compress_0_cpu',
'test_compress_1_cpu',
'test_compress_default_axis_cpu',
'test_constantofshape_float_ones_cpu',
'test_constantofshape_int_zeros_cpu',
'test_cosh_cpu',
'test_cosh_example_cpu',
'test_dynamic_slice_cpu',
'test_dynamic_slice_default_axes_cpu',
'test_dynamic_slice_end_out_of_bounds_cpu',
'test_dynamic_slice_neg_cpu',
'test_dynamic_slice_neg_cpu',
'test_dynamic_slice_start_out_of_bounds_cpu',
'test_eyelike_populate_off_main_diagonal_cpu',
'test_eyelike_with_dtype_cpu',
'test_eyelike_without_dtype_cpu',
'test_maxunpool_export_with_output_shape_cpu',
'test_maxunpool_export_without_output_shape_cpu',
'test_nonzero_example_cpu',
'test_onehot_with_axis_cpu',
'test_onehot_without_axis_cpu',
'test_scan9_sum_cpu',
'test_scan_sum_cpu',
'test_scatter_with_axis_cpu',
'test_scatter_without_axis_cpu',
'test_shrink_hard_cpu',
'test_shrink_soft_cpu',
'test_sinh_cpu',
'test_sinh_example_cpu',
'test_tfidfvectorizer_tf_batch_onlybigrams_skip0_cpu',
'test_tfidfvectorizer_tf_batch_onlybigrams_skip5_cpu',
'test_tfidfvectorizer_tf_batch_uniandbigrams_skip5_cpu',
'test_tfidfvectorizer_tf_only_bigrams_skip0_cpu',
'test_tfidfvectorizer_tf_onlybigrams_levelempty_cpu',
'test_tfidfvectorizer_tf_onlybigrams_skip5_cpu',
'test_tfidfvectorizer_tf_uniandbigrams_skip5_cpu',
'test_basic_convinteger_cpu',
# recurrent tests.
'test_operator_rnn_cpu',
'test_operator_rnn_single_layer_cpu',
'test_operator_lstm_cpu',
'test_gru_defaults_cpu',
'test_gru_with_initial_bias_cpu',
'test_lstm_defaults_cpu',
'test_lstm_with_initial_bias_cpu',
'test_lstm_with_peepholes_cpu',
'test_simple_rnn_defaults_cpu',
'test_simple_rnn_with_initial_bias_cpu',
# exclude all the model zoo tests. They are tested elsewhere.
'test_bvlc_alexnet',
'test_resnet50',
'test_vgg16',
'test_vgg19',
'test_densenet121',
'test_inception_v1',
'test_inception_v2',
'test_shufflenet',
'test_squeezenet',
'test_zfnet',
'test_maxunpool_export_with_output_shape_cpu',
'test_maxunpool_export_without_output_shape_cpu',
'test_nonzero_example_cpu',
'test_onehot_with_axis_cpu',
'test_onehot_without_axis_cpu',
'test_scan9_sum_cpu',
'test_scan_sum_cpu',
'test_shrink_hard_cpu',
'test_shrink_soft_cpu',
'test_sinh_cpu',
'test_sinh_example_cpu',
'test_tfidfvectorizer_tf_batch_onlybigrams_skip0_cpu',
'test_tfidfvectorizer_tf_batch_onlybigrams_skip5_cpu',
'test_tfidfvectorizer_tf_batch_uniandbigrams_skip5_cpu',
'test_tfidfvectorizer_tf_only_bigrams_skip0_cpu',
'test_tfidfvectorizer_tf_onlybigrams_levelempty_cpu',
'test_tfidfvectorizer_tf_onlybigrams_skip5_cpu',
'test_tfidfvectorizer_tf_uniandbigrams_skip5_cpu',
# Incorrect test setup
'test_mod_float_mixed_sign_example_cpu',
'test_mod_int64_mixed_sign_example_cpu',
# Need dead code elimination and removing unused inputs
'test_batchnorm_epsilon_cpu',
'test_batchnorm_example_cpu',
# recurrent tests.
'test_operator_rnn_cpu',
'test_operator_rnn_single_layer_cpu',
'test_operator_lstm_cpu',
'test_gru_defaults_cpu',
'test_gru_with_initial_bias_cpu',
'test_lstm_defaults_cpu',
'test_lstm_with_initial_bias_cpu',
'test_lstm_with_peepholes_cpu',
'test_simple_rnn_defaults_cpu',
'test_simple_rnn_with_initial_bias_cpu',
# exclude all the model zoo tests. They are tested elsewhere.
'test_bvlc_alexnet',
'test_resnet50',
'test_vgg16',
'test_vgg19',
'test_densenet121',
'test_inception_v1',
'test_inception_v2',
'test_shufflenet',
'test_squeezenet',
'test_zfnet'
'test_maxunpool_export_with_output_shape_cpu',
'test_maxunpool_export_without_output_shape_cpu',
'test_nonzero_example_cpu',
'test_onehot_with_axis_cpu',
'test_onehot_without_axis_cpu',
'test_scan9_sum_cpu',
'test_scan_sum_cpu',
'test_shrink_hard_cpu',
'test_shrink_soft_cpu',
'test_sinh_cpu',
'test_sinh_example_cpu',
'test_tfidfvectorizer_tf_batch_onlybigrams_skip0_cpu',
'test_tfidfvectorizer_tf_batch_onlybigrams_skip5_cpu',
'test_tfidfvectorizer_tf_batch_uniandbigrams_skip5_cpu',
'test_tfidfvectorizer_tf_only_bigrams_skip0_cpu',
'test_tfidfvectorizer_tf_onlybigrams_levelempty_cpu',
'test_tfidfvectorizer_tf_onlybigrams_skip5_cpu',
'test_tfidfvectorizer_tf_uniandbigrams_skip5_cpu',
'test_basic_convinteger_cpu',
'test_cast_FLOAT_to_STRING_cpu',
'test_cast_STRING_to_FLOAT_cpu',
'test_cast_DOUBLE_to_FLOAT16_cpu',
'test_cast_FLOAT16_to_DOUBLE_cpu',
'test_cast_FLOAT16_to_FLOAT_cpu',
'test_cast_FLOAT_to_FLOAT16_cpu',
'test_matmulinteger_cpu',
'test_prelu_broadcast_cpu',
'test_convinteger_with_padding_cpu',
'test_convtranspose_3d_cpu',
'test_dequantizelinear_cpu',
'test_instancenorm_epsilon_cpu',
'test_instancenorm_example_cpu',
'test_isinf_cpu',
'test_isinf_negative_cpu',
'test_isinf_positive_cpu',
'test_nonmaxsuppression_center_point_box_format_cpu',
'test_nonmaxsuppression_flipped_coordinates_cpu',
'test_nonmaxsuppression_identical_boxes_cpu',
'test_nonmaxsuppression_limit_output_size_cpu',
'test_nonmaxsuppression_single_box_cpu',
'test_nonmaxsuppression_suppress_by_IOU_and_scores_cpu',
'test_nonmaxsuppression_suppress_by_IOU_cpu',
'test_nonmaxsuppression_two_batches_cpu',
'test_nonmaxsuppression_two_classes_cpu',
'test_prelu_example_cpu',
'test_qlinearconv_cpu',
'test_qlinearmatmul_2D_cpu',
'test_qlinearmatmul_3D_cpu',
'test_quantizelinear_cpu',
'test_resize_downsample_linear_cpu',
'test_resize_downsample_nearest_cpu',
'test_resize_nearest_cpu',
'test_resize_upsample_linear_cpu',
'test_resize_upsample_nearest_cpu',
'test_reversesequence_batch_cpu',
'test_reversesequence_time_cpu',
'test_roialign_cpu',
'test_slice_cpu',
'test_slice_default_axes_cpu',
'test_slice_default_steps_cpu',
'test_slice_end_out_of_bounds_cpu',
'test_slice_neg_cpu',
'test_slice_neg_steps_cpu',
'test_strnormalizer_export_monday_casesensintive_lower_cpu',
'test_strnormalizer_export_monday_casesensintive_nochangecase_cpu',
'test_strnormalizer_export_monday_casesensintive_upper_cpu',
'test_strnormalizer_export_monday_empty_output_cpu',
'test_strnormalizer_export_monday_insensintive_upper_twodim_cpu',
'test_strnormalizer_nostopwords_nochangecase_cpu',
'test_PReLU_1d_cpu',
'test_PReLU_2d_cpu',
'test_strnorm_model_monday_casesensintive_lower_cpu',
'test_strnorm_model_monday_casesensintive_nochangecase_cpu',
'test_strnorm_model_monday_casesensintive_upper_cpu',
'test_strnorm_model_monday_empty_output_cpu',
'test_strnorm_model_monday_insensintive_upper_twodim_cpu',
'test_strnorm_model_nostopwords_nochangecase_cpu',
'test_averagepool_2d_ceil_cpu',
'test_convtranspose_pad_cpu',
'test_logsoftmax_large_number_cpu',
'test_maxpool_2d_ceil_cpu',
'test_maxpool_2d_dilations_cpu',
'test_softmax_large_number_cpu',
'test_mod_bcast_cpu',
'test_mod_fmod_mixed_sign_example_cpu',
'test_mvn_expanded_cpu',
'test_operator_non_float_params_cpu',
'test_operator_params_cpu',
]
## Test cases to be disabled in CoreML 3.0
unsupported_tests_coreml3 = [
# Failing due to tolerance (in particular due to the way outputs are compared,
'test_log_example_cpu',
'test_operator_add_broadcast_cpu',
'test_operator_add_size1_right_broadcast_cpu',
'test_operator_add_size1_singleton_broadcast_cpu',
'test_operator_addconstant_cpu',
'test_sign_cpu',
'test_sign_model_cpu',
# Dynamic ONNX ops not supported in CoreML
'test_reshape_extended_dims_cpu',
'test_reshape_negative_dim_cpu',
'test_reshape_one_dim_cpu',
'test_reshape_reduced_dims_cpu',
'test_reshape_reordered_dims_cpu',
'test_gemm_broadcast_cpu',
'test_gemm_nobroadcast_cpu',
'test_upsample_nearest_cpu',
# Failure due to axis alignment between ONNX and CoreML
'test_add_bcast_cpu', # (3,4,5, shaped tensor + (5,, shaped tensor
'test_div_bcast_cpu', # (3,4,5, shaped tensor + (5,, shaped tensor
'test_mul_bcast_cpu', # (3,4,5, shaped tensor + (5,, shaped tensor
'test_sub_bcast_cpu', # (3,4,5, shaped tensor + (5,, shaped tensor
'test_operator_index_cpu', # input: [1,1]: cannot slice along batch axis
'test_concat_2d_axis_0_cpu', # cannot slice along batch axis
'test_argmax_default_axis_example_cpu', # batch axis
'test_argmin_default_axis_example_cpu', # batch axis
'test_AvgPool1d_cpu', # 3-D tensor unsqueezed to 4D with axis = 3 and then pool
'test_AvgPool1d_stride_cpu', # same as above
# Possibly Fixable by the converter
'test_slice_start_out_of_bounds_cpu', # starts and ends exceed input size
# "Supported ops, but Unsupported parameters by CoreML"
'test_logsoftmax_axis_1_cpu', # this one converts but fails at runtime: must give error during conversion
'test_logsoftmax_default_axis_cpu', # this one converts but fails at runtime: must give error during conversion
'test_softmax_axis_1_cpu', # this one converts but fails at runtime: must give error during conversion
'test_softmax_default_axis_cpu', # this one converts but fails at runtime: must give error during conversion
'test_logsoftmax_axis_0_cpu',
'test_softmax_axis_0_cpu',
'test_PReLU_1d_multiparam_cpu', # this one converts but fails at runtime: must give error during conversion
'test_mvn_cpu', # not sure why there is numerical mismatch
'test_flatten_axis2_cpu', # 2,3,4,5 --> 6,20
'test_flatten_axis3_cpu', # 2,3,4,5 --> 24,5
# Unsupported ops by CoreML
'test_constant_cpu',
'test_hardmax_axis_0_cpu',
'test_hardmax_axis_1_cpu',
'test_hardmax_axis_2_cpu',
'test_hardmax_default_axis_cpu',
'test_hardmax_example_cpu',
'test_hardmax_one_hot_cpu',
'test_matmul_2d_cpu',
'test_matmul_3d_cpu',
'test_matmul_4d_cpu',
'test_AvgPool3d_cpu',
'test_AvgPool3d_stride_cpu',
'test_BatchNorm3d_eval_cpu',
'test_Conv3d_cpu',
'test_Conv3d_dilated_cpu',
'test_Conv3d_dilated_strided_cpu',
'test_Conv3d_groups_cpu',
'test_Conv3d_no_bias_cpu',
'test_Conv3d_stride_cpu',
'test_Conv3d_stride_padding_cpu',
'test_MaxPool3d_cpu',
'test_MaxPool3d_stride_cpu',
'test_MaxPool3d_stride_padding_cpu',
'test_PReLU_3d_cpu',
'test_PReLU_3d_multiparam_cpu',
'test_AvgPool3d_stride1_pad0_gpu_input_cpu',
'test_BatchNorm3d_momentum_eval_cpu',
'test_BatchNorm1d_3d_input_eval_cpu',
'test_averagepool_3d_default_cpu',
'test_maxpool_3d_default_cpu',
'test_split_variable_parts_1d_cpu',
'test_split_variable_parts_2d_cpu',
'test_split_variable_parts_default_axis_cpu',
'test_shape_cpu',
'test_shape_example_cpu',
'test_top_k_cpu',
'test_PoissonNLLLLoss_no_reduce_cpu',
'test_operator_mm_cpu',
'test_operator_addmm_cpu',
'test_tile_cpu',
'test_tile_precomputed_cpu',
'test_acos_cpu',
'test_acos_example_cpu',
'test_asin_cpu',
'test_asin_example_cpu',
'test_atan_cpu',
'test_atan_example_cpu',
'test_cos_cpu',
'test_cos_example_cpu',
'test_sin_cpu',
'test_sin_example_cpu',
'test_tan_cpu',
'test_tan_example_cpu',
'test_dropout_cpu',
'test_dropout_default_cpu',
'test_dropout_random_cpu',
'test_gru_seq_length_cpu',
'test_identity_cpu',
'test_asin_example_cpu',
'test_reduce_log_sum_exp_default_axes_keepdims_example_cpu',
'test_reduce_log_sum_exp_default_axes_keepdims_random_cpu',
'test_reduce_log_sum_exp_do_not_keepdims_example_cpu',
'test_reduce_log_sum_exp_do_not_keepdims_random_cpu',
'test_reduce_log_sum_exp_keepdims_example_cpu',
'test_reduce_log_sum_exp_keepdims_random_cpu',
'test_rnn_seq_length_cpu',
'test_operator_repeat_cpu',
'test_operator_repeat_dim_overflow_cpu',
'test_thresholdedrelu_example_cpu', #different convention for CoreML
'test_expand_shape_model1_cpu',
'test_expand_shape_model2_cpu',
'test_expand_shape_model3_cpu',
'test_expand_shape_model4_cpu',
'test_expand_dim_changed_cpu',
'test_operator_chunk_cpu', # unequal splits
'test_operator_permute2_cpu', # rank 6 input
'test_maxpool_with_argmax_2d_precomputed_pads_cpu',
'test_maxpool_with_argmax_2d_precomputed_strides_cpu',
'test_gather_1_cpu',
'test_gather_0_cpu',
'test_Embedding_cpu', # gather op
'test_Embedding_sparse_cpu', # gather op
'test_pow_bcast_scalar_cpu',
'test_pow_bcast_array_cpu',
'test_pow_cpu',
'test_pow_example_cpu',
'test_operator_pow_cpu',
'test_shrink_cpu',
'test_acosh_cpu',
'test_asinh_cpu',
'test_atanh_cpu',
'test_isnan_cpu',
'test_acosh_example_cpu',
'test_asinh_example_cpu',
'test_atanh_example_cpu',
'test_compress_0_cpu',
'test_compress_1_cpu',
'test_compress_default_axis_cpu',
'test_constantofshape_float_ones_cpu',
'test_constantofshape_int_zeros_cpu',
'test_cosh_cpu',
'test_cosh_example_cpu',
'test_dynamic_slice_cpu',
'test_dynamic_slice_default_axes_cpu',
'test_dynamic_slice_end_out_of_bounds_cpu',
'test_dynamic_slice_neg_cpu',
'test_dynamic_slice_neg_cpu',
'test_dynamic_slice_start_out_of_bounds_cpu',
'test_eyelike_populate_off_main_diagonal_cpu',
'test_eyelike_with_dtype_cpu',
'test_eyelike_without_dtype_cpu',
'test_maxunpool_export_with_output_shape_cpu',
'test_maxunpool_export_without_output_shape_cpu',
'test_nonzero_example_cpu',
'test_onehot_with_axis_cpu',
'test_onehot_without_axis_cpu',
'test_scan9_sum_cpu',
'test_scan_sum_cpu',
'test_shrink_hard_cpu',
'test_shrink_soft_cpu',
'test_sinh_cpu',
'test_sinh_example_cpu',
'test_tfidfvectorizer_tf_batch_onlybigrams_skip0_cpu',
'test_tfidfvectorizer_tf_batch_onlybigrams_skip5_cpu',
'test_tfidfvectorizer_tf_batch_uniandbigrams_skip5_cpu',
'test_tfidfvectorizer_tf_only_bigrams_skip0_cpu',
'test_tfidfvectorizer_tf_onlybigrams_levelempty_cpu',
'test_tfidfvectorizer_tf_onlybigrams_skip5_cpu',
'test_tfidfvectorizer_tf_uniandbigrams_skip5_cpu',
'test_basic_convinteger_cpu',
'test_cast_FLOAT_to_STRING_cpu',
'test_cast_STRING_to_FLOAT_cpu',
'test_cast_DOUBLE_to_FLOAT16_cpu',
'test_cast_FLOAT16_to_DOUBLE_cpu',
'test_cast_FLOAT16_to_FLOAT_cpu',
'test_cast_FLOAT_to_FLOAT16_cpu',
'test_matmulinteger_cpu',
'test_prelu_broadcast_cpu',
'test_convinteger_with_padding_cpu',
'test_convtranspose_3d_cpu',
'test_dequantizelinear_cpu',
'test_instancenorm_epsilon_cpu',
'test_instancenorm_example_cpu',
'test_isinf_cpu',
'test_isinf_negative_cpu',
'test_isinf_positive_cpu',
'test_nonmaxsuppression_center_point_box_format_cpu',
'test_nonmaxsuppression_flipped_coordinates_cpu',
'test_nonmaxsuppression_identical_boxes_cpu',
'test_nonmaxsuppression_limit_output_size_cpu',
'test_nonmaxsuppression_single_box_cpu',
'test_nonmaxsuppression_suppress_by_IOU_and_scores_cpu',
'test_nonmaxsuppression_suppress_by_IOU_cpu',
'test_nonmaxsuppression_two_batches_cpu',
'test_nonmaxsuppression_two_classes_cpu',
'test_prelu_example_cpu',
'test_qlinearconv_cpu',
'test_qlinearmatmul_2D_cpu',
'test_qlinearmatmul_3D_cpu',
'test_quantizelinear_cpu',
# Resize test fails due to unknown Scaling factor
# Requires custom layer/function to bypass
'test_resize_downsample_linear_cpu',
'test_resize_downsample_nearest_cpu',
'test_resize_nearest_cpu',
'test_resize_upsample_linear_cpu',
'test_resize_upsample_nearest_cpu',
'test_reversesequence_time_cpu',
'test_slice_cpu',
'test_slice_default_axes_cpu',
'test_slice_default_steps_cpu',
'test_slice_end_out_of_bounds_cpu',
'test_slice_neg_cpu',
'test_slice_neg_steps_cpu',
'test_strnormalizer_export_monday_casesensintive_lower_cpu',
'test_strnormalizer_export_monday_casesensintive_nochangecase_cpu',
'test_strnormalizer_export_monday_casesensintive_upper_cpu',
'test_strnormalizer_export_monday_empty_output_cpu',
'test_strnormalizer_export_monday_insensintive_upper_twodim_cpu',
'test_strnormalizer_nostopwords_nochangecase_cpu',
'test_PReLU_1d_cpu',
'test_PReLU_2d_cpu',
'test_strnorm_model_monday_casesensintive_lower_cpu',
'test_strnorm_model_monday_casesensintive_nochangecase_cpu',
'test_strnorm_model_monday_casesensintive_upper_cpu',
'test_strnorm_model_monday_empty_output_cpu',
'test_strnorm_model_monday_insensintive_upper_twodim_cpu',
'test_strnorm_model_nostopwords_nochangecase_cpu',
'test_averagepool_2d_ceil_cpu',
'test_convtranspose_pad_cpu',
'test_maxpool_2d_ceil_cpu',
'test_maxpool_2d_dilations_cpu',
# Incorrect test setup
'test_mod_float_mixed_sign_example_cpu',
'test_mod_int64_mixed_sign_example_cpu',
# Need dead code elimination and removing unused inputs
'test_batchnorm_epsilon_cpu',
'test_batchnorm_example_cpu',
# recurrent tests.
'test_operator_rnn_cpu',
'test_operator_rnn_single_layer_cpu',
'test_operator_lstm_cpu',
'test_gru_defaults_cpu',
'test_gru_with_initial_bias_cpu',
'test_lstm_defaults_cpu',
'test_lstm_with_initial_bias_cpu',
'test_lstm_with_peepholes_cpu',
'test_simple_rnn_defaults_cpu',
'test_simple_rnn_with_initial_bias_cpu',
# exclude all the model zoo tests. They are tested elsewhere.
'test_bvlc_alexnet',
'test_resnet50',
'test_vgg16',
'test_vgg19',
'test_densenet121',
'test_inception_v1',
'test_inception_v2',
'test_shufflenet',
'test_squeezenet',
'test_zfnet',
'test_maxunpool_export_with_output_shape_cpu',
'test_maxunpool_export_without_output_shape_cpu',
'test_nonzero_example_cpu',
'test_onehot_with_axis_cpu',
'test_onehot_without_axis_cpu',
'test_scan9_sum_cpu',
'test_scan_sum_cpu',
'test_shrink_hard_cpu',
'test_shrink_soft_cpu',
'test_sinh_cpu',
'test_sinh_example_cpu',
'test_tfidfvectorizer_tf_batch_onlybigrams_skip0_cpu',
'test_tfidfvectorizer_tf_batch_onlybigrams_skip5_cpu',
'test_tfidfvectorizer_tf_batch_uniandbigrams_skip5_cpu',
'test_tfidfvectorizer_tf_only_bigrams_skip0_cpu',
'test_tfidfvectorizer_tf_onlybigrams_levelempty_cpu',
'test_tfidfvectorizer_tf_onlybigrams_skip5_cpu',
'test_tfidfvectorizer_tf_uniandbigrams_skip5_cpu',
]
if SupportedVersion.is_nd_array_supported(MINIMUM_IOS_DEPLOYMENT_TARGET):
for each in unsupported_tests_coreml3:
backend_test.exclude(each)
else:
for each in unsupported_tests_till_coreml2:
backend_test.exclude(each)
from onnx_coreml._backend import CoreMLBackend, CoreMLBackendND
from onnx_coreml.converter import SupportedVersion
from coremltools.models.utils import macos_version
# Default target iOS
MINIMUM_IOS_DEPLOYMENT_TARGET = '13'
MIN_MACOS_VERSION_10_15 = (10, 15)
# If MACOS version is less than 10.15
# Then force testing on CoreML 2.0
if macos_version() < MIN_MACOS_VERSION_10_15:
MINIMUM_IOS_DEPLOYMENT_TARGET = '12'
if not SupportedVersion.ios_support_check(MINIMUM_IOS_DEPLOYMENT_TARGET):
raise ValueError(
"Invalid Target iOS version provided. Valid target iOS: {}".format(supported_ios_version)
)
class CoreMLTestingBackend(CoreMLBackendND if SupportedVersion.is_nd_array_supported(MINIMUM_IOS_DEPLOYMENT_TARGET) else CoreMLBackend):
@classmethod
def run_node(cls,
node, # type: onnx.NodeProto
inputs, # type: Sequence[numpy.ndarray[Any]]
device='CPU', # type: Text
outputs_info=None, # type: Optional[Sequence[Tuple[numpy.dtype, Tuple[int, ...]]]]
):
# type: (...) -> onnx.namedtupledict
'''
CoreML requires full model for prediction, not just single layer.
from onnx_coreml._backend import CoreMLBackend, CoreMLBackendND
from onnx_coreml.converter import SupportedVersion
from coremltools.models.utils import macos_version
# Default target iOS
TARGET_IOS = '13'
MIN_MACOS_VERSION_10_15 = (10, 15)
# If MACOS version is less than 10.15
# Then force testing on CoreML 2.0
if macos_version() < MIN_MACOS_VERSION_10_15:
TARGET_IOS = '12'
if not SupportedVersion.ios_support_check(TARGET_IOS):
raise ValueError(
"Invalid Target iOS version provided. Valid target iOS: {}".format(
supported_ios_version))
class CoreMLTestingBackend(
CoreMLBackendND if SupportedVersion.is_nd_array_supported(TARGET_IOS)
else CoreMLBackend):
@classmethod
def run_node(
cls,
node, # type: onnx.NodeProto
inputs, # type: Sequence[numpy.ndarray[Any]]
device='CPU', # type: Text
outputs_info=None, # type: Optional[Sequence[Tuple[numpy.dtype, Tuple[int, ...]]]]
from onnx_coreml._backend import CoreMLBackend, CoreMLBackendND
from onnx_coreml.converter import SupportedVersion
from coremltools.models.utils import macos_version
# Default target iOS
MINIMUM_IOS_DEPLOYMENT_TARGET = '13'
MIN_MACOS_VERSION_10_15 = (10, 15)
# If MACOS version is less than 10.15
# Then force testing on CoreML 2.0
if macos_version() < MIN_MACOS_VERSION_10_15:
MINIMUM_IOS_DEPLOYMENT_TARGET = '12'
if not SupportedVersion.ios_support_check(MINIMUM_IOS_DEPLOYMENT_TARGET):
raise ValueError(
"Invalid Target iOS version provided. Valid target iOS: {}".format(supported_ios_version)
)
# import all test cases at global scope to make them visible to python.unittest
backend_test = onnx.backend.test.BackendTest(CoreMLBackendND if SupportedVersion.is_nd_array_supported(MINIMUM_IOS_DEPLOYMENT_TARGET) else CoreMLBackend, __name__)
# Only include the big models tests
backend_test.include('test_resnet50')
backend_test.include('test_inception_v1')
backend_test.include('test_inception_v2')
backend_test.include('test_densenet121')
backend_test.include('test_shufflenet')
backend_test.include('test_squeezenet')
backend_test.include('test_bvlc_alexnet')
from onnx_coreml._backend import CoreMLBackend, CoreMLBackendND
from onnx_coreml.converter import SupportedVersion
from coremltools.models.utils import macos_version
# Default target iOS
TARGET_IOS = '13'
MIN_MACOS_VERSION_10_15 = (10, 15)
# If MACOS version is less than 10.15
# Then force testing on CoreML 2.0
if macos_version() < MIN_MACOS_VERSION_10_15:
TARGET_IOS = '12'
if not SupportedVersion.ios_support_check(TARGET_IOS):
raise ValueError(
"Invalid Target iOS version provided. Valid target iOS: {}".format(
supported_ios_version))
# import all test cases at global scope to make them visible to python.unittest
backend_test = onnx.backend.test.BackendTest(
CoreMLBackendND if SupportedVersion.is_nd_array_supported(TARGET_IOS) else
CoreMLBackend, __name__)
# Only include the big models tests
backend_test.include('test_resnet50')
backend_test.include('test_inception_v1')
backend_test.include('test_inception_v2')
backend_test.include('test_densenet121')
backend_test.include('test_shufflenet')
