def create_backend_test(testname=None):
backend_test = OrtBackendTest(c2, __name__)
# Type not supported
backend_test.exclude(r'(FLOAT16)')
if testname:
backend_test.include(testname + '.*')
else:
# Tests that are failing temporarily and should be fixed
current_failing_tests = (
'^test_cast_STRING_to_FLOAT_cpu.*',
'^test_cast_FLOAT_to_STRING_cpu.*',
'^test_qlinearconv_cpu.*',
'^test_gru_seq_length_cpu.*',
'^test_bitshift_right_uint16_cpu.*',
'^test_bitshift_right_uint32_cpu.*',
'^test_bitshift_right_uint64_cpu.*',
'^test_bitshift_right_uint8_cpu.*',
'^test_bitshift_left_uint16_cpu.*',
'^test_bitshift_left_uint32_cpu.*',
'^test_bitshift_left_uint64_cpu.*',
'^test_bitshift_left_uint8_cpu.*',
'^test_round_cpu.*',
'^test_cumsum_1d_cpu.*',
'^test_cumsum_1d_exclusive_cpu.*',
'^test_cumsum_1d_reverse_cpu.*',
'^test_cumsum_1d_reverse_exclusive_cpu.*',
'^test_cumsum_2d_axis_0_cpu.*',
'^test_cumsum_2d_axis_1_cpu.*',
'^test_dynamicquantizelinear_expanded*',
'^test_dynamicquantizelinear_max_adjusted_expanded*',
'^test_dynamicquantizelinear_min_adjusted_expanded*',
'^test_depthtospace*',
'^test_gather_elements*',
'^test_scatter_elements*',
'^test_top_k*',
'^test_unique_*',
'^test_mod_float_mixed_sign_example_cpu.*', #onnxruntime::Mod::Compute fmod_ was false. fmod attribute must be true for float, float16 and double types
'^test_shrink_cpu.*', #Invalid rank for input: x Got: 1 Expected: 2 Please fix either the inputs or the model.
)
# Example of how to disable tests for a specific provider.
# if c2.supports_device('NGRAPH'):
# current_failing_tests = current_failing_tests + ('|^test_operator_repeat_dim_overflow_cpu.*',)
if c2.supports_device('NGRAPH'):
current_failing_tests = current_failing_tests + ('|^test_clip*', )
filters = current_failing_tests + \
tests_with_pre_opset7_dependencies_filters() + \
unsupported_usages_filters()
backend_test.exclude('(' + '|'.join(filters) + ')')
print('excluded tests:', filters)
# import all test cases at global scope to make
# them visible to python.unittest.
globals().update(backend_test.enable_report().test_cases)
return backend_test
def create_backend_test(testname=None):
backend_test = OrtBackendTest(c2, __name__)
# Type not supported
backend_test.exclude(r'(FLOAT16)')
if testname:
backend_test.include(testname + '.*')
else:
# read filters data
with open(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
'testdata',
'onnx_backend_test_series_filters.jsonc')) as f:
filters_lines = f.readlines()
filters_lines = [x.split('//')[0] for x in filters_lines]
filters = json.loads('\n'.join(filters_lines))
current_failing_tests = filters['current_failing_tests']
if platform.architecture()[0] == '32bit':
current_failing_tests += filters['current_failing_tests_x86']
if c2.supports_device('NGRAPH'):
current_failing_tests += filters['current_failing_tests_NGRAPH']
if c2.supports_device('DNNL'):
current_failing_tests += filters['current_failing_tests_DNNL']
if c2.supports_device('NNAPI'):
current_failing_tests += filters['current_failing_tests_NNAPI']
if c2.supports_device('OPENVINO_GPU_FP32') or c2.supports_device(
'OPENVINO_GPU_FP16'):
current_failing_tests += filters[
'current_failing_tests_OPENVINO_GPU']
if c2.supports_device('OPENVINO_GPU_FP32'):
current_failing_tests += filters[
'current_failing_tests_OPENVINO_GPU_FP32']
if c2.supports_device('OPENVINO_CPU_FP32'):
current_failing_tests += filters[
'current_failing_tests_OPENVINO_CPU_FP32']
filters = current_failing_tests + \
filters['tests_with_pre_opset7_dependencies'] + \
filters['unsupported_usages'] + \
filters['failing_permanently'] + \
filters['test_with_types_disabled_due_to_binary_size_concerns']
backend_test.exclude('(' + '|'.join(filters) + ')')
print('excluded tests:', filters)
# import all test cases at global scope to make
# them visible to python.unittest.
globals().update(backend_test.enable_report().test_cases)
return backend_test
def create_backend_test(testname=None):
backend_test = OrtBackendTest(c2, __name__)
# Type not supported
backend_test.exclude(r'(FLOAT16)')
if testname:
backend_test.include(testname + '.*')
else:
# Tests that are failing temporarily and should be fixed
current_failing_tests = ('^test_cast_STRING_to_FLOAT_cpu.*',
'^test_cast_FLOAT_to_STRING_cpu.*',
'^test_dequantizelinear_cpu.*',
'^test_qlinearconv_cpu.*',
'^test_quantizelinear_cpu.*',
'^test_gru_seq_length_cpu.*')
global version_tag
if version_tag == 'onnx141' or onnx.__version__ == '1.4.1':
current_failing_tests = current_failing_tests + (
'^test_shrink_cpu.*',
'^test_constantofshape_*.*',
)
if version_tag == 'onnx150' or onnx.__version__ == '1.5.0':
current_failing_tests = current_failing_tests + (
'^test_constantofshape_*.*', )
# Failing for nGraph.
if c2.supports_device('NGRAPH'):
current_failing_tests = current_failing_tests + (
'|^test_operator_repeat_dim_overflow_cpu.*', )
filters = current_failing_tests + \
tests_with_pre_opset7_dependencies_filters() + \
unsupported_usages_filters()
backend_test.exclude('(' + '|'.join(filters) + ')')
print('excluded tests:', filters)
# import all test cases at global scope to make
# them visible to python.unittest.
globals().update(backend_test.enable_report().test_cases)
return backend_test
def create_backend_test(testname=None):
backend_test = OrtBackendTest(c2, __name__)
# Type not supported
backend_test.exclude(r'(FLOAT16)')
if testname:
backend_test.include(testname + '.*')
else:
# Tests that are failing temporarily and should be fixed
current_failing_tests = [ #'^test_cast_STRING_to_FLOAT_cpu', # old test data that is bad on Linux CI builds
'^test_unique_not_sorted_without_axis_cpu', # bad expected data. enable after https://github.com/onnx/onnx/pull/2381 is picked up
'^test_mod_float_mixed_sign_example_cpu', #onnxruntime::Mod::Compute fmod_ was false. fmod attribute must be true for float, float16 and double types
'^test_resize_downsample_scales_cubic_align_corners_cpu', # results mismatch with onnx tests
'^test_resize_downsample_scales_linear_align_corners_cpu', # results mismatch with onnx tests
'^test_resize_tf_crop_and_resize_cpu', # bad expected data, needs test fix
'^test_resize_upsample_sizes_nearest_ceil_half_pixel_cpu', # bad expected data, needs test fix
'^test_resize_upsample_sizes_nearest_floor_align_corners_cpu', # bad expected data, needs test fix
'^test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric_cpu', # bad expected data, needs test fix
'^test_maxunpool_export_with_output_shape_cpu', # Invalid output in ONNX test. See https://github.com/onnx/onnx/issues/2398'
]
# Example of how to disable tests for a specific provider.
# if c2.supports_device('NGRAPH'):
# current_failing_tests.append('^test_operator_repeat_dim_overflow_cpu')
if c2.supports_device('NGRAPH'):
current_failing_tests += [
'^test_clip.*', '^test_qlinearconv_cpu',
'^test_depthtospace_crd.*', '^test_argmax_negative_axis.*',
'^test_argmin_negative_axis.*',
'^test_hardmax_negative_axis.*',
'^test_gemm_default_no_bias_cpu',
'^test_flatten_negative_axis.*',
'^test_reduce_[a-z1-9_]*_negative_axes_.*',
'test_squeeze_negative_axes_cpu',
'test_unsqueeze_negative_axes_cpu', 'test_constant_pad_cpu',
'test_edge_pad_cpu', 'test_reflect_pad_cpu'
]
if c2.supports_device('MKL-DNN'):
current_failing_tests += [
'^test_range_float_type_positive_delta_expanded_cpu',
'^test_range_int32_type_negative_delta_expanded_cpu'
]
if c2.supports_device('OPENVINO_GPU_FP32') or c2.supports_device(
'OPENVINO_GPU_FP16'):
current_failing_tests.append('^test_div_cpu*')
if c2.supports_device('OPENVINO_CPU_FP32'):
current_failing_tests += [
'^test_scan9_sum_cpu', #sum_out output node not defined, temporarily disabling test
'^test_scan_sum_cpu'
] #sum_out output node not defined, temporarily disabling test
filters = current_failing_tests + \
tests_with_pre_opset7_dependencies_filters() + \
unsupported_usages_filters() + \
other_tests_failing_permanently_filters() + \
test_with_types_disabled_due_to_binary_size_concerns_filters()
backend_test.exclude('(' + '|'.join(filters) + ')')
print('excluded tests:', filters)
# import all test cases at global scope to make
# them visible to python.unittest.
globals().update(backend_test.enable_report().test_cases)
return backend_test
def create_backend_test(testname=None):
backend_test = OrtBackendTest(c2, __name__)
# Type not supported
backend_test.exclude(r'(FLOAT16)')
if testname:
backend_test.include(testname + '.*')
else:
# Tests that are failing temporarily and should be fixed
current_failing_tests = [
'^test_adagrad_cpu',
'^test_adagrad_multiple_cpu',
'^test_batchnorm_epsilon_old_cpu',
'^test_batchnorm_epsilon_training_mode_cpu',
'^test_batchnorm_example_old_cpu',
'^test_batchnorm_example_training_mode_cpu',
'^test_celu_cpu',
'^test_dropout_default_cpu',
'^test_dropout_random_cpu',
'^test_einsum_batch_diagonal_cpu',
'^test_einsum_batch_matmul_cpu',
'^test_einsum_inner_prod_cpu',
'^test_einsum_sum_cpu',
'^test_einsum_transpose_cpu',
'^test_gathernd_example_int32_batch_dim1_cpu',
'^test_inverse_batched_cpu',
'^test_inverse_cpu',
'^test_max_int16_cpu',
'^test_max_int8_cpu',
'^test_max_uint16_cpu',
'^test_max_uint8_cpu',
'^test_mean_square_distance_mean_3d_cpu',
'^test_mean_square_distance_mean_3d_expanded_cpu',
'^test_mean_square_distance_mean_4d_cpu',
'^test_mean_square_distance_mean_4d_expanded_cpu',
'^test_mean_square_distance_mean_cpu',
'^test_mean_square_distance_mean_expanded_cpu',
'^test_mean_square_distance_none_cpu',
'^test_mean_square_distance_none_expanded_cpu',
'^test_mean_square_distance_none_weights_cpu',
'^test_mean_square_distance_none_weights_expanded_cpu',
'^test_mean_square_distance_sum_cpu',
'^test_mean_square_distance_sum_expanded_cpu',
'^test_min_int16_cpu',
'^test_min_int8_cpu',
'^test_min_uint16_cpu',
'^test_min_uint8_cpu',
'^test_momentum_cpu',
'^test_momentum_multiple_cpu',
'^test_negative_log_likelihood_loss_input_shape_is_NC_cpu',
'^test_negative_log_likelihood_loss_input_shape_is_NCd1d2_cpu',
'^test_negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_mean_cpu',
'^test_negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_sum_cpu',
'^test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_cpu',
'^test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_mean_cpu',
'^test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum_cpu',
'^test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum_ignore_index_cpu',
'^test_nesterov_momentum_cpu',
'^test_pow_bcast_array_cpu',
'^test_pow_bcast_scalar_cpu',
'^test_pow_cpu',
'^test_pow_example_cpu',
'^test_pow_types_float32_int32_cpu',
'^test_pow_types_float32_int64_cpu',
'^test_pow_types_float32_uint32_cpu',
'^test_pow_types_float32_uint64_cpu',
'^test_pow_types_float_cpu',
'^test_pow_types_int32_float32_cpu',
'^test_pow_types_int32_int32_cpu',
'^test_pow_types_int64_float32_cpu',
'^test_pow_types_int64_int64_cpu',
'^test_pow_types_int_cpu',
'^test_softmax_cross_entropy_mean_3d_cpu',
'^test_softmax_cross_entropy_mean_3d_expanded_cpu',
'^test_softmax_cross_entropy_mean_cpu',
'^test_softmax_cross_entropy_mean_expanded_cpu',
'^test_softmax_cross_entropy_mean_weight_cpu',
'^test_softmax_cross_entropy_mean_weight_expanded_cpu',
'^test_softmax_cross_entropy_mean_weight_ignore_index_cpu',
'^test_softmax_cross_entropy_mean_weight_ignore_index_expanded_cpu',
'^test_softmax_cross_entropy_none_cpu',
'^test_softmax_cross_entropy_none_expanded_cpu',
'^test_softmax_cross_entropy_none_weights_cpu',
'^test_softmax_cross_entropy_none_weights_expanded_cpu',
'^test_softmax_cross_entropy_sum_cpu',
'^test_softmax_cross_entropy_sum_expanded_cpu',
'^test_unfoldtodepth_with_padding_cpu',
'^test_unfoldtodepth_with_padding_stride_cpu',
'^test_unfoldtodepth_without_padding_cpu',
'^test_gradient_of_add_and_mul_cpu',
'^test_gradient_of_add_cpu',
'^test_batchnorm_example_training_mode_cpu',
'^test_batchnorm_epsilon_training_mode_cpu',
'^test_maxunpool_export_with_output_shape_cpu', #result mismatch
'^test_resize_downsample_scales_cubic_align_corners_cpu', # results mismatch with onnx tests
'^test_resize_downsample_scales_linear_align_corners_cpu' # results mismatch with onnx tests
]
if platform.architecture()[0] == '32bit':
current_failing_tests += [
'^test_vgg19', '^test_zfnet512', '^test_bvlc_alexnet_cpu'
]
# Example of how to disable tests for a specific provider.
# if c2.supports_device('NGRAPH'):
# current_failing_tests.append('^test_operator_repeat_dim_overflow_cpu')
if c2.supports_device('NGRAPH'):
current_failing_tests += [
'^test_clip.*', '^test_qlinearconv_cpu',
'^test_depthtospace_crd.*', '^test_argmax_negative_axis.*',
'^test_argmin_negative_axis.*',
'^test_hardmax_negative_axis.*',
'^test_gemm_default_no_bias_cpu',
'^test_flatten_negative_axis.*',
'^test_reduce_[a-z1-9_]*_negative_axes_.*',
'test_squeeze_negative_axes_cpu',
'test_unsqueeze_negative_axes_cpu', 'test_constant_pad_cpu',
'test_edge_pad_cpu', 'test_reflect_pad_cpu',
'^test_split_zero_size_splits_.*',
'^test_argmax_keepdims_example_select_last_index_.*',
'^test_argmax_no_keepdims_example_select_last_index_.*',
'^test_argmin_no_keepdims_example_select_last_index_.*',
'^test_argmin_keepdims_example_select_last_index_.*'
]
if c2.supports_device('DNNL'):
current_failing_tests += [
'^test_range_float_type_positive_delta_expanded_cpu',
'^test_range_int32_type_negative_delta_expanded_cpu',
'^test_averagepool_2d_ceil_cpu', '^test_maxpool_2d_ceil_cpu',
'^test_maxpool_2d_dilations_cpu', '^test_maxpool_2d_uint8'
]
if c2.supports_device('NNAPI'):
current_failing_tests += ['^test_maxpool_2d_uint8']
if c2.supports_device('OPENVINO_GPU_FP32') or c2.supports_device(
'OPENVINO_GPU_FP16'):
current_failing_tests.append('^test_div_cpu')
# temporarily exclude vgg19 test which comsumes too much memory, run out of memory on Upsquared device.
# single test pass for vgg19, need furture investigation
current_failing_tests.append('^test_vgg19_cpu')
if c2.supports_device('OPENVINO_CPU_FP32'):
current_failing_tests += [
'^test_operator_permute2_cpu', '^test_operator_repeat_cpu',
'^test_operator_repeat_dim_overflow_cpu'
]
if c2.supports_device('OPENVINO_GPU_FP32'):
current_failing_tests += [
'^test_operator_permute2_cpu', '^test_operator_repeat_cpu',
'^test_operator_repeat_dim_overflow_cpu',
'^test_add_bcast_cpu', '^test_batchnorm_epsilon_cpu',
'^test_div_bcast_cpu', '^test_mul_bcast_cpu',
'^test_pow_bcast_array_cpu', '^test_sub_bcast_cpu',
'^test_batchnorm_example_cpu',
'^test_clip_default_inbounds_cpu',
'^test_resize_upsample_sizes_nearest_ceil_half_pixel_cpu',
'^test_resize_upsample_sizes_nearest_floor_align_corners_cpu',
'^test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric_cpu',
'^test_unique_not_sorted_without_axis_cpu'
]
filters = current_failing_tests + \
tests_with_pre_opset7_dependencies_filters() + \
unsupported_usages_filters() + \
other_tests_failing_permanently_filters() + \
test_with_types_disabled_due_to_binary_size_concerns_filters()
backend_test.exclude('(' + '|'.join(filters) + ')')
print('excluded tests:', filters)
# import all test cases at global scope to make
# them visible to python.unittest.
globals().update(backend_test.enable_report().test_cases)
return backend_test
def create_backend_test(testname=None):
backend_test = OrtBackendTest(c2, __name__)
# Type not supported
backend_test.exclude(r'(FLOAT16)')
if testname:
backend_test.include(testname + '.*')
else:
# read filters data
with open(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
'testdata',
'onnx_backend_test_series_filters.jsonc')) as f:
filters_lines = f.readlines()
filters_lines = [x.split('//')[0] for x in filters_lines]
filters = json.loads('\n'.join(filters_lines))
current_failing_tests = filters['current_failing_tests']
if platform.architecture()[0] == '32bit':
current_failing_tests += filters['current_failing_tests_x86']
if c2.supports_device('NGRAPH'):
current_failing_tests += filters['current_failing_tests_NGRAPH']
if c2.supports_device('DNNL'):
current_failing_tests += filters['current_failing_tests_DNNL']
if c2.supports_device('NNAPI'):
current_failing_tests += filters['current_failing_tests_NNAPI']
if c2.supports_device('OPENVINO_GPU_FP32') or c2.supports_device(
'OPENVINO_GPU_FP16'):
current_failing_tests += filters[
'current_failing_tests_OPENVINO_GPU']
if c2.supports_device('OPENVINO_MYRIAD'):
current_failing_tests += filters[
'current_failing_tests_OPENVINO_GPU']
current_failing_tests += filters[
'current_failing_tests_OPENVINO_MYRIAD']
if c2.supports_device('OPENVINO_CPU_FP32'):
current_failing_tests += filters[
'current_failing_tests_OPENVINO_CPU_FP32']
if c2.supports_device('MIGRAPHX'):
current_failing_tests += [
'^test_constant_pad_cpu', '^test_softmax_axis_1_cpu',
'^test_softmax_axis_0_cpu', '^test_softmax_default_axis_cpu',
'^test_round_cpu', '^test_lrn_default_cpu', '^test_lrn_cpu',
'^test_logsoftmax_axis_0_cpu', '^test_logsoftmax_axis_1_cpu',
'^test_logsoftmax_default_axis_cpu',
'^test_dynamicquantizelinear_expanded_cpu',
'^test_dynamicquantizelinear_max_adjusted_cpu',
'^test_dynamicquantizelinear_max_adjusted_expanded_cpu',
'^test_dynamicquantizelinear_min_adjusted_cpu',
'^test_dynamicquantizelinear_min_adjusted_expanded_cpu',
'^test_range_float_type_positive_delta_expanded_cpu',
'^test_range_int32_type_negative_delta_expanded_cpu',
'^test_operator_symbolic_override_nested_cpu',
'^test_negative_log_likelihood_loss',
'^test_softmax_cross_entropy', '^test_greater_equal',
'^test_less_equal'
]
# Skip these tests for a "pure" DML onnxruntime python wheel. We keep these tests enabled for instances where both DML and CUDA
# EPs are available (Windows GPU CI pipeline has this config) - these test will pass because CUDA has higher precendence than DML
# and the nodes are assigned to only the CUDA EP (which supports these tests)
if c2.supports_device('DML') and not c2.supports_device('GPU'):
current_failing_tests += [
'^test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_cpu',
'^test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_expanded_cpu',
'^test_softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_cpu',
'^test_softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_expanded_cpu',
'^test_softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_log_prob_cpu',
'^test_softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_log_prob_expanded_cpu',
'^test_asin_example_cpu',
'^test_dynamicquantizelinear_expanded_cpu',
'^test_resize_downsample_scales_linear_cpu',
'^test_resize_downsample_sizes_linear_pytorch_half_pixel_cpu',
'^test_resize_downsample_sizes_nearest_cpu',
'^test_resize_upsample_sizes_nearest_cpu', '^test_roialign_cpu'
]
filters = current_failing_tests + \
filters['tests_with_pre_opset7_dependencies'] + \
filters['unsupported_usages'] + \
filters['failing_permanently'] + \
filters['test_with_types_disabled_due_to_binary_size_concerns']
backend_test.exclude('(' + '|'.join(filters) + ')')
print('excluded tests:', filters)
# import all test cases at global scope to make
# them visible to python.unittest.
globals().update(backend_test.enable_report().test_cases)
return backend_test
def create_backend_test(testname=None):
backend_test = OrtBackendTest(c2, __name__)
# Type not supported
backend_test.exclude(r"(FLOAT16)")
if testname:
backend_test.include(testname + ".*")
else:
# read filters data
with open(
os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"testdata",
"onnx_backend_test_series_filters.jsonc",
)) as f:
filters_lines = f.readlines()
filters_lines = [x.split("//")[0] for x in filters_lines]
filters = json.loads("\n".join(filters_lines))
current_failing_tests = filters["current_failing_tests"]
if platform.architecture()[0] == "32bit":
current_failing_tests += filters["current_failing_tests_x86"]
if c2.supports_device("DNNL"):
current_failing_tests += filters["current_failing_tests_DNNL"]
if c2.supports_device("NNAPI"):
current_failing_tests += filters["current_failing_tests_NNAPI"]
if c2.supports_device("OPENVINO_GPU_FP32") or c2.supports_device(
"OPENVINO_GPU_FP16"):
current_failing_tests += filters[
"current_failing_tests_OPENVINO_GPU"]
if c2.supports_device("OPENVINO_MYRIAD"):
current_failing_tests += filters[
"current_failing_tests_OPENVINO_GPU"]
current_failing_tests += filters[
"current_failing_tests_OPENVINO_MYRIAD"]
if c2.supports_device("OPENVINO_CPU_FP32"):
current_failing_tests += filters[
"current_failing_tests_OPENVINO_CPU_FP32"]
if c2.supports_device("MIGRAPHX"):
current_failing_tests += [
"^test_constant_pad_cpu",
"^test_round_cpu",
"^test_lrn_default_cpu",
"^test_lrn_cpu",
"^test_dynamicquantizelinear_expanded_cpu",
"^test_dynamicquantizelinear_max_adjusted_cpu",
"^test_dynamicquantizelinear_max_adjusted_expanded_cpu",
"^test_dynamicquantizelinear_min_adjusted_cpu",
"^test_dynamicquantizelinear_min_adjusted_expanded_cpu",
"^test_range_float_type_positive_delta_expanded_cpu",
"^test_range_int32_type_negative_delta_expanded_cpu",
"^test_operator_symbolic_override_nested_cpu",
"^test_negative_log_likelihood_loss",
"^test_softmax_cross_entropy",
"^test_greater_equal",
"^test_if_seq_cpu",
"^test_loop11_cpu",
"^test_loop13_seq_cpu",
"^test_sequence_insert_at_back_cpu",
"^test_sequence_insert_at_front_cpu",
"^test_nonmaxsuppression_two_classes_cpu",
"^test_nonmaxsuppression_two_batches_cpu",
"^test_nonmaxsuppression_suppress_by_IOU_cpu",
"^test_nonmaxsuppression_suppress_by_IOU_and_scores_cpu",
"^test_nonmaxsuppression_limit_output_size_cpu",
"^test_nonmaxsuppression_identical_boxes_cpu",
"^test_nonmaxsuppression_flipped_coordinates_cpu",
"^test_nonmaxsuppression_center_point_box_format_cpu",
]
# Skip these tests for a "pure" DML onnxruntime python wheel. We keep these tests enabled for instances where both DML and CUDA
# EPs are available (Windows GPU CI pipeline has this config) - these test will pass because CUDA has higher precendence than DML
# and the nodes are assigned to only the CUDA EP (which supports these tests)
if c2.supports_device("DML") and not c2.supports_device("GPU"):
current_failing_tests += [
"^test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_cpu",
"^test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_expanded_cpu",
"^test_softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_cpu",
"^test_softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_expanded_cpu",
"^test_softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_log_prob_cpu",
"^test_softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_log_prob_expanded_cpu",
"^test_asin_example_cpu",
"^test_dynamicquantizelinear_cpu",
"^test_dynamicquantizelinear_expanded_cpu",
"^test_resize_downsample_scales_linear_cpu",
"^test_resize_downsample_sizes_linear_pytorch_half_pixel_cpu",
"^test_resize_downsample_sizes_nearest_cpu",
"^test_resize_upsample_sizes_nearest_cpu",
"^test_roialign_cpu",
]
filters = (
current_failing_tests +
filters["tests_with_pre_opset7_dependencies"] +
filters["unsupported_usages"] + filters["failing_permanently"] +
filters["test_with_types_disabled_due_to_binary_size_concerns"])
backend_test.exclude("(" + "|".join(filters) + ")")
print("excluded tests:", filters)
# exclude TRT EP temporarily and only test CUDA EP to retain previous behavior
os.environ[
"ORT_ONNX_BACKEND_EXCLUDE_PROVIDERS"] = "TensorrtExecutionProvider"
# import all test cases at global scope to make
# them visible to python.unittest.
globals().update(backend_test.enable_report().test_cases)
return backend_test
def create_backend_test(testname=None):
backend_test = OrtBackendTest(c2, __name__)
# Type not supported
backend_test.exclude(r'(FLOAT16)')
if testname:
backend_test.include(testname + '.*')
else:
# Tests that are failing temporarily and should be fixed
current_failing_tests = (
'^test_cast_STRING_to_FLOAT_cpu.*',
'^test_cast_FLOAT_to_STRING_cpu.*',
'^test_qlinearconv_cpu.*',
'^test_gru_seq_length_cpu.*',
'^test_bitshift_right_uint16_cpu.*',
'^test_bitshift_right_uint32_cpu.*',
'^test_bitshift_right_uint64_cpu.*',
'^test_bitshift_right_uint8_cpu.*',
'^test_bitshift_left_uint16_cpu.*',
'^test_bitshift_left_uint32_cpu.*',
'^test_bitshift_left_uint64_cpu.*',
'^test_bitshift_left_uint8_cpu.*',
'^test_round_cpu.*',
'^test_cumsum_1d_cpu.*',
'^test_cumsum_1d_exclusive_cpu.*',
'^test_cumsum_1d_reverse_cpu.*',
'^test_cumsum_1d_reverse_exclusive_cpu.*',
'^test_cumsum_2d_axis_0_cpu.*',
'^test_cumsum_2d_axis_1_cpu.*',
'^test_cumsum_2d_negative_axis_cpu.*',
'^test_dynamicquantizelinear_expanded*',
'^test_dynamicquantizelinear_max_adjusted_expanded*',
'^test_dynamicquantizelinear_min_adjusted_expanded*',
'^test_gather_elements*',
'^test_scatter_elements*',
'^test_top_k*',
'^test_unique_*',
'^test_mod_float_mixed_sign_example_cpu.*', #onnxruntime::Mod::Compute fmod_ was false. fmod attribute must be true for float, float16 and double types
'^test_shrink_cpu.*', #Invalid rank for input: x Got: 1 Expected: 2 Please fix either the inputs or the model.
'^test_range_float_type_positive_delta_cpu.*',
'^test_range_float_type_positive_delta_expanded_cpu.*',
'^test_range_int32_type_negative_delta_cpu.*',
'^test_range_int32_type_negative_delta_expanded_cpu.*',
'^test_det_2d_cpu.*',
'^test_det_nd_cpu.*',
'^test_gathernd_example_float32_cpu.*',
'^test_gathernd_example_int32_cpu.*',
'^test_resize_downsample_scales_cubic_A_n0p5_exclude_outside_cpu.*',
'^test_resize_downsample_scales_cubic_align_corners_cpu.*',
'^test_resize_downsample_scales_cubic_cpu.*',
'^test_resize_downsample_scales_linear_align_corners_cpu.*',
'^test_resize_downsample_scales_linear_cpu.*',
'^test_resize_downsample_scales_nearest_cpu.*',
'^test_resize_downsample_sizes_cubic_cpu.*',
'^test_resize_downsample_sizes_linear_pytorch_half_pixel_cpu.*',
'^test_resize_downsample_sizes_nearest_cpu.*',
'^test_resize_downsample_sizes_nearest_tf_half_pixel_for_nn_cpu.*',
'^test_resize_tf_crop_and_resize_cpu.*',
'^test_resize_upsample_scales_cubic_A_n0p5_exclude_outside_cpu.*',
'^test_resize_upsample_scales_cubic_align_corners_cpu.*',
'^test_resize_upsample_scales_cubic_asymmetric_cpu.*',
'^test_resize_upsample_scales_cubic_cpu.*',
'^test_resize_upsample_scales_linear_align_corners_cpu.*',
'^test_resize_upsample_scales_linear_cpu.*',
'^test_resize_upsample_scales_nearest_cpu.*',
'^test_resize_upsample_sizes_cubic_cpu.*',
'^test_resize_upsample_sizes_nearest_ceil_half_pixel_cpu.*',
'^test_resize_upsample_sizes_nearest_cpu.*',
'^test_resize_upsample_sizes_nearest_floor_align_corners_cpu.*',
'^test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric_cpu.*',
'^test_scatternd_cpu.*',
'^test_sequence_*',
'^test_unsqueeze_*',
'^test_squeeze_*',
'^test_slice_*',
'^test_scatter_*',
'^test_reduce_*',
'^test_onehot_*',
'^test_flatten_*',
'^test_concat_*',
'^test_compress_*',
'^test_constant_pad_cpu.*',
'^test_gemm_default_scalar_bias_cpu.*',
'^test_gather_negative_indices_cpu.*',
'^test_gemm_*',
'^test_edge_pad_cpu.*',
'^test_reflect_pad_cpu.*')
# Example of how to disable tests for a specific provider.
# if c2.supports_device('NGRAPH'):
# current_failing_tests = current_failing_tests + ('|^test_operator_repeat_dim_overflow_cpu.*',)
if c2.supports_device('NGRAPH'):
current_failing_tests = current_failing_tests + ('|^test_clip*', )
current_failing_tests = current_failing_tests + (
'|^test_depthtospace_crd*', )
current_failing_tests = current_failing_tests + (
'|^test_argmax_negative_axis*', )
current_failing_tests = current_failing_tests + (
'|^test_argmin_negative_axis*', )
current_failing_tests = current_failing_tests + (
'|^test_hadmax_negative_axis*', )
current_failing_tests = current_failing_tests + (
'|^test_gemm_default_no_bias_cpu.*', )
if c2.supports_device('OPENVINO_GPU_FP32') or c2.supports_device(
'OPENVINO_GPU_FP16'):
current_failing_tests = current_failing_tests + (
'^test_div_cpu*', )
filters = current_failing_tests + \
tests_with_pre_opset7_dependencies_filters() + \
unsupported_usages_filters()
backend_test.exclude('(' + '|'.join(filters) + ')')
print('excluded tests:', filters)
# import all test cases at global scope to make
# them visible to python.unittest.
globals().update(backend_test.enable_report().test_cases)
return backend_test
def create_backend_test(testname=None):
backend_test = OrtBackendTest(c2, __name__)
# Type not supported
backend_test.exclude(r'(FLOAT16)')
if testname:
backend_test.include(testname + '.*')
else:
# read filters data
with open(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
'testdata',
'onnx_backend_test_series_filters.jsonc')) as f:
filters_lines = f.readlines()
filters_lines = [x.split('//')[0] for x in filters_lines]
filters = json.loads('\n'.join(filters_lines))
current_failing_tests = filters['current_failing_tests']
if platform.architecture()[0] == '32bit':
current_failing_tests += filters['current_failing_tests_x86']
if c2.supports_device('NGRAPH'):
current_failing_tests += filters['current_failing_tests_NGRAPH']
if c2.supports_device('DNNL'):
current_failing_tests += filters['current_failing_tests_DNNL']
if c2.supports_device('NNAPI'):
current_failing_tests += filters['current_failing_tests_NNAPI']
if c2.supports_device('OPENVINO_GPU_FP32') or c2.supports_device(
'OPENVINO_GPU_FP16'):
current_failing_tests += filters[
'current_failing_tests_OPENVINO_GPU']
if c2.supports_device('OPENVINO_GPU_FP32'):
current_failing_tests += filters[
'current_failing_tests_OPENVINO_GPU_FP32']
if c2.supports_device('OPENVINO_CPU_FP32'):
current_failing_tests += filters[
'current_failing_tests_OPENVINO_CPU_FP32']
if c2.supports_device('MIGRAPHX'):
current_failing_tests += [
'^test_constant_pad_cpu', '^test_softmax_axis_1_cpu',
'^test_softmax_axis_0_cpu', '^test_softmax_default_axis_cpu',
'^test_round_cpu', '^test_lrn_default_cpu', '^test_lrn_cpu',
'^test_logsoftmax_axis_0_cpu', '^test_logsoftmax_axis_1_cpu',
'^test_logsoftmax_default_axis_cpu',
'^test_dynamicquantizelinear_expanded_cpu',
'^test_dynamicquantizelinear_max_adjusted_cpu',
'^test_dynamicquantizelinear_max_adjusted_expanded_cpu',
'^test_dynamicquantizelinear_min_adjusted_cpu',
'^test_dynamicquantizelinear_min_adjusted_expanded_cpu',
'^test_range_float_type_positive_delta_expanded_cpu',
'^test_range_int32_type_negative_delta_expanded_cpu',
'^test_operator_symbolic_override_nested_cpu'
]
filters = current_failing_tests + \
filters['tests_with_pre_opset7_dependencies'] + \
filters['unsupported_usages'] + \
filters['failing_permanently'] + \
filters['test_with_types_disabled_due_to_binary_size_concerns']
backend_test.exclude('(' + '|'.join(filters) + ')')
print('excluded tests:', filters)
# import all test cases at global scope to make
# them visible to python.unittest.
globals().update(backend_test.enable_report().test_cases)
return backend_test
def create_backend_test(testname=None):
backend_test = OrtBackendTest(c2, __name__)
# Type not supported
backend_test.exclude(r'(FLOAT16)')
if testname:
backend_test.include(testname + '.*')
else:
# Tests that are failing temporarily and should be fixed
current_failing_tests = [#'^test_cast_STRING_to_FLOAT_cpu', # old test data that is bad on Linux CI builds
'^test_qlinearconv_cpu',
'^test_gru_seq_length_cpu',
'^test_dynamicquantizelinear_expanded.*',
'^test_dynamicquantizelinear_max_adjusted_expanded.*',
'^test_dynamicquantizelinear_min_adjusted_expanded.*',
'^test_top_k.*',
'^test_unique_not_sorted_without_axis_cpu', # bad expected data. enable after https://github.com/onnx/onnx/pull/2381 is picked up
'^test_mod_float_mixed_sign_example_cpu', #onnxruntime::Mod::Compute fmod_ was false. fmod attribute must be true for float, float16 and double types
'^test_shrink_cpu', #Invalid rank for input: x Got: 1 Expected: 2 Please fix either the inputs or the model.
'^test_range_float_type_positive_delta_cpu',
'^test_range_float_type_positive_delta_expanded_cpu',
'^test_range_int32_type_negative_delta_cpu',
'^test_range_int32_type_negative_delta_expanded_cpu',
'^test_det_2d_cpu',
'^test_det_nd_cpu',
'^test_resize_downsample_scales_cubic_A_n0p5_exclude_outside_cpu',
'^test_resize_downsample_scales_cubic_align_corners_cpu',
'^test_resize_downsample_scales_cubic_cpu',
'^test_resize_downsample_scales_linear_align_corners_cpu',
'^test_resize_downsample_scales_linear_cpu',
'^test_resize_downsample_scales_nearest_cpu',
'^test_resize_downsample_sizes_cubic_cpu',
'^test_resize_downsample_sizes_linear_pytorch_half_pixel_cpu',
'^test_resize_downsample_sizes_nearest_cpu',
'^test_resize_downsample_sizes_nearest_tf_half_pixel_for_nn_cpu',
'^test_resize_tf_crop_and_resize_cpu',
'^test_resize_upsample_scales_cubic_A_n0p5_exclude_outside_cpu',
'^test_resize_upsample_scales_cubic_align_corners_cpu',
'^test_resize_upsample_scales_cubic_asymmetric_cpu',
'^test_resize_upsample_scales_cubic_cpu',
'^test_resize_upsample_scales_linear_align_corners_cpu',
'^test_resize_upsample_scales_linear_cpu',
'^test_resize_upsample_scales_nearest_cpu',
'^test_resize_upsample_sizes_cubic_cpu',
'^test_resize_upsample_sizes_nearest_ceil_half_pixel_cpu',
'^test_resize_upsample_sizes_nearest_cpu',
'^test_resize_upsample_sizes_nearest_floor_align_corners_cpu',
'^test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric_cpu',
'^test_sequence_*',
'^test_scatter_.*',
'^test_onehot_.*',
'^test_edge_pad_cpu.*', # test data type `int32_t` not supported yet, the `float` equivalent is covered via unit tests
'^test_reflect_pad_cpu.*' # test data type `int32_t` not supported yet, the `float` equivalent is covered via unit tests
]
# Example of how to disable tests for a specific provider.
# if c2.supports_device('NGRAPH'):
# current_failing_tests.append('^test_operator_repeat_dim_overflow_cpu')
if c2.supports_device('NGRAPH'):
current_failing_tests += ['^test_clip.*',
'^test_depthtospace_crd.*',
'^test_argmax_negative_axis.*',
'^test_argmin_negative_axis.*',
'^test_hardmax_negative_axis.*',
'^test_gemm_default_no_bias_cpu',
'^test_flatten_negative_axis.*',
'^test_reduce_[a-z1-9_]*_negative_axes_.*',
'test_squeeze_negative_axes_cpu',
'test_unsqueeze_negative_axes_cpu',
'test_constant_pad_cpu']
if c2.supports_device('OPENVINO_GPU_FP32') or c2.supports_device('OPENVINO_GPU_FP16'):
current_failing_tests.append('^test_div_cpu*')
filters = current_failing_tests + \
tests_with_pre_opset7_dependencies_filters() + \
unsupported_usages_filters() + \
other_tests_failing_permanently_filters() + \
test_with_types_disabled_due_to_binary_size_concerns_filters()
backend_test.exclude('(' + '|'.join(filters) + ')')
print('excluded tests:', filters)
# import all test cases at global scope to make
# them visible to python.unittest.
globals().update(backend_test.enable_report().test_cases)
return backend_test
def create_backend_test(test_name=None):
"""Creates an OrtBackendTest and adds its TestCase's to global scope so unittest will find them."""
overrides = load_jsonc("onnx_backend_test_series_overrides.jsonc")
rtol_default = overrides["rtol_default"]
atol_default = overrides["atol_default"]
rtol_overrides = collections.defaultdict(lambda: rtol_default)
rtol_overrides.update(overrides["rtol_overrides"])
atol_overrides = collections.defaultdict(lambda: atol_default)
atol_overrides.update(overrides["atol_overrides"])
backend_test = OrtBackendTest(rtol_overrides, atol_overrides)
# Type not supported
backend_test.exclude(r"(FLOAT16)")
if test_name:
backend_test.include(test_name + ".*")
else:
filters = load_jsonc("onnx_backend_test_series_filters.jsonc")
current_failing_tests = apply_filters(filters, "current_failing_tests")
if platform.architecture()[0] == "32bit":
current_failing_tests += apply_filters(filters, "current_failing_tests_x86")
if backend.supports_device("DNNL"):
current_failing_tests += apply_filters(filters, "current_failing_tests_DNNL")
if backend.supports_device("NNAPI"):
current_failing_tests += apply_filters(filters, "current_failing_tests_NNAPI")
if backend.supports_device("OPENVINO_GPU_FP32") or backend.supports_device("OPENVINO_GPU_FP16"):
current_failing_tests += apply_filters(filters, "current_failing_tests_OPENVINO_GPU")
if backend.supports_device("OPENVINO_MYRIAD"):
current_failing_tests += apply_filters(filters, "current_failing_tests_OPENVINO_GPU")
current_failing_tests += apply_filters(filters, "current_failing_tests_OPENVINO_MYRIAD")
if backend.supports_device("OPENVINO_CPU_FP32"):
current_failing_tests += apply_filters(filters, "current_failing_tests_OPENVINO_CPU_FP32")
if backend.supports_device("MIGRAPHX"):
current_failing_tests += apply_filters(filters, "current_failing_tests_MIGRAPHX")
# Skip these tests for a "pure" DML onnxruntime python wheel. We keep these tests enabled for instances where both DML and CUDA
# EPs are available (Windows GPU CI pipeline has this config) - these test will pass because CUDA has higher precedence than DML
# and the nodes are assigned to only the CUDA EP (which supports these tests)
if backend.supports_device("DML") and not backend.supports_device("GPU"):
current_failing_tests += apply_filters(filters, "current_failing_tests_pure_DML")
filters = (
current_failing_tests
+ apply_filters(filters, "tests_with_pre_opset7_dependencies")
+ apply_filters(filters, "unsupported_usages")
+ apply_filters(filters, "failing_permanently")
+ apply_filters(filters, "test_with_types_disabled_due_to_binary_size_concerns")
)
backend_test.exclude("(" + "|".join(filters) + ")")
print("excluded tests:", filters)
# exclude TRT EP temporarily and only test CUDA EP to retain previous behavior
os.environ["ORT_ONNX_BACKEND_EXCLUDE_PROVIDERS"] = "TensorrtExecutionProvider"
# import all test cases at global scope to make
# them visible to python.unittest.
globals().update(backend_test.enable_report().test_cases)
