from __future__ import print_function
from __future__ import unicode_literals
from collections import defaultdict
import io
import os
import numpy as np # type: ignore
from onnx import defs, FunctionProto, helper
from onnx.defs import OpSchema, ONNX_DOMAIN, ONNX_ML_DOMAIN
from onnx.backend.test.case import collect_snippets
from onnx.backend.sample.ops import collect_sample_implementations
from typing import Any, Text, Sequence, Dict, List, NamedTuple, Set, Tuple
SNIPPETS = collect_snippets()
SAMPLE_IMPLEMENTATIONS = collect_sample_implementations()
ONNX_ML = not bool(os.getenv('ONNX_ML') == '0')
ext = '-ml.md' if ONNX_ML else '.md'
def display_number(v: int) -> Text:
if defs.OpSchema.is_infinite(v):
return '∞'
return Text(v)
def should_render_domain(domain: Text) -> bool:
if domain == ONNX_ML_DOMAIN and not ONNX_ML:
return False
def gen_node_test_coverage(schemas, f, ml):
# type: (Sequence[defs.OpSchema], IO[Any], bool) -> None
global common_covered
global experimental_covered
generators = set({
'Multinomial',
'RandomNormal',
'RandomNormalLike',
'RandomUniform',
'RandomUniformLike',
})
node_tests = collect_snippets()
common_covered = sorted([
s.name for s in schemas if s.name in node_tests and s.support_level ==
defs.OpSchema.SupportType.COMMON and (s.domain == 'ai.onnx.ml') == ml
])
common_no_cover = sorted([
s.name for s in schemas
if s.name not in node_tests and s.support_level ==
defs.OpSchema.SupportType.COMMON and (s.domain == 'ai.onnx.ml') == ml
])
common_generator = sorted(
[name for name in common_no_cover if name in generators])
experimental_covered = sorted([
s.name for s in schemas
if s.name in node_tests and s.support_level == defs.OpSchema.
SupportType.EXPERIMENTAL and (s.domain == 'ai.onnx.ml') == ml
])
experimental_no_cover = sorted([
s.name for s in schemas
if s.name not in node_tests and s.support_level == defs.OpSchema.
SupportType.EXPERIMENTAL and (s.domain == 'ai.onnx.ml') == ml
])
experimental_generator = sorted(
[name for name in experimental_no_cover if name in generators])
num_common = len(common_covered) + len(common_no_cover) \
- len(common_generator)
num_experimental = len(experimental_covered) + len(experimental_no_cover) \
- len(experimental_generator)
f.write('# Node Test Coverage\n')
f.write('## Summary\n')
if num_common:
f.write(
'Node tests have covered {}/{} ({:.2f}%, {} generators excluded) '
'common operators.\n\n'.format(
len(common_covered), num_common,
(len(common_covered) / float(num_common) * 100),
len(common_generator)))
else:
f.write('Node tests have covered 0/0 (N/A) common operators. \n\n')
if num_experimental:
f.write(
'Node tests have covered {}/{} ({:.2f}%, {} generators excluded) '
'experimental operators.\n\n'.format(
len(experimental_covered), num_experimental,
(len(experimental_covered) / float(num_experimental) * 100),
len(experimental_generator)))
else:
f.write(
'Node tests have covered 0/0 (N/A) experimental operators.\n\n')
titles = [
'💚Covered Common Operators',
'💔No Cover Common Operators',
'💚Covered Experimental Operators',
'💔No Cover Experimental Operators',
]
all_lists = [
common_covered, common_no_cover, experimental_covered,
experimental_no_cover
]
for t in titles:
f.write('* [{}](#{})\n'.format(t[9:], t[9:].lower().replace(' ', '-')))
f.write('\n')
for t, l in zip(titles, all_lists):
f.write('## {}\n'.format(t))
for s in l:
f.write('### {}'.format(s))
if s in node_tests:
f.write(
'\nThere are {} test cases, listed as following:\n'.format(
len(node_tests[s])))
for summary, code in sorted(node_tests[s]):
f.write('<details>\n')
f.write('<summary>{}</summary>\n\n'.format(summary))
f.write('```python\n{}\n```\n\n'.format(code))
f.write('</details>\n')
else:
if s in generators:
f.write(' (random generator operator)\n')
else:
f.write(' (call for test cases)\n')
f.write('\n\n')
f.write('<br/>\n\n')
#!/usr/bin/env python
from __future__ import unicode_literals
import json
import io
import sys
from onnx import defs
from onnx.defs import OpSchema
from onnx.backend.test.case import collect_snippets
snippets = collect_snippets()
categories = {
'Constant': 'Constant',
'Conv': 'Layer',
'ConvTranspose': 'Layer',
'FC': 'Layer',
'RNN': 'Layer',
'LSTM': 'Layer',
'GRU': 'Layer',
'Dropout': 'Dropout',
'Elu': 'Activation',
'HardSigmoid': 'Activation',
'LeakyRelu': 'Activation',
'PRelu': 'Activation',
'ThresholdedRelu': 'Activation',
'Relu': 'Activation',
'Selu': 'Activation',
'Sigmoid': 'Activation',
def gen_node_test_coverage(schemas, f, ml):
# type: (Sequence[defs.OpSchema], IO[Any], bool) -> None
generators = set({
'Multinomial',
'RandomNormal',
'RandomNormalLike',
'RandomUniform',
'RandomUniformLike',
})
node_tests = collect_snippets()
common_covered = sorted([s.name for s in schemas
if s.name in node_tests
and s.support_level == defs.OpSchema.SupportType.COMMON
and (s.domain == 'ai.onnx.ml') == ml])
common_no_cover = sorted([s.name for s in schemas
if s.name not in node_tests
and s.support_level == defs.OpSchema.SupportType.COMMON
and (s.domain == 'ai.onnx.ml') == ml])
common_generator = sorted([name for name in common_no_cover
if name in generators])
experimental_covered = sorted([s.name for s in schemas
if s.name in node_tests
and s.support_level == defs.OpSchema.SupportType.EXPERIMENTAL
and (s.domain == 'ai.onnx.ml') == ml])
experimental_no_cover = sorted([s.name for s in schemas
if s.name not in node_tests
and s.support_level == defs.OpSchema.SupportType.EXPERIMENTAL
and (s.domain == 'ai.onnx.ml') == ml])
experimental_generator = sorted([name for name in experimental_no_cover
if name in generators])
num_common = len(common_covered) + len(common_no_cover) \
- len(common_generator)
num_experimental = len(experimental_covered) + len(experimental_no_cover) \
- len(experimental_generator)
f.write('# Node Test Coverage\n')
f.write('## Summary\n')
if num_common:
f.write('Node tests have covered {}/{} ({:.2f}%, {} generators excluded) '
'common operators.\n\n'.format(
len(common_covered), num_common,
(len(common_covered) / float(num_common) * 100),
len(common_generator)))
else:
f.write('Node tests have covered 0/0 (N/A) common operators. \n\n')
if num_experimental:
f.write('Node tests have covered {}/{} ({:.2f}%, {} generators excluded) '
'experimental operators.\n\n'.format(
len(experimental_covered), num_experimental,
(len(experimental_covered) / float(num_experimental) * 100),
len(experimental_generator)))
else:
f.write('Node tests have covered 0/0 (N/A) experimental operators.\n\n')
titles = ['💚Covered Common Operators',
'💔No Cover Common Operators',
'💚Covered Experimental Operators',
'💔No Cover Experimental Operators',
]
all_lists = [common_covered, common_no_cover,
experimental_covered, experimental_no_cover]
for t in titles:
f.write('* [{}](#{})\n'.format(t[9:], t[9:].lower().replace(' ', '-')))
f.write('\n')
for t, l in zip(titles, all_lists):
f.write('## {}\n'.format(t))
for s in l:
f.write('### {}'.format(s))
if s in node_tests:
f.write('\nThere are {} test cases, listed as following:\n'.format(
len(node_tests[s])))
for summary, code in sorted(node_tests[s]):
f.write('<details>\n')
f.write('<summary>{}</summary>\n\n'.format(summary))
f.write('```python\n{}\n```\n\n'.format(code))
f.write('</details>\n')
else:
if s in generators:
f.write(' (random generator operator)\n')
else:
f.write(' (call for test cases)\n')
f.write('\n\n')
f.write('<br/>\n\n')