def get_opset_status():
ops = []
onnx_ops = {}
for op_schema in defs.get_all_schemas_with_history():
domain = onnx_ops.setdefault(op_schema.domain, {})
domain[op_schema.name] = op_schema.deprecated
for domain in onnx_ops:
onnx_ops[domain] = len(onnx_ops[domain])
counts_by_domain = {}
for handler in BackendHandler.__subclasses__():
handler.check_cls()
counts_by_domain.setdefault(handler.DOMAIN, [0, onnx_ops.get(handler.DOMAIN, 0)])
counts_by_domain[handler.DOMAIN][0] += 1
ops.append([
handler.DOMAIN,
handler.ONNX_OP,
handler.get_versions(),
handler.PS_DESCRIPTION
])
ops.sort(key=lambda x: x[0] + x[1])
ops.insert(0, [
'Domain', 'Op', 'Versions', 'Notes'
])
return ops, counts_by_domain
def build_operator_schemas():
# domain -> support level -> name -> [schema]
index = defaultdict(lambda: defaultdict(lambda: defaultdict(
list))) # type: Dict[Text, Dict[int, Dict[Text, List[OpSchema]]]]
for schema in defs.get_all_schemas_with_history():
index[schema.domain][int(
schema.support_level)][schema.name].append(schema)
# Preprocess the Operator Schemas
# [(domain, [(support_level, [(schema name, current schema, all versions schemas)])])]
operator_schemas = list(
) # type: List[Tuple[Text, List[Tuple[int, List[Tuple[Text, OpSchema, List[OpSchema]]]]]]]
exsting_ops = set() # type: Set[Text]
for domain, _supportmap in sorted(index.items()):
if not should_render_domain(domain):
continue
processed_supportmap = list()
for _support, _namemap in sorted(_supportmap.items()):
processed_namemap = list()
for n, unsorted_versions in sorted(_namemap.items()):
versions = sorted(unsorted_versions,
key=lambda s: s.since_version)
schema = versions[-1]
if schema.name in exsting_ops:
continue
exsting_ops.add(schema.name)
processed_namemap.append((n, schema, versions))
processed_supportmap.append((_support, processed_namemap))
operator_schemas.append((domain, processed_supportmap))
return operator_schemas
def build_operator_schemas():
# domain -> support level -> name -> [schema]
index = defaultdict(lambda: defaultdict(lambda: defaultdict(
list))) # type: Dict[Text, Dict[int, Dict[Text, List[OpSchema]]]]
for schema in defs.get_all_schemas_with_history():
index[schema.domain][int(
schema.support_level)][schema.name].append(schema)
# Preprocess the Operator Schemas
# [(domain, [(support_level, [(schema name, current schema, all versions schemas)])])]
operator_schemas = list(
) # type: List[Tuple[Text, List[Tuple[int, List[Tuple[Text, OpSchema, List[OpSchema]]]]]]]
exsting_ops = set() # type: Set[Text]
for domain, _supportmap in sorted(index.items()):
if not should_render_domain(domain):
continue
processed_supportmap = list()
for _support, _namemap in sorted(_supportmap.items()):
processed_namemap = list()
for n, unsorted_versions in sorted(_namemap.items()):
versions = sorted(unsorted_versions,
key=lambda s: s.since_version)
schema = versions[-1]
if schema.name in exsting_ops:
continue
if check_operation_version :
# Generate operation of the latest version of your onnx.
exsting_ops.add(schema.name)
processed_namemap.append((n, schema, versions))
# Add checks against version_dict
if schema.name not in version_dict :
print("Check-operation-version: Operation {} is new with version {}"
.format(schema.name, schema.since_version))
elif schema.since_version > version_dict[schema.name]:
print("Check-operation-version: Operation {}"
.format(schema.name)+
" has a newer version {} over old version {}"
.format(schema.since_version, version_dict[schema.name]))
else:
# Generate operation according to the version in version_dict.
if schema.name not in version_dict :
continue
found = False
for schema in reversed(versions):
# Check the version number against the version_dict
if schema.since_version == version_dict[schema.name]:
exsting_ops.add(schema.name)
processed_namemap.append((n, schema, versions))
found = True
break
if not found:
print("Your onnx installation may be too old. "
"The desired version for operation {} is not found.".format(
schema.name))
sys.exit()
processed_supportmap.append((_support, processed_namemap))
operator_schemas.append((domain, processed_supportmap))
return operator_schemas
def getSchemas():
schemas = [x for x in defs.get_all_schemas_with_history() if not x.deprecated]
max_version = {}
for x in schemas:
if x.name in max_version:
max_version[x.name] = max(max_version[x.name],x.since_version)
else:
max_version[x.name] = x.since_version
return [x for x in schemas if x.since_version == max_version[x.name]]
def _build_op_version(self):
res = {}
for schema in get_all_schemas_with_history():
dom = schema.domain
name = schema.name
vers = schema.since_version
if (dom, name) not in res:
res[dom, name] = set()
res[dom, name].add(vers)
self._op_versions = {}
for k, v in res.items():
self._op_versions[k] = list(sorted(v))
def _register_all_schemas_with_history():
"""Register all schemas with history"""
onnx_schemas = defs.get_all_schemas_with_history()
name_domain_version_schema_map = defaultdict(lambda: defaultdict(dict))
for s in onnx_schemas:
schema = OnnxOpSchema.from_onnx_schema(s)
name_domain_version_schema_map[schema.name][schema.domain][schema.since_version] = schema
ordered_map = defaultdict(lambda: defaultdict(OrderedDict))
for name, domain_version_schema_map in name_domain_version_schema_map.items():
for domain, version_schema_map in domain_version_schema_map.items():
ordered_map[name][domain] = OrderedDict(
sorted(version_schema_map.items(), key=lambda x: -x[0])
)
return ordered_map
def main(args): # type: (Type[Args]) -> None
with io.open(args.changelog, 'w', newline='') as fout:
fout.write('## Operator Changelog\n')
fout.write(
"*This file is automatically generated from the\n"
" [def files](/onnx/defs) via [this script](/onnx/defs/gen_doc.py).\n"
" Do not modify directly and instead edit operator definitions.*\n"
)
# domain -> version -> [schema]
dv_index = defaultdict(lambda: defaultdict(list)
) # type: Dict[Text, Dict[int, List[OpSchema]]]
for schema in defs.get_all_schemas_with_history():
dv_index[schema.domain][schema.since_version].append(schema)
fout.write('\n')
for domain, versionmap in sorted(dv_index.items()):
if not should_render_domain(domain):
continue
if domain:
s = '# {}\n'.format(domain)
domain_prefix = '{}.'.format(domain)
else:
s = '# ai.onnx (default)\n'
domain_prefix = ''
for version, unsorted_schemas in sorted(versionmap.items()):
s += '## Version {} of {}\n'.format(version,
display_domain(domain))
for schema in sorted(unsorted_schemas, key=lambda s: s.name):
name_with_ver = '{}-{}'.format(domain_prefix + schema.name,
schema.since_version)
s += '### <a name="{}"></a>**{}**</a>\n'.format(
name_with_ver, name_with_ver)
s += display_schema(schema, [schema])
s += '\n'
fout.write(s)
with io.open(args.output, 'w', newline='') as fout:
fout.write('## Operator Schemas\n')
fout.write(
"*This file is automatically generated from the\n"
" [def files](/onnx/defs) via [this script](/onnx/defs/gen_doc.py).\n"
" Do not modify directly and instead edit operator definitions.*\n"
)
# domain -> support level -> name -> [schema]
index = defaultdict(lambda: defaultdict(lambda: defaultdict(
list))) # type: Dict[Text, Dict[int, Dict[Text, List[OpSchema]]]]
for schema in defs.get_all_schemas_with_history():
index[schema.domain][int(
schema.support_level)][schema.name].append(schema)
fout.write('\n')
# Table of contents
for domain, supportmap in sorted(index.items()):
if not should_render_domain(domain):
continue
if domain:
s = '* {}\n'.format(domain)
domain_prefix = '{}.'.format(domain)
else:
s = '* ai.onnx (default)\n'
domain_prefix = ''
fout.write(s)
for _, namemap in sorted(supportmap.items()):
for n, unsorted_versions in sorted(namemap.items()):
versions = sorted(unsorted_versions,
key=lambda s: s.since_version)
schema = versions[-1]
s = ' * {}<a href="#{}">{}</a>\n'.format(
support_level_str(schema.support_level),
domain_prefix + n, domain_prefix + n)
fout.write(s)
fout.write('\n')
for domain, supportmap in sorted(index.items()):
if not should_render_domain(domain):
continue
if domain:
s = '## {}\n'.format(domain)
domain_prefix = '{}.'.format(domain)
else:
s = '## ai.onnx (default)\n'
domain_prefix = ''
fout.write(s)
for _support, namemap in sorted(supportmap.items()):
for op_type, unsorted_versions in sorted(namemap.items()):
versions = sorted(unsorted_versions,
key=lambda s: s.since_version)
schema = versions[-1]
# op_type
s = '### {}<a name="{}"></a><a name="{}">**{}**</a>\n'.format(
support_level_str(schema.support_level),
domain_prefix + op_type,
domain_prefix + op_type.lower(),
domain_prefix + op_type)
s += display_schema(schema, versions)
s += '\n\n'
if op_type in SNIPPETS:
s += '#### Examples\n\n'
for summary, code in sorted(SNIPPETS[op_type]):
s += '<details>\n'
s += '<summary>{}</summary>\n\n'.format(summary)
s += '```python\n{}\n```\n\n'.format(code)
s += '</details>\n'
s += '\n\n'
fout.write(s)
def main(args): # type: (Type[Args]) -> None
with io.open(args.changelog, 'w', newline='') as fout:
fout.write('## Operator Changelog\n')
fout.write(
"*This file is automatically generated from the\n"
" [def files](/onnx/defs) via [this script](/onnx/defs/gen_doc.py).\n"
" Do not modify directly and instead edit operator definitions.*\n"
)
# domain -> version -> [schema]
dv_index = defaultdict(lambda: defaultdict(list)
) # type: Dict[Text, Dict[int, List[OpSchema]]]
for schema in defs.get_all_schemas_with_history():
dv_index[schema.domain][schema.since_version].append(schema)
fout.write('\n')
for domain, versionmap in sorted(dv_index.items()):
if not should_render_domain(domain):
continue
s = '# {}\n'.format(display_domain_short(domain))
for version, unsorted_schemas in sorted(versionmap.items()):
s += '## Version {} of {}\n'.format(version,
display_domain(domain))
for schema in sorted(unsorted_schemas, key=lambda s: s.name):
name_with_ver = '{}-{}'.format(
format_name_with_domain(domain, schema.name),
schema.since_version)
s += ('### <a name="{}"></a>**{}**' +
(' (deprecated)' if schema.deprecated else '') +
'</a>\n').format(name_with_ver, name_with_ver)
s += display_schema(schema, [schema])
s += '\n'
fout.write(s)
with io.open(args.output, 'w', newline='', encoding="utf-8") as fout:
fout.write('## Operator Schemas\n')
fout.write('#Table of Contents \n')
fout.write(
"*This file is automatically generated from the\n"
" [def files](/onnx/defs) via [this script](/onnx/defs/gen_doc.py).\n"
" Do not modify directly and instead edit operator definitions.*\n"
)
fout.write("|Operator |Input |Output |Type Constraint | Version |\n"
) #create table
fout.write("|-|-|-|-|-|-|\n")
# domain -> support level -> name -> [schema]
index = defaultdict(lambda: defaultdict(lambda: defaultdict(
list))) # type: Dict[Text, Dict[int, Dict[Text, List[OpSchema]]]]
for schema in defs.get_all_schemas_with_history():
index[schema.domain][int(
schema.support_level)][schema.name].append(schema)
# Preprocess the Operator Schemas
# [(domain, [(support_level, [(schema name, current schema, all versions schemas)])])]
operator_schemas = list(
) # type: List[Tuple[Text, List[Tuple[int, List[Tuple[Text, OpSchema, List[OpSchema]]]]]]]
exsting_ops = set() # type: Set[Text]
for domain, _supportmap in sorted(index.items()):
if not should_render_domain(domain):
continue
processed_supportmap = list()
for _support, _namemap in sorted(_supportmap.items()):
processed_namemap = list()
for n, unsorted_versions in sorted(_namemap.items()):
versions = sorted(unsorted_versions,
key=lambda s: s.since_version)
schema = versions[-1]
if schema.name in exsting_ops:
continue
exsting_ops.add(schema.name)
processed_namemap.append((n, schema, versions))
processed_supportmap.append((_support, processed_namemap))
operator_schemas.append((domain, processed_supportmap))
for domain, supportmap in operator_schemas:
for _, namemap in supportmap:
for op_type, schema, versions in namemap:
input_str = ""
if schema.min_input < schema.max_input:
input_str = "({} - {}) ".format(
display_number(schema.min_input),
display_number(schema.max_input))
isFirst = True
for input in schema.inputs:
if (isFirst):
isFirst = False
else:
input_str += "<br>"
input_str += input.description
if schema.min_output < schema.max_output:
input_str += ' ({} - {})'.format(
display_number(schema.min_output),
display_number(schema.max_output))
isFirstOut = True
output_str = ""
for output in schema.outputs:
if (isFirstOut):
isFirstOut = False
else:
output_str += "<br>"
output_str += output.description
tc_str = ""
if schema.type_constraints:
isFirstTc = True
for type_constraint in schema.type_constraints:
if (isFirstTc):
isFirstTc = False
else:
tc_str += "<br>"
allowedTypes = type_constraint.allowed_type_strs
allowedTypeStr = ', '.join(allowedTypes)
tc_str += allowedTypeStr
tc_str += type_constraint.description
version_str = ""
if schema.support_level == OpSchema.SupportType.EXPERIMENTAL:
version_str = '\nNo versioning maintained for experimental ops.'
else:
version_str = str(schema.since_version)
#s += ' of {}.\n'.format(display_domain(schema.domain))
op_link = '<a href="#{}">{}</a>'.format(op_type, op_type)
fout.write("|{}|{}|{}|{}|{}|\n".format(
format_name_with_domain(domain, op_link), input_str,
output_str, tc_str, version_str))
def __getattr__(self, item):
return getattr(defs.get_all_schemas_with_history()[self.i], item)
def main(args: Args) -> None:
with io.open(args.changelog, 'w', newline='') as fout:
fout.write('<!--- SPDX-License-Identifier: Apache-2.0 -->\n')
fout.write('## Operator Changelog\n')
fout.write(
"*This file is automatically generated from the\n"
" [def files](/onnx/defs) via [this script](/onnx/defs/gen_doc.py).\n"
" Do not modify directly and instead edit operator definitions.*\n"
"\n"
"For an operator input/output's differentiability, it can be differentiable,\n"
" non-differentiable, or undefined. If a variable's differentiability\n"
" is not specified, that variable has undefined differentiability.\n"
)
# domain -> version -> [schema]
dv_index: Dict[Text, Dict[int, List[OpSchema]]] = defaultdict(
lambda: defaultdict(list))
for schema in defs.get_all_schemas_with_history():
dv_index[schema.domain][schema.since_version].append(schema)
fout.write('\n')
for domain, versionmap in sorted(dv_index.items()):
if not should_render_domain(domain):
continue
s = '# {}\n'.format(display_domain_short(domain))
for version, unsorted_schemas in sorted(versionmap.items()):
s += '## Version {} of {}\n'.format(version,
display_domain(domain))
for schema in sorted(unsorted_schemas, key=lambda s: s.name):
name_with_ver = '{}-{}'.format(
format_name_with_domain(domain, schema.name),
schema.since_version)
s += ('### <a name="{}"></a>**{}**' +
(' (deprecated)' if schema.deprecated else '') +
'</a>\n').format(name_with_ver, name_with_ver)
s += display_schema(schema, [schema])
s += '\n'
fout.write(s)
with io.open(args.output, 'w', newline='', encoding="utf-8") as fout:
fout.write('<!--- SPDX-License-Identifier: Apache-2.0 -->\n')
fout.write('## Operator Schemas\n')
fout.write(
"*This file is automatically generated from the\n"
" [def files](/onnx/defs) via [this script](/onnx/defs/gen_doc.py).\n"
" Do not modify directly and instead edit operator definitions.*\n"
"\n"
"For an operator input/output's differentiability, it can be differentiable,\n"
" non-differentiable, or undefined. If a variable's differentiability\n"
" is not specified, that variable has undefined differentiability.\n"
)
# domain -> support level -> name -> [schema]
index: Dict[Text, Dict[int, Dict[Text, List[OpSchema]]]] = defaultdict(
lambda: defaultdict(lambda: defaultdict(list)))
for schema in defs.get_all_schemas_with_history():
index[schema.domain][int(
schema.support_level)][schema.name].append(schema)
fout.write('\n')
# Preprocess the Operator Schemas
# [(domain, [(support_level, [(schema name, current schema, all versions schemas)])])]
operator_schemas: List[Tuple[Text, List[Tuple[int, List[Tuple[
Text, OpSchema, List[OpSchema]]]]]]] = list()
existing_ops: Set[Text] = set()
for domain, _supportmap in sorted(index.items()):
if not should_render_domain(domain):
continue
processed_supportmap = list()
for _support, _namemap in sorted(_supportmap.items()):
processed_namemap = list()
for n, unsorted_versions in sorted(_namemap.items()):
versions = sorted(unsorted_versions,
key=lambda s: s.since_version)
schema = versions[-1]
if schema.name in existing_ops:
continue
existing_ops.add(schema.name)
processed_namemap.append((n, schema, versions))
processed_supportmap.append((_support, processed_namemap))
operator_schemas.append((domain, processed_supportmap))
# Table of contents
for domain, supportmap in operator_schemas:
s = '### {}\n'.format(display_domain_short(domain))
fout.write(s)
fout.write('|**Operator**|**Since version**|\n')
fout.write('|-|-|\n')
function_ops = list()
for _, namemap in supportmap:
for n, schema, versions in namemap:
if schema.has_function or schema.has_context_dependent_function: # type: ignore
function_ops.append((n, schema, versions))
continue
s = '|{}<a href="#{}">{}</a>{}|{}|\n'.format(
support_level_str(schema.support_level),
format_name_with_domain(domain, n),
format_name_with_domain(domain, n),
' (deprecated)' if schema.deprecated else '',
format_versions(versions))
fout.write(s)
if len(function_ops):
fout.write('|**Function**|**Since version**|\n')
for n, schema, versions in function_ops:
s = '|{}<a href="#{}">{}</a>|{}|\n'.format(
support_level_str(schema.support_level),
format_name_with_domain(domain, n),
format_name_with_domain(domain, n),
format_versions(versions))
fout.write(s)
fout.write('\n')
fout.write('\n')
for domain, supportmap in operator_schemas:
s = '## {}\n'.format(display_domain_short(domain))
fout.write(s)
for _, namemap in supportmap:
for op_type, schema, versions in namemap:
# op_type
s = ('### {}<a name="{}"></a><a name="{}">**{}**' +
(' (deprecated)' if schema.deprecated else '') +
'</a>\n').format(
support_level_str(schema.support_level),
format_name_with_domain(domain, op_type),
format_name_with_domain(domain, op_type.lower()),
format_name_with_domain(domain, op_type))
s += display_schema(schema, versions)
s += '\n\n'
if op_type in SNIPPETS:
s += '#### Examples\n\n'
for summary, code in sorted(SNIPPETS[op_type]):
s += '<details>\n'
s += '<summary>{}</summary>\n\n'.format(summary)
s += '```python\n{}\n```\n\n'.format(code)
s += '</details>\n'
s += '\n\n'
if op_type.lower() in SAMPLE_IMPLEMENTATIONS:
s += '#### Sample Implementation\n\n'
s += '<details>\n'
s += '<summary>{}</summary>\n\n'.format(op_type)
s += '```python\n{}\n```\n\n'.format(
SAMPLE_IMPLEMENTATIONS[op_type.lower()])
s += '</details>\n'
s += '\n\n'
fout.write(s)
def main(args): # type: (Type[Args]) -> None
with io.open(args.changelog, 'w', newline='') as fout:
fout.write('## Operator Changelog\n')
fout.write(
"*This file is automatically generated from the\n"
" [def files](/onnx/defs) via [this script](/onnx/defs/gen_doc.py).\n"
" Do not modify directly and instead edit operator definitions.*\n"
)
# domain -> version -> [schema]
dv_index = defaultdict(lambda: defaultdict(list)
) # type: Dict[Text, Dict[int, List[OpSchema]]]
for schema in defs.get_all_schemas_with_history():
dv_index[schema.domain][schema.since_version].append(schema)
fout.write('\n')
for domain, versionmap in sorted(dv_index.items()):
if not should_render_domain(domain):
continue
s = '# {}\n'.format(display_domain_short(domain))
for version, unsorted_schemas in sorted(versionmap.items()):
s += '## Version {} of {}\n'.format(version,
display_domain(domain))
for schema in sorted(unsorted_schemas, key=lambda s: s.name):
name_with_ver = '{}-{}'.format(
format_name_with_domain(domain, schema.name),
schema.since_version)
s += ('### <a name="{}"></a>**{}**' +
(' (deprecated)' if schema.deprecated else '') +
'</a>\n').format(name_with_ver, name_with_ver)
s += display_schema(schema, [schema])
s += '\n'
fout.write(s)
with io.open(args.fn_changelog, 'w', newline='') as fout:
fout.write('## Function Changelog\n')
fout.write(
"*This file is automatically generated from the\n"
" [def files](/onnx/defs) via [this script](/onnx/defs/gen_doc.py).\n"
" Do not modify directly and instead edit function definitions.*\n"
)
if os.getenv('ONNX_ML'):
all_functions = defs.get_functions(ONNX_ML_DOMAIN)
else:
all_functions = defs.get_functions('')
changelog_versionmap = defaultdict(
list) # type: Dict[int, List[FunctionProto]]
for fn_name, functions in sorted(all_functions.items()):
for func in functions:
changelog_versionmap[func.since_version].append(func)
if os.getenv('ONNX_ML'):
s = '## {}\n'.format(ONNX_ML_DOMAIN)
domain_display_name = ONNX_ML_DOMAIN
domain_prefix = '{}.'.format(ONNX_ML_DOMAIN)
else:
s = '# ai.onnx (default)\n'
domain_display_name = 'ai.onnx (default)'
domain_prefix = ''
fout.write(s)
for version, function_list in sorted(changelog_versionmap.items()):
s = ""
for function in function_list:
s += '## Version {} of domain {}\n'.format(
version, domain_display_name)
name_with_ver = '{}-{}'.format(domain_prefix + fn_name,
function.since_version)
s += '### <a name="{}"></a>**{}**</a>\n'.format(
name_with_ver, name_with_ver)
available_versions = [
func.since_version for func in all_functions[function.name]
]
s += display_function(function, available_versions,
domain_prefix)
s += '\n'
fout.write(s)
with io.open(args.output, 'w', newline='') as fout:
fout.write('## Operator Schemas\n')
fout.write(
"*This file is automatically generated from the\n"
" [def files](/onnx/defs) via [this script](/onnx/defs/gen_doc.py).\n"
" Do not modify directly and instead edit operator definitions.*\n"
)
# domain -> support level -> name -> [schema]
index = defaultdict(lambda: defaultdict(lambda: defaultdict(
list))) # type: Dict[Text, Dict[int, Dict[Text, List[OpSchema]]]]
for schema in defs.get_all_schemas_with_history():
index[schema.domain][int(
schema.support_level)][schema.name].append(schema)
fout.write('\n')
# Preprocess the Operator Schemas
# [(domain, [(support_level, [(schema name, current schema, all versions schemas)])])]
operator_schemas = list(
) # type: List[Tuple[Text, List[Tuple[int, List[Tuple[Text, OpSchema, List[OpSchema]]]]]]]
exsting_ops = set() # type: Set[Text]
for domain, _supportmap in sorted(index.items()):
if not should_render_domain(domain):
continue
processed_supportmap = list()
for _support, _namemap in sorted(_supportmap.items()):
processed_namemap = list()
for n, unsorted_versions in sorted(_namemap.items()):
versions = sorted(unsorted_versions,
key=lambda s: s.since_version)
schema = versions[-1]
if schema.name in exsting_ops:
continue
exsting_ops.add(schema.name)
processed_namemap.append((n, schema, versions))
processed_supportmap.append((_support, processed_namemap))
operator_schemas.append((domain, processed_supportmap))
# Table of contents
for domain, supportmap in operator_schemas:
s = '* {}\n'.format(display_domain_short(domain))
fout.write(s)
for _, namemap in supportmap:
for n, schema, versions in namemap:
s = ' * {}<a href="#{}">{}</a>\n'.format(
support_level_str(schema.support_level),
format_name_with_domain(domain, n),
format_name_with_domain(domain, n))
fout.write(s)
fout.write('\n')
for domain, supportmap in operator_schemas:
s = '## {}\n'.format(display_domain_short(domain))
fout.write(s)
for _, namemap in supportmap:
for op_type, schema, versions in namemap:
# op_type
s = ('### {}<a name="{}"></a><a name="{}">**{}**' +
(' (deprecated)' if schema.deprecated else '') +
'</a>\n').format(
support_level_str(schema.support_level),
format_name_with_domain(domain, op_type),
format_name_with_domain(domain, op_type.lower()),
format_name_with_domain(domain, op_type))
s += display_schema(schema, versions)
s += '\n\n'
if op_type in SNIPPETS:
s += '#### Examples\n\n'
for summary, code in sorted(SNIPPETS[op_type]):
s += '<details>\n'
s += '<summary>{}</summary>\n\n'.format(summary)
s += '```python\n{}\n```\n\n'.format(code)
s += '</details>\n'
s += '\n\n'
fout.write(s)
with io.open(args.function_output, 'w', newline='') as fout:
fout.write('## Functions\n')
fout.write(
"*This file is automatically generated from the\n"
" [def files](/onnx/defs) via [this script](/onnx/defs/gen_doc.py).\n"
" Do not modify directly and instead edit function definitions.*\n"
)
if os.getenv('ONNX_ML'):
all_functions = defs.get_functions(ONNX_ML_DOMAIN)
else:
all_functions = defs.get_functions('')
if all_functions:
if os.getenv('ONNX_ML'):
s = '## {}\n'.format(ONNX_ML_DOMAIN)
domain_prefix = '{}.'.format(ONNX_ML_DOMAIN)
else:
s = '## ai.onnx (default)\n'
domain_prefix = ''
fout.write(s)
existing_functions = set() # type: Set[Text]
for function_name, functions in sorted(all_functions.items()):
for function in sorted(functions,
key=lambda s: s.since_version,
reverse=True):
if function.name in existing_functions:
continue
existing_functions.add(function.name)
s = ' * {}<a href="#{}">{}</a>\n'.format(
function_status_str(function.status),
domain_prefix + function.name,
domain_prefix + function.name)
fout.write(s)
fout.write('\n')
fout.write('\n\n')
for function_name, functions in sorted(all_functions.items()):
available_versions = [func.since_version for func in functions]
function = sorted(functions,
key=lambda s: s.since_version,
reverse=True)[0]
s = '### {}<a name="{}"></a><a name="{}">**{}**</a>\n'.format(
function_status_str(function.status),
domain_prefix + function.name,
domain_prefix + function.name.lower(),
domain_prefix + function.name)
s += display_function(function, available_versions,
domain_prefix)
s += '\n\n'
fout.write(s)
def gen_support_status(docs_dir, onnx_version, onnx_path,
onnx_tf_release_build):
# set filename
if onnx_tf_release_build:
# get onnx-tf version from VERSION_NUMBER file
version_dir = os.path.dirname(
os.path.dirname(os.path.realpath('VERSION_NUMBER')))
version_file = os.path.join(version_dir, 'VERSION_NUMBER')
onnx_tf_version = subprocess.check_output('cat ' + version_file,
shell=True)
onnx_tf_version = 'v' + onnx_tf_version.decode().strip('\n')
filename = 'support_status_' + onnx_tf_version.replace('.',
'_') + '.md'
else: # onnx-tf = master
# get onnx-tf commit id
onnx_tf_commit_id = subprocess.check_output('git rev-parse HEAD',
shell=True)
onnx_tf_commit_id = onnx_tf_commit_id.decode().strip('\n')
onnx_tf_version = 'Master ( commit id: {} )'.format(onnx_tf_commit_id)
filename = 'support_status.md'
with open(os.path.join(docs_dir, filename), 'w') as status_file:
status_file.write('# ONNX-Tensorflow Support Status\n')
status_file.write('|||\n')
status_file.write('|-:|:-|\n')
status_file.write(
'|ONNX-Tensorflow Version|{}|\n'.format(onnx_tf_version))
# get onnx commit id
if onnx_version == 'master':
onnx_commit_id = subprocess.check_output('cd ' + onnx_path +
'; git rev-parse HEAD',
shell=True)
onnx_commit_id = onnx_commit_id.decode().strip('\n')
status_file.write(
'|ONNX Version|Master ( commit id: {} )|\n'.format(
onnx_commit_id))
else:
status_file.write('|ONNX Version|{}|\n'.format(onnx_version))
# get tf_version
status_file.write('|Tensorflow Version|v{}|\n\n'.format(
tf.__version__))
# display the table legend
status_file.write('Notes:\n')
status_file.write('* Values that are new or updated from a ')
status_file.write('previous opset version are in bold.\n')
status_file.write('* -: not defined in corresponding ONNX ')
status_file.write('opset version\n')
status_file.write('* \*: the operator is deprecated\n')
status_file.write('* :small_red_triangle:: not supported yet\n')
status_file.write('* :small_orange_diamond:: partially supported\n')
status_file.write('* the rest are all supported\n\n')
# get oll onnx ops
onnx_ops = {}
for schema in defs.get_all_schemas():
if schema.domain == '': # only get onnx ops
onnx_ops[schema.name] = {
'versions': [],
'deprecated':
schema.since_version if schema.deprecated else -1
}
for schema in defs.get_all_schemas_with_history():
if schema.domain == '': # only get onnx ops
op = onnx_ops[schema.name]
versions = op['versions']
versions.append(schema.since_version)
# get all onnx-tf supported ops
onnx_tf_ops = opset_version.backend_opset_version
onnx_tf_ops_ps = opset_version.backend_partial_support
# get the cureent opset version
current_opset = defs.onnx_opset_version()
# setup table header
status_file.write('||')
for i in range(current_opset):
status_file.write('|')
status_file.write('\n|:-:|')
for i in range(current_opset):
status_file.write(':-:|')
status_file.write('\n|**ONNX Operator**|')
for opset in range(1, current_opset + 1):
status_file.write('**Opset {}**|'.format(opset))
ops_count = len(onnx_ops)
# fill in data for the table
for key, val in sorted(onnx_ops.items()):
try:
status_file.write('\n|{}|'.format(key))
i = 0
vers = val['versions']
deprecated = val['deprecated']
for opset in range(1, current_opset + 1):
if i <= len(vers) - 1:
lb = vers[i]
ub = vers[i + 1] if i < len(vers) - 1 else vers[i]
if opset < lb:
if i == 0:
status_file.write('-')
elif opset == lb:
status_file.write('**{}**'.format(lb))
if lb == deprecated:
status_file.write('\*')
elif lb not in onnx_tf_ops[key]:
status_file.write(':small_red_triangle:')
if opset == current_opset:
ops_count -= 1
elif key in onnx_tf_ops_ps:
status_file.write(':small_orange_diamond:')
else: # opset > lb
if opset < ub:
status_file.write('{}'.format(lb))
if lb == deprecated:
status_file.write('\*')
elif lb not in onnx_tf_ops[key]:
status_file.write(':small_red_triangle:')
if opset == current_opset:
ops_count -= 1
elif key in onnx_tf_ops_ps:
status_file.write(':small_orange_diamond:')
elif opset == ub:
status_file.write('**{}**'.format(ub))
if ub == deprecated:
status_file.write('\*')
elif ub not in onnx_tf_ops[key]:
status_file.write(':small_red_triangle:')
if opset == current_opset:
ops_count -= 1
elif key in onnx_tf_ops_ps:
status_file.write(':small_orange_diamond:')
i += 1
else: #opset > ub
status_file.write('{}'.format(ub))
if ub == deprecated:
status_file.write('\*')
elif ub not in onnx_tf_ops[key]:
status_file.write(':small_red_triangle:')
if opset == current_opset:
ops_count -= 1
elif key in onnx_tf_ops_ps:
status_file.write(':small_orange_diamond:')
status_file.write('|')
except:
# ops defined in onnx but not in opset_version.backend_opset_versionn
status_file.write(':small_red_triangle:|')
status_file.write(
'\n\nONNX-TF Supported Operators / ONNX Operators: {} / {}'.format(
ops_count, len(onnx_ops)))
# display partial support footnote
status_file.write('\n\nNotes:\n')
index = 1
for key in onnx_tf_ops_ps:
status_file.write(
str(index) + '. ' + key + ': ' + onnx_tf_ops_ps[key] + '\n')
index += 1
def main(args): # type: (Type[Args]) -> None
with io.open(args.changelog, 'w', newline='') as fout:
fout.write('## Operator Changelog\n')
fout.write(
"*This file is automatically generated from the\n"
" [def files](/onnx/defs) via [this script](/onnx/defs/gen_doc.py).\n"
" Do not modify directly and instead edit operator definitions.*\n"
)
# domain -> version -> [schema]
dv_index = defaultdict(lambda: defaultdict(list)
) # type: Dict[Text, Dict[int, List[OpSchema]]]
for schema in defs.get_all_schemas_with_history():
dv_index[schema.domain][schema.since_version].append(schema)
fout.write('\n')
for domain, versionmap in sorted(dv_index.items()):
if not should_render_domain(domain):
continue
s = '# {}\n'.format(display_domain_short(domain))
for version, unsorted_schemas in sorted(versionmap.items()):
s += '## Version {} of {}\n'.format(version,
display_domain(domain))
for schema in sorted(unsorted_schemas, key=lambda s: s.name):
name_with_ver = '{}-{}'.format(
format_name_with_domain(domain, schema.name),
schema.since_version)
s += '### <a name="{}"></a>**{}**</a>\n'.format(
name_with_ver, name_with_ver)
s += display_schema(schema, [schema])
s += '\n'
fout.write(s)
with io.open(args.output, 'w', newline='') as fout:
fout.write('## Operator Schemas\n')
fout.write(
"*This file is automatically generated from the\n"
" [def files](/onnx/defs) via [this script](/onnx/defs/gen_doc.py).\n"
" Do not modify directly and instead edit operator definitions.*\n"
)
# domain -> support level -> name -> [schema]
index = defaultdict(lambda: defaultdict(lambda: defaultdict(
list))) # type: Dict[Text, Dict[int, Dict[Text, List[OpSchema]]]]
for schema in defs.get_all_schemas_with_history():
index[schema.domain][int(
schema.support_level)][schema.name].append(schema)
fout.write('\n')
# Preprocess the Operator Schemas
# [(domain, [(support_level, [(schema name, current schema, all versions schemas)])])]
operator_schemas = list(
) # type: List[Tuple[Text, List[Tuple[int, List[Tuple[Text, OpSchema, List[OpSchema]]]]]]]
exsting_ops = set() # type: Set[Text]
for domain, _supportmap in sorted(index.items()):
if not should_render_domain(domain):
continue
processed_supportmap = list()
for _support, _namemap in sorted(_supportmap.items()):
processed_namemap = list()
for n, unsorted_versions in sorted(_namemap.items()):
versions = sorted(unsorted_versions,
key=lambda s: s.since_version)
schema = versions[-1]
if schema.name in exsting_ops:
continue
exsting_ops.add(schema.name)
processed_namemap.append((n, schema, versions))
processed_supportmap.append((_support, processed_namemap))
operator_schemas.append((domain, processed_supportmap))
# Table of contents
for domain, supportmap in operator_schemas:
s = '* {}\n'.format(display_domain_short(domain))
fout.write(s)
for _, namemap in supportmap:
for n, schema, versions in namemap:
s = ' * {}<a href="#{}">{}</a>\n'.format(
support_level_str(schema.support_level),
format_name_with_domain(domain, n),
format_name_with_domain(domain, n))
fout.write(s)
fout.write('\n')
for domain, supportmap in operator_schemas:
s = '## {}\n'.format(display_domain_short(domain))
fout.write(s)
for _, namemap in supportmap:
for op_type, schema, versions in namemap:
# op_type
s = '### {}<a name="{}"></a><a name="{}">**{}**</a>\n'.format(
support_level_str(schema.support_level),
format_name_with_domain(domain, op_type),
format_name_with_domain(domain, op_type.lower()),
format_name_with_domain(domain, op_type))
s += display_schema(schema, versions)
s += '\n\n'
if op_type in SNIPPETS:
s += '#### Examples\n\n'
for summary, code in sorted(SNIPPETS[op_type]):
s += '<details>\n'
s += '<summary>{}</summary>\n\n'.format(summary)
s += '```python\n{}\n```\n\n'.format(code)
s += '</details>\n'
s += '\n\n'
fout.write(s)
def metadata():
schemas = defs.get_all_schemas_with_history()
schemas = sorted(schemas, key=lambda schema: schema.name)
generate_json(schemas, '../src/onnx-metadata.json')
json_schema['type_constraints'].append({
'description': type_constraint.description,
'type_param_str': type_constraint.type_param_str,
'allowed_type_strs': type_constraint.allowed_type_strs
})
if schema.name in snippets:
json_schema['snippets'] = []
for summary, code in sorted(snippets[schema.name]):
json_schema['snippets'].append({
'summary': summary,
'code': code
})
if schema.name in categories:
json_schema['category'] = categories[schema.name]
json_root.append({
'name': schema.name,
'schema': json_schema
})
with io.open(json_file, 'w', newline='') as fout:
json_root = json.dumps(json_root, sort_keys=True, indent=2)
for line in json_root.splitlines():
line = line.rstrip()
if sys.version_info[0] < 3:
line = unicode(line)
fout.write(line)
fout.write('\n')
if __name__ == '__main__':
schemas = sorted(defs.get_all_schemas_with_history(), key=lambda schema: schema.name)
generate_json(schemas, '../src/onnx-operator.json')
def main(args): # type: (Type[Args]) -> None
with io.open(args.changelog, 'w', newline='') as fout:
fout.write('## Operator Changelog\n')
fout.write(
"*This file is automatically generated from the\n"
" [def files](/onnx/defs) via [this script](/onnx/defs/gen_doc.py).\n"
" Do not modify directly and instead edit operator definitions.*\n"
)
# domain -> version -> [schema]
dv_index = defaultdict(lambda: defaultdict(list)
) # type: Dict[Text, Dict[int, List[OpSchema]]]
for schema in defs.get_all_schemas_with_history():
dv_index[schema.domain][schema.since_version].append(schema)
fout.write('\n')
for domain, versionmap in sorted(dv_index.items()):
print("domain", domain)
if not should_render_domain(domain):
continue
s = '# {}\n'.format(display_domain_short(domain))
for version, unsorted_schemas in sorted(versionmap.items()):
s += '## Version {} of {}\n'.format(version,
display_domain(domain))
for schema in sorted(unsorted_schemas, key=lambda s: s.name):
name_with_ver = '{}-{}'.format(
format_name_with_domain(domain, schema.name),
schema.since_version)
s += ('### <a name="{}"></a>**{}**' +
(' (deprecated)' if schema.deprecated else '') +
'</a>\n').format(name_with_ver, name_with_ver)
s += display_schema(schema, [schema])
s += '\n'
fout.write(s)
with io.open(args.output, 'w', newline='', encoding="utf-8") as fout:
fout.write('## Operator Schemas\n')
fout.write(
"*This file is automatically generated from the\n"
" [def files](/onnx/defs) via [this script](/onnx/defs/gen_doc.py).\n"
" Do not modify directly and instead edit operator definitions.*\n"
)
# domain -> support level -> name -> [schema]
index = defaultdict(lambda: defaultdict(lambda: defaultdict(
list))) # type: Dict[Text, Dict[int, Dict[Text, List[OpSchema]]]]
for schema in defs.get_all_schemas_with_history():
#print("check point 0", schema.name, schema.domain, schema.support_level)
#gen_schema(schema)
index[schema.domain][int(
schema.support_level)][schema.name].append(schema)
fout.write('\n')
# Preprocess the Operator Schemas
# [(domain, [(support_level, [(schema name, current schema, all versions schemas)])])]
operator_schemas = list(
) # type: List[Tuple[Text, List[Tuple[int, List[Tuple[Text, OpSchema, List[OpSchema]]]]]]]
exsting_ops = set() # type: Set[Text]
for domain, _supportmap in sorted(index.items()):
if not should_render_domain(domain):
continue
processed_supportmap = list()
for _support, _namemap in sorted(_supportmap.items()):
processed_namemap = list()
for n, unsorted_versions in sorted(_namemap.items()):
versions = sorted(unsorted_versions,
key=lambda s: s.since_version)
schema = versions[-1]
#print("check point 2", schema)
if schema.name in exsting_ops:
continue
exsting_ops.add(schema.name)
processed_namemap.append((n, schema, versions))
processed_supportmap.append((_support, processed_namemap))
operator_schemas.append((domain, processed_supportmap))
# Table of contents
for domain, supportmap in operator_schemas:
s = '* {}\n'.format(display_domain_short(domain))
fout.write(s)
function_ops = list()
for _, namemap in supportmap:
for n, schema, versions in namemap:
if schema.has_function: # type: ignore
function_ops.append((n, schema, versions))
continue
s = ' * {}<a href="#{}">{}</a>\n'.format(
support_level_str(schema.support_level),
format_name_with_domain(domain, n),
format_name_with_domain(domain, n))
fout.write(s)
if len(function_ops):
fout.write('\n')
fout.write(' **Operators with function registered:**\n')
for n, schema, versions in function_ops:
s = ' * {}<a href="#{}">{}</a>\n'.format(
support_level_str(schema.support_level),
format_name_with_domain(domain, n),
format_name_with_domain(domain, n))
fout.write(s)
fout.write('\n')
tdfile = io.open(args.tdfile, 'w', newline='')
fefile = io.open('op_build_table.inc', 'w', newline='')
firstfunc = True
fefile.write(' ' + 'if (OpName == "DUMMY") {\n')
for domain, supportmap in operator_schemas:
s = '## {}\n'.format(display_domain_short(domain))
fout.write(s)
for _, namemap in supportmap:
for op_type, schema, versions in namemap:
# op_type
#print("check point 1", schema.name, len(schema.inputs), len(schema.outputs))
gen_code(schema, fefile)
r = gen_schema(schema)
tdfile.write(r)
s = ('### {}<a name="{}"></a><a name="{}">**{}**' +
(' (deprecated)' if schema.deprecated else '') +
'</a>\n').format(
support_level_str(schema.support_level),
format_name_with_domain(domain, op_type),
format_name_with_domain(domain, op_type.lower()),
format_name_with_domain(domain, op_type))
s += display_schema(schema, versions)
s += '\n\n'
if op_type in SNIPPETS:
s += '#### Examples\n\n'
for summary, code in sorted(SNIPPETS[op_type]):
s += '<details>\n'
s += '<summary>{}</summary>\n\n'.format(summary)
s += '```python\n{}\n```\n\n'.format(code)
s += '</details>\n'
s += '\n\n'
if op_type.lower() in SAMPLE_IMPLEMENTATIONS:
s += '#### Sample Implementation\n\n'
s += '<details>\n'
s += '<summary>{}</summary>\n\n'.format(op_type)
s += '```python\n{}\n```\n\n'.format(
SAMPLE_IMPLEMENTATIONS[op_type.lower()])
s += '</details>\n'
s += '\n\n'
fout.write(s)
fefile.write(' }')
fefile.close()
type_constraint.allowed_type_strs
})
if schema.name in SNIPPETS:
json_schema['snippets'] = []
for summary, code in sorted(SNIPPETS[schema.name]):
json_schema['snippets'].append({
'summary': summary,
'code': code
})
json_root.append({"name": schema.name, "schema": json_schema})
with io.open(file, 'w', newline='') as fout:
json_root = json.dumps(json_root, sort_keys=True, indent=2)
for line in json_root.splitlines():
line = line.rstrip()
if sys.version_info[0] < 3:
line = unicode(line)
fout.write(line)
fout.write('\n')
if __name__ == '__main__':
schemas = sorted(defs.get_all_schemas_with_history(),
key=lambda schema: schema.name)
generate_json(schemas, '../src/onnx-operator.json')
# print(schema.name + "|" + schema.domain + "|" + str(schema.since_version))
# sorted_ops = sorted(
# (int(schema.support_level), op_type, schema)
# for (op_type, schema) in defs.get_all_schemas().items())
type_constraint.description,
'type_param_str':
type_constraint.type_param_str,
'allowed_type_strs':
type_constraint.allowed_type_strs
})
if schema.name in snippets:
json_schema['examples'] = []
for summary, code in sorted(snippets[schema.name]):
json_schema['examples'].append({
'summary': summary,
'code': code
})
if schema.name in categories:
json_schema['category'] = categories[schema.name]
json_root.append({'name': schema.name, 'schema': json_schema})
with io.open(json_file, 'w', newline='') as fout:
json_root = json.dumps(json_root, sort_keys=True, indent=2)
for line in json_root.splitlines():
line = line.rstrip()
if sys.version_info[0] < 3:
line = unicode(line)
fout.write(line)
fout.write('\n')
if __name__ == '__main__':
schemas = defs.get_all_schemas_with_history()
schemas = sorted(schemas, key=lambda schema: schema.name)
generate_json(schemas, '../src/onnx-metadata.json')
#!/usr/bin/env python3
#
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import os.path as path
from sys import argv
from onnx import defs
out_file = path.join(path.dirname(path.realpath(__file__)), 'opgen',
'onnxops.py')
onnx_ops = {}
for schema in defs.get_all_schemas_with_history():
key = schema.name.lower()
if schema.deprecated:
continue
if key not in onnx_ops or \
onnx_ops[key].since_version < schema.since_version:
onnx_ops[key] = schema
with open(out_file, 'wt') as fp:
def write(s):
fp.write(s)
def writeline(s=''):
fp.write(s + '\n')
writeline(f'# AUTO-GENERATED CODE! - DO NOT EDIT!')
writeline(f'# $ python {" ".join(argv)}')
def metadata():
schemas = defs.get_all_schemas_with_history()
schemas = sorted(schemas, key=lambda schema: schema.name)
json_file = os.path.join(os.path.dirname(__file__),
'../src/onnx-metadata.json')
generate_json(schemas, json_file)
