def generate_functions_pkl():
import pickle
yaml_data = {}
d = utils.load_yaml_ordered(open(join(here, 'functions.yaml'), 'r'))
for cat_name, cat_info in d.items():
for func_name, func_info in d[cat_name].items():
if 'doc' in func_info:
del func_info['doc']
for a in ['inputs', 'arguments', 'outputs']:
if a in func_info:
for b in func_info[a]:
if 'doc' in func_info[a][b]:
del func_info[a][b]['doc']
fmt = ''
if 'arguments' in func_info:
fmt = '_'
for a, a_info in func_info['arguments'].items():
fmt += type_to_pack_format(a_info['type'])
func_info['uniq_name'] = func_name + fmt
func_info['id'] = list(func_info['function_ids'].items()).pop()[1]
yaml_data['nnabla_func_info'] = d
o = utils.load_yaml_ordered(open(join(base, 'python/test/utils/conversion/exporter_funcs_opset.yaml'), 'r'))
yaml_data['onnx_func_info'] = {}
for func, func_info in o.items():
if 'Not implemented' in func_info:
continue
else:
yaml_data['onnx_func_info'][func] = func_info
with open(join(base, 'python/src/nnabla/utils/converter/functions.pkl'), 'wb') as f:
pickle.dump(yaml_data, f, 2)
def generate():
function_info = utils.load_function_info(flatten=True)
solver_info = utils.load_solver_info()
function_types = utils.load_yaml_ordered(
open(join(here, 'function_types.yaml'), 'r'))
solver_types = utils.load_yaml_ordered(
open(join(here, 'solver_types.yaml'), 'r'))
utils.generate_init(function_info, function_types, solver_info,
solver_types)
utils.generate_function_types(function_info, function_types)
utils.generate_solver_types(solver_info, solver_types)
utils.generate_version()
generate_solver_python_interface(solver_info)
generate_function_python_interface(function_info)
generate_python_utils(function_info)
generate_proto(function_info, solver_info)
generate_cpp_utils(function_info)
generate_function_order(function_info)
# Generate function skeletons if new ones are added to functions.yaml and function_types.yaml.
utils.generate_skeleton_function_impl(function_info, function_types)
func_header_template = join(
base, 'include/nbla/function/function_impl.hpp.tmpl')
utils.generate_skeleton_function_impl(function_info,
function_types,
template=func_header_template,
output_format='%s.hpp')
def generate():
version = sys.argv[1]
update_function_order_in_functsions_yaml()
generate_functions_pkl()
function_info = utils.load_function_info(flatten=True)
solver_info = utils.load_solver_info()
function_types = utils.load_yaml_ordered(open(join(here, 'function_types.yaml'), 'r'))
solver_types = utils.load_yaml_ordered(open(join(here, 'solver_types.yaml'), 'r'))
utils.generate_init(function_info, function_types, solver_info, solver_types)
utils.generate_function_types(function_info, function_types)
utils.generate_solver_types(solver_info, solver_types)
utils.generate_version(join(base, 'python/src/nnabla/_version.py.tmpl'), base, version=version)
utils.generate_version(join(base, 'src/nbla/version.cpp.tmpl'), base, version=version)
utils.generate_version(join(base, 'doc/requirements.txt.tmpl'), base, version=version)
generate_solver_python_interface(solver_info)
generate_function_python_interface(function_info)
generate_python_utils(function_info)
generate_proto(function_info, solver_info)
generate_cpp_utils(function_info)
# Generate function skeletons if new ones are added to functions.yaml and function_types.yaml.
utils.generate_skeleton_function_impl(
function_info, function_types)
func_header_template = join(
base,
'include/nbla/function/function_impl.hpp.tmpl')
utils.generate_skeleton_function_impl(
function_info, function_types,
template=func_header_template, output_format='%s.hpp')
def format_file(file_ext, input):
cmd = None
file_ext = file_ext.lower()
if file_ext in c_extensions:
cmd = [search_clang_format(), '--style=llvm']
elif file_ext in python_extensions:
cmd = [search_autopep8(), '--ignore={}'.format(pep8_ignores), '-']
elif file_ext in cython_extensions:
cmd = [
search_autopep8(), '--ignore={}'.format(pep8_cython_ignores), '-'
]
if cmd is not None:
p = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
output, output_err = p.communicate(input.encode('utf-8'))
return output.decode('utf_8')
else:
from six import StringIO
import sys
sys.path.append(
os.path.join(os.path.dirname(__file__), '../code_generator'))
import code_generator_utils as utils
d = utils.load_yaml_ordered(StringIO(input))
output = StringIO()
utils.dump_yaml(d, output, default_flow_style=False, width=80)
return output.getvalue()
return input
def open_api_level_yaml():
class ApiLevelManager:
def __init__(self, d):
self.changed = False
self.api_level_d = d
self.current_level = sorted(self.api_level_d.keys()).pop() + 1
def append_new_id(self, uniq_name, id):
if self.current_level not in self.api_level_d:
self.api_level_d[self.current_level] = {uniq_name: id}
else:
self.api_level_d[self.current_level][uniq_name] = id
self.changed = True
api_level_yaml = join(here, 'api_levels.yaml')
if not exists(api_level_yaml):
regenerate_api_level_yaml()
d = utils.load_yaml_ordered(open(api_level_yaml, 'r'))
api_level_man = ApiLevelManager(d)
yield api_level_man
if api_level_man.changed:
utils.dump_yaml(api_level_man.api_level_d,
open(join(here, 'api_levels.yaml'), 'w'),
default_flow_style=False,
width=80)
def regenerate_api_level_yaml():
'''
regenerate api_level.yaml if there is no this file before.
This is only called when api_level.yaml is missing.
We re-constructed the revision history by its incremental id.
We supposed that the duplicated function id is the fork point
of API level. But once this file is created and committed,
this function should never be called so that api_levels.yaml
trace the revision history of nnabla APIs.
'''
d = utils.load_yaml_ordered(open(join(here, 'functions.yaml'), 'r'))
class APIInstance:
pass
api_instance_list = []
for cat_name, cat_info in d.items():
for func_name, func_info in d[cat_name].items():
fids = func_info['function_ids']
for param, id in fids.items():
api_inst = APIInstance()
api_inst.func = func_name
api_inst.param = param
api_inst.id = id
api_instance_list.append(api_inst)
api_instance_list = sorted(api_instance_list, key=lambda api: api.id)
level = 1
api_level = {}
for api in api_instance_list:
if level not in api_level:
api_level[level] = {}
api_level[level][api.func] = (api.param, api.id)
elif api.func not in api_level[level]:
api_level[level][api.func] = (api.param, api.id)
else:
level += 1
api_level[level] = {}
api_level[level][api.func] = (api.param, api.id)
yaml_api_level = {}
for level, funcs in api_level.items():
if level not in yaml_api_level:
yaml_api_level[level] = {}
for func_name, (param, id) in funcs.items():
yaml_api_level[level][func_name + '_' + param] = id
utils.dump_yaml(yaml_api_level,
open(join(here, 'api_levels.yaml'), 'w'),
default_flow_style=False,
width=80)
def generate_functions_pkl():
import pickle
d = utils.load_yaml_ordered(open(join(here, 'functions.yaml'), 'r'))
for cat_name, cat_info in d.items():
for func_name, func_info in d[cat_name].items():
if 'doc' in func_info:
del func_info['doc']
for a in ['inputs', 'arguments', 'outputs']:
if a in func_info:
for b in func_info[a]:
if 'doc' in func_info[a][b]:
del func_info[a][b]['doc']
with open(join(base, 'python/src/nnabla/utils/converter/functions.pkl'), 'wb') as f:
pickle.dump(d, f, 2)
def main():
args = get_args()
func_info = utils.load_function_info(flatten=True)
if exists(args.path_types):
func_types = utils.load_yaml_ordered(open(args.path_types, 'r'))
else:
func_types = OrderedDict()
for name, func in func_info.items():
if name in func_types:
continue
print("Processing %s..." % name)
types = OrderedDict()
if args.default_type is not None:
types[args.default_type] = [args.default_type]
func_types[name] = types
utils.dump_yaml(func_types, open(args.path_types, 'w'))
def generate_function_order(function_info):
with open(join(base, 'python/src/nnabla/utils/converter/utils.py')) as f:
exec(f.read())
order_yaml = join(base,
'python/src/nnabla/utils/converter/function_order.yaml')
order_info = None
if exists(order_yaml):
with open(order_yaml, 'r') as f:
order_info = utils.load_yaml_ordered(f)
if order_info is None:
order_info = OrderedDict()
func_id = 0
for func, func_info in function_info.items():
fmt = ''
if 'arguments' in func_info:
fmt = '_'
for arg, arg_info in func_info['arguments'].items():
fmt += type_to_pack_format(arg_info['type'])
name = func + fmt
if name not in order_info:
while func_id in order_info.values():
func_id += 1
order_info[name] = func_id
func_id += 1
with open(order_yaml, 'w') as f:
f.write(
'# Copyright (c) 2017 Sony Corporation. All Rights Reserved.\n')
f.write('#\n')
f.write(
'# Licensed under the Apache License, Version 2.0 (the "License");\n'
)
f.write(
'# you may not use this file except in compliance with the License.\n'
)
f.write('# You may obtain a copy of the License at\n')
f.write('#\n')
f.write('# http://www.apache.org/licenses/LICENSE-2.0\n')
f.write('#\n')
f.write(
'# Unless required by applicable law or agreed to in writing, software\n'
)
f.write(
'# distributed under the License is distributed on an "AS IS" BASIS,\n'
)
f.write(
'# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n'
)
f.write(
'# See the License for the specific language governing permissions and\n'
)
f.write('# limitations under the License.\n')
f.write('#\n')
f.write('# DO NOT EDIT THIS FILE!\n')
f.write(
'# THIS FILE IS GENERATED BY CODE GENERATOR BUT ITS COMMITTED INTO\n'
)
f.write('# SOURCE TREE TO MAKE FUNCTION ID PERSIST.\n')
f.write('\n')
f.write(yaml.dump(order_info, default_flow_style=False))
def update_function_order_in_functsions_yaml():
d = utils.load_yaml_ordered(open(join(here, 'functions.yaml'), 'r'))
order_info_by_id = {}
order_info = OrderedDict()
duplicated = {}
missing = {}
for cat_name, cat_info in d.items():
for func_name, func_info in d[cat_name].items():
order_info[func_name] = OrderedDict()
default_full_name = func_name
default_arg = ''
if 'arguments' in func_info:
for arg, arg_info in func_info['arguments'].items():
default_arg += type_to_pack_format(arg_info['type'])
if default_arg == '':
default_arg = 'Empty'
else:
default_full_name = func_name + '_' + default_arg
if 'function_ids' in func_info and func_info['function_ids'] is not None:
for func_arg, func_id in func_info['function_ids'].items():
full_name = func_name
if func_arg != 'Empty':
full_name = func_name + '_' + func_arg
if func_id in order_info_by_id:
if func_id not in duplicated:
duplicated[func_id] = [order_info_by_id[func_id]]
duplicated[func_id].append(full_name)
order_info_by_id[func_id] = full_name
order_info[func_name][full_name] = func_id
if default_full_name not in order_info[func_name]:
if cat_name not in missing:
missing[cat_name] = {}
if func_name not in missing[cat_name]:
missing[cat_name][func_name] = []
missing[cat_name][func_name].append(default_arg)
else:
if cat_name not in missing:
missing[cat_name] = {}
if func_name not in missing[cat_name]:
missing[cat_name][func_name] = []
missing[cat_name][func_name].append(default_arg)
current_id = sorted(order_info_by_id.keys()).pop() + 1
for cat_name in missing:
for func_name in missing[cat_name]:
for arg in missing[cat_name][func_name]:
if 'function_ids' not in d[cat_name][func_name] or d[cat_name][func_name]['function_ids'] is None:
d[cat_name][func_name]['function_ids'] = OrderedDict()
d[cat_name][func_name]['function_ids'][arg] = current_id
current_id += 1
if len(duplicated):
print('')
print('############################################## Errors in functions.yaml(START)')
for func_id, functions in duplicated.items():
if len(functions) > 1:
print('ID {} duplicated between {}.'.format(func_id, functions))
print('Correct ID in "build-tools/code_generator/functions.yaml" manually.')
print('############################################## Errors in functions.yaml(END)')
print('')
import sys
sys.exit(-1)
utils.dump_yaml(d, open(join(here, 'functions.yaml'), 'w'), default_flow_style=False, width=80)
def generate(argv):
function_info = utils.load_function_info(flatten=True)
solver_info = utils.load_solver_info()
function_types = utils.load_yaml_ordered(
open(join(here, 'function_types.yaml'), 'r'))
function_types_cudnn = utils.load_yaml_ordered(
open(join(here, 'function_types_cudnn.yaml'), 'r'))
solver_types = utils.load_yaml_ordered(
open(join(here, 'solver_types.yaml'), 'r'))
function_template = join(base,
'src/nbla/cuda/function/function_types.cu.tmpl')
function_template_cudnn = join(
base, 'src/nbla/cuda/cudnn/function/function_types.cu.tmpl')
solver_template = join(base, 'src/nbla/cuda/solver/solver_types.cu.tmpl')
init_template = join(base, 'src/nbla/cuda/init.cpp.tmpl')
init_template_cudnn = join(base, 'src/nbla/cuda/cudnn/init.cpp.tmpl')
utils.generate_init(function_info,
function_types,
solver_info,
solver_types,
ext_info={},
template=init_template)
utils.generate_init(function_info,
function_types_cudnn,
solver_info,
solver_types,
ext_info={},
template=init_template_cudnn)
utils.generate_function_types(function_info,
function_types,
ext_info={},
template=function_template,
output_format='%s.cu')
utils.generate_function_types(function_info,
function_types_cudnn,
ext_info={},
template=function_template_cudnn,
output_format='%s.cu')
utils.generate_solver_types(solver_info,
solver_types,
ext_info={},
template=solver_template,
output_format='%s.cu')
if len(argv) == 4:
version = argv[1]
cuda_version = argv[2]
cudnn_version = argv[3]
else:
version = 'unknown'
cuda_version = 'unknown'
cudnn_version = 'unknown'
utils.generate_version(template=join(
base, 'python/src/nnabla_ext/cuda/_version.py.tmpl'),
rootdir=base,
version=version,
cuda_version=cuda_version,
cudnn_version=cudnn_version)
utils.generate_version(template=join(
base, 'python/src/nnabla_ext/cudnn/_version.py.tmpl'),
rootdir=base,
version=version,
cuda_version=cuda_version,
cudnn_version=cudnn_version)
utils.generate_version(template=join(base,
'src/nbla/cuda/version.cpp.tmpl'),
rootdir=base,
version=version,
cuda_version=cuda_version,
cudnn_version=cudnn_version)
# Generate function skeletons
func_src_template = join(
base, 'src/nbla/cuda/function/generic/function_impl.cu.tmpl')
func_src_template_cudnn = join(
base, 'src/nbla/cuda/cudnn/function/generic/function_impl.cu.tmpl')
func_header_template = join(
base, 'include/nbla/cuda/function/function_impl.hpp.tmpl')
func_header_template_cudnn = join(
base, 'include/nbla/cuda/cudnn/function/function_impl.hpp.tmpl')
utils.generate_skeleton_function_impl(function_info,
function_types,
ext_info={},
template=func_src_template,
output_format='%s.cu')
utils.generate_skeleton_function_impl(function_info,
function_types,
ext_info={},
template=func_header_template,
output_format='%s.hpp')
utils.generate_skeleton_function_impl(function_info,
function_types_cudnn,
ext_info={},
template=func_src_template_cudnn,
output_format='%s.cu')
utils.generate_skeleton_function_impl(function_info,
function_types_cudnn,
ext_info={},
template=func_header_template_cudnn,
output_format='%s.hpp')
def generate_function_order(function_info):
with open(join(base, 'python/src/nnabla/utils/converter/utils.py')) as f:
exec(f.read())
order_yaml = join(base,
'python/src/nnabla/utils/converter/function_order.yaml')
order_info = None
if exists(order_yaml):
with open(order_yaml, 'r') as f:
order_info = utils.load_yaml_ordered(f)
if order_info is None:
order_info = OrderedDict()
func_id = 0
for func, func_info in function_info.items():
fmt = ''
if 'arguments' in func_info:
fmt = '_'
for arg, arg_info in func_info['arguments'].items():
fmt += type_to_pack_format(arg_info['type'])
name = func + fmt
if name not in order_info:
while func_id in order_info.values():
func_id += 1
order_info[name] = func_id
func_id += 1
############### Check duplicated function ID.
orders = {}
result = True
for func, func_id in order_info.items():
if func_id not in orders:
orders[func_id] = []
else:
result = False
orders[func_id].append(func)
if not result:
print('')
print(
'############################################## Errors in function_order.yaml(START)'
)
for func_id, functions in orders.items():
if len(functions) > 1:
print('ID {} duplicated between [{}].'.format(func_id, functions))
if not result:
print(
'Correct ID in "python/src/nnabla/utils/converter/function_order.yaml" manually.'
)
print(
'############################################## Errors in function_order.yaml(END)'
)
print('')
import sys
sys.exit(-1)
with open(order_yaml, 'w') as f:
f.write(
'# Copyright (c) 2017 Sony Corporation. All Rights Reserved.\n')
f.write('#\n')
f.write(
'# Licensed under the Apache License, Version 2.0 (the "License");\n'
)
f.write(
'# you may not use this file except in compliance with the License.\n'
)
f.write('# You may obtain a copy of the License at\n')
f.write('#\n')
f.write('# http://www.apache.org/licenses/LICENSE-2.0\n')
f.write('#\n')
f.write(
'# Unless required by applicable law or agreed to in writing, software\n'
)
f.write(
'# distributed under the License is distributed on an "AS IS" BASIS,\n'
)
f.write(
'# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n'
)
f.write(
'# See the License for the specific language governing permissions and\n'
)
f.write('# limitations under the License.\n')
f.write('#\n')
f.write('# DO NOT EDIT THIS FILE!\n')
f.write(
'# THIS FILE IS GENERATED BY CODE GENERATOR BUT ITS COMMITTED INTO\n'
)
f.write('# SOURCE TREE TO MAKE FUNCTION ID PERSIST.\n')
f.write('\n')
for func_id in sorted(orders.keys()):
f.write('{}: {}\n'.format(orders[func_id][0], func_id))
