def generate_functions_pkl(): import pickle yaml_data = {} d = utils.load_yaml_ordered(open(join(here, 'functions.yaml'), 'r')) for cat_name, cat_info in d.items(): for func_name, func_info in d[cat_name].items(): if 'doc' in func_info: del func_info['doc'] for a in ['inputs', 'arguments', 'outputs']: if a in func_info: for b in func_info[a]: if 'doc' in func_info[a][b]: del func_info[a][b]['doc'] fmt = '' if 'arguments' in func_info: fmt = '_' for a, a_info in func_info['arguments'].items(): fmt += type_to_pack_format(a_info['type']) func_info['uniq_name'] = func_name + fmt func_info['id'] = list(func_info['function_ids'].items()).pop()[1] yaml_data['nnabla_func_info'] = d o = utils.load_yaml_ordered(open(join(base, 'python/test/utils/conversion/exporter_funcs_opset.yaml'), 'r')) yaml_data['onnx_func_info'] = {} for func, func_info in o.items(): if 'Not implemented' in func_info: continue else: yaml_data['onnx_func_info'][func] = func_info with open(join(base, 'python/src/nnabla/utils/converter/functions.pkl'), 'wb') as f: pickle.dump(yaml_data, f, 2)
def generate(): function_info = utils.load_function_info(flatten=True) solver_info = utils.load_solver_info() function_types = utils.load_yaml_ordered( open(join(here, 'function_types.yaml'), 'r')) solver_types = utils.load_yaml_ordered( open(join(here, 'solver_types.yaml'), 'r')) utils.generate_init(function_info, function_types, solver_info, solver_types) utils.generate_function_types(function_info, function_types) utils.generate_solver_types(solver_info, solver_types) utils.generate_version() generate_solver_python_interface(solver_info) generate_function_python_interface(function_info) generate_python_utils(function_info) generate_proto(function_info, solver_info) generate_cpp_utils(function_info) generate_function_order(function_info) # Generate function skeletons if new ones are added to functions.yaml and function_types.yaml. utils.generate_skeleton_function_impl(function_info, function_types) func_header_template = join( base, 'include/nbla/function/function_impl.hpp.tmpl') utils.generate_skeleton_function_impl(function_info, function_types, template=func_header_template, output_format='%s.hpp')
def generate(): version = sys.argv[1] update_function_order_in_functsions_yaml() generate_functions_pkl() function_info = utils.load_function_info(flatten=True) solver_info = utils.load_solver_info() function_types = utils.load_yaml_ordered(open(join(here, 'function_types.yaml'), 'r')) solver_types = utils.load_yaml_ordered(open(join(here, 'solver_types.yaml'), 'r')) utils.generate_init(function_info, function_types, solver_info, solver_types) utils.generate_function_types(function_info, function_types) utils.generate_solver_types(solver_info, solver_types) utils.generate_version(join(base, 'python/src/nnabla/_version.py.tmpl'), base, version=version) utils.generate_version(join(base, 'src/nbla/version.cpp.tmpl'), base, version=version) utils.generate_version(join(base, 'doc/requirements.txt.tmpl'), base, version=version) generate_solver_python_interface(solver_info) generate_function_python_interface(function_info) generate_python_utils(function_info) generate_proto(function_info, solver_info) generate_cpp_utils(function_info) # Generate function skeletons if new ones are added to functions.yaml and function_types.yaml. utils.generate_skeleton_function_impl( function_info, function_types) func_header_template = join( base, 'include/nbla/function/function_impl.hpp.tmpl') utils.generate_skeleton_function_impl( function_info, function_types, template=func_header_template, output_format='%s.hpp')
def format_file(file_ext, input): cmd = None file_ext = file_ext.lower() if file_ext in c_extensions: cmd = [search_clang_format(), '--style=llvm'] elif file_ext in python_extensions: cmd = [search_autopep8(), '--ignore={}'.format(pep8_ignores), '-'] elif file_ext in cython_extensions: cmd = [ search_autopep8(), '--ignore={}'.format(pep8_cython_ignores), '-' ] if cmd is not None: p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) output, output_err = p.communicate(input.encode('utf-8')) return output.decode('utf_8') else: from six import StringIO import sys sys.path.append( os.path.join(os.path.dirname(__file__), '../code_generator')) import code_generator_utils as utils d = utils.load_yaml_ordered(StringIO(input)) output = StringIO() utils.dump_yaml(d, output, default_flow_style=False, width=80) return output.getvalue() return input
def open_api_level_yaml(): class ApiLevelManager: def __init__(self, d): self.changed = False self.api_level_d = d self.current_level = sorted(self.api_level_d.keys()).pop() + 1 def append_new_id(self, uniq_name, id): if self.current_level not in self.api_level_d: self.api_level_d[self.current_level] = {uniq_name: id} else: self.api_level_d[self.current_level][uniq_name] = id self.changed = True api_level_yaml = join(here, 'api_levels.yaml') if not exists(api_level_yaml): regenerate_api_level_yaml() d = utils.load_yaml_ordered(open(api_level_yaml, 'r')) api_level_man = ApiLevelManager(d) yield api_level_man if api_level_man.changed: utils.dump_yaml(api_level_man.api_level_d, open(join(here, 'api_levels.yaml'), 'w'), default_flow_style=False, width=80)
def regenerate_api_level_yaml(): ''' regenerate api_level.yaml if there is no this file before. This is only called when api_level.yaml is missing. We re-constructed the revision history by its incremental id. We supposed that the duplicated function id is the fork point of API level. But once this file is created and committed, this function should never be called so that api_levels.yaml trace the revision history of nnabla APIs. ''' d = utils.load_yaml_ordered(open(join(here, 'functions.yaml'), 'r')) class APIInstance: pass api_instance_list = [] for cat_name, cat_info in d.items(): for func_name, func_info in d[cat_name].items(): fids = func_info['function_ids'] for param, id in fids.items(): api_inst = APIInstance() api_inst.func = func_name api_inst.param = param api_inst.id = id api_instance_list.append(api_inst) api_instance_list = sorted(api_instance_list, key=lambda api: api.id) level = 1 api_level = {} for api in api_instance_list: if level not in api_level: api_level[level] = {} api_level[level][api.func] = (api.param, api.id) elif api.func not in api_level[level]: api_level[level][api.func] = (api.param, api.id) else: level += 1 api_level[level] = {} api_level[level][api.func] = (api.param, api.id) yaml_api_level = {} for level, funcs in api_level.items(): if level not in yaml_api_level: yaml_api_level[level] = {} for func_name, (param, id) in funcs.items(): yaml_api_level[level][func_name + '_' + param] = id utils.dump_yaml(yaml_api_level, open(join(here, 'api_levels.yaml'), 'w'), default_flow_style=False, width=80)
def generate_functions_pkl(): import pickle d = utils.load_yaml_ordered(open(join(here, 'functions.yaml'), 'r')) for cat_name, cat_info in d.items(): for func_name, func_info in d[cat_name].items(): if 'doc' in func_info: del func_info['doc'] for a in ['inputs', 'arguments', 'outputs']: if a in func_info: for b in func_info[a]: if 'doc' in func_info[a][b]: del func_info[a][b]['doc'] with open(join(base, 'python/src/nnabla/utils/converter/functions.pkl'), 'wb') as f: pickle.dump(d, f, 2)
def main(): args = get_args() func_info = utils.load_function_info(flatten=True) if exists(args.path_types): func_types = utils.load_yaml_ordered(open(args.path_types, 'r')) else: func_types = OrderedDict() for name, func in func_info.items(): if name in func_types: continue print("Processing %s..." % name) types = OrderedDict() if args.default_type is not None: types[args.default_type] = [args.default_type] func_types[name] = types utils.dump_yaml(func_types, open(args.path_types, 'w'))
def generate_function_order(function_info): with open(join(base, 'python/src/nnabla/utils/converter/utils.py')) as f: exec(f.read()) order_yaml = join(base, 'python/src/nnabla/utils/converter/function_order.yaml') order_info = None if exists(order_yaml): with open(order_yaml, 'r') as f: order_info = utils.load_yaml_ordered(f) if order_info is None: order_info = OrderedDict() func_id = 0 for func, func_info in function_info.items(): fmt = '' if 'arguments' in func_info: fmt = '_' for arg, arg_info in func_info['arguments'].items(): fmt += type_to_pack_format(arg_info['type']) name = func + fmt if name not in order_info: while func_id in order_info.values(): func_id += 1 order_info[name] = func_id func_id += 1 with open(order_yaml, 'w') as f: f.write( '# Copyright (c) 2017 Sony Corporation. All Rights Reserved.\n') f.write('#\n') f.write( '# Licensed under the Apache License, Version 2.0 (the "License");\n' ) f.write( '# you may not use this file except in compliance with the License.\n' ) f.write('# You may obtain a copy of the License at\n') f.write('#\n') f.write('# http://www.apache.org/licenses/LICENSE-2.0\n') f.write('#\n') f.write( '# Unless required by applicable law or agreed to in writing, software\n' ) f.write( '# distributed under the License is distributed on an "AS IS" BASIS,\n' ) f.write( '# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n' ) f.write( '# See the License for the specific language governing permissions and\n' ) f.write('# limitations under the License.\n') f.write('#\n') f.write('# DO NOT EDIT THIS FILE!\n') f.write( '# THIS FILE IS GENERATED BY CODE GENERATOR BUT ITS COMMITTED INTO\n' ) f.write('# SOURCE TREE TO MAKE FUNCTION ID PERSIST.\n') f.write('\n') f.write(yaml.dump(order_info, default_flow_style=False))
def update_function_order_in_functsions_yaml(): d = utils.load_yaml_ordered(open(join(here, 'functions.yaml'), 'r')) order_info_by_id = {} order_info = OrderedDict() duplicated = {} missing = {} for cat_name, cat_info in d.items(): for func_name, func_info in d[cat_name].items(): order_info[func_name] = OrderedDict() default_full_name = func_name default_arg = '' if 'arguments' in func_info: for arg, arg_info in func_info['arguments'].items(): default_arg += type_to_pack_format(arg_info['type']) if default_arg == '': default_arg = 'Empty' else: default_full_name = func_name + '_' + default_arg if 'function_ids' in func_info and func_info['function_ids'] is not None: for func_arg, func_id in func_info['function_ids'].items(): full_name = func_name if func_arg != 'Empty': full_name = func_name + '_' + func_arg if func_id in order_info_by_id: if func_id not in duplicated: duplicated[func_id] = [order_info_by_id[func_id]] duplicated[func_id].append(full_name) order_info_by_id[func_id] = full_name order_info[func_name][full_name] = func_id if default_full_name not in order_info[func_name]: if cat_name not in missing: missing[cat_name] = {} if func_name not in missing[cat_name]: missing[cat_name][func_name] = [] missing[cat_name][func_name].append(default_arg) else: if cat_name not in missing: missing[cat_name] = {} if func_name not in missing[cat_name]: missing[cat_name][func_name] = [] missing[cat_name][func_name].append(default_arg) current_id = sorted(order_info_by_id.keys()).pop() + 1 for cat_name in missing: for func_name in missing[cat_name]: for arg in missing[cat_name][func_name]: if 'function_ids' not in d[cat_name][func_name] or d[cat_name][func_name]['function_ids'] is None: d[cat_name][func_name]['function_ids'] = OrderedDict() d[cat_name][func_name]['function_ids'][arg] = current_id current_id += 1 if len(duplicated): print('') print('############################################## Errors in functions.yaml(START)') for func_id, functions in duplicated.items(): if len(functions) > 1: print('ID {} duplicated between {}.'.format(func_id, functions)) print('Correct ID in "build-tools/code_generator/functions.yaml" manually.') print('############################################## Errors in functions.yaml(END)') print('') import sys sys.exit(-1) utils.dump_yaml(d, open(join(here, 'functions.yaml'), 'w'), default_flow_style=False, width=80)
def generate(argv): function_info = utils.load_function_info(flatten=True) solver_info = utils.load_solver_info() function_types = utils.load_yaml_ordered( open(join(here, 'function_types.yaml'), 'r')) function_types_cudnn = utils.load_yaml_ordered( open(join(here, 'function_types_cudnn.yaml'), 'r')) solver_types = utils.load_yaml_ordered( open(join(here, 'solver_types.yaml'), 'r')) function_template = join(base, 'src/nbla/cuda/function/function_types.cu.tmpl') function_template_cudnn = join( base, 'src/nbla/cuda/cudnn/function/function_types.cu.tmpl') solver_template = join(base, 'src/nbla/cuda/solver/solver_types.cu.tmpl') init_template = join(base, 'src/nbla/cuda/init.cpp.tmpl') init_template_cudnn = join(base, 'src/nbla/cuda/cudnn/init.cpp.tmpl') utils.generate_init(function_info, function_types, solver_info, solver_types, ext_info={}, template=init_template) utils.generate_init(function_info, function_types_cudnn, solver_info, solver_types, ext_info={}, template=init_template_cudnn) utils.generate_function_types(function_info, function_types, ext_info={}, template=function_template, output_format='%s.cu') utils.generate_function_types(function_info, function_types_cudnn, ext_info={}, template=function_template_cudnn, output_format='%s.cu') utils.generate_solver_types(solver_info, solver_types, ext_info={}, template=solver_template, output_format='%s.cu') if len(argv) == 4: version = argv[1] cuda_version = argv[2] cudnn_version = argv[3] else: version = 'unknown' cuda_version = 'unknown' cudnn_version = 'unknown' utils.generate_version(template=join( base, 'python/src/nnabla_ext/cuda/_version.py.tmpl'), rootdir=base, version=version, cuda_version=cuda_version, cudnn_version=cudnn_version) utils.generate_version(template=join( base, 'python/src/nnabla_ext/cudnn/_version.py.tmpl'), rootdir=base, version=version, cuda_version=cuda_version, cudnn_version=cudnn_version) utils.generate_version(template=join(base, 'src/nbla/cuda/version.cpp.tmpl'), rootdir=base, version=version, cuda_version=cuda_version, cudnn_version=cudnn_version) # Generate function skeletons func_src_template = join( base, 'src/nbla/cuda/function/generic/function_impl.cu.tmpl') func_src_template_cudnn = join( base, 'src/nbla/cuda/cudnn/function/generic/function_impl.cu.tmpl') func_header_template = join( base, 'include/nbla/cuda/function/function_impl.hpp.tmpl') func_header_template_cudnn = join( base, 'include/nbla/cuda/cudnn/function/function_impl.hpp.tmpl') utils.generate_skeleton_function_impl(function_info, function_types, ext_info={}, template=func_src_template, output_format='%s.cu') utils.generate_skeleton_function_impl(function_info, function_types, ext_info={}, template=func_header_template, output_format='%s.hpp') utils.generate_skeleton_function_impl(function_info, function_types_cudnn, ext_info={}, template=func_src_template_cudnn, output_format='%s.cu') utils.generate_skeleton_function_impl(function_info, function_types_cudnn, ext_info={}, template=func_header_template_cudnn, output_format='%s.hpp')
def generate_function_order(function_info): with open(join(base, 'python/src/nnabla/utils/converter/utils.py')) as f: exec(f.read()) order_yaml = join(base, 'python/src/nnabla/utils/converter/function_order.yaml') order_info = None if exists(order_yaml): with open(order_yaml, 'r') as f: order_info = utils.load_yaml_ordered(f) if order_info is None: order_info = OrderedDict() func_id = 0 for func, func_info in function_info.items(): fmt = '' if 'arguments' in func_info: fmt = '_' for arg, arg_info in func_info['arguments'].items(): fmt += type_to_pack_format(arg_info['type']) name = func + fmt if name not in order_info: while func_id in order_info.values(): func_id += 1 order_info[name] = func_id func_id += 1 ############### Check duplicated function ID. orders = {} result = True for func, func_id in order_info.items(): if func_id not in orders: orders[func_id] = [] else: result = False orders[func_id].append(func) if not result: print('') print( '############################################## Errors in function_order.yaml(START)' ) for func_id, functions in orders.items(): if len(functions) > 1: print('ID {} duplicated between [{}].'.format(func_id, functions)) if not result: print( 'Correct ID in "python/src/nnabla/utils/converter/function_order.yaml" manually.' ) print( '############################################## Errors in function_order.yaml(END)' ) print('') import sys sys.exit(-1) with open(order_yaml, 'w') as f: f.write( '# Copyright (c) 2017 Sony Corporation. All Rights Reserved.\n') f.write('#\n') f.write( '# Licensed under the Apache License, Version 2.0 (the "License");\n' ) f.write( '# you may not use this file except in compliance with the License.\n' ) f.write('# You may obtain a copy of the License at\n') f.write('#\n') f.write('# http://www.apache.org/licenses/LICENSE-2.0\n') f.write('#\n') f.write( '# Unless required by applicable law or agreed to in writing, software\n' ) f.write( '# distributed under the License is distributed on an "AS IS" BASIS,\n' ) f.write( '# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n' ) f.write( '# See the License for the specific language governing permissions and\n' ) f.write('# limitations under the License.\n') f.write('#\n') f.write('# DO NOT EDIT THIS FILE!\n') f.write( '# THIS FILE IS GENERATED BY CODE GENERATOR BUT ITS COMMITTED INTO\n' ) f.write('# SOURCE TREE TO MAKE FUNCTION ID PERSIST.\n') f.write('\n') for func_id in sorted(orders.keys()): f.write('{}: {}\n'.format(orders[func_id][0], func_id))