def format_file(file_ext, input): cmd = None file_ext = file_ext.lower() if file_ext in c_extensions: cmd = [search_clang_format(), '--style=llvm'] elif file_ext in python_extensions: cmd = [search_autopep8(), '--ignore={}'.format(pep8_ignores), '-'] elif file_ext in cython_extensions: cmd = [ search_autopep8(), '--ignore={}'.format(pep8_cython_ignores), '-' ] if cmd is not None: p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) output, output_err = p.communicate(input.encode('utf-8')) return output.decode('utf_8') else: from six import StringIO import sys sys.path.append( os.path.join(os.path.dirname(__file__), '../code_generator')) import code_generator_utils as utils d = utils.load_yaml_ordered(StringIO(input)) output = StringIO() utils.dump_yaml(d, output, default_flow_style=False, width=80) return output.getvalue() return input
def open_api_level_yaml(): class ApiLevelManager: def __init__(self, d): self.changed = False self.api_level_d = d self.current_level = sorted(self.api_level_d.keys()).pop() + 1 def append_new_id(self, uniq_name, id): if self.current_level not in self.api_level_d: self.api_level_d[self.current_level] = {uniq_name: id} else: self.api_level_d[self.current_level][uniq_name] = id self.changed = True api_level_yaml = join(here, 'api_levels.yaml') if not exists(api_level_yaml): regenerate_api_level_yaml() d = utils.load_yaml_ordered(open(api_level_yaml, 'r')) api_level_man = ApiLevelManager(d) yield api_level_man if api_level_man.changed: utils.dump_yaml(api_level_man.api_level_d, open(join(here, 'api_levels.yaml'), 'w'), default_flow_style=False, width=80)
def main(): functions = Functions() info = functions.info root_yaml = OrderedDict() for cat_name, cat in info['Functions'].items(): root_yaml[cat_name] = category_yaml(cat, info['Names']) utils.dump_yaml(root_yaml, open('functions.yaml', 'w'), default_flow_style=False)
def regenerate_api_level_yaml(): ''' regenerate api_level.yaml if there is no this file before. This is only called when api_level.yaml is missing. We re-constructed the revision history by its incremental id. We supposed that the duplicated function id is the fork point of API level. But once this file is created and committed, this function should never be called so that api_levels.yaml trace the revision history of nnabla APIs. ''' d = utils.load_yaml_ordered(open(join(here, 'functions.yaml'), 'r')) class APIInstance: pass api_instance_list = [] for cat_name, cat_info in d.items(): for func_name, func_info in d[cat_name].items(): fids = func_info['function_ids'] for param, id in fids.items(): api_inst = APIInstance() api_inst.func = func_name api_inst.param = param api_inst.id = id api_instance_list.append(api_inst) api_instance_list = sorted(api_instance_list, key=lambda api: api.id) level = 1 api_level = {} for api in api_instance_list: if level not in api_level: api_level[level] = {} api_level[level][api.func] = (api.param, api.id) elif api.func not in api_level[level]: api_level[level][api.func] = (api.param, api.id) else: level += 1 api_level[level] = {} api_level[level][api.func] = (api.param, api.id) yaml_api_level = {} for level, funcs in api_level.items(): if level not in yaml_api_level: yaml_api_level[level] = {} for func_name, (param, id) in funcs.items(): yaml_api_level[level][func_name + '_' + param] = id utils.dump_yaml(yaml_api_level, open(join(here, 'api_levels.yaml'), 'w'), default_flow_style=False, width=80)
def main(): args = get_args() func_info = utils.load_function_info(flatten=True) if exists(args.path_types): func_types = utils.load_yaml_ordered(open(args.path_types, 'r')) else: func_types = OrderedDict() for name, func in func_info.items(): if name in func_types: continue print("Processing %s..." % name) types = OrderedDict() if args.default_type is not None: types[args.default_type] = [args.default_type] func_types[name] = types utils.dump_yaml(func_types, open(args.path_types, 'w'))
def update_function_order_in_functsions_yaml(): d = utils.load_yaml_ordered(open(join(here, 'functions.yaml'), 'r')) order_info_by_id = {} order_info = OrderedDict() duplicated = {} missing = {} for cat_name, cat_info in d.items(): for func_name, func_info in d[cat_name].items(): order_info[func_name] = OrderedDict() default_full_name = func_name default_arg = '' if 'arguments' in func_info: for arg, arg_info in func_info['arguments'].items(): default_arg += type_to_pack_format(arg_info['type']) if default_arg == '': default_arg = 'Empty' else: default_full_name = func_name + '_' + default_arg if 'function_ids' in func_info and func_info['function_ids'] is not None: for func_arg, func_id in func_info['function_ids'].items(): full_name = func_name if func_arg != 'Empty': full_name = func_name + '_' + func_arg if func_id in order_info_by_id: if func_id not in duplicated: duplicated[func_id] = [order_info_by_id[func_id]] duplicated[func_id].append(full_name) order_info_by_id[func_id] = full_name order_info[func_name][full_name] = func_id if default_full_name not in order_info[func_name]: if cat_name not in missing: missing[cat_name] = {} if func_name not in missing[cat_name]: missing[cat_name][func_name] = [] missing[cat_name][func_name].append(default_arg) else: if cat_name not in missing: missing[cat_name] = {} if func_name not in missing[cat_name]: missing[cat_name][func_name] = [] missing[cat_name][func_name].append(default_arg) current_id = sorted(order_info_by_id.keys()).pop() + 1 for cat_name in missing: for func_name in missing[cat_name]: for arg in missing[cat_name][func_name]: if 'function_ids' not in d[cat_name][func_name] or d[cat_name][func_name]['function_ids'] is None: d[cat_name][func_name]['function_ids'] = OrderedDict() d[cat_name][func_name]['function_ids'][arg] = current_id current_id += 1 if len(duplicated): print('') print('############################################## Errors in functions.yaml(START)') for func_id, functions in duplicated.items(): if len(functions) > 1: print('ID {} duplicated between {}.'.format(func_id, functions)) print('Correct ID in "build-tools/code_generator/functions.yaml" manually.') print('############################################## Errors in functions.yaml(END)') print('') import sys sys.exit(-1) utils.dump_yaml(d, open(join(here, 'functions.yaml'), 'w'), default_flow_style=False, width=80)