def convert_loss_scale_api(node): """Convert loss scale related Tensorflow APIs""" if isinstance(node.func, ast.Attribute): if node.func.attr == "FixedLossScale": log_msg( getattr(node, 'lineno', 'None'), "change tf.train.experimental.FixedLossScale" " to FixedLossScaleManager") node.func = ast.Name(id="FixedLossScaleManager", ctx=ast.Load()) if len(node.keywords) == 1: node.keywords[0].arg = "loss_scale" util_global.set_value('need_conver', True) return node if node.func.attr == "DynamicLossScale": return convert_dynamic_loss_scale(node) if node.func.attr == "MixedPrecisionLossScaleOptimizer": log_msg( getattr(node, 'lineno', 'None'), "change tf.train.experimental.MixedPrecisionLossScaleOptimizer" " to NPULossScaleOptimizer") node.func = ast.Name(id="NPULossScaleOptimizer", ctx=ast.Load()) for keyword in node.keywords: if keyword.arg == "loss_scale": keyword.arg = "loss_scale_manager" if (len(util_global.get_value("distributed_mode", "")) != 0): node.keywords.append( ast.keyword(arg="is_distributed", value=pasta.parse("True"))) util_global.set_value('need_conver', True) return node
def conver(): print("Begin conver, input file: " + util_global.get_value('input')) out_path = util_global.get_value('output') dst_path = os.path.split(util_global.get_value('input').rstrip('\\/'))[-1] conver_path = os.walk(util_global.get_value('input')) for path, dir_list, file_list in conver_path: for file_name in file_list: out_path_dst = abs_join(dst_path, path.split(dst_path)[1]) if file_name.endswith(".py"): util_global.set_value('path', os.path.join(path, file_name)) mkdir(os.path.join(out_path, out_path_dst)) conver_ast(path, out_path_dst, file_name) if util_global.get_value('need_conver', False): content = "Finish conver file: " + os.path.join( path, file_name) print(content) write_report_terminator(content) else: mkdir_and_copyfile(path, abs_join(out_path, out_path_dst), file_name) else: mkdir_and_copyfile(path, abs_join(out_path, out_path_dst), file_name) print("Finish conver, output file: " + out_path + "; report file: " + util_global.get_value('report'))
def ast_import(node): for value in node.names: if isinstance(value, ast.alias): values = value.name.split(".") if "keras" in values: log_migration_report(getattr(node, "lineno", "None"), "keras") util_global.set_value('need_conver', True)
def scan_file(path, file_name, api, lineno): api_list = pd.read_excel(util_global.get_value('list'), sheet_name=0) api_module = api_list['模块名'].values.tolist() api_name = api_list['API名'].values.tolist() api_support = api_list['工具迁移API支持度'].values.tolist() api_advice = api_list['说明'].values.tolist() script_name = [] code_line = [] code_module = [] code_api = [] support_type = [] migrate_advice = [] for i in range(len(api)): name = api[i] if name in api_name: script_name.append(file_name) code_api.append(name) code_line.append(lineno[i]) code_module.append(api_module[api_name.index(name)]) support_type.append(api_support[api_name.index(name)]) migrate_advice.append(api_advice[api_name.index(name)]) # search for tf enumeration enume_list = pd.read_excel(util_global.get_value('list'), sheet_name=1) enume_name = enume_list['API名'].values.tolist() (enume, lineno) = get_tf_enume(os.path.join(path, file_name), enume_name) for i in range(len(enume)): name = enume[i] class_name = '.'.join(name.split('.')[:-1]) if name not in code_api and class_name not in code_api: if class_name in api_name: script_name.append(file_name) code_api.append(class_name) code_line.append(lineno[i]) code_module.append(api_module[api_name.index(class_name)]) support_type.append(api_support[api_name.index(class_name)]) migrate_advice.append(api_advice[api_name.index(class_name)]) # record unsupported api (unsupport, unsupport_module, lineno) = get_unsupport_api(os.path.join(path, file_name)) for i in range(len(unsupport)): script_name.append(file_name) code_api.append(unsupport[i]) code_line.append(lineno[i]) code_module.append(unsupport_module[i]) support_type.append('不支持(无迁移方案,建议用户不使用)') migrate_advice.append('第三方非TF官网API,暂不支持') analyse_result = pd.DataFrame({'脚本文件名': script_name, '代码行': code_line, '模块名': code_module, 'API名': code_api, '工具迁移API支持度': support_type, '说明': migrate_advice}) # when there are tf apis used in script, analysis report will be generated report = util_global.get_value('generate_dir_report') if len(script_name): report = report.append(analyse_result) util_global.set_value('generate_dir_report', report)
def convert_origin_func_to_npu(node, origin_func, org_func_name, params_list, is_class_func=None): """Convert original Tensorflow function to NPU function""" if not check_func_arguments(origin_func, node.args, node.keywords, is_class_func): return node if org_func_name == "Estimator.train": content = "".join([ util_global.get_value('path'), ":", str(getattr(node, "lineno", "None")) ]) while True: message = input( "Check if the train function in " + content + " is the Estimator train function. If yes, " "enter 'y' to perform distributed porting on the train function. if no, enter 'n': " ) if message == "y": break if message == "n": log_warning("".join([ "The train func in ", content, " is user-defined functions, will not perform distributed porting" ])) return node print("Input is error, Please enter 'y' or 'n'.") for param_name in params_list: node = match_func_params_and_convert(node, origin_func, org_func_name, param_name, is_class_func) util_global.set_value('need_conver', True) return node
def conver(): """The entry point to convert Tensorflow script""" print("Begin conver, input file: " + util_global.get_value('input') + '\n') out_path = util_global.get_value('output') dst_path = os.path.split(util_global.get_value('input').rstrip('\\/'))[-1] dst_path_new = dst_path + util_global.get_value('timestap') conver_path = os.walk(util_global.get_value('input')) report_dir = util_global.get_value('report') mkdir(report_dir) report_xlsx = os.path.join(report_dir, 'api_analysis_report.xlsx') util_global.set_value('generate_dir_report', pd.DataFrame()) for path, _, file_list in conver_path: for file_name in file_list: out_path_dst = abs_join( dst_path_new, path.split(util_global.get_value('input'))[1]) file_path = os.path.join(path, file_name).replace('\\', '/') if not check_path_length(file_path): content = "".join([ "The file:", file_path, " length is invalid, skip convert." ]) log_warning(content) continue content = "".join(["Begin conver file: ", file_path]) print(content) threshold_file_size = 10 * 1024 * 1024 if file_name.endswith(".py"): if os.path.getsize(file_path) > threshold_file_size: content = "".join([ "The file:", file_path, " size is over 10M, skip convert." ]) log_warning(content) continue util_global.set_value('path', file_path) mkdir(os.path.join(out_path, out_path_dst)) conver_ast(path, out_path_dst, file_name) if util_global.get_value('need_conver', False): content = "".join( ["Finish conver file: ", file_path, '\n']) print(content) write_report_terminator(content) else: mkdir_and_copyfile(path, abs_join(out_path, out_path_dst), file_name) else: mkdir_and_copyfile(path, abs_join(out_path, out_path_dst), file_name) adjust_index() analysis_report = util_global.get_value('generate_dir_report') if analysis_report.empty: print('No api data in the report') else: analysis_report.to_excel(report_xlsx, index=True) get_api_statistic(analysis_report) print("Finish conver, output file: " + out_path + "; report file: " + util_global.get_value('report'))
def adjust_index(): report = util_global.get_value('generate_dir_report') index_column = [] for i in range(len(report)): index_column.append(i + 1) report.index = index_column report.index.name = '序号' util_global.set_value('generate_dir_report', report)
def attribute(node): log_success_report(getattr(node, "lineno", "None"), node.attr) if node.attr == 'dropout': node.value = ast.Name(id=util_global.get_value(node.attr)[0], ctx=ast.Load()) else: node = ast.Name(id=util_global.get_value(node.attr)[0], ctx=ast.Load()) util_global.set_value('need_conver', True) return node
def log_success_report(lineno, msg): content = (util_global.get_value('path', '') + ':' + str(lineno) + ' change ' + util_global.get_value(msg)[1] + ' to ' + util_global.get_value(msg)[2]) print(content) write_conver_report(content, util_global.get_value('report_file')[0]) util_global.set_value('report_file_status', (util_global.get_value('report_file_status') | 0b1))
def convert_dynamic_loss_scale(node): """Convert dynamic loss scale related Tensorflow APIs""" log_msg( getattr(node, 'lineno', 'None'), "change tf.train.experimental.DynamicLossScale" " to ExponentialUpdateLossScaleManager") node.func = ast.Name(id="ExponentialUpdateLossScaleManager", ctx=ast.Load()) def check_arg(node): initial_loss_scale = None increment_period = None multiplier = None for index, arg in enumerate(node.args): if index == 0: initial_loss_scale = arg if index == 1: increment_period = arg if index == 2: multiplier = arg for keyword in node.keywords: if keyword.arg == "initial_loss_scale": keyword.arg = "init_loss_scale" initial_loss_scale = keyword if keyword.arg == "increment_period": keyword.arg = "incr_every_n_steps" increment_period = keyword if keyword.arg == "multiplier": keyword.arg = "incr_ratio" multiplier = keyword return (initial_loss_scale, increment_period, multiplier) (initial_loss_scale, increment_period, multiplier) = check_arg(node) if initial_loss_scale: if not isinstance(initial_loss_scale, ast.keyword): node.keywords.append( ast.keyword(arg="init_loss_scale", value=initial_loss_scale)) else: node.keywords.append( ast.keyword(arg="init_loss_scale", value=pasta.parse("2**15"))) if increment_period: if not isinstance(increment_period, ast.keyword): node.keywords.append( ast.keyword(arg="incr_every_n_steps", value=increment_period)) else: node.keywords.append( ast.keyword(arg="incr_every_n_steps", value=pasta.parse("2000"))) if multiplier: if not isinstance(multiplier, ast.keyword): node.keywords.append( ast.keyword(arg="incr_ratio", value=multiplier)) else: node.keywords.append( ast.keyword(arg="incr_ratio", value=pasta.parse("2"))) node.args = [] util_global.set_value('need_conver', True) return node
def log_migration_report(lineno, msg): content = ( util_global.get_value('path', '') + ':' + str(lineno) + ' "' + msg + '" feature needs to be migrated manually, Please refer to the migration guide.' + util_global.get_value(msg)[0]) print(content) write_conver_report(content, util_global.get_value('report_file')[2]) util_global.set_value( 'report_file_status', (util_global.get_value('report_file_status') | 0b100))
def ast_function_def(node): log_success_report(getattr(node, "lineno", "None"), node.name) node.body = [ast.Return(value=ast.Call( func=ast.Attribute(value=ast.Name(id=util_global.get_value(node.name)[0], ctx=ast.Load()), attr='gelu', ctx=ast.Load()), args=[ast.Name(id='x', ctx=ast.Load())], keywords=[]))] util_global.set_value('need_conver', True) return node
def visit_Attribute(self, node): self.generic_visit(node) if node.attr == "keras": util_global.set_value('is_keras_net', True) if node.attr in util_global.get_value('hvd'): if isinstance(node.value, ast.Name): if 'hvd' in str(node.value.id): return attribute(node) if isinstance(node.value, ast.Attribute): if 'hvd' in str(node.value.attr): return attribute(node) return node
def visit_Attribute(self, node): """Visit and transform attr node""" self.generic_visit(node) if node.attr == "keras": util_global.set_value('is_keras_net', True) if node.attr in util_global.get_value('hvd'): distributed_mode = util_global.get_value("distributed_mode", "") if isinstance(node.value, ast.Name) and 'hvd' in str(node.value.id): if distributed_mode in ("tf_strategy", ""): log_strategy_distributed_mode_error(node) return node return attribute(node) return node
def ast_if(node): if isinstance(node.test, ast.Compare): if len(node.test.comparators) == 1 and isinstance( node.test.comparators[0], ast.Str): if node.test.comparators[0].s == "__main__": util_global.set_value("is_main_file", False) util_global.set_value("has_main_func", True) if util_global.get_value("is_keras_net", False): log_msg(getattr(node, "lineno", "None"), " add keras session npu config") close_sess_call = ast.Call( func=ast.Name(id="close_session", ctx=ast.Load()), args=[ast.Name(id="npu_keras_sess", ctx=ast.Load())], keywords=[]) keras_sess_assign = ast.Assign( targets=[ ast.Name(id="npu_keras_sess", ctx=ast.Store()) ], value=ast.Call(func=ast.Name( id="set_keras_session_npu_config", ctx=ast.Load()), args=[], keywords=[])) node.body = [keras_sess_assign] + node.body + [ ast.Expr(value=close_sess_call) ] util_global.set_value('need_conver', True) if util_global.get_value("has_hccl_api", False): log_msg(getattr(node, "lineno", "None"), " add npu resource init api") close_sess_call = ast.Call( func=ast.Name(id="close_session", ctx=ast.Load()), args=[ast.Name(id="npu_sess", ctx=ast.Load())], keywords=[]) init_assign = ast.Assign(targets=[ ast.Tuple(elts=[ ast.Name(id="npu_sess", ctx=ast.Store()), ast.Name(id="npu_shutdown", ctx=ast.Store()) ], ctx=ast.Store()) ], value=ast.Call(func=ast.Name( id="init_resource", ctx=ast.Load()), args=[], keywords=[])) shutdown_call = ast.Call(func=ast.Name( id="shutdown_resource", ctx=ast.Load()), args=[ ast.Name(id="npu_sess", ctx=ast.Load()), ast.Name(id="npu_shutdown", ctx=ast.Load()) ], keywords=[]) node.body = [init_assign] + node.body + [ ast.Expr(value=shutdown_call), ast.Expr(value=close_sess_call) ] util_global.set_value('need_conver', True) return node
def write_report_terminator(content): report_path = util_global.get_value('report') value = util_global.get_value('report_file_status') times = value.bit_length() while times > 0: if get_bit_val(value, times - 1): file = util_global.get_value('report_file')[times - 1] if os.path.exists(os.path.join(report_path, file)): file = open(os.path.join(report_path, file), 'a') file.write(content) file.write("\r\n") file.write("\r\n") file.close() times = times - 1 util_global.set_value('report_file_status', 0)
def convert_tf_gradient_distributed(node): """Convert Tensorflow gradient APIs in distributed mode""" content = "".join([ util_global.get_value('path'), ":", str(getattr(node, "lineno", "None")), " is tf.gradient api, tool inserts npu_allreduce after computing grads by default.", " You can adjust the allreduce position according to the algorithm" ]) log_warning(content) new_node = ast.Call(func=ast.Name(id="npu_allreduce", ctx=ast.Load()), args=[node], keywords=[]) ast.copy_location(new_node, node) util_global.set_value("need_conver", True) return new_node
def write_report_terminator(content): """Write content to report and update global variable""" report_path = util_global.get_value('report') value = util_global.get_value('report_file_status') times = value.bit_length() while times > 0: if get_bit_val(value, times - 1): file = util_global.get_value('report_file')[times - 1] if os.path.exists(os.path.join(report_path, file)): with open(os.path.join(report_path, file), 'a') as file: file.write(content) file.write("\r\n") file.write("\r\n") times = times - 1 util_global.set_value('report_file_status', 0)
def import_from(node): if node.module != None: values = node.module.split(".") if "keras" in values: util_global.set_value('is_keras_net', True) if "horovod" in values: util_global.set_value('has_hccl_api', True) for value in node.names: if isinstance(value, ast.alias): values = value.name.split(".") if "keras" in values: util_global.set_value('is_keras_net', True) if "horovod" in values: util_global.set_value('has_hccl_api', True) util_global.set_value('need_conver', True)
def conver(): print("Begin conver, input file: " + util_global.get_value('input') + '\n') out_path = util_global.get_value('output') dst_path = os.path.split(util_global.get_value('input').rstrip('\\/'))[-1] dst_path_new = dst_path + util_global.get_value('timestap') conver_path = os.walk(util_global.get_value('input')) report_dir = util_global.get_value('report') mkdir(report_dir) report_xlsx = os.path.join(report_dir, 'api_analysis_report.xlsx') util_global.set_value('generate_dir_report', pd.DataFrame()) for path, dir_list, file_list in conver_path: for file_name in file_list: out_path_dst = abs_join( dst_path_new, path.split(util_global.get_value('input'))[1]) file_path = os.path.join(path, file_name).replace('\\', '/') content = "Begin conver file: " + file_path print(content) if file_name.endswith(".py"): util_global.set_value('path', file_path) mkdir(os.path.join(out_path, out_path_dst)) conver_ast(path, out_path_dst, file_name) if util_global.get_value('need_conver', False): content = "Finish conver file: " + file_path + '\n' print(content) write_report_terminator(content) else: mkdir_and_copyfile(path, abs_join(out_path, out_path_dst), file_name) else: mkdir_and_copyfile(path, abs_join(out_path, out_path_dst), file_name) adjust_index() analysis_report = util_global.get_value('generate_dir_report') if analysis_report.empty: print('No api data in the report') else: analysis_report.to_excel(report_xlsx, index=True) get_api_statistic(analysis_report) print("Finish conver, output file: " + out_path + "; report file: " + util_global.get_value('report'))
def convert_hvd_distributed_api(node): """Convert horovod distributed APIs""" log_msg( getattr(node, "lineno", "None"), 'change hvd.DistributedOptimizer to npu_distributed_optimizer_wrapper') node.func = ast.Name(id="npu_distributed_optimizer_wrapper", ctx=ast.Load()) opt_keyword = None for keyword in node.keywords: if keyword.arg == "optimizer": opt_keyword = keyword node.keywords.clear() if opt_keyword is None: opt_arg = node.args[0] node.args.clear() node.args.append(opt_arg) else: node.keywords.append(opt_keyword) util_global.set_value('need_conver', True) return node
def conver_ast(path, out_path_dst, file_name): util_global.set_value('need_conver', False) file = open(os.path.join(path, file_name), "r") source = file.read() r_node = ast.parse(source) sys.setrecursionlimit(10000) visitor = ConverByAst() visitor.visit(r_node) ast.fix_missing_locations(r_node) if util_global.get_value('need_conver', False): insert_npu_import(r_node) dst_content = astunparse.unparse(r_node) write_output_after_conver( os.path.join(util_global.get_value('output'), out_path_dst, file_name), dst_content) if file_name.endswith("a.py"): write_report_after_conver("only_for_test", file_name, node_tree(ast.dump(r_node)))
def get_tf_api(file_name): """ Args: file_name: script file Returns: list objects containing all the tensorflow apis extracted from file_name and corrosponding line number """ util_global.set_value('use_keras_dropout', False) with open(file_name, 'r', encoding='utf-8') as file: source = file.read() tree = ast.parse(source) visitor = VisitCall() visitor.visit(tree) # get tensorflow related api api = [] lineno = [] import_list = ['tf', 'hvd'] keras_dropout_api = [ 'tf.layers.dropout', 'tf.layers.Dropout', 'tf.keras.layers.Dropout', 'tf.keras.layers.SpatialDropout1D', 'tf.keras.layers.SpatialDropout2D', 'tf.keras.layers.SpatialDropout3D' ] for module in import_list: for i in range(len(visitor.call_nodes)): if "".join([ module, '.' ]) in visitor.call_nodes[i] and visitor.call_nodes[i].split( '.')[0] == module: api.append(visitor.call_nodes[i]) lineno.append(visitor.line_nos[i]) # get tf api using keras dropout if visitor.call_nodes[i] in keras_dropout_api: util_global.set_value('use_keras_dropout', True) return api, lineno
def ast_import(node): for value in node.names: if isinstance(value, ast.alias): values = value.name.split(".") if "keras" in values: util_global.set_value('is_keras_net', True) if "horovod" in values: util_global.set_value('has_hccl_api', True) util_global.set_value('need_conver', True)
def ast_import(node): """Modify import module""" for value in node.names: if isinstance(value, ast.alias): values = value.name.split(".") if "keras" in values: util_global.set_value('is_keras_net', True) if "horovod" in values: log_msg(getattr(node, "lineno", "None"), "remove horovod import line to None") util_global.set_value('has_hvd_api', True) new_node = ast.Expr(value=ast.NameConstant(value=None)) ast.copy_location(new_node, node) util_global.set_value('need_conver', True) return new_node util_global.set_value('need_conver', True) return node
def ast_call(node): if isinstance(node.func, ast.Attribute): if len(node.args) > 0: if isinstance(node.args[0], ast.Call): if isinstance(node.args[0].func, ast.Attribute): if node.args[0].func.attr == 'BroadcastGlobalVariablesHook': log_success_report(getattr(node, "lineno", "None"), 'BroadcastGlobalVariablesHook') node.func = ast.Name(id=util_global.get_value('BroadcastGlobalVariablesHook')[0], ctx=ast.Load()) node.args = [] util_global.set_value('need_conver', True) if isinstance(node.func, ast.Attribute) and node.func.attr == 'shard': log_success_report(getattr(node, "lineno", "None"), 'shard') node.args = [ast.Call(func=ast.Name(id='get_rank_size', ctx=ast.Load()), args=[], keywords=[]), ast.Call(func=ast.Name(id='get_rank_id', ctx=ast.Load()), args=[], keywords=[])] util_global.set_value('need_conver', True) if isinstance(node.func, ast.Attribute) and (node.func.attr == 'batch' or node.func.attr == 'map_and_batch'): exist = False for keyword in node.keywords: if keyword.arg == 'drop_remainder': exist = True if ((isinstance(keyword.value, ast.NameConstant) and keyword.value.value != True) or (not isinstance(keyword.value, ast.NameConstant))): log_success_report(getattr(node, "lineno", "None"), node.func.attr) keyword.value = ast.NameConstant(value=True) util_global.set_value('need_conver', True) if not exist: log_success_report(getattr(node, "lineno", "None"), node.func.attr) keyword = ast.keyword(arg='drop_remainder', value=ast.NameConstant(value=True)) node.keywords.insert(0, keyword) util_global.set_value('need_conver', True) if (isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name) and node.func.value.id == 'tf' and node.func.attr == 'device'): log_success_report(getattr(node, "lineno", "None"), node.func.attr) node.args = [ast.Str(s='/cpu:0')] util_global.set_value('need_conver', True) return node
def para_check_and_set(argv): input = "input" output = "output" report = "report" try: opts, args = getopt.getopt(argv, "hi:o:r:", ["help", "input=", "output=", "report="]) except getopt.GetoptError: print('Parameter error, please check.') print(' main.py -i <input> -o <output> -r <report>') print( 'or: main.py --input=<input> --output=<output> --report=<report>') print( '-i or --input: The source script to be converted, Default value: input/' ) print( '-o or --output: The destination script after converted, Default value: output/' ) print('-r or --report: Conversion report, Default value: report/') sys.exit(2) for opt, arg in opts: if opt in ("-h", "--help"): print(' main.py -i <input> -o <output> -r <report>') print( 'or: main.py --input=<input> --output=<output> --report=<report>' ) print( '-i or --input: The source script to be converted, Default value: input/' ) print( '-o or --output: The destination script after converted, Default value: output/' ) print('-r or --report: Conversion report, Default value: report/') sys.exit() elif opt in ("-i", "--input"): input = arg elif opt in ("-o", "--output"): output = arg elif opt in ("-r", "--report"): report = arg util_global.set_value('input', input) util_global.set_value('output', output) util_global.set_value('report', report)
def para_check_and_set(argv): input_dir = "npu_input" support_list = "tf1.15_api_support_list.xlsx" output = "output" + util_global.get_value('timestap') report = "report" + util_global.get_value('timestap') report_suffix = report main_file = "" try: opts, args = getopt.getopt(argv, "hi:l:o:r:m:", ["help", "input=", "list=", "output=", "report=", "main="]) except getopt.GetoptError: print('Parameter error, please check.') print(' this tool just support to convert tf-1.15 scripts.') print(' main.py -i <input> -l <list> -o <output> -r <report> -m <main>') print('or: main.py --input=<input> --list=<list> --output=<output> --report=<report> --main=<main>') print('-i or --input: The source script to be converted.') print('-l or --list: The list of supported api, Default value: tf1.15_api_support_list.xlsx') print('-o or --output: The destination script after converted, Default value: output_npu_***/') print('-r or --report: Conversion report, Default value: report_npu_***/') print('-m or --main: the executed entry *.py file, default:None') sys.exit(2) for opt, arg in opts: if opt in ("-h", "--help"): print(' this tool just support to convert tf-1.15 scripts.') print(' main.py -i <input> -l <list> -o <output> -r <report> -m <main>') print('or: main.py --input=<input> --list=<list> --output=<output> --report=<report> --main=<main>') print('-i or --input: The source script to be converted') print('-l or --list: The list of supported api, Default value: tf1.15_api_support_list.xlsx') print('-o or --output: The destination script after converted, Default value: output_npu_***/') print('-r or --report: Conversion report, Default value: report_npu_***/') print('-m or --main: the executed entry *.py file, default:None') sys.exit() elif opt in ("-i", "--input"): input_dir = os.path.abspath(arg) if str(input_dir).endswith('/'): input_dir = input_dir[0:len(input_dir)-1] input_dir = input_dir.replace('\\', '/') elif opt in ("-l", "--list"): support_list = arg elif opt in ("-o", "--output"): output = os.path.abspath(arg) if str(output).endswith('/'): output = output[0:len(output)-1] output = output.replace('\\', '/') elif opt in ("-r", "--report"): report = os.path.abspath(arg) if str(report).endswith('/'): report = report[0:len(report)-1] report = os.path.join(report, report_suffix) report = report.replace('\\', '/') elif opt in ("-m", "--main"): if os.path.isfile(arg): main_file = os.path.abspath(arg) main_path = os.path.dirname(main_file) file = os.path.basename(main_file) main_path = main_path.replace('\\', '/') main_file = os.path.join(main_path, file) else: raise ValueError("--main args must be exited files") if input_dir == "npu_input": raise ValueError("Please check -i or --input.") if input_dir + '/' in output + '/' or input_dir + '/' in report + '/': print("<output> or <report> could not be the subdirectory of <input>, please try another option.") sys.exit(2) util_global.set_value('input', input_dir) util_global.set_value('list', support_list) util_global.set_value('output', output) util_global.set_value('report', report) util_global.set_value('main', main_file)
def conver_ast(path, out_path_dst, file_name): util_global.set_value('need_conver', False) util_global.set_value('is_keras_net', False) util_global.set_value('has_hccl_api', False) util_global.set_value('is_main_file', False) util_global.set_value('has_main_func', False) if os.path.join(path, file_name) == util_global.get_value('main', ""): util_global.set_value('is_main_file', True) with open(os.path.join(path, file_name), "r", encoding='utf-8') as file: source = file.read() try: r_node = pasta.parse(source) except Exception as e: print(repr(e)) return sys.setrecursionlimit(10000) visitor = ConverByAst() visitor.visit(r_node) ast.fix_missing_locations(r_node) (api, lineno) = get_tf_api(os.path.join(path, file_name)) if len(api) == 0: print( "No Tensorflow module is imported in script {}.".format(file_name)) scan_file(path, file_name, api, lineno) if util_global.get_value('need_conver', False): insert_npu_import(r_node) if not util_global.get_value('has_main_func', False) and ( util_global.get_value('has_hccl_api', False) or util_global.get_value('is_keras_net', False)): log_warning( 'the network of keras and horovod, or using dataset.shard script do not have main func, ' 'should set -m or --main parameter') if util_global.get_value('is_main_file', False) and util_global.get_value( 'has_hccl_api', False): insert_npu_resource_init(r_node) insert_npu_resource_shutdown(r_node) if util_global.get_value('is_main_file', False) and util_global.get_value( 'is_keras_net', False): insert_keras_sess_npu_config(r_node) insert_keras_sess_close(r_node) dst_content = pasta.dump(r_node) write_output_after_conver( os.path.join(util_global.get_value('output'), out_path_dst, file_name), dst_content) if file_name.endswith("a.py"): write_report_after_conver("only_for_test", file_name, node_tree(ast.dump(r_node)))
def import_from(node): if node.module != None: values = node.module.split(".") if "keras" in values: log_migration_report(getattr(node, "lineno", "None"), "keras") util_global.set_value('need_conver', True)