def convert_loss_scale_api(node):
"""Convert loss scale related Tensorflow APIs"""
if isinstance(node.func, ast.Attribute):
if node.func.attr == "FixedLossScale":
log_msg(
getattr(node, 'lineno', 'None'),
"change tf.train.experimental.FixedLossScale"
" to FixedLossScaleManager")
node.func = ast.Name(id="FixedLossScaleManager", ctx=ast.Load())
if len(node.keywords) == 1:
node.keywords[0].arg = "loss_scale"
util_global.set_value('need_conver', True)
return node
if node.func.attr == "DynamicLossScale":
return convert_dynamic_loss_scale(node)
if node.func.attr == "MixedPrecisionLossScaleOptimizer":
log_msg(
getattr(node, 'lineno', 'None'),
"change tf.train.experimental.MixedPrecisionLossScaleOptimizer"
" to NPULossScaleOptimizer")
node.func = ast.Name(id="NPULossScaleOptimizer", ctx=ast.Load())
for keyword in node.keywords:
if keyword.arg == "loss_scale":
keyword.arg = "loss_scale_manager"
if (len(util_global.get_value("distributed_mode", "")) != 0):
node.keywords.append(
ast.keyword(arg="is_distributed",
value=pasta.parse("True")))
util_global.set_value('need_conver', True)
return node
def conver():
print("Begin conver, input file: " + util_global.get_value('input'))
out_path = util_global.get_value('output')
dst_path = os.path.split(util_global.get_value('input').rstrip('\\/'))[-1]
conver_path = os.walk(util_global.get_value('input'))
for path, dir_list, file_list in conver_path:
for file_name in file_list:
out_path_dst = abs_join(dst_path, path.split(dst_path)[1])
if file_name.endswith(".py"):
util_global.set_value('path', os.path.join(path, file_name))
mkdir(os.path.join(out_path, out_path_dst))
conver_ast(path, out_path_dst, file_name)
if util_global.get_value('need_conver', False):
content = "Finish conver file: " + os.path.join(
path, file_name)
print(content)
write_report_terminator(content)
else:
mkdir_and_copyfile(path, abs_join(out_path, out_path_dst),
file_name)
else:
mkdir_and_copyfile(path, abs_join(out_path, out_path_dst),
file_name)
print("Finish conver, output file: " + out_path + "; report file: " +
util_global.get_value('report'))
def ast_import(node):
for value in node.names:
if isinstance(value, ast.alias):
values = value.name.split(".")
if "keras" in values:
log_migration_report(getattr(node, "lineno", "None"), "keras")
util_global.set_value('need_conver', True)
def scan_file(path, file_name, api, lineno):
api_list = pd.read_excel(util_global.get_value('list'), sheet_name=0)
api_module = api_list['模块名'].values.tolist()
api_name = api_list['API名'].values.tolist()
api_support = api_list['工具迁移API支持度'].values.tolist()
api_advice = api_list['说明'].values.tolist()
script_name = []
code_line = []
code_module = []
code_api = []
support_type = []
migrate_advice = []
for i in range(len(api)):
name = api[i]
if name in api_name:
script_name.append(file_name)
code_api.append(name)
code_line.append(lineno[i])
code_module.append(api_module[api_name.index(name)])
support_type.append(api_support[api_name.index(name)])
migrate_advice.append(api_advice[api_name.index(name)])
# search for tf enumeration
enume_list = pd.read_excel(util_global.get_value('list'), sheet_name=1)
enume_name = enume_list['API名'].values.tolist()
(enume, lineno) = get_tf_enume(os.path.join(path, file_name), enume_name)
for i in range(len(enume)):
name = enume[i]
class_name = '.'.join(name.split('.')[:-1])
if name not in code_api and class_name not in code_api:
if class_name in api_name:
script_name.append(file_name)
code_api.append(class_name)
code_line.append(lineno[i])
code_module.append(api_module[api_name.index(class_name)])
support_type.append(api_support[api_name.index(class_name)])
migrate_advice.append(api_advice[api_name.index(class_name)])
# record unsupported api
(unsupport, unsupport_module, lineno) = get_unsupport_api(os.path.join(path, file_name))
for i in range(len(unsupport)):
script_name.append(file_name)
code_api.append(unsupport[i])
code_line.append(lineno[i])
code_module.append(unsupport_module[i])
support_type.append('不支持（无迁移方案，建议用户不使用）')
migrate_advice.append('第三方非TF官网API，暂不支持')
analyse_result = pd.DataFrame({'脚本文件名': script_name, '代码行': code_line,
'模块名': code_module, 'API名': code_api,
'工具迁移API支持度': support_type, '说明': migrate_advice})
# when there are tf apis used in script, analysis report will be generated
report = util_global.get_value('generate_dir_report')
if len(script_name):
report = report.append(analyse_result)
util_global.set_value('generate_dir_report', report)
def convert_origin_func_to_npu(node,
origin_func,
org_func_name,
params_list,
is_class_func=None):
"""Convert original Tensorflow function to NPU function"""
if not check_func_arguments(origin_func, node.args, node.keywords,
is_class_func):
return node
if org_func_name == "Estimator.train":
content = "".join([
util_global.get_value('path'), ":",
str(getattr(node, "lineno", "None"))
])
while True:
message = input(
"Check if the train function in " + content +
" is the Estimator train function. If yes, "
"enter 'y' to perform distributed porting on the train function. if no, enter 'n': "
)
if message == "y":
break
if message == "n":
log_warning("".join([
"The train func in ", content,
" is user-defined functions, will not perform distributed porting"
]))
return node
print("Input is error, Please enter 'y' or 'n'.")
for param_name in params_list:
node = match_func_params_and_convert(node, origin_func, org_func_name,
param_name, is_class_func)
util_global.set_value('need_conver', True)
return node
def conver():
"""The entry point to convert Tensorflow script"""
print("Begin conver, input file: " + util_global.get_value('input') + '\n')
out_path = util_global.get_value('output')
dst_path = os.path.split(util_global.get_value('input').rstrip('\\/'))[-1]
dst_path_new = dst_path + util_global.get_value('timestap')
conver_path = os.walk(util_global.get_value('input'))
report_dir = util_global.get_value('report')
mkdir(report_dir)
report_xlsx = os.path.join(report_dir, 'api_analysis_report.xlsx')
util_global.set_value('generate_dir_report', pd.DataFrame())
for path, _, file_list in conver_path:
for file_name in file_list:
out_path_dst = abs_join(
dst_path_new,
path.split(util_global.get_value('input'))[1])
file_path = os.path.join(path, file_name).replace('\\', '/')
if not check_path_length(file_path):
content = "".join([
"The file:", file_path, " length is invalid, skip convert."
])
log_warning(content)
continue
content = "".join(["Begin conver file: ", file_path])
print(content)
threshold_file_size = 10 * 1024 * 1024
if file_name.endswith(".py"):
if os.path.getsize(file_path) > threshold_file_size:
content = "".join([
"The file:", file_path,
" size is over 10M, skip convert."
])
log_warning(content)
continue
util_global.set_value('path', file_path)
mkdir(os.path.join(out_path, out_path_dst))
conver_ast(path, out_path_dst, file_name)
if util_global.get_value('need_conver', False):
content = "".join(
["Finish conver file: ", file_path, '\n'])
print(content)
write_report_terminator(content)
else:
mkdir_and_copyfile(path, abs_join(out_path, out_path_dst),
file_name)
else:
mkdir_and_copyfile(path, abs_join(out_path, out_path_dst),
file_name)
adjust_index()
analysis_report = util_global.get_value('generate_dir_report')
if analysis_report.empty:
print('No api data in the report')
else:
analysis_report.to_excel(report_xlsx, index=True)
get_api_statistic(analysis_report)
print("Finish conver, output file: " + out_path + "; report file: " +
util_global.get_value('report'))
def adjust_index():
report = util_global.get_value('generate_dir_report')
index_column = []
for i in range(len(report)):
index_column.append(i + 1)
report.index = index_column
report.index.name = '序号'
util_global.set_value('generate_dir_report', report)
def attribute(node):
log_success_report(getattr(node, "lineno", "None"), node.attr)
if node.attr == 'dropout':
node.value = ast.Name(id=util_global.get_value(node.attr)[0], ctx=ast.Load())
else:
node = ast.Name(id=util_global.get_value(node.attr)[0], ctx=ast.Load())
util_global.set_value('need_conver', True)
return node
def log_success_report(lineno, msg):
content = (util_global.get_value('path', '') + ':' + str(lineno) +
' change ' + util_global.get_value(msg)[1] + ' to ' +
util_global.get_value(msg)[2])
print(content)
write_conver_report(content, util_global.get_value('report_file')[0])
util_global.set_value('report_file_status',
(util_global.get_value('report_file_status') | 0b1))
def convert_dynamic_loss_scale(node):
"""Convert dynamic loss scale related Tensorflow APIs"""
log_msg(
getattr(node, 'lineno', 'None'),
"change tf.train.experimental.DynamicLossScale"
" to ExponentialUpdateLossScaleManager")
node.func = ast.Name(id="ExponentialUpdateLossScaleManager",
ctx=ast.Load())
def check_arg(node):
initial_loss_scale = None
increment_period = None
multiplier = None
for index, arg in enumerate(node.args):
if index == 0:
initial_loss_scale = arg
if index == 1:
increment_period = arg
if index == 2:
multiplier = arg
for keyword in node.keywords:
if keyword.arg == "initial_loss_scale":
keyword.arg = "init_loss_scale"
initial_loss_scale = keyword
if keyword.arg == "increment_period":
keyword.arg = "incr_every_n_steps"
increment_period = keyword
if keyword.arg == "multiplier":
keyword.arg = "incr_ratio"
multiplier = keyword
return (initial_loss_scale, increment_period, multiplier)
(initial_loss_scale, increment_period, multiplier) = check_arg(node)
if initial_loss_scale:
if not isinstance(initial_loss_scale, ast.keyword):
node.keywords.append(
ast.keyword(arg="init_loss_scale", value=initial_loss_scale))
else:
node.keywords.append(
ast.keyword(arg="init_loss_scale", value=pasta.parse("2**15")))
if increment_period:
if not isinstance(increment_period, ast.keyword):
node.keywords.append(
ast.keyword(arg="incr_every_n_steps", value=increment_period))
else:
node.keywords.append(
ast.keyword(arg="incr_every_n_steps", value=pasta.parse("2000")))
if multiplier:
if not isinstance(multiplier, ast.keyword):
node.keywords.append(
ast.keyword(arg="incr_ratio", value=multiplier))
else:
node.keywords.append(
ast.keyword(arg="incr_ratio", value=pasta.parse("2")))
node.args = []
util_global.set_value('need_conver', True)
return node
def log_migration_report(lineno, msg):
content = (
util_global.get_value('path', '') + ':' + str(lineno) + ' "' + msg +
'" feature needs to be migrated manually, Please refer to the migration guide.'
+ util_global.get_value(msg)[0])
print(content)
write_conver_report(content, util_global.get_value('report_file')[2])
util_global.set_value(
'report_file_status',
(util_global.get_value('report_file_status') | 0b100))
def ast_function_def(node):
log_success_report(getattr(node, "lineno", "None"), node.name)
node.body = [ast.Return(value=ast.Call(
func=ast.Attribute(value=ast.Name(id=util_global.get_value(node.name)[0],
ctx=ast.Load()), attr='gelu',
ctx=ast.Load()),
args=[ast.Name(id='x', ctx=ast.Load())],
keywords=[]))]
util_global.set_value('need_conver', True)
return node
def visit_Attribute(self, node):
self.generic_visit(node)
if node.attr == "keras":
util_global.set_value('is_keras_net', True)
if node.attr in util_global.get_value('hvd'):
if isinstance(node.value, ast.Name):
if 'hvd' in str(node.value.id):
return attribute(node)
if isinstance(node.value, ast.Attribute):
if 'hvd' in str(node.value.attr):
return attribute(node)
return node
def visit_Attribute(self, node):
"""Visit and transform attr node"""
self.generic_visit(node)
if node.attr == "keras":
util_global.set_value('is_keras_net', True)
if node.attr in util_global.get_value('hvd'):
distributed_mode = util_global.get_value("distributed_mode", "")
if isinstance(node.value, ast.Name) and 'hvd' in str(node.value.id):
if distributed_mode in ("tf_strategy", ""):
log_strategy_distributed_mode_error(node)
return node
return attribute(node)
return node
def ast_if(node):
if isinstance(node.test, ast.Compare):
if len(node.test.comparators) == 1 and isinstance(
node.test.comparators[0], ast.Str):
if node.test.comparators[0].s == "__main__":
util_global.set_value("is_main_file", False)
util_global.set_value("has_main_func", True)
if util_global.get_value("is_keras_net", False):
log_msg(getattr(node, "lineno", "None"),
" add keras session npu config")
close_sess_call = ast.Call(
func=ast.Name(id="close_session", ctx=ast.Load()),
args=[ast.Name(id="npu_keras_sess", ctx=ast.Load())],
keywords=[])
keras_sess_assign = ast.Assign(
targets=[
ast.Name(id="npu_keras_sess", ctx=ast.Store())
],
value=ast.Call(func=ast.Name(
id="set_keras_session_npu_config", ctx=ast.Load()),
args=[],
keywords=[]))
node.body = [keras_sess_assign] + node.body + [
ast.Expr(value=close_sess_call)
]
util_global.set_value('need_conver', True)
if util_global.get_value("has_hccl_api", False):
log_msg(getattr(node, "lineno", "None"),
" add npu resource init api")
close_sess_call = ast.Call(
func=ast.Name(id="close_session", ctx=ast.Load()),
args=[ast.Name(id="npu_sess", ctx=ast.Load())],
keywords=[])
init_assign = ast.Assign(targets=[
ast.Tuple(elts=[
ast.Name(id="npu_sess", ctx=ast.Store()),
ast.Name(id="npu_shutdown", ctx=ast.Store())
],
ctx=ast.Store())
],
value=ast.Call(func=ast.Name(
id="init_resource",
ctx=ast.Load()),
args=[],
keywords=[]))
shutdown_call = ast.Call(func=ast.Name(
id="shutdown_resource", ctx=ast.Load()),
args=[
ast.Name(id="npu_sess",
ctx=ast.Load()),
ast.Name(id="npu_shutdown",
ctx=ast.Load())
],
keywords=[])
node.body = [init_assign] + node.body + [
ast.Expr(value=shutdown_call),
ast.Expr(value=close_sess_call)
]
util_global.set_value('need_conver', True)
return node
def write_report_terminator(content):
report_path = util_global.get_value('report')
value = util_global.get_value('report_file_status')
times = value.bit_length()
while times > 0:
if get_bit_val(value, times - 1):
file = util_global.get_value('report_file')[times - 1]
if os.path.exists(os.path.join(report_path, file)):
file = open(os.path.join(report_path, file), 'a')
file.write(content)
file.write("\r\n")
file.write("\r\n")
file.close()
times = times - 1
util_global.set_value('report_file_status', 0)
def convert_tf_gradient_distributed(node):
"""Convert Tensorflow gradient APIs in distributed mode"""
content = "".join([
util_global.get_value('path'), ":",
str(getattr(node, "lineno", "None")),
" is tf.gradient api, tool inserts npu_allreduce after computing grads by default.",
" You can adjust the allreduce position according to the algorithm"
])
log_warning(content)
new_node = ast.Call(func=ast.Name(id="npu_allreduce", ctx=ast.Load()),
args=[node],
keywords=[])
ast.copy_location(new_node, node)
util_global.set_value("need_conver", True)
return new_node
def write_report_terminator(content):
"""Write content to report and update global variable"""
report_path = util_global.get_value('report')
value = util_global.get_value('report_file_status')
times = value.bit_length()
while times > 0:
if get_bit_val(value, times - 1):
file = util_global.get_value('report_file')[times - 1]
if os.path.exists(os.path.join(report_path, file)):
with open(os.path.join(report_path, file), 'a') as file:
file.write(content)
file.write("\r\n")
file.write("\r\n")
times = times - 1
util_global.set_value('report_file_status', 0)
def import_from(node):
if node.module != None:
values = node.module.split(".")
if "keras" in values:
util_global.set_value('is_keras_net', True)
if "horovod" in values:
util_global.set_value('has_hccl_api', True)
for value in node.names:
if isinstance(value, ast.alias):
values = value.name.split(".")
if "keras" in values:
util_global.set_value('is_keras_net', True)
if "horovod" in values:
util_global.set_value('has_hccl_api', True)
util_global.set_value('need_conver', True)
def conver():
print("Begin conver, input file: " + util_global.get_value('input') + '\n')
out_path = util_global.get_value('output')
dst_path = os.path.split(util_global.get_value('input').rstrip('\\/'))[-1]
dst_path_new = dst_path + util_global.get_value('timestap')
conver_path = os.walk(util_global.get_value('input'))
report_dir = util_global.get_value('report')
mkdir(report_dir)
report_xlsx = os.path.join(report_dir, 'api_analysis_report.xlsx')
util_global.set_value('generate_dir_report', pd.DataFrame())
for path, dir_list, file_list in conver_path:
for file_name in file_list:
out_path_dst = abs_join(
dst_path_new,
path.split(util_global.get_value('input'))[1])
file_path = os.path.join(path, file_name).replace('\\', '/')
content = "Begin conver file: " + file_path
print(content)
if file_name.endswith(".py"):
util_global.set_value('path', file_path)
mkdir(os.path.join(out_path, out_path_dst))
conver_ast(path, out_path_dst, file_name)
if util_global.get_value('need_conver', False):
content = "Finish conver file: " + file_path + '\n'
print(content)
write_report_terminator(content)
else:
mkdir_and_copyfile(path, abs_join(out_path, out_path_dst),
file_name)
else:
mkdir_and_copyfile(path, abs_join(out_path, out_path_dst),
file_name)
adjust_index()
analysis_report = util_global.get_value('generate_dir_report')
if analysis_report.empty:
print('No api data in the report')
else:
analysis_report.to_excel(report_xlsx, index=True)
get_api_statistic(analysis_report)
print("Finish conver, output file: " + out_path + "; report file: " +
util_global.get_value('report'))
def convert_hvd_distributed_api(node):
"""Convert horovod distributed APIs"""
log_msg(
getattr(node, "lineno", "None"),
'change hvd.DistributedOptimizer to npu_distributed_optimizer_wrapper')
node.func = ast.Name(id="npu_distributed_optimizer_wrapper",
ctx=ast.Load())
opt_keyword = None
for keyword in node.keywords:
if keyword.arg == "optimizer":
opt_keyword = keyword
node.keywords.clear()
if opt_keyword is None:
opt_arg = node.args[0]
node.args.clear()
node.args.append(opt_arg)
else:
node.keywords.append(opt_keyword)
util_global.set_value('need_conver', True)
return node
def conver_ast(path, out_path_dst, file_name):
util_global.set_value('need_conver', False)
file = open(os.path.join(path, file_name), "r")
source = file.read()
r_node = ast.parse(source)
sys.setrecursionlimit(10000)
visitor = ConverByAst()
visitor.visit(r_node)
ast.fix_missing_locations(r_node)
if util_global.get_value('need_conver', False):
insert_npu_import(r_node)
dst_content = astunparse.unparse(r_node)
write_output_after_conver(
os.path.join(util_global.get_value('output'), out_path_dst,
file_name), dst_content)
if file_name.endswith("a.py"):
write_report_after_conver("only_for_test", file_name,
node_tree(ast.dump(r_node)))
def get_tf_api(file_name):
"""
Args:
file_name: script file
Returns:
list objects containing all the tensorflow apis extracted from file_name
and corrosponding line number
"""
util_global.set_value('use_keras_dropout', False)
with open(file_name, 'r', encoding='utf-8') as file:
source = file.read()
tree = ast.parse(source)
visitor = VisitCall()
visitor.visit(tree)
# get tensorflow related api
api = []
lineno = []
import_list = ['tf', 'hvd']
keras_dropout_api = [
'tf.layers.dropout', 'tf.layers.Dropout', 'tf.keras.layers.Dropout',
'tf.keras.layers.SpatialDropout1D', 'tf.keras.layers.SpatialDropout2D',
'tf.keras.layers.SpatialDropout3D'
]
for module in import_list:
for i in range(len(visitor.call_nodes)):
if "".join([
module, '.'
]) in visitor.call_nodes[i] and visitor.call_nodes[i].split(
'.')[0] == module:
api.append(visitor.call_nodes[i])
lineno.append(visitor.line_nos[i])
# get tf api using keras dropout
if visitor.call_nodes[i] in keras_dropout_api:
util_global.set_value('use_keras_dropout', True)
return api, lineno
def ast_import(node):
for value in node.names:
if isinstance(value, ast.alias):
values = value.name.split(".")
if "keras" in values:
util_global.set_value('is_keras_net', True)
if "horovod" in values:
util_global.set_value('has_hccl_api', True)
util_global.set_value('need_conver', True)
def ast_import(node):
"""Modify import module"""
for value in node.names:
if isinstance(value, ast.alias):
values = value.name.split(".")
if "keras" in values:
util_global.set_value('is_keras_net', True)
if "horovod" in values:
log_msg(getattr(node, "lineno", "None"),
"remove horovod import line to None")
util_global.set_value('has_hvd_api', True)
new_node = ast.Expr(value=ast.NameConstant(value=None))
ast.copy_location(new_node, node)
util_global.set_value('need_conver', True)
return new_node
util_global.set_value('need_conver', True)
return node
def ast_call(node):
if isinstance(node.func, ast.Attribute):
if len(node.args) > 0:
if isinstance(node.args[0], ast.Call):
if isinstance(node.args[0].func, ast.Attribute):
if node.args[0].func.attr == 'BroadcastGlobalVariablesHook':
log_success_report(getattr(node, "lineno", "None"), 'BroadcastGlobalVariablesHook')
node.func = ast.Name(id=util_global.get_value('BroadcastGlobalVariablesHook')[0], ctx=ast.Load())
node.args = []
util_global.set_value('need_conver', True)
if isinstance(node.func, ast.Attribute) and node.func.attr == 'shard':
log_success_report(getattr(node, "lineno", "None"), 'shard')
node.args = [ast.Call(func=ast.Name(id='get_rank_size', ctx=ast.Load()), args=[], keywords=[]),
ast.Call(func=ast.Name(id='get_rank_id', ctx=ast.Load()), args=[], keywords=[])]
util_global.set_value('need_conver', True)
if isinstance(node.func, ast.Attribute) and (node.func.attr == 'batch' or node.func.attr == 'map_and_batch'):
exist = False
for keyword in node.keywords:
if keyword.arg == 'drop_remainder':
exist = True
if ((isinstance(keyword.value, ast.NameConstant) and keyword.value.value != True) or
(not isinstance(keyword.value, ast.NameConstant))):
log_success_report(getattr(node, "lineno", "None"), node.func.attr)
keyword.value = ast.NameConstant(value=True)
util_global.set_value('need_conver', True)
if not exist:
log_success_report(getattr(node, "lineno", "None"), node.func.attr)
keyword = ast.keyword(arg='drop_remainder', value=ast.NameConstant(value=True))
node.keywords.insert(0, keyword)
util_global.set_value('need_conver', True)
if (isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name) and
node.func.value.id == 'tf' and node.func.attr == 'device'):
log_success_report(getattr(node, "lineno", "None"), node.func.attr)
node.args = [ast.Str(s='/cpu:0')]
util_global.set_value('need_conver', True)
return node
def para_check_and_set(argv):
input = "input"
output = "output"
report = "report"
try:
opts, args = getopt.getopt(argv, "hi:o:r:",
["help", "input=", "output=", "report="])
except getopt.GetoptError:
print('Parameter error, please check.')
print(' main.py -i <input> -o <output> -r <report>')
print(
'or: main.py --input=<input> --output=<output> --report=<report>')
print(
'-i or --input: The source script to be converted, Default value: input/'
)
print(
'-o or --output: The destination script after converted, Default value: output/'
)
print('-r or --report: Conversion report, Default value: report/')
sys.exit(2)
for opt, arg in opts:
if opt in ("-h", "--help"):
print(' main.py -i <input> -o <output> -r <report>')
print(
'or: main.py --input=<input> --output=<output> --report=<report>'
)
print(
'-i or --input: The source script to be converted, Default value: input/'
)
print(
'-o or --output: The destination script after converted, Default value: output/'
)
print('-r or --report: Conversion report, Default value: report/')
sys.exit()
elif opt in ("-i", "--input"):
input = arg
elif opt in ("-o", "--output"):
output = arg
elif opt in ("-r", "--report"):
report = arg
util_global.set_value('input', input)
util_global.set_value('output', output)
util_global.set_value('report', report)
def para_check_and_set(argv):
input_dir = "npu_input"
support_list = "tf1.15_api_support_list.xlsx"
output = "output" + util_global.get_value('timestap')
report = "report" + util_global.get_value('timestap')
report_suffix = report
main_file = ""
try:
opts, args = getopt.getopt(argv, "hi:l:o:r:m:", ["help", "input=", "list=", "output=", "report=", "main="])
except getopt.GetoptError:
print('Parameter error, please check.')
print(' this tool just support to convert tf-1.15 scripts.')
print(' main.py -i <input> -l <list> -o <output> -r <report> -m <main>')
print('or: main.py --input=<input> --list=<list> --output=<output> --report=<report> --main=<main>')
print('-i or --input: The source script to be converted.')
print('-l or --list: The list of supported api, Default value: tf1.15_api_support_list.xlsx')
print('-o or --output: The destination script after converted, Default value: output_npu_***/')
print('-r or --report: Conversion report, Default value: report_npu_***/')
print('-m or --main: the executed entry *.py file, default:None')
sys.exit(2)
for opt, arg in opts:
if opt in ("-h", "--help"):
print(' this tool just support to convert tf-1.15 scripts.')
print(' main.py -i <input> -l <list> -o <output> -r <report> -m <main>')
print('or: main.py --input=<input> --list=<list> --output=<output> --report=<report> --main=<main>')
print('-i or --input: The source script to be converted')
print('-l or --list: The list of supported api, Default value: tf1.15_api_support_list.xlsx')
print('-o or --output: The destination script after converted, Default value: output_npu_***/')
print('-r or --report: Conversion report, Default value: report_npu_***/')
print('-m or --main: the executed entry *.py file, default:None')
sys.exit()
elif opt in ("-i", "--input"):
input_dir = os.path.abspath(arg)
if str(input_dir).endswith('/'):
input_dir = input_dir[0:len(input_dir)-1]
input_dir = input_dir.replace('\\', '/')
elif opt in ("-l", "--list"):
support_list = arg
elif opt in ("-o", "--output"):
output = os.path.abspath(arg)
if str(output).endswith('/'):
output = output[0:len(output)-1]
output = output.replace('\\', '/')
elif opt in ("-r", "--report"):
report = os.path.abspath(arg)
if str(report).endswith('/'):
report = report[0:len(report)-1]
report = os.path.join(report, report_suffix)
report = report.replace('\\', '/')
elif opt in ("-m", "--main"):
if os.path.isfile(arg):
main_file = os.path.abspath(arg)
main_path = os.path.dirname(main_file)
file = os.path.basename(main_file)
main_path = main_path.replace('\\', '/')
main_file = os.path.join(main_path, file)
else:
raise ValueError("--main args must be exited files")
if input_dir == "npu_input":
raise ValueError("Please check -i or --input.")
if input_dir + '/' in output + '/' or input_dir + '/' in report + '/':
print("<output> or <report> could not be the subdirectory of <input>, please try another option.")
sys.exit(2)
util_global.set_value('input', input_dir)
util_global.set_value('list', support_list)
util_global.set_value('output', output)
util_global.set_value('report', report)
util_global.set_value('main', main_file)
def conver_ast(path, out_path_dst, file_name):
util_global.set_value('need_conver', False)
util_global.set_value('is_keras_net', False)
util_global.set_value('has_hccl_api', False)
util_global.set_value('is_main_file', False)
util_global.set_value('has_main_func', False)
if os.path.join(path, file_name) == util_global.get_value('main', ""):
util_global.set_value('is_main_file', True)
with open(os.path.join(path, file_name), "r", encoding='utf-8') as file:
source = file.read()
try:
r_node = pasta.parse(source)
except Exception as e:
print(repr(e))
return
sys.setrecursionlimit(10000)
visitor = ConverByAst()
visitor.visit(r_node)
ast.fix_missing_locations(r_node)
(api, lineno) = get_tf_api(os.path.join(path, file_name))
if len(api) == 0:
print(
"No Tensorflow module is imported in script {}.".format(file_name))
scan_file(path, file_name, api, lineno)
if util_global.get_value('need_conver', False):
insert_npu_import(r_node)
if not util_global.get_value('has_main_func', False) and (
util_global.get_value('has_hccl_api', False)
or util_global.get_value('is_keras_net', False)):
log_warning(
'the network of keras and horovod, or using dataset.shard script do not have main func, '
'should set -m or --main parameter')
if util_global.get_value('is_main_file',
False) and util_global.get_value(
'has_hccl_api', False):
insert_npu_resource_init(r_node)
insert_npu_resource_shutdown(r_node)
if util_global.get_value('is_main_file',
False) and util_global.get_value(
'is_keras_net', False):
insert_keras_sess_npu_config(r_node)
insert_keras_sess_close(r_node)
dst_content = pasta.dump(r_node)
write_output_after_conver(
os.path.join(util_global.get_value('output'), out_path_dst,
file_name), dst_content)
if file_name.endswith("a.py"):
write_report_after_conver("only_for_test", file_name,
node_tree(ast.dump(r_node)))
def import_from(node):
if node.module != None:
values = node.module.split(".")
if "keras" in values:
log_migration_report(getattr(node, "lineno", "None"), "keras")
util_global.set_value('need_conver', True)
