def warn_check_compile(cmd, checked_path, headers_list):
checked_path = replace_env(checked_path)
if not os.path.exists(checked_path):
cilog.cilog_error(THIS_FILE_NAME, "can not find cpp list file: %s",
checked_path)
return False
checked_file_cmd = "find " + checked_path + " -name \"*.cpp\" -o -name \"*.h\""
ret = util.execute(checked_file_cmd)
if ret[0] is False:
cilog.cilog_error(THIS_FILE_NAME, "can not find cpp list file: %s",
checked_path)
return False
checked_file = ret[1]
headers = ""
if headers_list is not None:
headers = " -I".join(headers_list)
headers = "-I" + headers
cmd = re.sub("__WARN_CHECK_HEADERS__", headers, cmd)
for file in checked_file:
file_names = os.path.split(file)
temp_cmd = re.sub("__WARN_CHECK_FILE__", file, cmd)
temp_cmd = re.sub("__WARN_CHECK_FILE_NAME__", file_names[1], temp_cmd)
util.execute(temp_cmd, print_output_flag=True)
return True
def static_check_cmd(command):
cmd = command.get("cmd")
args = command.get("args")
result = True
if args is None:
ret = util.execute(cmd, print_output_flag=True)
result = ret[0]
else:
for each_arg in args:
# replace all parameters in the command
for param_key, param_value in each_arg.items():
if param_value is None:
param_value = ""
temp_cmd = re.sub(param_key, param_value, cmd)
ret = util.execute(temp_cmd, print_output_flag=True)
# not return here, check every args
if not ret[0]:
result = False
if not result:
cilog.cilog_error(THIS_FILE_NAME, "static check failed.")
return result
def single_warn_check_compile(cmd, mind_file, oi_engine_config_dict):
'''single warm_check path in compile mode'''
mind_file_paths = os.path.split(mind_file)
mind_file_path = mind_file_paths[0]
try:
mind_file_stream = open(mind_file, 'r')
mind_file_info = json.load(mind_file_stream)
mind_nodes = mind_file_info.get("node")
except OSError as reason:
cilog.cilog_error(THIS_FILE_NAME, "read %s failed: %s", mind_file,
reason)
return False
result = True
checked_file_path = ""
for mind_node in mind_nodes:
if mind_node.get("group") == "MyModel":
continue
name = mind_node.get("name")
run_side = mind_node.get("params").get("runSide")
checked_file_path = os.path.join(mind_file_path, name)
header_list = oi_engine_config_dict.get(
run_side.lower()).get("includes").get("include")
header_list = list(
map(lambda x: re.sub(r"\$\(SRC_DIR\)", checked_file_path, x),
header_list))
header_list = list(
map(lambda x: re.sub(r"\.\.", mind_file_path, x), header_list))
header_list = list(map(lambda x: re.sub(r" \\", "", x), header_list))
header_list = list(
map(lambda x: re.sub(r"\$\(DDK_HOME\)", os.getenv("DDK_HOME"), x),
header_list))
replaced_cmd = re.sub("__WARN_CHECK_HEADERS__", " ".join(header_list),
cmd)
checked_file_cmd = "find " + checked_file_path + \
" -name \"*.cpp\" -o -name \"*.h\""
ret = util.execute(checked_file_cmd, print_output_flag=True)
if ret[0] is False:
result = False
continue
checked_files = ret[1]
for file in checked_files:
file_path_name = os.path.split(file)
file_names = os.path.splitext(file_path_name[1])
file_name = file_names[0] + ".o"
temp_cmd = re.sub(r"__WARN_CHECK_FILE__", file, replaced_cmd)
temp_cmd = re.sub(r"__WARN_CHECK_FILE_NAME__", file_name, temp_cmd)
ret = util.execute(temp_cmd, print_output_flag=True)
if ret[0] is False:
result = False
return result
def warn_check_makefile(cmd):
'''warm check in makefile mode'''
# find path which need to be checked
ret = sc_util.find_checked_path()
if ret[0] is False:
return False
checked_path = ret[1]
if checked_path is None:
cilog.cilog_info(THIS_FILE_NAME, "no path to check in makefile mode")
return True
makefile_path_list = []
for each_path in checked_path:
makefile_cmd = "find " + each_path + " -name \"Makefile\""
ret = util.execute(makefile_cmd, print_output_flag=True)
if ret[0] is False:
return False
makefile_path_list.extend(ret[1])
if "" in makefile_path_list:
makefile_path_list.remove("")
# base so should be executed fist in sequence and copy to DDK path
make_install_path = "make install -C __INSTALL_MAKEFILE_PATH__"
base_so_path = sc_util.get_base_list()
for each_path in base_so_path:
each_path = sc_util.replace_env(each_path)
makefile_path_list.remove(each_path)
ret = single_warn_check_makefile(cmd, each_path)
if ret is False:
return False
temp_copy_cmd = re.sub(r"(__[\w+_\w+]*__)",
os.path.split(each_path)[0], make_install_path)
ret = util.execute(temp_copy_cmd, print_output_flag=True)
if ret[0] is False:
return False
if makefile_path_list is None:
cilog.cilog_info(THIS_FILE_NAME,
"no Makefile to check in makefile mode")
return True
result = True
with ProcessPoolExecutor(max_workers=5) as executor:
futures_pool = {executor.submit(
single_warn_check_makefile, cmd, mind_file_path): \
mind_file_path for mind_file_path in makefile_path_list}
for future in as_completed(futures_pool):
if future.result() is False:
result = False
return result
def single_warn_check_makefile(cmd, makefile_path):
'''single warn check in makefile mode'''
ret = validate_makefile(makefile_path)
if ret is False:
return False
makefile_path = os.path.split(makefile_path)[0]
cmd = re.sub(r"(__[\w+_\w+]*__)", makefile_path, cmd)
ret = util.execute(cmd, print_output_flag=True)
return ret[0]
def find_checked_path():
'''find static check base path'''
checked_path_cmd = "find " + \
os.path.join(ASCEND_ROOT_PATH, "ascenddk") + \
" -maxdepth 1 -mindepth 1 -type d -print"
ret = util.execute(checked_path_cmd, print_output_flag=True)
if ret[0] is False:
return False, []
found_path = []
for each_path in ret[1]:
if each_path not in GLOBAL_IGNORE_PATH:
found_path.append(each_path)
return True, found_path
def main():
command_file_name = None
if len(os.sys.argv) >= 2:
command_file_name = os.sys.argv[1]
install_commands = InstallationCommands(command_file_name)
ret, commands = install_commands.get_install_commands()
if not ret:
exit(-1)
for command in commands:
ret = util.execute(command, print_output_flag=True)
if not ret[0]:
exit(-1)
exit(0)
def warn_check_compile(cmd):
'''warm check in compile mode'''
engine_mind_cmd = "find " + \
os.path.join(sc_util.ASCEND_ROOT_PATH,
"ascenddk/engine") + " -name \"*.mind\""
ret = util.execute(engine_mind_cmd, print_output_flag=True)
if ret[0] is False:
return False
mind_files = ret[1]
ddk_engine_config_path = os.path.join(os.getenv("DDK_HOME"),
"conf/settings_engine.conf")
try:
ddk_engine_config_file = open(ddk_engine_config_path, 'r')
ddk_engine_config_info = json.load(ddk_engine_config_file)
oi_engine_config_dict = ddk_engine_config_info.get(
"configuration").get("OI")
except OSError as reason:
cilog.cilog_error(THIS_FILE_NAME,
"read ddk conf/settings_engine.conf failed: %s",
reason)
return False
finally:
if ddk_engine_config_file in locals():
ddk_engine_config_file.close()
oi_lower_dict = {k.lower(): v for k, v in oi_engine_config_dict.items()}
result = True
with ProcessPoolExecutor(max_workers=5) as executor:
futures_pool = {executor.submit(
single_warn_check_compile, cmd, mind_file, oi_lower_dict): \
mind_file for mind_file in mind_files}
for future in as_completed(futures_pool):
if future.result() is False:
result = False
return result
def run_cmd(command, sub_params):
'''static check in command mode'''
cmd = command.get("cmd")
sub_commands = None
if "sub_params" in command.keys():
sub_commands = sub_params.get(command.get("sub_params"))
replaced_vars = re.findall(r"(__[\w+_\w+]*__)", cmd)
replaced_vars.sort()
vars_length = len(replaced_vars)
result = True
if sub_commands is None and vars_length != 0:
cilog.cilog_error(
THIS_FILE_NAME, "sub commands %s is not match" \
" replaced vars length %s", sub_commands, vars_length)
result = False
elif sub_commands is None and vars_length == 0:
ret = util.execute(cmd, print_output_flag=True)
result = ret[0]
else:
if isinstance(sub_commands, list):
for each_arg in sub_commands:
args = each_arg.split()
if vars_length != len(args):
cilog.cilog_error(
THIS_FILE_NAME, "sub commands %s is not match" \
" replaced vars length %s", sub_commands, vars_length)
return False
temp_cmd = cmd
for index in range(0, vars_length):
temp_cmd = re.sub(replaced_vars[index], args[index],
temp_cmd)
ret = util.execute(temp_cmd, print_output_flag=True)
# not return here, check every args
if not ret[0]:
result = False
elif isinstance(sub_commands, dict):
for each_arg in sub_commands:
if len(replaced_vars) != len(each_arg):
cilog.cilog_error(
THIS_FILE_NAME, "sub commands %s is not match" \
" replaced vars length %s", sub_commands, vars_length)
return False
key_list = each_arg.keys().sort()
if not operator.eq(replaced_vars, key_list):
cilog.cilog_error(
THIS_FILE_NAME, "sub commands %s is not match" \
" replaced vars %s", key_list, replaced_vars)
return False
temp_cmd = cmd
# replace all parameters in the command
for param_key, param_value in each_arg.items():
if param_value is None:
param_value = ""
temp_cmd = re.sub(param_key, param_value, temp_cmd)
ret = util.execute(temp_cmd, print_output_flag=True)
# not return here, check every args
if not ret[0]:
result = False
else:
cilog.cilog_error(THIS_FILE_NAME, "unsuppoted sub commands type")
result = False
if not result:
cilog.cilog_error(THIS_FILE_NAME, "scripts run failed.")
return result
def main():
base_path = os.path.dirname(os.path.realpath(__file__))
ret = util.execute("git branch", cwd=base_path)
if ret[0] is False:
exit(-1)
branch_info = ret[1]
code_branch = branch_info[0].split(" ")[1]
cilog.print_in_color(code_branch, cilog.COLOR_F_YELLOW)
file_path = os.path.join(base_path, "../../.travis.yml")
try:
file_stream = open(file_path, 'r')
travis_dict = yaml.load(file_stream)
env_variables = travis_dict.get("env")
except OSError as reason:
print(reason)
exit(-1)
finally:
if file_stream in locals():
file_stream.close()
try:
env_variables_list = env_variables.split(" ")
env_variables_list.append("TRAVIS_BRANCH=" + code_branch)
env_variables_list = list(
map(lambda x: "export " + x + "\n", env_variables_list))
cilog.print_in_color("env list: %s" % env_variables_list,
cilog.COLOR_F_YELLOW)
env_file = os.path.join(os.getenv("HOME"), ".bashrc_ascend")
env_stream = open(env_file, 'w')
env_stream.writelines(env_variables_list)
except OSError as reason:
print(reason)
exit(-1)
finally:
if env_stream in locals():
env_stream.close()
# add env to bashrc
bashrc_file = os.path.join(os.getenv("HOME"), ".bashrc")
try:
bashrc_read_stream = open(bashrc_file, 'r')
all_lines = []
while True:
lines = bashrc_read_stream.readlines(10000)
if not lines:
break
all_lines.extend(lines)
except OSError as reason:
print(reason)
exit(-1)
finally:
if bashrc_read_stream in locals():
bashrc_read_stream.close()
# if bashrc haven been added, skip it
if ". ~/.bashrc_ascend" not in all_lines:
try:
bashrc_write_stream = open(bashrc_file, 'a')
bashrc_write_stream.write(". ~/.bashrc_ascend")
except OSError as reason:
print(reason)
finally:
if bashrc_write_stream in locals():
bashrc_write_stream.close()
