def main():
'''static check'''
check_type = os.sys.argv[1]
commahd_type = os.sys.argv[2]
scripts_commands = ScriptsCommands(check_type, commahd_type)
ret, commands = scripts_commands.get_commands()
if not ret:
exit(-1)
ret, sub_params = scripts_commands.get_sub_params()
if not ret:
exit(-1)
for command_dict in commands:
comand_type = command_dict.get("type")
if comand_type == "command":
ret = run_cmd(command_dict, sub_params)
if not ret:
exit(-1)
elif comand_type == "function":
ret = exec_func(command_dict, sub_params)
if not ret:
exit(-1)
else:
cilog.cilog_error(THIS_FILE_NAME, "unsupported command: %s",
command_dict)
exit(-1)
exit(0)
def get_commands(self):
'''get static check commands'''
if self.error is True:
cilog.cilog_error(
THIS_FILE_NAME, "get command failed.")
return False, None
return True, self.commands
def validate_commands(self):
for each_command in self.commands:
command_type = each_command.get("type")
if command_type is None:
cilog.cilog_error(THIS_FILE_NAME, "type is invalid: %s",
each_command)
self.error = True
break
command = each_command.get("command")
if command is None:
cilog.cilog_error(THIS_FILE_NAME, "command is invalid: %s",
each_command)
self.error = True
break
if command_type == "command":
if "cmd" not in command.keys():
cilog.cilog_error(THIS_FILE_NAME, "cmd is invalid: %s",
each_command)
self.error = True
break
elif command_type == "function":
if "function_name" not in command.keys():
cilog.cilog_error(THIS_FILE_NAME,
"function_name is invalid: %s",
each_command)
self.error = True
break
else:
cilog.cilog_error(THIS_FILE_NAME, "type is invalid: %s",
each_command)
self.error = True
break
def validate_makefile(makefile_path):
'''validate -Wall in makefile'''
result = False
try:
makefile_stream = open(makefile_path, "r")
while True:
lines = makefile_stream.readlines(1000)
if not lines:
break
for line in lines:
if "-Wall" in line:
result = True
break
except OSError as reason:
cilog.cilog_error(THIS_FILE_NAME, "read makefile %s failed: %s",
makefile_path, reason)
finally:
if makefile_stream in locals():
makefile_stream.close()
if result is False:
cilog.cilog_error(
THIS_FILE_NAME, "makefile %s is invalid:"\
" no -Wall compile parameters", makefile_path)
return result
def __init__(self, command_file_name=None):
self.commands = {}
self.command_file_name = command_file_name
self.error = False
if self.command_file_name is None:
self.command_file_name = "default"
self.command_file = os.path.join(CONFIG_PATH,
self.command_file_name + ".yaml")
if not os.path.exists(self.command_file):
self.error = True
cilog.cilog_error(THIS_FILE_NAME,
"command yaml file is not exist: %s",
self.command_file)
return
cilog.cilog_debug(THIS_FILE_NAME, "read command yaml file: %s",
self.command_file)
try:
stream = open(self.command_file, 'r')
self.commands = yaml.load(stream)
except OSError as reason:
self.error = True
cilog.cilog_error(THIS_FILE_NAME, "read command file failed: %s",
reason)
finally:
# if stream in current symbol table:locals(), close it
if stream in locals():
stream.close()
def warn_check_compile(cmd, checked_path, headers_list):
checked_path = replace_env(checked_path)
if not os.path.exists(checked_path):
cilog.cilog_error(THIS_FILE_NAME, "can not find cpp list file: %s",
checked_path)
return False
checked_file_cmd = "find " + checked_path + " -name \"*.cpp\" -o -name \"*.h\""
ret = util.execute(checked_file_cmd)
if ret[0] is False:
cilog.cilog_error(THIS_FILE_NAME, "can not find cpp list file: %s",
checked_path)
return False
checked_file = ret[1]
headers = ""
if headers_list is not None:
headers = " -I".join(headers_list)
headers = "-I" + headers
cmd = re.sub("__WARN_CHECK_HEADERS__", headers, cmd)
for file in checked_file:
file_names = os.path.split(file)
temp_cmd = re.sub("__WARN_CHECK_FILE__", file, cmd)
temp_cmd = re.sub("__WARN_CHECK_FILE_NAME__", file_names[1], temp_cmd)
util.execute(temp_cmd, print_output_flag=True)
return True
def get_install_commands(self):
if self.error:
cilog.cilog_error(
THIS_FILE_NAME, "no command yaml or config yaml not exist.")
return False, None
commands = self.commands.get("install")
return True, commands
def get_sub_params(self):
'''get static check sub params'''
if self.error is True:
cilog.cilog_error(
THIS_FILE_NAME, "get sub params failed.")
return False, None
return True, self.sub_params
def static_check_cmd(command):
cmd = command.get("cmd")
args = command.get("args")
result = True
if args is None:
ret = util.execute(cmd, print_output_flag=True)
result = ret[0]
else:
for each_arg in args:
# replace all parameters in the command
for param_key, param_value in each_arg.items():
if param_value is None:
param_value = ""
temp_cmd = re.sub(param_key, param_value, cmd)
ret = util.execute(temp_cmd, print_output_flag=True)
# not return here, check every args
if not ret[0]:
result = False
if not result:
cilog.cilog_error(THIS_FILE_NAME, "static check failed.")
return result
def single_warn_check_compile(cmd, mind_file, oi_engine_config_dict):
'''single warm_check path in compile mode'''
mind_file_paths = os.path.split(mind_file)
mind_file_path = mind_file_paths[0]
try:
mind_file_stream = open(mind_file, 'r')
mind_file_info = json.load(mind_file_stream)
mind_nodes = mind_file_info.get("node")
except OSError as reason:
cilog.cilog_error(THIS_FILE_NAME, "read %s failed: %s", mind_file,
reason)
return False
result = True
checked_file_path = ""
for mind_node in mind_nodes:
if mind_node.get("group") == "MyModel":
continue
name = mind_node.get("name")
run_side = mind_node.get("params").get("runSide")
checked_file_path = os.path.join(mind_file_path, name)
header_list = oi_engine_config_dict.get(
run_side.lower()).get("includes").get("include")
header_list = list(
map(lambda x: re.sub(r"\$\(SRC_DIR\)", checked_file_path, x),
header_list))
header_list = list(
map(lambda x: re.sub(r"\.\.", mind_file_path, x), header_list))
header_list = list(map(lambda x: re.sub(r" \\", "", x), header_list))
header_list = list(
map(lambda x: re.sub(r"\$\(DDK_HOME\)", os.getenv("DDK_HOME"), x),
header_list))
replaced_cmd = re.sub("__WARN_CHECK_HEADERS__", " ".join(header_list),
cmd)
checked_file_cmd = "find " + checked_file_path + \
" -name \"*.cpp\" -o -name \"*.h\""
ret = util.execute(checked_file_cmd, print_output_flag=True)
if ret[0] is False:
result = False
continue
checked_files = ret[1]
for file in checked_files:
file_path_name = os.path.split(file)
file_names = os.path.splitext(file_path_name[1])
file_name = file_names[0] + ".o"
temp_cmd = re.sub(r"__WARN_CHECK_FILE__", file, replaced_cmd)
temp_cmd = re.sub(r"__WARN_CHECK_FILE_NAME__", file_name, temp_cmd)
ret = util.execute(temp_cmd, print_output_flag=True)
if ret[0] is False:
result = False
return result
def static_check_func(command):
function_name = command.get("function_name")
parameter_list = command.get("args")
cmd = command.get("cmd")
if parameter_list is None:
parameter_list = []
parameter_list.append({"arg_value": []})
result = True
for parameter in parameter_list:
args = []
kwargs = {}
# each time either mode-values or mode-key-values works
mode_values = parameter.get("arg_value")
mode_key_values = parameter.get("arg_key_value")
if mode_values is not None and mode_key_values is None:
if cmd is not None:
args.append(cmd)
args.extend(mode_values)
elif mode_values is None and mode_key_values is not None:
if cmd is not None:
kwargs["cmd"] = cmd
kwargs.update(mode_key_values)
else:
cilog.cilog_error(
THIS_FILE_NAME,
"mode_value: %s and mode_key_value: %s are invalid",
mode_values, mode_key_values)
return False
cilog.cilog_info(THIS_FILE_NAME, "execute the func: %s(%s, %s)",
function_name, args, kwargs)
ret = eval(function_name)(*args, **kwargs)
expected_result = command.get("expected_result")
# not break here, finish every check
if ret == expected_result:
cilog.cilog_info(THIS_FILE_NAME,
"execute the func as expected_result: %s",
expected_result)
else:
cilog.cilog_error(
THIS_FILE_NAME, "execute the func failed: actural result: %s,"
" expected_result: %s", ret, expected_result)
result = False
return result
def __init__(self, command_type):
self.commands = {}
self.command_type = command_type
self.error = False
self.static_check_command_file = os.path.join(CONFIG_PATH,
"static_check.yaml")
if not os.path.exists(self.static_check_command_file):
self.error = True
cilog.cilog_error(
THIS_FILE_NAME,
"static check file is not exist: static_check.yaml")
self.command_file = os.path.join(
CONFIG_PATH, "static_check_" + self.command_type + ".yaml")
if not os.path.exists(self.command_file):
self.error = True
cilog.cilog_error(THIS_FILE_NAME,
"command yaml file is not exist: %s",
self.command_file)
return
cilog.cilog_debug(THIS_FILE_NAME, "read command yaml file: %s__%s",
self.static_check_command_file, self.command_file)
try:
static_stream = open(self.static_check_command_file, 'r')
sub_stream = open(self.command_file, 'r')
static_dict = yaml.load(static_stream)
sub_dict = yaml.load(sub_stream)
self.commands = static_dict.get(command_type)
self.validate_commands()
if self.error:
return
for commands in self.commands:
command = commands.get("command")
arg_name = command.get("arg_name")
if arg_name is None:
continue
args = sub_dict.get(arg_name)
if args is None:
self.error = True
cilog.cilog_error(THIS_FILE_NAME, "args in %s is invalid",
self.command_file)
return
for key, value in args.items():
command[key] = value
except OSError as reason:
self.error = True
cilog.cilog_error(THIS_FILE_NAME, "read command file failed: %s",
reason)
finally:
# if stream in current symbol table:locals(), close it
if static_stream in locals():
static_stream.close()
if sub_stream in locals():
sub_stream.close()
def filter_warn_check_is_none(file_name):
# replace env in the file_name
file_name = replace_env(file_name)
try:
file_stream = open(file_name, 'r')
while True:
lines = file_stream.readlines(100000)
if not lines:
break
for line in lines:
if "warning" in line:
return False
except OSError as reason:
cilog.cilog_error(THIS_FILE_NAME, "read file failed: %s", reason)
return False
finally:
if file_stream in locals():
file_stream.close()
return True
def exec_func(command, sub_params):
'''static check in function mode'''
function_name = command.get("function_name")
params = command.get("params")
sub_commands = None
if "sub_params" in command.keys():
sub_commands = sub_params.get(command.get("sub_params"))
args = []
kwargs = {}
if isinstance(params, list):
args.extend(params)
if sub_commands is not None:
args.append(sub_commands)
elif isinstance(params, dict):
if sub_commands is not None:
kwargs["sub_params"] = sub_commands
kwargs.update(params)
else:
cilog.cilog_error(THIS_FILE_NAME, "unsuppoted sub commands type")
return False
cilog.cilog_info(THIS_FILE_NAME, "execute the func: %s(%s, %s)",
function_name, args, kwargs)
ret = eval(function_name)(*args, **kwargs)
expected_result = command.get("expected_result")
result = True
# not break here, finish every check
if ret == expected_result:
cilog.cilog_info(THIS_FILE_NAME,
"execute the func as expected_result: %s",
expected_result)
else:
cilog.cilog_error(
THIS_FILE_NAME, "execute the func failed: actural result: %s,"
" expected_result: %s", ret, expected_result)
result = False
return result
def warn_check_compile(cmd):
'''warm check in compile mode'''
engine_mind_cmd = "find " + \
os.path.join(sc_util.ASCEND_ROOT_PATH,
"ascenddk/engine") + " -name \"*.mind\""
ret = util.execute(engine_mind_cmd, print_output_flag=True)
if ret[0] is False:
return False
mind_files = ret[1]
ddk_engine_config_path = os.path.join(os.getenv("DDK_HOME"),
"conf/settings_engine.conf")
try:
ddk_engine_config_file = open(ddk_engine_config_path, 'r')
ddk_engine_config_info = json.load(ddk_engine_config_file)
oi_engine_config_dict = ddk_engine_config_info.get(
"configuration").get("OI")
except OSError as reason:
cilog.cilog_error(THIS_FILE_NAME,
"read ddk conf/settings_engine.conf failed: %s",
reason)
return False
finally:
if ddk_engine_config_file in locals():
ddk_engine_config_file.close()
oi_lower_dict = {k.lower(): v for k, v in oi_engine_config_dict.items()}
result = True
with ProcessPoolExecutor(max_workers=5) as executor:
futures_pool = {executor.submit(
single_warn_check_compile, cmd, mind_file, oi_lower_dict): \
mind_file for mind_file in mind_files}
for future in as_completed(futures_pool):
if future.result() is False:
result = False
return result
def check_coverage(coverage_result_path, threshold = 0):
'''check coverage'''
coverage_result_path = replace_env(coverage_result_path)
html_file = os.path.join(coverage_result_path, "index.html")
with open(html_file, 'r') as index_html:
index_html_content = index_html.read()
pattern = r'<span class="pc_cov">(.*?)</span>'
coverage_data = re.findall(pattern, index_html_content)
cilog.cilog_info(THIS_FILE_NAME, "line coverage_data is %s", coverage_data)
line_coverage = "0.0%"
if len(coverage_data) > 0:
line_coverage = coverage_data[0].replace("%", "")
cilog.cilog_info(THIS_FILE_NAME, "line coverage is %s", line_coverage)
coverage_number = int(line_coverage.split(".")[0])
threshold_num = int(threshold)
if coverage_number >= threshold_num:
cilog.cilog_info(THIS_FILE_NAME, "line coverage is equal or over %d, coverage result is succ!",
threshold_num)
return True
else:
cilog.cilog_error(THIS_FILE_NAME, "line coverage is below %d, coverage result is fail!",
threshold_num)
return False
def main():
check_type = os.sys.argv[1]
static_check_commands = StaticCheckCommands(check_type)
ret, commands = static_check_commands.get_commands()
if not ret:
exit(-1)
for command_dict in commands:
comand_type = command_dict.get("type")
command = command_dict.get("command")
if comand_type == "command":
ret = static_check_cmd(command)
if not ret:
exit(-1)
elif comand_type == "function":
ret = static_check_func(command)
if not ret:
exit(-1)
else:
cilog.cilog_error(THIS_FILE_NAME, "unsupported command: %s",
command_dict)
exit(-1)
exit(0)
def __init__(self, check_type, command_type):
'''init function'''
self.commands = {}
self.sub_params = None
self.check_type = check_type
self.command_type = command_type
self.error = False
self.static_check_command_file = os.path.join(
CONFIG_PATH, self.check_type + ".yaml")
if not os.path.exists(self.static_check_command_file):
self.error = True
cilog.cilog_error(
THIS_FILE_NAME, "static check file is not exist: static_check.yaml")
cilog.cilog_debug(
THIS_FILE_NAME, "read command yaml file: %s", self.static_check_command_file)
try:
static_stream = open(self.static_check_command_file, 'r')
static_dict = yaml.load(static_stream)
self.commands = static_dict.get(command_type)
self.validate_commands()
if self.error:
return
self.command_file = os.path.join(
CONFIG_PATH, self.check_type + "_" + self.command_type + ".yaml")
if os.path.exists(self.command_file):
cilog.cilog_debug(
THIS_FILE_NAME, "read sub command yaml file: %s", self.command_file)
try:
sub_stream = open(self.command_file, 'r')
self.sub_params = yaml.load(sub_stream)
except OSError as reason:
self.error = True
cilog.cilog_error(
THIS_FILE_NAME, "read command file failed: %s", reason)
finally:
# if stream in current symbol table:locals(), close it
if sub_stream in locals():
sub_stream.close()
except OSError as reason:
self.error = True
cilog.cilog_error(
THIS_FILE_NAME, "read command file failed: %s", reason)
finally:
# if stream in current symbol table:locals(), close it
if static_stream in locals():
static_stream.close()
def execute(cmd,
timeout=3600,
print_output_flag=False,
print_cmd=True,
cwd=""):
if print_cmd:
if len(cmd) > 200:
cilog.print_in_color("%s ... %s" % (cmd[0:100], cmd[-100:]),
cilog.COLOR_F_YELLOW)
else:
cilog.print_in_color(cmd, cilog.COLOR_F_YELLOW)
is_linux = platform.system() == 'Linux'
# 生成一个子进程，执行cmd命令
if not cwd:
cwd = os.getcwd()
p = subprocess.Popen(cmd,
cwd=cwd,
bufsize=32768,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=True,
preexec_fn=os.setsid if is_linux else None)
# 判断子进程执行时间是否超时
t_beginning = time.time()
# 计算循环次数
time_gap = 0.01
loop = timeout * 100 + 200
str_std_output = ""
for loop_index in range(0, loop):
# 检查子进程是否结束
str_out = str(p.stdout.read().decode())
str_std_output = str_std_output + str_out
if print_output_flag:
cilog.print_in_color(str_out, cilog.COLOR_F_YELLOW)
if p.poll() is not None:
break
seconds_passed = time.time() - t_beginning
if timeout and seconds_passed > timeout:
# 杀掉命令进程
if is_linux:
os.kill(p.pid, signal.SIGTERM)
else:
p.terminate()
cilog.cilog_error(
THIS_FILE_NAME,
"execute %s timeout! excute seconds passed :%s, timer length:%s, return code %s",
cmd, seconds_passed, timeout, p.returncode)
return False, p.stdout.readlines()
time.sleep(time_gap)
str_std_output = str_std_output.strip()
std_output_lines_last = []
std_output_lines = str_std_output.split("\n")
for i in std_output_lines:
std_output_lines_last.append(i)
if p.returncode != 0 or "Traceback" in str_std_output:
cilog.print_in_color(str_std_output, cilog.COLOR_F_RED)
return False, std_output_lines_last
return True, std_output_lines_last
def execute(cmd,
timeout=3600,
print_output_flag=False,
print_cmd=True,
cwd=""):
'''execute os command'''
if print_cmd:
if len(cmd) > 2000:
cilog.print_in_color("%s ... %s" % (cmd[0:100], cmd[-100:]),
cilog.COLOR_F_YELLOW)
else:
cilog.print_in_color(cmd, cilog.COLOR_F_YELLOW)
is_linux = platform.system() == 'Linux'
if not cwd:
cwd = os.getcwd()
process = subprocess.Popen(cmd,
cwd=cwd,
bufsize=32768,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=True,
preexec_fn=os.setsid if is_linux else None)
t_beginning = time.time()
time_gap = 0.01
str_std_output = ""
while True:
str_out = str(process.stdout.read().decode())
str_std_output = str_std_output + str_out
if process.poll() is not None:
break
seconds_passed = time.time() - t_beginning
if timeout and seconds_passed > timeout:
if is_linux:
os.kill(process.pid, signal.SIGTERM)
else:
process.terminate()
cilog.cilog_error(THIS_FILE_NAME,
"execute %s timeout! excute seconds passed " \
" :%s, timer length:%s, return code %s",
cmd, seconds_passed, timeout, process.returncode)
return False, process.stdout.readlines()
time.sleep(time_gap)
str_std_output = str_std_output.strip()
std_output_lines_last = []
std_output_lines = str_std_output.split("\n")
for i in std_output_lines:
std_output_lines_last.append(i)
if process.returncode != 0 or "Traceback" in str_std_output:
cilog.print_in_color(str_std_output, cilog.COLOR_F_RED)
return False, std_output_lines_last
if print_output_flag:
cilog.print_in_color(str_std_output, cilog.COLOR_F_YELLOW)
return True, std_output_lines_last
def run_cmd(command, sub_params):
'''static check in command mode'''
cmd = command.get("cmd")
sub_commands = None
if "sub_params" in command.keys():
sub_commands = sub_params.get(command.get("sub_params"))
replaced_vars = re.findall(r"(__[\w+_\w+]*__)", cmd)
replaced_vars.sort()
vars_length = len(replaced_vars)
result = True
if sub_commands is None and vars_length != 0:
cilog.cilog_error(
THIS_FILE_NAME, "sub commands %s is not match" \
" replaced vars length %s", sub_commands, vars_length)
result = False
elif sub_commands is None and vars_length == 0:
ret = util.execute(cmd, print_output_flag=True)
result = ret[0]
else:
if isinstance(sub_commands, list):
for each_arg in sub_commands:
args = each_arg.split()
if vars_length != len(args):
cilog.cilog_error(
THIS_FILE_NAME, "sub commands %s is not match" \
" replaced vars length %s", sub_commands, vars_length)
return False
temp_cmd = cmd
for index in range(0, vars_length):
temp_cmd = re.sub(replaced_vars[index], args[index],
temp_cmd)
ret = util.execute(temp_cmd, print_output_flag=True)
# not return here, check every args
if not ret[0]:
result = False
elif isinstance(sub_commands, dict):
for each_arg in sub_commands:
if len(replaced_vars) != len(each_arg):
cilog.cilog_error(
THIS_FILE_NAME, "sub commands %s is not match" \
" replaced vars length %s", sub_commands, vars_length)
return False
key_list = each_arg.keys().sort()
if not operator.eq(replaced_vars, key_list):
cilog.cilog_error(
THIS_FILE_NAME, "sub commands %s is not match" \
" replaced vars %s", key_list, replaced_vars)
return False
temp_cmd = cmd
# replace all parameters in the command
for param_key, param_value in each_arg.items():
if param_value is None:
param_value = ""
temp_cmd = re.sub(param_key, param_value, temp_cmd)
ret = util.execute(temp_cmd, print_output_flag=True)
# not return here, check every args
if not ret[0]:
result = False
else:
cilog.cilog_error(THIS_FILE_NAME, "unsuppoted sub commands type")
result = False
if not result:
cilog.cilog_error(THIS_FILE_NAME, "scripts run failed.")
return result
