def main(): '''static check''' check_type = os.sys.argv[1] commahd_type = os.sys.argv[2] scripts_commands = ScriptsCommands(check_type, commahd_type) ret, commands = scripts_commands.get_commands() if not ret: exit(-1) ret, sub_params = scripts_commands.get_sub_params() if not ret: exit(-1) for command_dict in commands: comand_type = command_dict.get("type") if comand_type == "command": ret = run_cmd(command_dict, sub_params) if not ret: exit(-1) elif comand_type == "function": ret = exec_func(command_dict, sub_params) if not ret: exit(-1) else: cilog.cilog_error(THIS_FILE_NAME, "unsupported command: %s", command_dict) exit(-1) exit(0)
def get_commands(self): '''get static check commands''' if self.error is True: cilog.cilog_error( THIS_FILE_NAME, "get command failed.") return False, None return True, self.commands
def validate_commands(self): for each_command in self.commands: command_type = each_command.get("type") if command_type is None: cilog.cilog_error(THIS_FILE_NAME, "type is invalid: %s", each_command) self.error = True break command = each_command.get("command") if command is None: cilog.cilog_error(THIS_FILE_NAME, "command is invalid: %s", each_command) self.error = True break if command_type == "command": if "cmd" not in command.keys(): cilog.cilog_error(THIS_FILE_NAME, "cmd is invalid: %s", each_command) self.error = True break elif command_type == "function": if "function_name" not in command.keys(): cilog.cilog_error(THIS_FILE_NAME, "function_name is invalid: %s", each_command) self.error = True break else: cilog.cilog_error(THIS_FILE_NAME, "type is invalid: %s", each_command) self.error = True break
def validate_makefile(makefile_path): '''validate -Wall in makefile''' result = False try: makefile_stream = open(makefile_path, "r") while True: lines = makefile_stream.readlines(1000) if not lines: break for line in lines: if "-Wall" in line: result = True break except OSError as reason: cilog.cilog_error(THIS_FILE_NAME, "read makefile %s failed: %s", makefile_path, reason) finally: if makefile_stream in locals(): makefile_stream.close() if result is False: cilog.cilog_error( THIS_FILE_NAME, "makefile %s is invalid:"\ " no -Wall compile parameters", makefile_path) return result
def __init__(self, command_file_name=None): self.commands = {} self.command_file_name = command_file_name self.error = False if self.command_file_name is None: self.command_file_name = "default" self.command_file = os.path.join(CONFIG_PATH, self.command_file_name + ".yaml") if not os.path.exists(self.command_file): self.error = True cilog.cilog_error(THIS_FILE_NAME, "command yaml file is not exist: %s", self.command_file) return cilog.cilog_debug(THIS_FILE_NAME, "read command yaml file: %s", self.command_file) try: stream = open(self.command_file, 'r') self.commands = yaml.load(stream) except OSError as reason: self.error = True cilog.cilog_error(THIS_FILE_NAME, "read command file failed: %s", reason) finally: # if stream in current symbol table:locals(), close it if stream in locals(): stream.close()
def warn_check_compile(cmd, checked_path, headers_list): checked_path = replace_env(checked_path) if not os.path.exists(checked_path): cilog.cilog_error(THIS_FILE_NAME, "can not find cpp list file: %s", checked_path) return False checked_file_cmd = "find " + checked_path + " -name \"*.cpp\" -o -name \"*.h\"" ret = util.execute(checked_file_cmd) if ret[0] is False: cilog.cilog_error(THIS_FILE_NAME, "can not find cpp list file: %s", checked_path) return False checked_file = ret[1] headers = "" if headers_list is not None: headers = " -I".join(headers_list) headers = "-I" + headers cmd = re.sub("__WARN_CHECK_HEADERS__", headers, cmd) for file in checked_file: file_names = os.path.split(file) temp_cmd = re.sub("__WARN_CHECK_FILE__", file, cmd) temp_cmd = re.sub("__WARN_CHECK_FILE_NAME__", file_names[1], temp_cmd) util.execute(temp_cmd, print_output_flag=True) return True
def get_install_commands(self): if self.error: cilog.cilog_error( THIS_FILE_NAME, "no command yaml or config yaml not exist.") return False, None commands = self.commands.get("install") return True, commands
def get_sub_params(self): '''get static check sub params''' if self.error is True: cilog.cilog_error( THIS_FILE_NAME, "get sub params failed.") return False, None return True, self.sub_params
def static_check_cmd(command): cmd = command.get("cmd") args = command.get("args") result = True if args is None: ret = util.execute(cmd, print_output_flag=True) result = ret[0] else: for each_arg in args: # replace all parameters in the command for param_key, param_value in each_arg.items(): if param_value is None: param_value = "" temp_cmd = re.sub(param_key, param_value, cmd) ret = util.execute(temp_cmd, print_output_flag=True) # not return here, check every args if not ret[0]: result = False if not result: cilog.cilog_error(THIS_FILE_NAME, "static check failed.") return result
def single_warn_check_compile(cmd, mind_file, oi_engine_config_dict): '''single warm_check path in compile mode''' mind_file_paths = os.path.split(mind_file) mind_file_path = mind_file_paths[0] try: mind_file_stream = open(mind_file, 'r') mind_file_info = json.load(mind_file_stream) mind_nodes = mind_file_info.get("node") except OSError as reason: cilog.cilog_error(THIS_FILE_NAME, "read %s failed: %s", mind_file, reason) return False result = True checked_file_path = "" for mind_node in mind_nodes: if mind_node.get("group") == "MyModel": continue name = mind_node.get("name") run_side = mind_node.get("params").get("runSide") checked_file_path = os.path.join(mind_file_path, name) header_list = oi_engine_config_dict.get( run_side.lower()).get("includes").get("include") header_list = list( map(lambda x: re.sub(r"\$\(SRC_DIR\)", checked_file_path, x), header_list)) header_list = list( map(lambda x: re.sub(r"\.\.", mind_file_path, x), header_list)) header_list = list(map(lambda x: re.sub(r" \\", "", x), header_list)) header_list = list( map(lambda x: re.sub(r"\$\(DDK_HOME\)", os.getenv("DDK_HOME"), x), header_list)) replaced_cmd = re.sub("__WARN_CHECK_HEADERS__", " ".join(header_list), cmd) checked_file_cmd = "find " + checked_file_path + \ " -name \"*.cpp\" -o -name \"*.h\"" ret = util.execute(checked_file_cmd, print_output_flag=True) if ret[0] is False: result = False continue checked_files = ret[1] for file in checked_files: file_path_name = os.path.split(file) file_names = os.path.splitext(file_path_name[1]) file_name = file_names[0] + ".o" temp_cmd = re.sub(r"__WARN_CHECK_FILE__", file, replaced_cmd) temp_cmd = re.sub(r"__WARN_CHECK_FILE_NAME__", file_name, temp_cmd) ret = util.execute(temp_cmd, print_output_flag=True) if ret[0] is False: result = False return result
def static_check_func(command): function_name = command.get("function_name") parameter_list = command.get("args") cmd = command.get("cmd") if parameter_list is None: parameter_list = [] parameter_list.append({"arg_value": []}) result = True for parameter in parameter_list: args = [] kwargs = {} # each time either mode-values or mode-key-values works mode_values = parameter.get("arg_value") mode_key_values = parameter.get("arg_key_value") if mode_values is not None and mode_key_values is None: if cmd is not None: args.append(cmd) args.extend(mode_values) elif mode_values is None and mode_key_values is not None: if cmd is not None: kwargs["cmd"] = cmd kwargs.update(mode_key_values) else: cilog.cilog_error( THIS_FILE_NAME, "mode_value: %s and mode_key_value: %s are invalid", mode_values, mode_key_values) return False cilog.cilog_info(THIS_FILE_NAME, "execute the func: %s(%s, %s)", function_name, args, kwargs) ret = eval(function_name)(*args, **kwargs) expected_result = command.get("expected_result") # not break here, finish every check if ret == expected_result: cilog.cilog_info(THIS_FILE_NAME, "execute the func as expected_result: %s", expected_result) else: cilog.cilog_error( THIS_FILE_NAME, "execute the func failed: actural result: %s," " expected_result: %s", ret, expected_result) result = False return result
def __init__(self, command_type): self.commands = {} self.command_type = command_type self.error = False self.static_check_command_file = os.path.join(CONFIG_PATH, "static_check.yaml") if not os.path.exists(self.static_check_command_file): self.error = True cilog.cilog_error( THIS_FILE_NAME, "static check file is not exist: static_check.yaml") self.command_file = os.path.join( CONFIG_PATH, "static_check_" + self.command_type + ".yaml") if not os.path.exists(self.command_file): self.error = True cilog.cilog_error(THIS_FILE_NAME, "command yaml file is not exist: %s", self.command_file) return cilog.cilog_debug(THIS_FILE_NAME, "read command yaml file: %s__%s", self.static_check_command_file, self.command_file) try: static_stream = open(self.static_check_command_file, 'r') sub_stream = open(self.command_file, 'r') static_dict = yaml.load(static_stream) sub_dict = yaml.load(sub_stream) self.commands = static_dict.get(command_type) self.validate_commands() if self.error: return for commands in self.commands: command = commands.get("command") arg_name = command.get("arg_name") if arg_name is None: continue args = sub_dict.get(arg_name) if args is None: self.error = True cilog.cilog_error(THIS_FILE_NAME, "args in %s is invalid", self.command_file) return for key, value in args.items(): command[key] = value except OSError as reason: self.error = True cilog.cilog_error(THIS_FILE_NAME, "read command file failed: %s", reason) finally: # if stream in current symbol table:locals(), close it if static_stream in locals(): static_stream.close() if sub_stream in locals(): sub_stream.close()
def filter_warn_check_is_none(file_name): # replace env in the file_name file_name = replace_env(file_name) try: file_stream = open(file_name, 'r') while True: lines = file_stream.readlines(100000) if not lines: break for line in lines: if "warning" in line: return False except OSError as reason: cilog.cilog_error(THIS_FILE_NAME, "read file failed: %s", reason) return False finally: if file_stream in locals(): file_stream.close() return True
def exec_func(command, sub_params): '''static check in function mode''' function_name = command.get("function_name") params = command.get("params") sub_commands = None if "sub_params" in command.keys(): sub_commands = sub_params.get(command.get("sub_params")) args = [] kwargs = {} if isinstance(params, list): args.extend(params) if sub_commands is not None: args.append(sub_commands) elif isinstance(params, dict): if sub_commands is not None: kwargs["sub_params"] = sub_commands kwargs.update(params) else: cilog.cilog_error(THIS_FILE_NAME, "unsuppoted sub commands type") return False cilog.cilog_info(THIS_FILE_NAME, "execute the func: %s(%s, %s)", function_name, args, kwargs) ret = eval(function_name)(*args, **kwargs) expected_result = command.get("expected_result") result = True # not break here, finish every check if ret == expected_result: cilog.cilog_info(THIS_FILE_NAME, "execute the func as expected_result: %s", expected_result) else: cilog.cilog_error( THIS_FILE_NAME, "execute the func failed: actural result: %s," " expected_result: %s", ret, expected_result) result = False return result
def warn_check_compile(cmd): '''warm check in compile mode''' engine_mind_cmd = "find " + \ os.path.join(sc_util.ASCEND_ROOT_PATH, "ascenddk/engine") + " -name \"*.mind\"" ret = util.execute(engine_mind_cmd, print_output_flag=True) if ret[0] is False: return False mind_files = ret[1] ddk_engine_config_path = os.path.join(os.getenv("DDK_HOME"), "conf/settings_engine.conf") try: ddk_engine_config_file = open(ddk_engine_config_path, 'r') ddk_engine_config_info = json.load(ddk_engine_config_file) oi_engine_config_dict = ddk_engine_config_info.get( "configuration").get("OI") except OSError as reason: cilog.cilog_error(THIS_FILE_NAME, "read ddk conf/settings_engine.conf failed: %s", reason) return False finally: if ddk_engine_config_file in locals(): ddk_engine_config_file.close() oi_lower_dict = {k.lower(): v for k, v in oi_engine_config_dict.items()} result = True with ProcessPoolExecutor(max_workers=5) as executor: futures_pool = {executor.submit( single_warn_check_compile, cmd, mind_file, oi_lower_dict): \ mind_file for mind_file in mind_files} for future in as_completed(futures_pool): if future.result() is False: result = False return result
def check_coverage(coverage_result_path, threshold = 0): '''check coverage''' coverage_result_path = replace_env(coverage_result_path) html_file = os.path.join(coverage_result_path, "index.html") with open(html_file, 'r') as index_html: index_html_content = index_html.read() pattern = r'<span class="pc_cov">(.*?)</span>' coverage_data = re.findall(pattern, index_html_content) cilog.cilog_info(THIS_FILE_NAME, "line coverage_data is %s", coverage_data) line_coverage = "0.0%" if len(coverage_data) > 0: line_coverage = coverage_data[0].replace("%", "") cilog.cilog_info(THIS_FILE_NAME, "line coverage is %s", line_coverage) coverage_number = int(line_coverage.split(".")[0]) threshold_num = int(threshold) if coverage_number >= threshold_num: cilog.cilog_info(THIS_FILE_NAME, "line coverage is equal or over %d, coverage result is succ!", threshold_num) return True else: cilog.cilog_error(THIS_FILE_NAME, "line coverage is below %d, coverage result is fail!", threshold_num) return False
def main(): check_type = os.sys.argv[1] static_check_commands = StaticCheckCommands(check_type) ret, commands = static_check_commands.get_commands() if not ret: exit(-1) for command_dict in commands: comand_type = command_dict.get("type") command = command_dict.get("command") if comand_type == "command": ret = static_check_cmd(command) if not ret: exit(-1) elif comand_type == "function": ret = static_check_func(command) if not ret: exit(-1) else: cilog.cilog_error(THIS_FILE_NAME, "unsupported command: %s", command_dict) exit(-1) exit(0)
def __init__(self, check_type, command_type): '''init function''' self.commands = {} self.sub_params = None self.check_type = check_type self.command_type = command_type self.error = False self.static_check_command_file = os.path.join( CONFIG_PATH, self.check_type + ".yaml") if not os.path.exists(self.static_check_command_file): self.error = True cilog.cilog_error( THIS_FILE_NAME, "static check file is not exist: static_check.yaml") cilog.cilog_debug( THIS_FILE_NAME, "read command yaml file: %s", self.static_check_command_file) try: static_stream = open(self.static_check_command_file, 'r') static_dict = yaml.load(static_stream) self.commands = static_dict.get(command_type) self.validate_commands() if self.error: return self.command_file = os.path.join( CONFIG_PATH, self.check_type + "_" + self.command_type + ".yaml") if os.path.exists(self.command_file): cilog.cilog_debug( THIS_FILE_NAME, "read sub command yaml file: %s", self.command_file) try: sub_stream = open(self.command_file, 'r') self.sub_params = yaml.load(sub_stream) except OSError as reason: self.error = True cilog.cilog_error( THIS_FILE_NAME, "read command file failed: %s", reason) finally: # if stream in current symbol table:locals(), close it if sub_stream in locals(): sub_stream.close() except OSError as reason: self.error = True cilog.cilog_error( THIS_FILE_NAME, "read command file failed: %s", reason) finally: # if stream in current symbol table:locals(), close it if static_stream in locals(): static_stream.close()
def execute(cmd, timeout=3600, print_output_flag=False, print_cmd=True, cwd=""): if print_cmd: if len(cmd) > 200: cilog.print_in_color("%s ... %s" % (cmd[0:100], cmd[-100:]), cilog.COLOR_F_YELLOW) else: cilog.print_in_color(cmd, cilog.COLOR_F_YELLOW) is_linux = platform.system() == 'Linux' # 生成一个子进程,执行cmd命令 if not cwd: cwd = os.getcwd() p = subprocess.Popen(cmd, cwd=cwd, bufsize=32768, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, preexec_fn=os.setsid if is_linux else None) # 判断子进程执行时间是否超时 t_beginning = time.time() # 计算循环次数 time_gap = 0.01 loop = timeout * 100 + 200 str_std_output = "" for loop_index in range(0, loop): # 检查子进程是否结束 str_out = str(p.stdout.read().decode()) str_std_output = str_std_output + str_out if print_output_flag: cilog.print_in_color(str_out, cilog.COLOR_F_YELLOW) if p.poll() is not None: break seconds_passed = time.time() - t_beginning if timeout and seconds_passed > timeout: # 杀掉命令进程 if is_linux: os.kill(p.pid, signal.SIGTERM) else: p.terminate() cilog.cilog_error( THIS_FILE_NAME, "execute %s timeout! excute seconds passed :%s, timer length:%s, return code %s", cmd, seconds_passed, timeout, p.returncode) return False, p.stdout.readlines() time.sleep(time_gap) str_std_output = str_std_output.strip() std_output_lines_last = [] std_output_lines = str_std_output.split("\n") for i in std_output_lines: std_output_lines_last.append(i) if p.returncode != 0 or "Traceback" in str_std_output: cilog.print_in_color(str_std_output, cilog.COLOR_F_RED) return False, std_output_lines_last return True, std_output_lines_last
def execute(cmd, timeout=3600, print_output_flag=False, print_cmd=True, cwd=""): '''execute os command''' if print_cmd: if len(cmd) > 2000: cilog.print_in_color("%s ... %s" % (cmd[0:100], cmd[-100:]), cilog.COLOR_F_YELLOW) else: cilog.print_in_color(cmd, cilog.COLOR_F_YELLOW) is_linux = platform.system() == 'Linux' if not cwd: cwd = os.getcwd() process = subprocess.Popen(cmd, cwd=cwd, bufsize=32768, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, preexec_fn=os.setsid if is_linux else None) t_beginning = time.time() time_gap = 0.01 str_std_output = "" while True: str_out = str(process.stdout.read().decode()) str_std_output = str_std_output + str_out if process.poll() is not None: break seconds_passed = time.time() - t_beginning if timeout and seconds_passed > timeout: if is_linux: os.kill(process.pid, signal.SIGTERM) else: process.terminate() cilog.cilog_error(THIS_FILE_NAME, "execute %s timeout! excute seconds passed " \ " :%s, timer length:%s, return code %s", cmd, seconds_passed, timeout, process.returncode) return False, process.stdout.readlines() time.sleep(time_gap) str_std_output = str_std_output.strip() std_output_lines_last = [] std_output_lines = str_std_output.split("\n") for i in std_output_lines: std_output_lines_last.append(i) if process.returncode != 0 or "Traceback" in str_std_output: cilog.print_in_color(str_std_output, cilog.COLOR_F_RED) return False, std_output_lines_last if print_output_flag: cilog.print_in_color(str_std_output, cilog.COLOR_F_YELLOW) return True, std_output_lines_last
def run_cmd(command, sub_params): '''static check in command mode''' cmd = command.get("cmd") sub_commands = None if "sub_params" in command.keys(): sub_commands = sub_params.get(command.get("sub_params")) replaced_vars = re.findall(r"(__[\w+_\w+]*__)", cmd) replaced_vars.sort() vars_length = len(replaced_vars) result = True if sub_commands is None and vars_length != 0: cilog.cilog_error( THIS_FILE_NAME, "sub commands %s is not match" \ " replaced vars length %s", sub_commands, vars_length) result = False elif sub_commands is None and vars_length == 0: ret = util.execute(cmd, print_output_flag=True) result = ret[0] else: if isinstance(sub_commands, list): for each_arg in sub_commands: args = each_arg.split() if vars_length != len(args): cilog.cilog_error( THIS_FILE_NAME, "sub commands %s is not match" \ " replaced vars length %s", sub_commands, vars_length) return False temp_cmd = cmd for index in range(0, vars_length): temp_cmd = re.sub(replaced_vars[index], args[index], temp_cmd) ret = util.execute(temp_cmd, print_output_flag=True) # not return here, check every args if not ret[0]: result = False elif isinstance(sub_commands, dict): for each_arg in sub_commands: if len(replaced_vars) != len(each_arg): cilog.cilog_error( THIS_FILE_NAME, "sub commands %s is not match" \ " replaced vars length %s", sub_commands, vars_length) return False key_list = each_arg.keys().sort() if not operator.eq(replaced_vars, key_list): cilog.cilog_error( THIS_FILE_NAME, "sub commands %s is not match" \ " replaced vars %s", key_list, replaced_vars) return False temp_cmd = cmd # replace all parameters in the command for param_key, param_value in each_arg.items(): if param_value is None: param_value = "" temp_cmd = re.sub(param_key, param_value, temp_cmd) ret = util.execute(temp_cmd, print_output_flag=True) # not return here, check every args if not ret[0]: result = False else: cilog.cilog_error(THIS_FILE_NAME, "unsuppoted sub commands type") result = False if not result: cilog.cilog_error(THIS_FILE_NAME, "scripts run failed.") return result