def do_vulture() -> str: """ This also finds code you are working on today! """ check_command_exists("vulture") # TODO: check if whitelist.py exists? command_text = f"{VENV_SHELL} vulture {PROJECT_NAME} whitelist.py" command_text = command_text.strip().replace(" ", " ") inform(command_text) command = shlex.split(command_text) output_file_name = f"{PROBLEMS_FOLDER}/dead_code.txt" with open(output_file_name, "w") as outfile: env = config_pythonpath() subprocess.call(command, stdout=outfile, env=env) if total_loc() > SMALL_CODE_BASE_CUTOFF: cutoff = MAXIMUM_DEAD_CODE else: cutoff = 0 with open(output_file_name) as file_handle: num_lines = sum(1 for line in file_handle if line) if num_lines > cutoff: say_and_exit( f"Too many lines of dead code : {num_lines}, max {cutoff}", "vulture" ) sys.exit(-1) return "dead-code (vulture) succeeded"
def wrapper(*args: Any, **kwargs: Any) -> Callable: """Wrapper""" start = time.time() result = func(*args, **kwargs) end = time.time() inform("{} ran in {}s".format(func.__name__, round(end - start, 2))) return result
def wrapper(*args: Any, **kwargs: Any) -> Callable: """Wrapper""" state_file = ( f"{settings.CONFIG_FOLDER}/.build_state/file_hash_" + name + ".txt" ) previous_hash = "catdog" if os.path.exists(state_file): with open(state_file) as old_file: previous_hash = old_file.read() new_hash = hash_it(file) if new_hash == previous_hash: inform("Nothing changed, won't re-" + name) return lambda x: f"Skipping {name}, no change" if not os.path.exists(f"{settings.CONFIG_FOLDER}/.build_state"): os.makedirs(f"{settings.CONFIG_FOLDER}/.build_state") with open(state_file, "w+") as state: state.write(new_hash) try: return func(*args, **kwargs) except: # noqa: B001 # reset if step fails os.remove(state_file) raise
def do_count_lines_of_code() -> None: """ Scale failure cut offs based on Lines of Code """ command_name = "pygount" check_command_exists(command_name) command_text = prepinform_simple(command_name) # keep out of src tree, causes extraneous change detections if not os.path.exists(f"{REPORTS_FOLDER}"): os.makedirs(f"{REPORTS_FOLDER}") output_file_name = f"{REPORTS_FOLDER}/line_counts.txt" command = shlex.split(command_text) with open(output_file_name, "w") as outfile: subprocess.call(command, stdout=outfile) with open(output_file_name) as file_handle: lines = sum( int(line.split("\t")[0]) for line in file_handle if line != "\n") total_loc_local = lines if not os.path.exists(f"{settings.CONFIG_FOLDER}/.build_state"): os.makedirs(f"{settings.CONFIG_FOLDER}/.build_state") with open(f"{settings.CONFIG_FOLDER}/.build_state/pygount_total_loc.txt", "w+") as state_file: state_file.write(str(total_loc_local)) inform(f"Lines of code: {total_loc_local}") if total_loc_local == 0: say_and_exit( "No code found to build or package. Maybe the PROJECT_NAME is wrong?", "lines of code", )
def do_openapi_check() -> None: """ Does swagger/openapi file parse """ if not os.path.exists(f"{PROJECT_NAME}/api.yaml"): inform("No api.yaml file, assuming this is not a microservice") # TODO: should be able to check all y?ml files and look at header. return command_text = (f"{VENV_SHELL} " "openapi-spec-validator" f" {PROJECT_NAME}/api.yaml".strip().replace(" ", " ")) inform(command_text) command = shlex.split(command_text) execute(*command) if IS_JENKINS or IS_GITLAB: inform("Jenkins/Gitlab and apistar don't work together, skipping") return command_text = (f"{VENV_SHELL} apistar validate " f"--path {PROJECT_NAME}/api.yaml " "--format openapi " "--encoding yaml".strip().replace(" ", " ")) inform(command_text) # subprocess.check_call(command.split(" "), shell=False) command = shlex.split(command_text) result = execute_get_text(command, ignore_error=True, env=config_pythonpath()) if "OK" not in result and "2713" not in result and "✓" not in result: inform(result) say_and_exit("apistar didn't like this", "apistar") sys.exit(-1)
def wrapper(*args: Any, **kwargs: Any) -> Callable: """Wrapper""" if not has_source_code_tree_changed(name, expect_files): inform("Nothing changed, won't re-" + name) return lambda x: None try: return func(*args, **kwargs) except: # noqa: B001 oh_never_mind(name) raise
def do_yamllint() -> str: """ Check yaml files for problems """ command = "yamllint" check_command_exists(command) command = f"{VENV_SHELL} yamllint {PROJECT_NAME}".strip().replace( " ", " ") inform(command) execute(*(command.split(" "))) return "yamllint succeeded"
def evaluated_mypy_results(mypy_file: str, small_code_base_cutoff: int, maximum_mypy: int, skips: List[str]) -> str: """ Decided if the mypy is bad enough to stop the build. """ with open(mypy_file) as out_file: out = out_file.read() def contains_a_skip(line_value: str) -> bool: """ Should this line be skipped """ # skips is a closure for skip in skips: if skip in line_value or line_value.startswith(skip): return True return False actually_bad_lines: List[str] = [] total_lines = 0 with open(mypy_file, "w+") as lint_file: lines = out.split("\n") for line in lines: total_lines += 1 if contains_a_skip(line): continue if not line.startswith(PROJECT_NAME): continue actually_bad_lines.append(line) lint_file.writelines([line]) num_lines = len(actually_bad_lines) if total_loc() > small_code_base_cutoff: max_lines = maximum_mypy else: max_lines = 2 # off by 1 right now if num_lines > max_lines: for line in actually_bad_lines: inform(line) say_and_exit(f"Too many lines of mypy : {num_lines}, max {max_lines}", "mypy") sys.exit(-1) if num_lines == 0 and total_lines == 0: # should always have at least 'found 0 errors' in output say_and_exit( "No mypy warnings at all, did mypy fail to run or is it installed?", "mypy") sys.exit(-1) return "mypy succeeded"
def do_liccheck() -> str: """ Make an explicit decision about license of referenced packages """ check_command_exists("liccheck") if not os.path.exists(f"{settings.CONFIG_FOLDER}/requirements.txt"): inform("No requirements.txt file, assuming we have no external deps") return "Skipping, not requirements.txt" command = ("liccheck " f"-r {settings.CONFIG_FOLDER}/requirements.txt " f"-s {settings.CONFIG_FOLDER}/.license_rules " "-l paranoid") command = prepinform_simple(command, no_project=True) execute(*(command.split(" "))) return "liccheck succeeded"
def do_mypy() -> str: """ Are types ok? """ check_command_exists("mypy") if sys.version_info < (3, 4): inform("Mypy doesn't work on python < 3.4") return "command is missing" command = (f"{VENV_SHELL} mypy {PROJECT_NAME} " "--ignore-missing-imports " "--strict".strip().replace(" ", " ")) inform(command) bash_process = subprocess.Popen(command.split(" "), stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _ = bash_process.communicate() # wait mypy_file = f"{PROBLEMS_FOLDER}/all_mypy_errors.txt" with open(mypy_file, "w", encoding="utf-8") as out_file: out_file.write(out.decode()) return mypy_file
def do_precommit(is_interactive: bool) -> None: """ Build time execution of pre-commit checks. Modifies code so run before linter. """ if not is_interactive: inform("Not running precommit because it changes files") return check_command_exists("pre-commit") if is_git_repo("."): # don't try to install because it isn't a git repo command_text = f"{VENV_SHELL} pre-commit install".strip().replace(" ", " ") inform(command_text) command = shlex.split(command_text) execute(*command) command_text = f"{VENV_SHELL} pre-commit run --all-files".strip().replace(" ", " ") inform(command_text) command = shlex.split(command_text) result = execute_get_text(command, ignore_error=True, env=config_pythonpath()) assert result changed = [] for line in result.split("\n"): if "changed " in line: file = line[len("reformatted ") :].strip() changed.append(file) if "FAILED" in result: inform(result) say_and_exit("Pre-commit Failed", "pre-commit") sys.exit(-1) if is_interactive: if not is_git_repo("."): # don't need to git add anything because this isn't a git repo return for change in changed: command_text = f"git add {change}" inform(command_text) # this breaks on windows! # command = shlex.split(command_text) execute(*command_text.split())