def do_vulture() -> str:
"""
This also finds code you are working on today!
"""
check_command_exists("vulture")
# TODO: check if whitelist.py exists?
command_text = f"{VENV_SHELL} vulture {PROJECT_NAME} whitelist.py"
command_text = command_text.strip().replace(" ", " ")
inform(command_text)
command = shlex.split(command_text)
output_file_name = f"{PROBLEMS_FOLDER}/dead_code.txt"
with open(output_file_name, "w") as outfile:
env = config_pythonpath()
subprocess.call(command, stdout=outfile, env=env)
if total_loc() > SMALL_CODE_BASE_CUTOFF:
cutoff = MAXIMUM_DEAD_CODE
else:
cutoff = 0
with open(output_file_name) as file_handle:
num_lines = sum(1 for line in file_handle if line)
if num_lines > cutoff:
say_and_exit(
f"Too many lines of dead code : {num_lines}, max {cutoff}", "vulture"
)
sys.exit(-1)
return "dead-code (vulture) succeeded"
def wrapper(*args: Any, **kwargs: Any) -> Callable:
"""Wrapper"""
start = time.time()
result = func(*args, **kwargs)
end = time.time()
inform("{} ran in {}s".format(func.__name__, round(end - start, 2)))
return result
def wrapper(*args: Any, **kwargs: Any) -> Callable:
"""Wrapper"""
state_file = (
f"{settings.CONFIG_FOLDER}/.build_state/file_hash_" + name + ".txt"
)
previous_hash = "catdog"
if os.path.exists(state_file):
with open(state_file) as old_file:
previous_hash = old_file.read()
new_hash = hash_it(file)
if new_hash == previous_hash:
inform("Nothing changed, won't re-" + name)
return lambda x: f"Skipping {name}, no change"
if not os.path.exists(f"{settings.CONFIG_FOLDER}/.build_state"):
os.makedirs(f"{settings.CONFIG_FOLDER}/.build_state")
with open(state_file, "w+") as state:
state.write(new_hash)
try:
return func(*args, **kwargs)
except: # noqa: B001
# reset if step fails
os.remove(state_file)
raise
def do_count_lines_of_code() -> None:
"""
Scale failure cut offs based on Lines of Code
"""
command_name = "pygount"
check_command_exists(command_name)
command_text = prepinform_simple(command_name)
# keep out of src tree, causes extraneous change detections
if not os.path.exists(f"{REPORTS_FOLDER}"):
os.makedirs(f"{REPORTS_FOLDER}")
output_file_name = f"{REPORTS_FOLDER}/line_counts.txt"
command = shlex.split(command_text)
with open(output_file_name, "w") as outfile:
subprocess.call(command, stdout=outfile)
with open(output_file_name) as file_handle:
lines = sum(
int(line.split("\t")[0]) for line in file_handle if line != "\n")
total_loc_local = lines
if not os.path.exists(f"{settings.CONFIG_FOLDER}/.build_state"):
os.makedirs(f"{settings.CONFIG_FOLDER}/.build_state")
with open(f"{settings.CONFIG_FOLDER}/.build_state/pygount_total_loc.txt",
"w+") as state_file:
state_file.write(str(total_loc_local))
inform(f"Lines of code: {total_loc_local}")
if total_loc_local == 0:
say_and_exit(
"No code found to build or package. Maybe the PROJECT_NAME is wrong?",
"lines of code",
)
def do_openapi_check() -> None:
"""
Does swagger/openapi file parse
"""
if not os.path.exists(f"{PROJECT_NAME}/api.yaml"):
inform("No api.yaml file, assuming this is not a microservice")
# TODO: should be able to check all y?ml files and look at header.
return
command_text = (f"{VENV_SHELL} "
"openapi-spec-validator"
f" {PROJECT_NAME}/api.yaml".strip().replace(" ", " "))
inform(command_text)
command = shlex.split(command_text)
execute(*command)
if IS_JENKINS or IS_GITLAB:
inform("Jenkins/Gitlab and apistar don't work together, skipping")
return
command_text = (f"{VENV_SHELL} apistar validate "
f"--path {PROJECT_NAME}/api.yaml "
"--format openapi "
"--encoding yaml".strip().replace(" ", " "))
inform(command_text)
# subprocess.check_call(command.split(" "), shell=False)
command = shlex.split(command_text)
result = execute_get_text(command,
ignore_error=True,
env=config_pythonpath())
if "OK" not in result and "2713" not in result and "✓" not in result:
inform(result)
say_and_exit("apistar didn't like this", "apistar")
sys.exit(-1)
def wrapper(*args: Any, **kwargs: Any) -> Callable:
"""Wrapper"""
if not has_source_code_tree_changed(name, expect_files):
inform("Nothing changed, won't re-" + name)
return lambda x: None
try:
return func(*args, **kwargs)
except: # noqa: B001
oh_never_mind(name)
raise
def do_yamllint() -> str:
"""
Check yaml files for problems
"""
command = "yamllint"
check_command_exists(command)
command = f"{VENV_SHELL} yamllint {PROJECT_NAME}".strip().replace(
" ", " ")
inform(command)
execute(*(command.split(" ")))
return "yamllint succeeded"
def evaluated_mypy_results(mypy_file: str, small_code_base_cutoff: int,
maximum_mypy: int, skips: List[str]) -> str:
"""
Decided if the mypy is bad enough to stop the build.
"""
with open(mypy_file) as out_file:
out = out_file.read()
def contains_a_skip(line_value: str) -> bool:
"""
Should this line be skipped
"""
# skips is a closure
for skip in skips:
if skip in line_value or line_value.startswith(skip):
return True
return False
actually_bad_lines: List[str] = []
total_lines = 0
with open(mypy_file, "w+") as lint_file:
lines = out.split("\n")
for line in lines:
total_lines += 1
if contains_a_skip(line):
continue
if not line.startswith(PROJECT_NAME):
continue
actually_bad_lines.append(line)
lint_file.writelines([line])
num_lines = len(actually_bad_lines)
if total_loc() > small_code_base_cutoff:
max_lines = maximum_mypy
else:
max_lines = 2 # off by 1 right now
if num_lines > max_lines:
for line in actually_bad_lines:
inform(line)
say_and_exit(f"Too many lines of mypy : {num_lines}, max {max_lines}",
"mypy")
sys.exit(-1)
if num_lines == 0 and total_lines == 0:
# should always have at least 'found 0 errors' in output
say_and_exit(
"No mypy warnings at all, did mypy fail to run or is it installed?",
"mypy")
sys.exit(-1)
return "mypy succeeded"
def do_liccheck() -> str:
"""
Make an explicit decision about license of referenced packages
"""
check_command_exists("liccheck")
if not os.path.exists(f"{settings.CONFIG_FOLDER}/requirements.txt"):
inform("No requirements.txt file, assuming we have no external deps")
return "Skipping, not requirements.txt"
command = ("liccheck "
f"-r {settings.CONFIG_FOLDER}/requirements.txt "
f"-s {settings.CONFIG_FOLDER}/.license_rules "
"-l paranoid")
command = prepinform_simple(command, no_project=True)
execute(*(command.split(" ")))
return "liccheck succeeded"
def do_mypy() -> str:
"""
Are types ok?
"""
check_command_exists("mypy")
if sys.version_info < (3, 4):
inform("Mypy doesn't work on python < 3.4")
return "command is missing"
command = (f"{VENV_SHELL} mypy {PROJECT_NAME} "
"--ignore-missing-imports "
"--strict".strip().replace(" ", " "))
inform(command)
bash_process = subprocess.Popen(command.split(" "),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, _ = bash_process.communicate() # wait
mypy_file = f"{PROBLEMS_FOLDER}/all_mypy_errors.txt"
with open(mypy_file, "w", encoding="utf-8") as out_file:
out_file.write(out.decode())
return mypy_file
def do_precommit(is_interactive: bool) -> None:
"""
Build time execution of pre-commit checks. Modifies code so run before linter.
"""
if not is_interactive:
inform("Not running precommit because it changes files")
return
check_command_exists("pre-commit")
if is_git_repo("."):
# don't try to install because it isn't a git repo
command_text = f"{VENV_SHELL} pre-commit install".strip().replace(" ", " ")
inform(command_text)
command = shlex.split(command_text)
execute(*command)
command_text = f"{VENV_SHELL} pre-commit run --all-files".strip().replace(" ", " ")
inform(command_text)
command = shlex.split(command_text)
result = execute_get_text(command, ignore_error=True, env=config_pythonpath())
assert result
changed = []
for line in result.split("\n"):
if "changed " in line:
file = line[len("reformatted ") :].strip()
changed.append(file)
if "FAILED" in result:
inform(result)
say_and_exit("Pre-commit Failed", "pre-commit")
sys.exit(-1)
if is_interactive:
if not is_git_repo("."):
# don't need to git add anything because this isn't a git repo
return
for change in changed:
command_text = f"git add {change}"
inform(command_text)
# this breaks on windows!
# command = shlex.split(command_text)
execute(*command_text.split())
