def do_vulture() -> str:
"""
This also finds code you are working on today!
"""
check_command_exists("vulture")
# TODO: check if whitelist.py exists?
command_text = f"{VENV_SHELL} vulture {PROJECT_NAME} whitelist.py"
command_text = command_text.strip().replace(" ", " ")
inform(command_text)
command = shlex.split(command_text)
output_file_name = f"{PROBLEMS_FOLDER}/dead_code.txt"
with open(output_file_name, "w") as outfile:
env = config_pythonpath()
subprocess.call(command, stdout=outfile, env=env)
if total_loc() > SMALL_CODE_BASE_CUTOFF:
cutoff = MAXIMUM_DEAD_CODE
else:
cutoff = 0
with open(output_file_name) as file_handle:
num_lines = sum(1 for line in file_handle if line)
if num_lines > cutoff:
say_and_exit(
f"Too many lines of dead code : {num_lines}, max {cutoff}", "vulture"
)
sys.exit(-1)
return "dead-code (vulture) succeeded"
def do_count_lines_of_code() -> None:
"""
Scale failure cut offs based on Lines of Code
"""
command_name = "pygount"
check_command_exists(command_name)
command_text = prepinform_simple(command_name)
# keep out of src tree, causes extraneous change detections
if not os.path.exists(f"{REPORTS_FOLDER}"):
os.makedirs(f"{REPORTS_FOLDER}")
output_file_name = f"{REPORTS_FOLDER}/line_counts.txt"
command = shlex.split(command_text)
with open(output_file_name, "w") as outfile:
subprocess.call(command, stdout=outfile)
with open(output_file_name) as file_handle:
lines = sum(
int(line.split("\t")[0]) for line in file_handle if line != "\n")
total_loc_local = lines
if not os.path.exists(f"{settings.CONFIG_FOLDER}/.build_state"):
os.makedirs(f"{settings.CONFIG_FOLDER}/.build_state")
with open(f"{settings.CONFIG_FOLDER}/.build_state/pygount_total_loc.txt",
"w+") as state_file:
state_file.write(str(total_loc_local))
inform(f"Lines of code: {total_loc_local}")
if total_loc_local == 0:
say_and_exit(
"No code found to build or package. Maybe the PROJECT_NAME is wrong?",
"lines of code",
)
def do_openapi_check() -> None:
"""
Does swagger/openapi file parse
"""
if not os.path.exists(f"{PROJECT_NAME}/api.yaml"):
inform("No api.yaml file, assuming this is not a microservice")
# TODO: should be able to check all y?ml files and look at header.
return
command_text = (f"{VENV_SHELL} "
"openapi-spec-validator"
f" {PROJECT_NAME}/api.yaml".strip().replace(" ", " "))
inform(command_text)
command = shlex.split(command_text)
execute(*command)
if IS_JENKINS or IS_GITLAB:
inform("Jenkins/Gitlab and apistar don't work together, skipping")
return
command_text = (f"{VENV_SHELL} apistar validate "
f"--path {PROJECT_NAME}/api.yaml "
"--format openapi "
"--encoding yaml".strip().replace(" ", " "))
inform(command_text)
# subprocess.check_call(command.split(" "), shell=False)
command = shlex.split(command_text)
result = execute_get_text(command,
ignore_error=True,
env=config_pythonpath())
if "OK" not in result and "2713" not in result and "✓" not in result:
inform(result)
say_and_exit("apistar didn't like this", "apistar")
sys.exit(-1)
def do_detect_secrets() -> str:
"""
Call detect-secrets tool
I think this is the problem:
# Code expects to stream output to file and then expects
# interactive person, so code hangs. But also hangs in git-bash
detect-secrets scan test_data/config.env > foo.txt
detect-secrets audit foo.txt
"""
inform("Detect secrets broken ... can't figure out why")
return "nope"
# pylint: disable=unreachable
check_command_exists("detect-secrets")
errors_file = f"{PROBLEMS_FOLDER}/detect-secrets-results.txt"
command_text = (
f"{VENV_SHELL} detect-secrets scan "
"--base64-limit 4 "
# f"--exclude-files .idea|.min.js|.html|.xsd|"
# f"lock.json|.scss|Pipfile.lock|.secrets.baseline|"
# f"{PROBLEMS_FOLDER}/lint.txt|{errors_file}".strip().replace(" ", " ")
)
inform(command_text)
command = shlex.split(command_text)
with open(errors_file, "w") as outfile:
env = config_pythonpath()
output = execute_get_text(command, ignore_error=False, env=env)
outfile.write(output)
# subprocess.call(command, stdout=outfile, env=env)
with open(errors_file, "w+") as file_handle:
text = file_handle.read()
if not text:
say_and_exit("Failed to check for secrets", "detect-secrets")
sys.exit(-1)
file_handle.write(text)
try:
with open(errors_file) as json_file:
data = json.load(json_file)
if data["results"]:
for result in data["results"]:
inform(result)
say_and_exit(
"detect-secrets has discovered high entropy strings, "
"possibly passwords?",
"detect-secrets",
)
except json.JSONDecodeError:
pass
return "Detect secrets completed."
def evaluated_mypy_results(mypy_file: str, small_code_base_cutoff: int,
maximum_mypy: int, skips: List[str]) -> str:
"""
Decided if the mypy is bad enough to stop the build.
"""
with open(mypy_file) as out_file:
out = out_file.read()
def contains_a_skip(line_value: str) -> bool:
"""
Should this line be skipped
"""
# skips is a closure
for skip in skips:
if skip in line_value or line_value.startswith(skip):
return True
return False
actually_bad_lines: List[str] = []
total_lines = 0
with open(mypy_file, "w+") as lint_file:
lines = out.split("\n")
for line in lines:
total_lines += 1
if contains_a_skip(line):
continue
if not line.startswith(PROJECT_NAME):
continue
actually_bad_lines.append(line)
lint_file.writelines([line])
num_lines = len(actually_bad_lines)
if total_loc() > small_code_base_cutoff:
max_lines = maximum_mypy
else:
max_lines = 2 # off by 1 right now
if num_lines > max_lines:
for line in actually_bad_lines:
inform(line)
say_and_exit(f"Too many lines of mypy : {num_lines}, max {max_lines}",
"mypy")
sys.exit(-1)
if num_lines == 0 and total_lines == 0:
# should always have at least 'found 0 errors' in output
say_and_exit(
"No mypy warnings at all, did mypy fail to run or is it installed?",
"mypy")
sys.exit(-1)
return "mypy succeeded"
def do_dodgy() -> str:
"""
Checks for AWS keys, diffs, pem keys
"""
# Not using the shell command version because it mysteriously failed
# and this seems to work fine.
warnings = dodgy_runner.run_checks(os.getcwd())
if warnings:
for message in warnings:
inform(message)
say_and_exit("Dodgy found problems", "dodgy")
sys.exit(-1)
return "dodgy succeeded"
def do_sonar() -> str:
"""
Upload code to sonar for review
"""
sonar_key = os.environ["SONAR_KEY"]
if is_windows():
command_name = "sonar-scanner.bat"
else:
command_name = "sonar-scanner"
command = (f"{VENV_SHELL} {command_name} "
f"-Dsonar.login={sonar_key} "
"-Dproject.settings="
"sonar-project.properties".strip().replace(" ",
" ").split(" "))
inform(command)
execute(*command)
url = ("https://code-quality-test.loc.gov/api/issues/search?"
f"componentKeys=public_record_{PROJECT_NAME}&resolved=false")
session = requests.Session()
session.auth = (sonar_key, "")
response = session.get(url)
errors_file = "sonar.json"
with open(errors_file, "w+") as file_handle:
inform(response.text)
text = response.text
if not text:
say_and_exit("Failed to check for sonar", "sonar")
sys.exit(-1)
file_handle.write(text)
try:
with open(errors_file) as file_handle:
data = json.load(file_handle)
if data["issues"]:
for result in data["issues"]:
inform("{} : {} line {}-{}".format(
result["component"],
result["message"],
result["textRange"]["startLine"],
result["textRange"]["endLine"],
))
say_and_exit("sonar has issues with this code", "sonar")
except json.JSONDecodeError:
pass
return "Sonar done"
def do_precommit(is_interactive: bool) -> None:
"""
Build time execution of pre-commit checks. Modifies code so run before linter.
"""
if not is_interactive:
inform("Not running precommit because it changes files")
return
check_command_exists("pre-commit")
if is_git_repo("."):
# don't try to install because it isn't a git repo
command_text = f"{VENV_SHELL} pre-commit install".strip().replace(" ", " ")
inform(command_text)
command = shlex.split(command_text)
execute(*command)
command_text = f"{VENV_SHELL} pre-commit run --all-files".strip().replace(" ", " ")
inform(command_text)
command = shlex.split(command_text)
result = execute_get_text(command, ignore_error=True, env=config_pythonpath())
assert result
changed = []
for line in result.split("\n"):
if "changed " in line:
file = line[len("reformatted ") :].strip()
changed.append(file)
if "FAILED" in result:
inform(result)
say_and_exit("Pre-commit Failed", "pre-commit")
sys.exit(-1)
if is_interactive:
if not is_git_repo("."):
# don't need to git add anything because this isn't a git repo
return
for change in changed:
command_text = f"git add {change}"
inform(command_text)
# this breaks on windows!
# command = shlex.split(command_text)
execute(*command_text.split())
def do_check_manifest() -> str:
"""
Require all files to be explicitly included/excluded from package
"""
if not os.path.exists("setup.py") and not os.path.exists("setup.cfg"):
inform("setup.py doesn't exists, not packaging.")
return "Nope"
env = config_pythonpath()
output_file_name = f"{PROBLEMS_FOLDER}/manifest_errors.txt"
call_check_manifest_command(output_file_name, env)
with open(output_file_name) as outfile_reader:
text = outfile_reader.read()
inform(text)
if not os.path.isfile(
"MANIFEST.in") and "no MANIFEST.in found" in text:
command_text = f"{VENV_SHELL} check-manifest -c".strip().replace(
" ", " ")
command = shlex.split(command_text)
subprocess.call(command, env=env)
# inform("Had to create MANIFEST.in, please review and redo")
call_check_manifest_command(output_file_name, env)
if total_loc() > SMALL_CODE_BASE_CUTOFF:
cutoff = 0
else:
cutoff = MAXIMUM_MANIFEST_ERRORS
with open(output_file_name) as file_handle:
num_lines = sum(
1 for line in file_handle if line and line.strip() != "" and
"lists of files in version control and sdist match" not in line)
if num_lines > cutoff:
say_and_exit(
f"Too many lines of manifest problems : {num_lines}, max {cutoff}",
"check-manifest",
)
sys.exit(-1)
return "manifest check succeeded"
def do_project_validation_for_setup_py() -> None:
"""
Verify that all projects/modules are explicitly declared
"""
found = setuptools.find_packages()
# Just care about root
found = [name for name in found if "." not in name and name != "test"]
problems = 0
if not found:
inform(
"Found more than no modules at all, did you forget __init__.py?")
problems += 1
# this is okay.
# if len(found) > 1:
# inform(f"Found more than one module, found {found}")
# problems += 1
if PROJECT_NAME not in found:
inform(f"Can't find {PROJECT_NAME}, found {found}")
problems += 1
if problems > 0:
say_and_exit("Modules not as expected, can't package", "setup.py")
sys.exit(-1)
def do_package() -> None:
"""
don't do anything that is potentially really slow or that modifies files.
"""
if not os.path.exists("setup.py") and not os.path.exists("setup.cfg"):
inform("setup.py doesn't exists, not packaging.")
return "Nope"
check_command_exists("twine")
for folder in ["build", "dist", PROJECT_NAME + ".egg-info"]:
if os.path.exists(folder):
shutil.rmtree(folder)
original_umask = os.umask(0)
try:
try:
os.makedirs(folder, 0o770)
except PermissionError:
execute("cmd", "mkdir", folder)
finally:
os.umask(original_umask)
# command = f"{PYTHON} setup.py sdist --formats=gztar,zip"
# bdist_wheel
command_text = f"{PYTHON} setup.py sdist --formats=gztar,zip"
command_text = prepinform_simple(command_text, no_project=True)
command = shlex.split(command_text)
result = execute_get_text(command,
env=config_pythonpath()).replace("\r", "")
error_count = 0
for row in result.split("\n"):
check_row = str(row).lower()
if check_row.startswith("adding") or check_row.startswith("copying"):
# adding a file named error/warning isn't a problem
continue
if "no previously-included files found matching" in check_row:
# excluding a file that already doesn't exist is wonderful!
# why this is a warning boggles the mind.
continue
if "lib2to3" in check_row:
# python 3.9 has deprecated lib2to3, which shows up as a warning which
# causes the build to fail. Seems ignorable as we don
continue
# sometimes to avoid pyc getting out of sync with .py, on
# dev workstations you PYTHONDONTWRITEBYTECODE=1 which just disables
# pyc altogether. Why wheel cares, I don't know.
has_error = any(
value in check_row
for value in ["Errno", "Error", "failed", "error", "warning"])
if has_error and "byte-compiling is disabled" not in check_row:
inform(row)
error_count += 1
if error_count > 0:
say_and_exit("Package failed", "setup.py")
sys.exit(-1)
# pylint: disable=broad-except
try:
# Twine check must run after package creation. Supersedes setup.py check
command_text = f"{VENV_SHELL} twine check dist/*".strip().replace(
" ", " ")
inform(command_text)
command = shlex.split(command_text)
execute(*command)
except Exception as ex:
inform(ex)
command_text = (f"{VENV_SHELL} setup.py "
"sdist "
"--formats=gztar,zip".strip().replace(" ", " "))
command = shlex.split(command_text)
execute(*command)
def list_files(startpath: str) -> None:
"""
List all files, handy for remote build servers
"""
for root, _, files in os.walk(startpath):
level = root.replace(startpath, "").count(os.sep)
indent = " " * 4 * level
inform("{}{}/".format(indent, os.path.basename(root)))
subindent = " " * 4 * (level + 1)
for file in files:
inform(f"{subindent}{file}")
inform("skipping twine check until I figure out what is up")
list_files(startpath=".")
return "Ok"
def evaluated_lint_results(
lint_output_file_name: str,
small_code_base_cut_off: int,
maximum_lint: int,
fatals: List[str],
) -> str:
"""Deciding if the lint is bad enough to fail
Also treats certain errors as fatal even if under the maximum cutoff.
"""
with open(lint_output_file_name) as file_handle:
full_text = file_handle.read()
lint_did_indeed_run = "Your code has been rated at" in full_text
with open(lint_output_file_name) as file_handle:
fatal_errors = sum(1 for line in file_handle
if ": E" in line or ": F" in line)
for fatal in fatals:
for line in file_handle:
if fatal in file_handle or ": E" in line or ": F" in line:
fatal_errors += 1
if fatal_errors > 0:
with open(lint_output_file_name) as file_handle:
for line in file_handle:
if "*************" in line:
continue
if not line or not line.strip("\n "):
continue
inform(line.strip("\n "))
message = f"Fatal lint errors and possibly others, too : {fatal_errors}"
if IS_GITLAB:
with open(lint_output_file_name) as error_file:
inform(error_file.read())
say_and_exit(message, "lint")
return message
with open(lint_output_file_name) as lint_file_handle:
for line in [
line for line in lint_file_handle if not (
"*************" in line or "---------------------" in line
or "Your code has been rated at" in line or line == "\n")
]:
inform(line)
if total_loc() > small_code_base_cut_off:
cutoff = maximum_lint
else:
cutoff = 0
with open(lint_output_file_name) as lint_file_handle:
num_lines = sum(
1 for line in lint_file_handle
if not ("*************" in line or "---------------------" in line
or "Your code has been rated at" in line or line == "\n"))
if num_lines > cutoff:
say_and_exit(f"Too many lines of lint : {num_lines}, max {cutoff}",
"pylint")
sys.exit(-1)
with open(lint_output_file_name) as lint_file_handle:
num_lines_all_output = sum(1 for _ in lint_file_handle)
if (not lint_did_indeed_run and num_lines_all_output == 0
and os.path.isfile(lint_output_file_name)):
# should always have at least 'found 0 errors' in output
# force lint to re-run, because empty file will be missing
os.remove(lint_output_file_name)
say_and_exit(
"No lint messages at all, did pylint fail to run or is it installed?",
"pylint",
)
sys.exit(-1)
return "pylint succeeded"