def do_pip_check() -> str:
"""
Call as normal function
"""
# TODO
result = execute_get_text(command=["pip", "check"],
ignore_error=True,
env=config_pythonpath())
problems = []
if PIP_CHECK_SKIPS:
print(f"Ignoring pip incompat problems for : {PIP_CHECK_SKIPS}")
for line in result.split("\n"):
if not line.strip():
continue
for skip in PIP_CHECK_SKIPS:
if skip in line:
pass
else:
problems.append(line)
if problems:
for problem in problems:
print(problem)
print("pip check problems, even after skips")
sys.exit(-1)
environment = config_pythonpath()
environment["PIPENV_PYUP_API_KEY"] = ""
if PIPENV_ACTIVE:
# ignore 38414 until aws fixes awscli
execute_with_environment("pipenv check --ignore 38414", environment)
return "Pip(env) check run"
def do_openapi_check() -> None:
"""
Does swagger/openapi file parse
"""
if not os.path.exists(f"{PROJECT_NAME}/api.yaml"):
inform("No api.yaml file, assuming this is not a microservice")
# TODO: should be able to check all y?ml files and look at header.
return
command_text = (f"{VENV_SHELL} "
"openapi-spec-validator"
f" {PROJECT_NAME}/api.yaml".strip().replace(" ", " "))
inform(command_text)
command = shlex.split(command_text)
execute(*command)
if IS_JENKINS or IS_GITLAB:
inform("Jenkins/Gitlab and apistar don't work together, skipping")
return
command_text = (f"{VENV_SHELL} apistar validate "
f"--path {PROJECT_NAME}/api.yaml "
"--format openapi "
"--encoding yaml".strip().replace(" ", " "))
inform(command_text)
# subprocess.check_call(command.split(" "), shell=False)
command = shlex.split(command_text)
result = execute_get_text(command,
ignore_error=True,
env=config_pythonpath())
if "OK" not in result and "2713" not in result and "✓" not in result:
inform(result)
say_and_exit("apistar didn't like this", "apistar")
sys.exit(-1)
def do_detect_secrets() -> str:
"""
Call detect-secrets tool
I think this is the problem:
# Code expects to stream output to file and then expects
# interactive person, so code hangs. But also hangs in git-bash
detect-secrets scan test_data/config.env > foo.txt
detect-secrets audit foo.txt
"""
inform("Detect secrets broken ... can't figure out why")
return "nope"
# pylint: disable=unreachable
check_command_exists("detect-secrets")
errors_file = f"{PROBLEMS_FOLDER}/detect-secrets-results.txt"
command_text = (
f"{VENV_SHELL} detect-secrets scan "
"--base64-limit 4 "
# f"--exclude-files .idea|.min.js|.html|.xsd|"
# f"lock.json|.scss|Pipfile.lock|.secrets.baseline|"
# f"{PROBLEMS_FOLDER}/lint.txt|{errors_file}".strip().replace(" ", " ")
)
inform(command_text)
command = shlex.split(command_text)
with open(errors_file, "w") as outfile:
env = config_pythonpath()
output = execute_get_text(command, ignore_error=False, env=env)
outfile.write(output)
# subprocess.call(command, stdout=outfile, env=env)
with open(errors_file, "w+") as file_handle:
text = file_handle.read()
if not text:
say_and_exit("Failed to check for secrets", "detect-secrets")
sys.exit(-1)
file_handle.write(text)
try:
with open(errors_file) as json_file:
data = json.load(json_file)
if data["results"]:
for result in data["results"]:
inform(result)
say_and_exit(
"detect-secrets has discovered high entropy strings, "
"possibly passwords?",
"detect-secrets",
)
except json.JSONDecodeError:
pass
return "Detect secrets completed."
def do_gitchangelog() -> None:
"""
Extract commit comments from git to a report. Makes for a lousy CHANGELOG.md
"""
# TODO: this app has lots of features for cleaning up comments
command_name = "gitchangelog"
check_command_exists(command_name)
command_text = f"{VENV_SHELL} {command_name}".strip().replace(" ", " ")
inform(command_text)
command = shlex.split(command_text)
with open("ChangeLog", "w+") as change_log:
result = execute_get_text(command, env=config_pythonpath()).replace("\r", "")
change_log.write(result)
def do_compile_py(python: str) -> str:
"""
Catch only the worst syntax errors in the currently python version
"""
command_text = f"{python} -m compileall"
command_text = prepinform_simple(command_text)
command = shlex.split(command_text)
result = execute_get_text(command, env=config_pythonpath())
for line in result.split("\n"):
if line and (line.startswith("Compiling")
or line.startswith("Listing")):
pass
else:
inform(line)
return "compileall succeeded"
def do_spell_check() -> None:
"""
Check spelling using scspell (pip install scspell3k)
"""
# tool can't recurse through files
# tool returns a hard to parse format
# tool has a really cumbersome way of adding values to dictionary
walk_dir = PROJECT_NAME
files_to_check = []
inform(walk_dir)
for root, _, files in os.walk(walk_dir):
if "pycache" in root:
continue
for file in files:
inform(root + "/" + file)
if file.endswith(".py"):
files_to_check.append(root + "/" + file)
files_to_check_string = " ".join(files_to_check)
command_text = (
f"{VENV_SHELL} scspell --report-only "
"--override-dictionary=spelling_dictionary.txt "
f"--use-builtin-base-dict {files_to_check_string}".strip().replace(
" ", " "))
inform(command_text)
command = shlex.split(command_text)
result = execute_get_text(command,
ignore_error=True,
env=config_pythonpath())
with open(f"{PROBLEMS_FOLDER}/spelling.txt", "w+") as outfile:
outfile.write("\n".join([
row for row in result.replace("\r", "").split("\n")
if "dictionary" in row
]))
def read_file() -> None:
with open(f"{settings.CONFIG_FOLDER}/spelling_dictionary.txt"
) as reading_file:
reading_result = reading_file.read()
inform("\n".join([
row for row in reading_result.split("\n")
if "dictionary" in row
]))
read_file()
def do_formatting(check: str, state: Dict[str, bool]) -> None:
"""
Format with black - this will not modify code if check is --check
"""
# check & format should be merged & use an arg
# global FORMATTING_CHECK_DONE
if state["check_already_done"]:
inform("Formatting check says black will not reformat, so no need to repeat")
return
if sys.version_info < (3, 6):
inform("Black doesn't work on python 2")
return
check_command_exists("black")
command_text = f"{VENV_SHELL} black {PROJECT_NAME} {check}".strip().replace(
" ", " "
)
inform(command_text)
command = shlex.split(command_text)
if check:
_ = execute(*command)
state["check_already_done"] = True
return
result = execute_get_text(command, env=config_pythonpath())
assert result
changed = []
for line in result.split("\n"):
if "reformatted " in line:
file = line[len("reformatted ") :].strip()
changed.append(file)
if not IS_GITLAB:
if not is_git_repo("."):
# don't need to git add anything because this isn't a git repo
return
for change in changed:
if is_windows():
change = change.replace("\\", "/")
command_text = f"git add {change}"
inform(command_text)
command = shlex.split(command_text)
execute(*command)
def do_precommit(is_interactive: bool) -> None:
"""
Build time execution of pre-commit checks. Modifies code so run before linter.
"""
if not is_interactive:
inform("Not running precommit because it changes files")
return
check_command_exists("pre-commit")
if is_git_repo("."):
# don't try to install because it isn't a git repo
command_text = f"{VENV_SHELL} pre-commit install".strip().replace(" ", " ")
inform(command_text)
command = shlex.split(command_text)
execute(*command)
command_text = f"{VENV_SHELL} pre-commit run --all-files".strip().replace(" ", " ")
inform(command_text)
command = shlex.split(command_text)
result = execute_get_text(command, ignore_error=True, env=config_pythonpath())
assert result
changed = []
for line in result.split("\n"):
if "changed " in line:
file = line[len("reformatted ") :].strip()
changed.append(file)
if "FAILED" in result:
inform(result)
say_and_exit("Pre-commit Failed", "pre-commit")
sys.exit(-1)
if is_interactive:
if not is_git_repo("."):
# don't need to git add anything because this isn't a git repo
return
for change in changed:
command_text = f"git add {change}"
inform(command_text)
# this breaks on windows!
# command = shlex.split(command_text)
execute(*command_text.split())
def do_package() -> None:
"""
don't do anything that is potentially really slow or that modifies files.
"""
if not os.path.exists("setup.py") and not os.path.exists("setup.cfg"):
inform("setup.py doesn't exists, not packaging.")
return "Nope"
check_command_exists("twine")
for folder in ["build", "dist", PROJECT_NAME + ".egg-info"]:
if os.path.exists(folder):
shutil.rmtree(folder)
original_umask = os.umask(0)
try:
try:
os.makedirs(folder, 0o770)
except PermissionError:
execute("cmd", "mkdir", folder)
finally:
os.umask(original_umask)
# command = f"{PYTHON} setup.py sdist --formats=gztar,zip"
# bdist_wheel
command_text = f"{PYTHON} setup.py sdist --formats=gztar,zip"
command_text = prepinform_simple(command_text, no_project=True)
command = shlex.split(command_text)
result = execute_get_text(command,
env=config_pythonpath()).replace("\r", "")
error_count = 0
for row in result.split("\n"):
check_row = str(row).lower()
if check_row.startswith("adding") or check_row.startswith("copying"):
# adding a file named error/warning isn't a problem
continue
if "no previously-included files found matching" in check_row:
# excluding a file that already doesn't exist is wonderful!
# why this is a warning boggles the mind.
continue
if "lib2to3" in check_row:
# python 3.9 has deprecated lib2to3, which shows up as a warning which
# causes the build to fail. Seems ignorable as we don
continue
# sometimes to avoid pyc getting out of sync with .py, on
# dev workstations you PYTHONDONTWRITEBYTECODE=1 which just disables
# pyc altogether. Why wheel cares, I don't know.
has_error = any(
value in check_row
for value in ["Errno", "Error", "failed", "error", "warning"])
if has_error and "byte-compiling is disabled" not in check_row:
inform(row)
error_count += 1
if error_count > 0:
say_and_exit("Package failed", "setup.py")
sys.exit(-1)
# pylint: disable=broad-except
try:
# Twine check must run after package creation. Supersedes setup.py check
command_text = f"{VENV_SHELL} twine check dist/*".strip().replace(
" ", " ")
inform(command_text)
command = shlex.split(command_text)
execute(*command)
except Exception as ex:
inform(ex)
command_text = (f"{VENV_SHELL} setup.py "
"sdist "
"--formats=gztar,zip".strip().replace(" ", " "))
command = shlex.split(command_text)
execute(*command)
def list_files(startpath: str) -> None:
"""
List all files, handy for remote build servers
"""
for root, _, files in os.walk(startpath):
level = root.replace(startpath, "").count(os.sep)
indent = " " * 4 * level
inform("{}{}/".format(indent, os.path.basename(root)))
subindent = " " * 4 * (level + 1)
for file in files:
inform(f"{subindent}{file}")
inform("skipping twine check until I figure out what is up")
list_files(startpath=".")
return "Ok"