def do_jiggle_version(
is_interactive: bool,
target_branch: str = "",
increase_version_on_all_branches: bool = False,
) -> str:
"""
Increase version number, but only the last number in a semantic version triplet
"""
if not is_interactive:
inform(
"Not an interactive session, skipping jiggle_version, which changes files."
)
return "Skipping"
# rorepo is a Repo instance pointing to the git-python repository.
# For all you know, the first argument to Repo is a path to the repository
# you want to work with
try:
repo = Repo(".")
active_branch = str(repo.active_branch)
except InvalidGitRepositoryError:
inform("Can't detect what branch we are on. Is this a git repo?")
active_branch = "don't know what branch we are on"
if active_branch == target_branch or increase_version_on_all_branches:
check_command_exists("jiggle_version")
command = f"jiggle_version here --module={PROJECT_NAME}"
parts = shlex.split(command)
execute(*parts)
else:
inform("Not master branch, not incrementing version")
return "ok"
def do_upload_package() -> None:
"""
Send to private package repo
"""
# devpi use http://localhost:3141
# login with root...
# devpi login root --password=
# get indexes
# devpi use -l
# make an index
# devpi index -c dev bases=root/pypi
# devpi use root/dev
# Must register (may go away with newer version of devpi), must be 1 file!
# twine register --config-file .pypirc -r devpi-root -u root
# -p PASSWORD dist/search_service-0.1.0.zip
# can be all files!
# twine upload --config-file .pypirc -r devpi-root -u root -p PASSWORD dist/*
# which is installable using...
# pip install search-service --index-url=http://localhost:3141/root/dev/
check_command_exists("devpi")
password = os.environ["DEVPI_PASSWORD"]
any_zip = [file for file in os.listdir("dist") if file.endswith(".zip")][0]
register_command = (
"twine register --config-file .pypirc -r devpi-root -u root"
f" -p {password} dist/{any_zip}")
upload_command = ("twine upload --config-file .pypirc -r devpi-root "
f"-u root -p {password} dist/*")
execute(*(register_command.strip().split(" ")))
execute(*(upload_command.strip().split(" ")))
def do_vulture() -> str:
"""
This also finds code you are working on today!
"""
check_command_exists("vulture")
# TODO: check if whitelist.py exists?
command_text = f"{VENV_SHELL} vulture {PROJECT_NAME} whitelist.py"
command_text = command_text.strip().replace(" ", " ")
inform(command_text)
command = shlex.split(command_text)
output_file_name = f"{PROBLEMS_FOLDER}/dead_code.txt"
with open(output_file_name, "w") as outfile:
env = config_pythonpath()
subprocess.call(command, stdout=outfile, env=env)
if total_loc() > SMALL_CODE_BASE_CUTOFF:
cutoff = MAXIMUM_DEAD_CODE
else:
cutoff = 0
with open(output_file_name) as file_handle:
num_lines = sum(1 for line in file_handle if line)
if num_lines > cutoff:
say_and_exit(
f"Too many lines of dead code : {num_lines}, max {cutoff}", "vulture"
)
sys.exit(-1)
return "dead-code (vulture) succeeded"
def convert_pipenv_to_requirements(pipenv: bool) -> None:
"""
Create requirement*.txt
"""
if not pipenv:
raise TypeError(
"Can't pin dependencies this way, we are only converting "
"pipfile to requirements.txt")
check_command_exists("pipenv_to_requirements")
execute(*(f"{VENV_SHELL} pipenv_to_requirements "
f"--dev-output {settings.CONFIG_FOLDER}/requirements-dev.txt "
f"--output {settings.CONFIG_FOLDER}/requirements.txt".strip().
split(" ")))
if not os.path.exists(f"{settings.CONFIG_FOLDER}/requirements.txt"):
inform(
"Warning: no requirements.txt found, assuming it is because there are"
"no external dependencies yet")
else:
with open(f"{settings.CONFIG_FOLDER}/requirements.txt", "r+") as file:
lines = file.readlines()
file.seek(0)
for line in lines:
if line.find("-e .") == -1:
file.write(line)
file.truncate()
with open(f"{settings.CONFIG_FOLDER}/requirements-dev.txt", "r+") as file:
lines = file.readlines()
file.seek(0)
for line in lines:
if line.find("-e .") == -1:
file.write(line)
file.truncate()
def do_count_lines_of_code() -> None:
"""
Scale failure cut offs based on Lines of Code
"""
command_name = "pygount"
check_command_exists(command_name)
command_text = prepinform_simple(command_name)
# keep out of src tree, causes extraneous change detections
if not os.path.exists(f"{REPORTS_FOLDER}"):
os.makedirs(f"{REPORTS_FOLDER}")
output_file_name = f"{REPORTS_FOLDER}/line_counts.txt"
command = shlex.split(command_text)
with open(output_file_name, "w") as outfile:
subprocess.call(command, stdout=outfile)
with open(output_file_name) as file_handle:
lines = sum(
int(line.split("\t")[0]) for line in file_handle if line != "\n")
total_loc_local = lines
if not os.path.exists(f"{settings.CONFIG_FOLDER}/.build_state"):
os.makedirs(f"{settings.CONFIG_FOLDER}/.build_state")
with open(f"{settings.CONFIG_FOLDER}/.build_state/pygount_total_loc.txt",
"w+") as state_file:
state_file.write(str(total_loc_local))
inform(f"Lines of code: {total_loc_local}")
if total_loc_local == 0:
say_and_exit(
"No code found to build or package. Maybe the PROJECT_NAME is wrong?",
"lines of code",
)
def do_flake8() -> str:
"""
Flake8 Checks
"""
command = "flake8"
check_command_exists(command)
command_text = f"flake8 --config {settings.CONFIG_FOLDER}/.flake8"
command_text = prepinform_simple(command_text)
execute(*(command_text.split(" ")))
return "flake 8 succeeded"
def do_detect_secrets() -> str:
"""
Call detect-secrets tool
I think this is the problem:
# Code expects to stream output to file and then expects
# interactive person, so code hangs. But also hangs in git-bash
detect-secrets scan test_data/config.env > foo.txt
detect-secrets audit foo.txt
"""
inform("Detect secrets broken ... can't figure out why")
return "nope"
# pylint: disable=unreachable
check_command_exists("detect-secrets")
errors_file = f"{PROBLEMS_FOLDER}/detect-secrets-results.txt"
command_text = (
f"{VENV_SHELL} detect-secrets scan "
"--base64-limit 4 "
# f"--exclude-files .idea|.min.js|.html|.xsd|"
# f"lock.json|.scss|Pipfile.lock|.secrets.baseline|"
# f"{PROBLEMS_FOLDER}/lint.txt|{errors_file}".strip().replace(" ", " ")
)
inform(command_text)
command = shlex.split(command_text)
with open(errors_file, "w") as outfile:
env = config_pythonpath()
output = execute_get_text(command, ignore_error=False, env=env)
outfile.write(output)
# subprocess.call(command, stdout=outfile, env=env)
with open(errors_file, "w+") as file_handle:
text = file_handle.read()
if not text:
say_and_exit("Failed to check for secrets", "detect-secrets")
sys.exit(-1)
file_handle.write(text)
try:
with open(errors_file) as json_file:
data = json.load(json_file)
if data["results"]:
for result in data["results"]:
inform(result)
say_and_exit(
"detect-secrets has discovered high entropy strings, "
"possibly passwords?",
"detect-secrets",
)
except json.JSONDecodeError:
pass
return "Detect secrets completed."
def do_docs() -> str:
"""
Generate docs based on rst.
"""
check_command_exists("make")
my_env = config_pythonpath()
command = f"{VENV_SHELL} make html".strip().replace(" ", " ")
inform(command)
execute_with_environment(command, env=my_env)
return "Docs generated"
def do_yamllint() -> str:
"""
Check yaml files for problems
"""
command = "yamllint"
check_command_exists(command)
command = f"{VENV_SHELL} yamllint {PROJECT_NAME}".strip().replace(
" ", " ")
inform(command)
execute(*(command.split(" ")))
return "yamllint succeeded"
def do_isort() -> str:
"""Sort the imports to discover import order bugs and prevent import order bugs"""
# This must run before black. black doesn't change import order but it wins
# any arguments about formatting.
# isort MUST be installed with pipx! It is not compatible with pylint in the same
# venv. Maybe someday, but it just isn't worth the effort.
check_command_exists("isort")
command = "isort --profile black"
command = prepinform_simple(command)
execute(*(command.split(" ")))
return "isort succeeded"
def do_safety() -> str:
"""
Check free database for vulnerabilities in pinned libraries.
"""
requirements_file_name = f"{settings.CONFIG_FOLDER}/requirements_for_safety.txt"
with open(requirements_file_name, "w+") as out:
subprocess.run(["pip", "freeze"], stdout=out, stderr=out, check=True)
check_command_exists("safety")
# ignore 38414 until aws fixes awscli
execute("safety", "check", "--ignore", "38414", "--file",
requirements_file_name)
return "Package safety checked"
def do_git_secrets() -> str:
"""
Install git secrets if possible.
"""
if is_cmd_exe():
inform("git secrets is a bash script, only works in bash (or maybe PS")
return "skipped git secrets, this is cmd.exe shell"
# not sure how to check for a git subcommand
if not is_git_repo("."):
inform("This is not a git repo, won't run git-secrets")
return "Not a git repo, skipped"
check_command_exists("git")
if check_is_aws():
# no easy way to install git secrets on ubuntu.
return "This is AWS, not doing git-secrets"
if IS_GITLAB:
inform("Nothing is edited on gitlab build server")
return "This is gitlab, not doing git-secrets"
try:
# check to see if secrets even is a git command
commands = ["git secrets --install", "git secrets --register-aws"]
for command in commands:
command_parts = shlex.split(command)
command_process = subprocess.run(
command_parts,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=True,
)
for stream in [command_process.stdout, command_process.stderr]:
if stream:
for line in stream.decode().split("\n"):
inform("*" + line)
except subprocess.CalledProcessError as cpe:
inform(cpe)
installed = False
for stream in [cpe.stdout, cpe.stderr]:
if stream:
for line in stream.decode().split("\n"):
inform("-" + line)
if "commit-msg already exists" in line:
inform("git secrets installed.")
installed = True
break
if not installed:
raise
command_text = "git secrets --scan -r ./".strip().replace(" ", " ")
command_parts = shlex.split(command_text)
execute(*command_parts)
return "git-secrets succeeded"
def do_pyroma_regardless() -> str:
"""
Check package goodness (essentially lints setup.py)
"""
if not os.path.exists("setup.py") and not os.path.exists("setup.cfg"):
inform("setup.py doesn't exists, not packaging.")
return "Nope"
command = "pyroma"
check_command_exists(command)
command = f"{VENV_SHELL} pyroma --directory --min=8 .".strip().replace(" ", " ")
inform(command)
execute(*(command.split(" ")))
return "pyroma succeeded"
def call_check_manifest_command(output_file_name: str, env: Dict[str,
str]) -> None:
"""
To allow for checking in multiple passes
"""
check_command_exists("check-manifest")
command_text = f"{VENV_SHELL} check-manifest".strip().replace(" ", " ")
with open(output_file_name, "w") as outfile:
inform(command_text)
command = shlex.split(command_text)
subprocess.call(command, stdout=outfile, env=env)
def do_mccabe() -> str:
"""
Complexity Checker
"""
check_command_exists("flake8") # yes, flake8, this is a plug in.
# mccabe doesn't have a direct way to run it
command_text = (f"flake8 --max-complexity {COMPLEXITY_CUT_OFF} "
f"--config {settings.CONFIG_FOLDER}/.flake8")
command_text = prepinform_simple(command_text)
command = shlex.split(command_text)
execute(*command)
return "mccabe succeeded"
def do_python_taint() -> str:
"""
Security Checks
"""
if sys.version_info.major == 3 and sys.version_info.minor > 8:
# pyt only works with python <3.8
return "skipping python taint"
command = "pyt"
check_command_exists(command)
command = "pyt -r"
command = prepinform_simple(command)
execute(*(command.split(" ")))
return "ok"
def do_gitchangelog() -> None:
"""
Extract commit comments from git to a report. Makes for a lousy CHANGELOG.md
"""
# TODO: this app has lots of features for cleaning up comments
command_name = "gitchangelog"
check_command_exists(command_name)
command_text = f"{VENV_SHELL} {command_name}".strip().replace(" ", " ")
inform(command_text)
command = shlex.split(command_text)
with open("ChangeLog", "w+") as change_log:
result = execute_get_text(command, env=config_pythonpath()).replace("\r", "")
change_log.write(result)
def do_jake() -> str:
"""
Check free database for vulnerabilities in active venv.
"""
# TODO: get an API key and start using
# .oss-index-config
command_name = "jake"
check_command_exists(command_name)
command_text = f"{VENV_SHELL} {command_name} ddt".strip().replace(
" ", " ")
inform(command_text)
command = shlex.split(command_text)
execute(*command)
return "jake succeeded"
def do_liccheck() -> str:
"""
Make an explicit decision about license of referenced packages
"""
check_command_exists("liccheck")
if not os.path.exists(f"{settings.CONFIG_FOLDER}/requirements.txt"):
inform("No requirements.txt file, assuming we have no external deps")
return "Skipping, not requirements.txt"
command = ("liccheck "
f"-r {settings.CONFIG_FOLDER}/requirements.txt "
f"-s {settings.CONFIG_FOLDER}/.license_rules "
"-l paranoid")
command = prepinform_simple(command, no_project=True)
execute(*(command.split(" ")))
return "liccheck succeeded"
def do_bandit(is_shell_script_like: bool) -> str:
"""
Security Checks
Generally returns a small number of problems to fix.
"""
if is_shell_script_like:
return (
"Skipping bandit, this code is shell script-like so it has security"
"issues on purpose.")
command = "bandit"
check_command_exists(command)
command = "bandit -r"
command = prepinform_simple(command)
execute(*(command.split(" ")))
return "bandit succeeded"
def do_register_scripts() -> None:
"""
Without this console_scripts in the entrypoints section of setup.py aren't
available
:return:
"""
check_command_exists("pip")
# This doesn't work, it can't tell if "install -e ." has already run
if dist_is_editable():
inform("console_scripts already registered")
return
# install in "editable" mode
command_text = f"{VENV_SHELL} pip install -e ."
inform(command_text)
command_text = command_text.strip().replace(" ", " ")
command = shlex.split(command_text)
execute(*command)
def say_and_exit(message: str, source: str) -> None:
"""
Audibly notify the developer that the build is
done so that long builds wouldn't cause an attention problem
"""
inform(f"{source}:{message}")
if (
settings.SPEAK_WHEN_BUILD_FAILS
and settings.IS_INTERACTIVE
and not check_is_aws()
):
# TODO: check a profile option or something.
if check_command_exists("say", throw_on_missing=False, exit_on_missing=False):
subprocess.call(["say", message])
elif check_command_exists(
"wsay.exe", throw_on_missing=False, exit_on_missing=False
):
subprocess.call(["wsay", message])
sys.exit(-1)
def do_formatting(check: str, state: Dict[str, bool]) -> None:
"""
Format with black - this will not modify code if check is --check
"""
# check & format should be merged & use an arg
# global FORMATTING_CHECK_DONE
if state["check_already_done"]:
inform("Formatting check says black will not reformat, so no need to repeat")
return
if sys.version_info < (3, 6):
inform("Black doesn't work on python 2")
return
check_command_exists("black")
command_text = f"{VENV_SHELL} black {PROJECT_NAME} {check}".strip().replace(
" ", " "
)
inform(command_text)
command = shlex.split(command_text)
if check:
_ = execute(*command)
state["check_already_done"] = True
return
result = execute_get_text(command, env=config_pythonpath())
assert result
changed = []
for line in result.split("\n"):
if "reformatted " in line:
file = line[len("reformatted ") :].strip()
changed.append(file)
if not IS_GITLAB:
if not is_git_repo("."):
# don't need to git add anything because this isn't a git repo
return
for change in changed:
if is_windows():
change = change.replace("\\", "/")
command_text = f"git add {change}"
inform(command_text)
command = shlex.split(command_text)
execute(*command)
def do_mypy() -> str:
"""
Are types ok?
"""
check_command_exists("mypy")
if sys.version_info < (3, 4):
inform("Mypy doesn't work on python < 3.4")
return "command is missing"
command = (f"{VENV_SHELL} mypy {PROJECT_NAME} "
"--ignore-missing-imports "
"--strict".strip().replace(" ", " "))
inform(command)
bash_process = subprocess.Popen(command.split(" "),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, _ = bash_process.communicate() # wait
mypy_file = f"{PROBLEMS_FOLDER}/all_mypy_errors.txt"
with open(mypy_file, "w", encoding="utf-8") as out_file:
out_file.write(out.decode())
return mypy_file
def do_precommit(is_interactive: bool) -> None:
"""
Build time execution of pre-commit checks. Modifies code so run before linter.
"""
if not is_interactive:
inform("Not running precommit because it changes files")
return
check_command_exists("pre-commit")
if is_git_repo("."):
# don't try to install because it isn't a git repo
command_text = f"{VENV_SHELL} pre-commit install".strip().replace(" ", " ")
inform(command_text)
command = shlex.split(command_text)
execute(*command)
command_text = f"{VENV_SHELL} pre-commit run --all-files".strip().replace(" ", " ")
inform(command_text)
command = shlex.split(command_text)
result = execute_get_text(command, ignore_error=True, env=config_pythonpath())
assert result
changed = []
for line in result.split("\n"):
if "changed " in line:
file = line[len("reformatted ") :].strip()
changed.append(file)
if "FAILED" in result:
inform(result)
say_and_exit("Pre-commit Failed", "pre-commit")
sys.exit(-1)
if is_interactive:
if not is_git_repo("."):
# don't need to git add anything because this isn't a git repo
return
for change in changed:
command_text = f"git add {change}"
inform(command_text)
# this breaks on windows!
# command = shlex.split(command_text)
execute(*command_text.split())
def do_pyupgrade(is_interactive: bool, minimum_python: str) -> str:
"""Update syntax to most recent variety."""
if not is_interactive:
inform(
"Not an interactive session, skipping pyupgrade wihch changes files."
)
return "Skipping"
command = "pyupgrade"
check_command_exists(command)
all_files = " ".join(
f for f in glob.glob(f"{PROJECT_NAME}/**/*.py", recursive=True))
# as of 2021, still doesn't appear to support recursive globs natively.
command = (f"{VENV_SHELL} pyupgrade "
f"--{minimum_python}-plus "
f"--exit-zero-even-if-changed {all_files}".strip().replace(
" ", " "))
inform(command)
execute(*(command.split(" ")))
return f"{command} succeeded"
def do_tox() -> str:
"""
See if everything works with python 3.8 and upcoming libraries
"""
# If tox fails the build with 3.8 or some future library, that means
# we can't migrate to 3.8 yet, or that we should stay with currently pinned
# libraries. We should fail the overall build.
#
# Because we control our python version we don't have to support cross ver
# compatibility, i.e. we are not supporting 2.7 & 3.x!
if settings.TOX_ACTIVE:
# this happens when testing the build script itself.
return "tox already, not nesting"
command_name = "tox"
check_command_exists(command_name)
command_text = f"{VENV_SHELL} {command_name}".strip().replace(" ", " ")
inform(command_text)
command = shlex.split(command_text)
execute(*command)
return "tox succeeded"
def do_pytest() -> None:
"""
Pytest and coverage, which replaces nose tests
"""
check_command_exists("pytest")
# Somedays VPN just isn't there.
if IS_INTERNAL_NETWORK or RUN_ALL_TESTS_REGARDLESS_TO_NETWORK:
fast_only = False
else:
fast_only = True
if fast_only:
test_folder = "test/test_fast"
minimum_coverage = MINIMUM_TEST_COVERAGE_FOR_FAST_TESTS
else:
test_folder = "test"
minimum_coverage = MINIMUM_TEST_COVERAGE
my_env = config_pythonpath()
command = (
f"{VENV_SHELL} pytest {test_folder} -v "
f"--junitxml={REPORTS_FOLDER}/sonar-unit-test-results.xml "
"--cov-report xml "
f"--cov={PROJECT_NAME} "
f"--cov-fail-under {minimum_coverage}".strip().replace(" ", " ") +
" --quiet" # 15000 pages of call stack don't help anyone
)
# when it works, it is FAST. when it doesn't, we get lots of timeouts.
# if not IS_GITLAB:
# command += f" -n {multiprocessing.cpu_count()} "
if not IS_GITLAB:
command += " -n 2 "
inform(command)
execute_with_environment(command, my_env)
inform(
"Tests will not be re-run until code changes. Run pynt reset to force."
)
def do_lint(folder_type: str) -> str:
"""
Execute pylint
"""
# pylint: disable=too-many-locals
check_command_exists("pylint")
if folder_type == PROJECT_NAME:
pylintrc = f"{settings.CONFIG_FOLDER}/.pylintrc"
lint_output_file_name = f"{PROBLEMS_FOLDER}/lint.txt"
else:
pylintrc = f"{settings.CONFIG_FOLDER}/.pylintrc_{folder_type}"
lint_output_file_name = f"{PROBLEMS_FOLDER}/lint_{folder_type}.txt"
if os.path.isfile(lint_output_file_name):
os.remove(lint_output_file_name)
if IS_DJANGO:
django_bits = "--load-plugins pylint_django "
else:
django_bits = ""
# pylint: disable=pointless-string-statement
command_text = (f"{VENV_SHELL} pylint {django_bits} "
f"--rcfile={pylintrc} {folder_type} ")
command_text += " "
"--msg-template={path}:{line}: [{msg_id}({symbol}), {obj}] {msg}"
"".strip().replace(" ", " ")
inform(command_text)
command = shlex.split(command_text)
with open(lint_output_file_name, "w") as outfile:
env = config_pythonpath()
subprocess.call(command, stdout=outfile, env=env)
return lint_output_file_name
def do_dependency_installs(pipenv: bool, poetry: bool, pip: bool) -> None:
"""Catch up with missing deps"""
if pipenv:
check_command_exists("pipenv")
command_text = "pipenv install --dev --skip-lock"
inform(command_text)
command = shlex.split(command_text)
execute(*command)
elif poetry:
check_command_exists("poetry")
command_text = "poetry install"
inform(command_text)
command = shlex.split(command_text)
execute(*command)
elif pip:
# TODO: move code to deprecated section?
# TODO: Check for poetry.
if os.path.exists("Pipfile"):
raise TypeError("Found Pipfile, settings imply we aren't using Pipenv.")
if os.path.exists("requirements.txt"):
command_text = "pip install -r requirements.txt"
inform(command_text)
command = shlex.split(command_text)
execute(*command)
else:
inform("no requirements.txt file yet, can't install dependencies")
if os.path.exists("requirements-dev.txt"):
command_text = "pip install -r requirements-dev.txt"
inform(command_text)
command = shlex.split(command_text)
execute(*command)
else:
inform("no requirements-dev.txt file yet, can't install dependencies")
else:
inform("VENV not previously activated, won't attempt to catch up on installs")
