def do_upload_package() -> None:
"""
Send to private package repo
"""
# devpi use http://localhost:3141
# login with root...
# devpi login root --password=
# get indexes
# devpi use -l
# make an index
# devpi index -c dev bases=root/pypi
# devpi use root/dev
# Must register (may go away with newer version of devpi), must be 1 file!
# twine register --config-file .pypirc -r devpi-root -u root
# -p PASSWORD dist/search_service-0.1.0.zip
# can be all files!
# twine upload --config-file .pypirc -r devpi-root -u root -p PASSWORD dist/*
# which is installable using...
# pip install search-service --index-url=http://localhost:3141/root/dev/
check_command_exists("devpi")
password = os.environ["DEVPI_PASSWORD"]
any_zip = [file for file in os.listdir("dist") if file.endswith(".zip")][0]
register_command = (
"twine register --config-file .pypirc -r devpi-root -u root"
f" -p {password} dist/{any_zip}")
upload_command = ("twine upload --config-file .pypirc -r devpi-root "
f"-u root -p {password} dist/*")
execute(*(register_command.strip().split(" ")))
execute(*(upload_command.strip().split(" ")))
def do_jiggle_version(
is_interactive: bool,
target_branch: str = "",
increase_version_on_all_branches: bool = False,
) -> str:
"""
Increase version number, but only the last number in a semantic version triplet
"""
if not is_interactive:
inform(
"Not an interactive session, skipping jiggle_version, which changes files."
)
return "Skipping"
# rorepo is a Repo instance pointing to the git-python repository.
# For all you know, the first argument to Repo is a path to the repository
# you want to work with
try:
repo = Repo(".")
active_branch = str(repo.active_branch)
except InvalidGitRepositoryError:
inform("Can't detect what branch we are on. Is this a git repo?")
active_branch = "don't know what branch we are on"
if active_branch == target_branch or increase_version_on_all_branches:
check_command_exists("jiggle_version")
command = f"jiggle_version here --module={PROJECT_NAME}"
parts = shlex.split(command)
execute(*parts)
else:
inform("Not master branch, not incrementing version")
return "ok"
def convert_pipenv_to_requirements(pipenv: bool) -> None:
"""
Create requirement*.txt
"""
if not pipenv:
raise TypeError(
"Can't pin dependencies this way, we are only converting "
"pipfile to requirements.txt")
check_command_exists("pipenv_to_requirements")
execute(*(f"{VENV_SHELL} pipenv_to_requirements "
f"--dev-output {settings.CONFIG_FOLDER}/requirements-dev.txt "
f"--output {settings.CONFIG_FOLDER}/requirements.txt".strip().
split(" ")))
if not os.path.exists(f"{settings.CONFIG_FOLDER}/requirements.txt"):
inform(
"Warning: no requirements.txt found, assuming it is because there are"
"no external dependencies yet")
else:
with open(f"{settings.CONFIG_FOLDER}/requirements.txt", "r+") as file:
lines = file.readlines()
file.seek(0)
for line in lines:
if line.find("-e .") == -1:
file.write(line)
file.truncate()
with open(f"{settings.CONFIG_FOLDER}/requirements-dev.txt", "r+") as file:
lines = file.readlines()
file.seek(0)
for line in lines:
if line.find("-e .") == -1:
file.write(line)
file.truncate()
def do_openapi_check() -> None:
"""
Does swagger/openapi file parse
"""
if not os.path.exists(f"{PROJECT_NAME}/api.yaml"):
inform("No api.yaml file, assuming this is not a microservice")
# TODO: should be able to check all y?ml files and look at header.
return
command_text = (f"{VENV_SHELL} "
"openapi-spec-validator"
f" {PROJECT_NAME}/api.yaml".strip().replace(" ", " "))
inform(command_text)
command = shlex.split(command_text)
execute(*command)
if IS_JENKINS or IS_GITLAB:
inform("Jenkins/Gitlab and apistar don't work together, skipping")
return
command_text = (f"{VENV_SHELL} apistar validate "
f"--path {PROJECT_NAME}/api.yaml "
"--format openapi "
"--encoding yaml".strip().replace(" ", " "))
inform(command_text)
# subprocess.check_call(command.split(" "), shell=False)
command = shlex.split(command_text)
result = execute_get_text(command,
ignore_error=True,
env=config_pythonpath())
if "OK" not in result and "2713" not in result and "✓" not in result:
inform(result)
say_and_exit("apistar didn't like this", "apistar")
sys.exit(-1)
def do_flake8() -> str:
"""
Flake8 Checks
"""
command = "flake8"
check_command_exists(command)
command_text = f"flake8 --config {settings.CONFIG_FOLDER}/.flake8"
command_text = prepinform_simple(command_text)
execute(*(command_text.split(" ")))
return "flake 8 succeeded"
def do_pip_check() -> str:
"""
Call as normal function
"""
execute("pip", "check")
environment = config_pythonpath()
environment["PIPENV_PYUP_API_KEY"] = ""
if PIPENV_ACTIVE:
# ignore 38414 until aws fixes awscli
execute_with_environment("pipenv check --ignore 38414", environment)
return "Pip(env) check run"
def do_safety() -> str:
"""
Check free database for vulnerabilities in pinned libraries.
"""
requirements_file_name = f"{settings.CONFIG_FOLDER}/requirements_for_safety.txt"
with open(requirements_file_name, "w+") as out:
subprocess.run(["pip", "freeze"], stdout=out, stderr=out, check=True)
check_command_exists("safety")
# ignore 38414 until aws fixes awscli
execute("safety", "check", "--ignore", "38414", "--file",
requirements_file_name)
return "Package safety checked"
def do_isort() -> str:
"""Sort the imports to discover import order bugs and prevent import order bugs"""
# This must run before black. black doesn't change import order but it wins
# any arguments about formatting.
# isort MUST be installed with pipx! It is not compatible with pylint in the same
# venv. Maybe someday, but it just isn't worth the effort.
check_command_exists("isort")
command = "isort --profile black"
command = prepinform_simple(command)
execute(*(command.split(" ")))
return "isort succeeded"
def do_yamllint() -> str:
"""
Check yaml files for problems
"""
command = "yamllint"
check_command_exists(command)
command = f"{VENV_SHELL} yamllint {PROJECT_NAME}".strip().replace(
" ", " ")
inform(command)
execute(*(command.split(" ")))
return "yamllint succeeded"
def do_git_secrets() -> str:
"""
Install git secrets if possible.
"""
if is_cmd_exe():
inform("git secrets is a bash script, only works in bash (or maybe PS")
return "skipped git secrets, this is cmd.exe shell"
# not sure how to check for a git subcommand
if not is_git_repo("."):
inform("This is not a git repo, won't run git-secrets")
return "Not a git repo, skipped"
check_command_exists("git")
if check_is_aws():
# no easy way to install git secrets on ubuntu.
return "This is AWS, not doing git-secrets"
if IS_GITLAB:
inform("Nothing is edited on gitlab build server")
return "This is gitlab, not doing git-secrets"
try:
# check to see if secrets even is a git command
commands = ["git secrets --install", "git secrets --register-aws"]
for command in commands:
command_parts = shlex.split(command)
command_process = subprocess.run(
command_parts,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=True,
)
for stream in [command_process.stdout, command_process.stderr]:
if stream:
for line in stream.decode().split("\n"):
inform("*" + line)
except subprocess.CalledProcessError as cpe:
inform(cpe)
installed = False
for stream in [cpe.stdout, cpe.stderr]:
if stream:
for line in stream.decode().split("\n"):
inform("-" + line)
if "commit-msg already exists" in line:
inform("git secrets installed.")
installed = True
break
if not installed:
raise
command_text = "git secrets --scan -r ./".strip().replace(" ", " ")
command_parts = shlex.split(command_text)
execute(*command_parts)
return "git-secrets succeeded"
def do_pyroma_regardless() -> str:
"""
Check package goodness (essentially lints setup.py)
"""
if not os.path.exists("setup.py") and not os.path.exists("setup.cfg"):
inform("setup.py doesn't exists, not packaging.")
return "Nope"
command = "pyroma"
check_command_exists(command)
command = f"{VENV_SHELL} pyroma --directory --min=8 .".strip().replace(" ", " ")
inform(command)
execute(*(command.split(" ")))
return "pyroma succeeded"
def do_python_taint() -> str:
"""
Security Checks
"""
if sys.version_info.major == 3 and sys.version_info.minor > 8:
# pyt only works with python <3.8
return "skipping python taint"
command = "pyt"
check_command_exists(command)
command = "pyt -r"
command = prepinform_simple(command)
execute(*(command.split(" ")))
return "ok"
def do_mccabe() -> str:
"""
Complexity Checker
"""
check_command_exists("flake8") # yes, flake8, this is a plug in.
# mccabe doesn't have a direct way to run it
command_text = (f"flake8 --max-complexity {COMPLEXITY_CUT_OFF} "
f"--config {settings.CONFIG_FOLDER}/.flake8")
command_text = prepinform_simple(command_text)
command = shlex.split(command_text)
execute(*command)
return "mccabe succeeded"
def do_sonar() -> str:
"""
Upload code to sonar for review
"""
sonar_key = os.environ["SONAR_KEY"]
if is_windows():
command_name = "sonar-scanner.bat"
else:
command_name = "sonar-scanner"
command = (f"{VENV_SHELL} {command_name} "
f"-Dsonar.login={sonar_key} "
"-Dproject.settings="
"sonar-project.properties".strip().replace(" ",
" ").split(" "))
inform(command)
execute(*command)
url = ("https://code-quality-test.loc.gov/api/issues/search?"
f"componentKeys=public_record_{PROJECT_NAME}&resolved=false")
session = requests.Session()
session.auth = (sonar_key, "")
response = session.get(url)
errors_file = "sonar.json"
with open(errors_file, "w+") as file_handle:
inform(response.text)
text = response.text
if not text:
say_and_exit("Failed to check for sonar", "sonar")
sys.exit(-1)
file_handle.write(text)
try:
with open(errors_file) as file_handle:
data = json.load(file_handle)
if data["issues"]:
for result in data["issues"]:
inform("{} : {} line {}-{}".format(
result["component"],
result["message"],
result["textRange"]["startLine"],
result["textRange"]["endLine"],
))
say_and_exit("sonar has issues with this code", "sonar")
except json.JSONDecodeError:
pass
return "Sonar done"
def do_jake() -> str:
"""
Check free database for vulnerabilities in active venv.
"""
# TODO: get an API key and start using
# .oss-index-config
command_name = "jake"
check_command_exists(command_name)
command_text = f"{VENV_SHELL} {command_name} ddt".strip().replace(
" ", " ")
inform(command_text)
command = shlex.split(command_text)
execute(*command)
return "jake succeeded"
def do_pyright() -> str:
"""
Execute pyright
"""
command = "pyright"
if check_command_exists(command, throw_on_missing=False):
# subprocess.check_call(("npm install -g pyright").split(" "), shell=True)
inform(
"You must install pyright before doing pyright checks: "
"npm install -g pyright"
)
command = prepinform_simple(command)
command += "/**/*.py"
execute(*(command.split(" ")))
return "Pyright finished"
def do_bandit(is_shell_script_like: bool) -> str:
"""
Security Checks
Generally returns a small number of problems to fix.
"""
if is_shell_script_like:
return (
"Skipping bandit, this code is shell script-like so it has security"
"issues on purpose.")
command = "bandit"
check_command_exists(command)
command = "bandit -r"
command = prepinform_simple(command)
execute(*(command.split(" ")))
return "bandit succeeded"
def do_liccheck() -> str:
"""
Make an explicit decision about license of referenced packages
"""
check_command_exists("liccheck")
if not os.path.exists(f"{settings.CONFIG_FOLDER}/requirements.txt"):
inform("No requirements.txt file, assuming we have no external deps")
return "Skipping, not requirements.txt"
command = ("liccheck "
f"-r {settings.CONFIG_FOLDER}/requirements.txt "
f"-s {settings.CONFIG_FOLDER}/.license_rules "
"-l paranoid")
command = prepinform_simple(command, no_project=True)
execute(*(command.split(" ")))
return "liccheck succeeded"
def do_register_scripts() -> None:
"""
Without this console_scripts in the entrypoints section of setup.py aren't
available
:return:
"""
check_command_exists("pip")
# This doesn't work, it can't tell if "install -e ." has already run
if dist_is_editable():
inform("console_scripts already registered")
return
# install in "editable" mode
command_text = f"{VENV_SHELL} pip install -e ."
inform(command_text)
command_text = command_text.strip().replace(" ", " ")
command = shlex.split(command_text)
execute(*command)
def do_formatting(check: str, state: Dict[str, bool]) -> None:
"""
Format with black - this will not modify code if check is --check
"""
# check & format should be merged & use an arg
# global FORMATTING_CHECK_DONE
if state["check_already_done"]:
inform("Formatting check says black will not reformat, so no need to repeat")
return
if sys.version_info < (3, 6):
inform("Black doesn't work on python 2")
return
check_command_exists("black")
command_text = f"{VENV_SHELL} black {PROJECT_NAME} {check}".strip().replace(
" ", " "
)
inform(command_text)
command = shlex.split(command_text)
if check:
_ = execute(*command)
state["check_already_done"] = True
return
result = execute_get_text(command, env=config_pythonpath())
assert result
changed = []
for line in result.split("\n"):
if "reformatted " in line:
file = line[len("reformatted ") :].strip()
changed.append(file)
if not IS_GITLAB:
if not is_git_repo("."):
# don't need to git add anything because this isn't a git repo
return
for change in changed:
if is_windows():
change = change.replace("\\", "/")
command_text = f"git add {change}"
inform(command_text)
command = shlex.split(command_text)
execute(*command)
def do_precommit(is_interactive: bool) -> None:
"""
Build time execution of pre-commit checks. Modifies code so run before linter.
"""
if not is_interactive:
inform("Not running precommit because it changes files")
return
check_command_exists("pre-commit")
if is_git_repo("."):
# don't try to install because it isn't a git repo
command_text = f"{VENV_SHELL} pre-commit install".strip().replace(" ", " ")
inform(command_text)
command = shlex.split(command_text)
execute(*command)
command_text = f"{VENV_SHELL} pre-commit run --all-files".strip().replace(" ", " ")
inform(command_text)
command = shlex.split(command_text)
result = execute_get_text(command, ignore_error=True, env=config_pythonpath())
assert result
changed = []
for line in result.split("\n"):
if "changed " in line:
file = line[len("reformatted ") :].strip()
changed.append(file)
if "FAILED" in result:
inform(result)
say_and_exit("Pre-commit Failed", "pre-commit")
sys.exit(-1)
if is_interactive:
if not is_git_repo("."):
# don't need to git add anything because this isn't a git repo
return
for change in changed:
command_text = f"git add {change}"
inform(command_text)
# this breaks on windows!
# command = shlex.split(command_text)
execute(*command_text.split())
def do_pyupgrade(is_interactive: bool, minimum_python: str) -> str:
"""Update syntax to most recent variety."""
if not is_interactive:
inform(
"Not an interactive session, skipping pyupgrade wihch changes files."
)
return "Skipping"
command = "pyupgrade"
check_command_exists(command)
all_files = " ".join(
f for f in glob.glob(f"{PROJECT_NAME}/**/*.py", recursive=True))
# as of 2021, still doesn't appear to support recursive globs natively.
command = (f"{VENV_SHELL} pyupgrade "
f"--{minimum_python}-plus "
f"--exit-zero-even-if-changed {all_files}".strip().replace(
" ", " "))
inform(command)
execute(*(command.split(" ")))
return f"{command} succeeded"
def do_tox() -> str:
"""
See if everything works with python 3.8 and upcoming libraries
"""
# If tox fails the build with 3.8 or some future library, that means
# we can't migrate to 3.8 yet, or that we should stay with currently pinned
# libraries. We should fail the overall build.
#
# Because we control our python version we don't have to support cross ver
# compatibility, i.e. we are not supporting 2.7 & 3.x!
if settings.TOX_ACTIVE:
# this happens when testing the build script itself.
return "tox already, not nesting"
command_name = "tox"
check_command_exists(command_name)
command_text = f"{VENV_SHELL} {command_name}".strip().replace(" ", " ")
inform(command_text)
command = shlex.split(command_text)
execute(*command)
return "tox succeeded"
def do_dependency_installs(pipenv: bool, poetry: bool, pip: bool) -> None:
"""Catch up with missing deps"""
if pipenv:
check_command_exists("pipenv")
command_text = "pipenv install --dev --skip-lock"
inform(command_text)
command = shlex.split(command_text)
execute(*command)
elif poetry:
check_command_exists("poetry")
command_text = "poetry install"
inform(command_text)
command = shlex.split(command_text)
execute(*command)
elif pip:
# TODO: move code to deprecated section?
# TODO: Check for poetry.
if os.path.exists("Pipfile"):
raise TypeError("Found Pipfile, settings imply we aren't using Pipenv.")
if os.path.exists("requirements.txt"):
command_text = "pip install -r requirements.txt"
inform(command_text)
command = shlex.split(command_text)
execute(*command)
else:
inform("no requirements.txt file yet, can't install dependencies")
if os.path.exists("requirements-dev.txt"):
command_text = "pip install -r requirements-dev.txt"
inform(command_text)
command = shlex.split(command_text)
execute(*command)
else:
inform("no requirements-dev.txt file yet, can't install dependencies")
else:
inform("VENV not previously activated, won't attempt to catch up on installs")
def check_package() -> None:
"""
Run twine check
"""
check_command_exists("twine")
execute(*(f"{VENV_SHELL} twine check dist/*".strip().split(" ")))
def do_package() -> None:
"""
don't do anything that is potentially really slow or that modifies files.
"""
if not os.path.exists("setup.py") and not os.path.exists("setup.cfg"):
inform("setup.py doesn't exists, not packaging.")
return "Nope"
check_command_exists("twine")
for folder in ["build", "dist", PROJECT_NAME + ".egg-info"]:
if os.path.exists(folder):
shutil.rmtree(folder)
original_umask = os.umask(0)
try:
try:
os.makedirs(folder, 0o770)
except PermissionError:
execute("cmd", "mkdir", folder)
finally:
os.umask(original_umask)
# command = f"{PYTHON} setup.py sdist --formats=gztar,zip"
# bdist_wheel
command_text = f"{PYTHON} setup.py sdist --formats=gztar,zip"
command_text = prepinform_simple(command_text, no_project=True)
command = shlex.split(command_text)
result = execute_get_text(command,
env=config_pythonpath()).replace("\r", "")
error_count = 0
for row in result.split("\n"):
check_row = str(row).lower()
if check_row.startswith("adding") or check_row.startswith("copying"):
# adding a file named error/warning isn't a problem
continue
if "no previously-included files found matching" in check_row:
# excluding a file that already doesn't exist is wonderful!
# why this is a warning boggles the mind.
continue
if "lib2to3" in check_row:
# python 3.9 has deprecated lib2to3, which shows up as a warning which
# causes the build to fail. Seems ignorable as we don
continue
# sometimes to avoid pyc getting out of sync with .py, on
# dev workstations you PYTHONDONTWRITEBYTECODE=1 which just disables
# pyc altogether. Why wheel cares, I don't know.
has_error = any(
value in check_row
for value in ["Errno", "Error", "failed", "error", "warning"])
if has_error and "byte-compiling is disabled" not in check_row:
inform(row)
error_count += 1
if error_count > 0:
say_and_exit("Package failed", "setup.py")
sys.exit(-1)
# pylint: disable=broad-except
try:
# Twine check must run after package creation. Supersedes setup.py check
command_text = f"{VENV_SHELL} twine check dist/*".strip().replace(
" ", " ")
inform(command_text)
command = shlex.split(command_text)
execute(*command)
except Exception as ex:
inform(ex)
command_text = (f"{VENV_SHELL} setup.py "
"sdist "
"--formats=gztar,zip".strip().replace(" ", " "))
command = shlex.split(command_text)
execute(*command)
def list_files(startpath: str) -> None:
"""
List all files, handy for remote build servers
"""
for root, _, files in os.walk(startpath):
level = root.replace(startpath, "").count(os.sep)
indent = " " * 4 * level
inform("{}{}/".format(indent, os.path.basename(root)))
subindent = " " * 4 * (level + 1)
for file in files:
inform(f"{subindent}{file}")
inform("skipping twine check until I figure out what is up")
list_files(startpath=".")
return "Ok"
