def do_upload_package() -> None: """ Send to private package repo """ # devpi use http://localhost:3141 # login with root... # devpi login root --password= # get indexes # devpi use -l # make an index # devpi index -c dev bases=root/pypi # devpi use root/dev # Must register (may go away with newer version of devpi), must be 1 file! # twine register --config-file .pypirc -r devpi-root -u root # -p PASSWORD dist/search_service-0.1.0.zip # can be all files! # twine upload --config-file .pypirc -r devpi-root -u root -p PASSWORD dist/* # which is installable using... # pip install search-service --index-url=http://localhost:3141/root/dev/ check_command_exists("devpi") password = os.environ["DEVPI_PASSWORD"] any_zip = [file for file in os.listdir("dist") if file.endswith(".zip")][0] register_command = ( "twine register --config-file .pypirc -r devpi-root -u root" f" -p {password} dist/{any_zip}") upload_command = ("twine upload --config-file .pypirc -r devpi-root " f"-u root -p {password} dist/*") execute(*(register_command.strip().split(" "))) execute(*(upload_command.strip().split(" ")))
def do_jiggle_version( is_interactive: bool, target_branch: str = "", increase_version_on_all_branches: bool = False, ) -> str: """ Increase version number, but only the last number in a semantic version triplet """ if not is_interactive: inform( "Not an interactive session, skipping jiggle_version, which changes files." ) return "Skipping" # rorepo is a Repo instance pointing to the git-python repository. # For all you know, the first argument to Repo is a path to the repository # you want to work with try: repo = Repo(".") active_branch = str(repo.active_branch) except InvalidGitRepositoryError: inform("Can't detect what branch we are on. Is this a git repo?") active_branch = "don't know what branch we are on" if active_branch == target_branch or increase_version_on_all_branches: check_command_exists("jiggle_version") command = f"jiggle_version here --module={PROJECT_NAME}" parts = shlex.split(command) execute(*parts) else: inform("Not master branch, not incrementing version") return "ok"
def convert_pipenv_to_requirements(pipenv: bool) -> None: """ Create requirement*.txt """ if not pipenv: raise TypeError( "Can't pin dependencies this way, we are only converting " "pipfile to requirements.txt") check_command_exists("pipenv_to_requirements") execute(*(f"{VENV_SHELL} pipenv_to_requirements " f"--dev-output {settings.CONFIG_FOLDER}/requirements-dev.txt " f"--output {settings.CONFIG_FOLDER}/requirements.txt".strip(). split(" "))) if not os.path.exists(f"{settings.CONFIG_FOLDER}/requirements.txt"): inform( "Warning: no requirements.txt found, assuming it is because there are" "no external dependencies yet") else: with open(f"{settings.CONFIG_FOLDER}/requirements.txt", "r+") as file: lines = file.readlines() file.seek(0) for line in lines: if line.find("-e .") == -1: file.write(line) file.truncate() with open(f"{settings.CONFIG_FOLDER}/requirements-dev.txt", "r+") as file: lines = file.readlines() file.seek(0) for line in lines: if line.find("-e .") == -1: file.write(line) file.truncate()
def do_openapi_check() -> None: """ Does swagger/openapi file parse """ if not os.path.exists(f"{PROJECT_NAME}/api.yaml"): inform("No api.yaml file, assuming this is not a microservice") # TODO: should be able to check all y?ml files and look at header. return command_text = (f"{VENV_SHELL} " "openapi-spec-validator" f" {PROJECT_NAME}/api.yaml".strip().replace(" ", " ")) inform(command_text) command = shlex.split(command_text) execute(*command) if IS_JENKINS or IS_GITLAB: inform("Jenkins/Gitlab and apistar don't work together, skipping") return command_text = (f"{VENV_SHELL} apistar validate " f"--path {PROJECT_NAME}/api.yaml " "--format openapi " "--encoding yaml".strip().replace(" ", " ")) inform(command_text) # subprocess.check_call(command.split(" "), shell=False) command = shlex.split(command_text) result = execute_get_text(command, ignore_error=True, env=config_pythonpath()) if "OK" not in result and "2713" not in result and "✓" not in result: inform(result) say_and_exit("apistar didn't like this", "apistar") sys.exit(-1)
def do_flake8() -> str: """ Flake8 Checks """ command = "flake8" check_command_exists(command) command_text = f"flake8 --config {settings.CONFIG_FOLDER}/.flake8" command_text = prepinform_simple(command_text) execute(*(command_text.split(" "))) return "flake 8 succeeded"
def do_pip_check() -> str: """ Call as normal function """ execute("pip", "check") environment = config_pythonpath() environment["PIPENV_PYUP_API_KEY"] = "" if PIPENV_ACTIVE: # ignore 38414 until aws fixes awscli execute_with_environment("pipenv check --ignore 38414", environment) return "Pip(env) check run"
def do_safety() -> str: """ Check free database for vulnerabilities in pinned libraries. """ requirements_file_name = f"{settings.CONFIG_FOLDER}/requirements_for_safety.txt" with open(requirements_file_name, "w+") as out: subprocess.run(["pip", "freeze"], stdout=out, stderr=out, check=True) check_command_exists("safety") # ignore 38414 until aws fixes awscli execute("safety", "check", "--ignore", "38414", "--file", requirements_file_name) return "Package safety checked"
def do_isort() -> str: """Sort the imports to discover import order bugs and prevent import order bugs""" # This must run before black. black doesn't change import order but it wins # any arguments about formatting. # isort MUST be installed with pipx! It is not compatible with pylint in the same # venv. Maybe someday, but it just isn't worth the effort. check_command_exists("isort") command = "isort --profile black" command = prepinform_simple(command) execute(*(command.split(" "))) return "isort succeeded"
def do_yamllint() -> str: """ Check yaml files for problems """ command = "yamllint" check_command_exists(command) command = f"{VENV_SHELL} yamllint {PROJECT_NAME}".strip().replace( " ", " ") inform(command) execute(*(command.split(" "))) return "yamllint succeeded"
def do_git_secrets() -> str: """ Install git secrets if possible. """ if is_cmd_exe(): inform("git secrets is a bash script, only works in bash (or maybe PS") return "skipped git secrets, this is cmd.exe shell" # not sure how to check for a git subcommand if not is_git_repo("."): inform("This is not a git repo, won't run git-secrets") return "Not a git repo, skipped" check_command_exists("git") if check_is_aws(): # no easy way to install git secrets on ubuntu. return "This is AWS, not doing git-secrets" if IS_GITLAB: inform("Nothing is edited on gitlab build server") return "This is gitlab, not doing git-secrets" try: # check to see if secrets even is a git command commands = ["git secrets --install", "git secrets --register-aws"] for command in commands: command_parts = shlex.split(command) command_process = subprocess.run( command_parts, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, ) for stream in [command_process.stdout, command_process.stderr]: if stream: for line in stream.decode().split("\n"): inform("*" + line) except subprocess.CalledProcessError as cpe: inform(cpe) installed = False for stream in [cpe.stdout, cpe.stderr]: if stream: for line in stream.decode().split("\n"): inform("-" + line) if "commit-msg already exists" in line: inform("git secrets installed.") installed = True break if not installed: raise command_text = "git secrets --scan -r ./".strip().replace(" ", " ") command_parts = shlex.split(command_text) execute(*command_parts) return "git-secrets succeeded"
def do_pyroma_regardless() -> str: """ Check package goodness (essentially lints setup.py) """ if not os.path.exists("setup.py") and not os.path.exists("setup.cfg"): inform("setup.py doesn't exists, not packaging.") return "Nope" command = "pyroma" check_command_exists(command) command = f"{VENV_SHELL} pyroma --directory --min=8 .".strip().replace(" ", " ") inform(command) execute(*(command.split(" "))) return "pyroma succeeded"
def do_python_taint() -> str: """ Security Checks """ if sys.version_info.major == 3 and sys.version_info.minor > 8: # pyt only works with python <3.8 return "skipping python taint" command = "pyt" check_command_exists(command) command = "pyt -r" command = prepinform_simple(command) execute(*(command.split(" "))) return "ok"
def do_mccabe() -> str: """ Complexity Checker """ check_command_exists("flake8") # yes, flake8, this is a plug in. # mccabe doesn't have a direct way to run it command_text = (f"flake8 --max-complexity {COMPLEXITY_CUT_OFF} " f"--config {settings.CONFIG_FOLDER}/.flake8") command_text = prepinform_simple(command_text) command = shlex.split(command_text) execute(*command) return "mccabe succeeded"
def do_sonar() -> str: """ Upload code to sonar for review """ sonar_key = os.environ["SONAR_KEY"] if is_windows(): command_name = "sonar-scanner.bat" else: command_name = "sonar-scanner" command = (f"{VENV_SHELL} {command_name} " f"-Dsonar.login={sonar_key} " "-Dproject.settings=" "sonar-project.properties".strip().replace(" ", " ").split(" ")) inform(command) execute(*command) url = ("https://code-quality-test.loc.gov/api/issues/search?" f"componentKeys=public_record_{PROJECT_NAME}&resolved=false") session = requests.Session() session.auth = (sonar_key, "") response = session.get(url) errors_file = "sonar.json" with open(errors_file, "w+") as file_handle: inform(response.text) text = response.text if not text: say_and_exit("Failed to check for sonar", "sonar") sys.exit(-1) file_handle.write(text) try: with open(errors_file) as file_handle: data = json.load(file_handle) if data["issues"]: for result in data["issues"]: inform("{} : {} line {}-{}".format( result["component"], result["message"], result["textRange"]["startLine"], result["textRange"]["endLine"], )) say_and_exit("sonar has issues with this code", "sonar") except json.JSONDecodeError: pass return "Sonar done"
def do_jake() -> str: """ Check free database for vulnerabilities in active venv. """ # TODO: get an API key and start using # .oss-index-config command_name = "jake" check_command_exists(command_name) command_text = f"{VENV_SHELL} {command_name} ddt".strip().replace( " ", " ") inform(command_text) command = shlex.split(command_text) execute(*command) return "jake succeeded"
def do_pyright() -> str: """ Execute pyright """ command = "pyright" if check_command_exists(command, throw_on_missing=False): # subprocess.check_call(("npm install -g pyright").split(" "), shell=True) inform( "You must install pyright before doing pyright checks: " "npm install -g pyright" ) command = prepinform_simple(command) command += "/**/*.py" execute(*(command.split(" "))) return "Pyright finished"
def do_bandit(is_shell_script_like: bool) -> str: """ Security Checks Generally returns a small number of problems to fix. """ if is_shell_script_like: return ( "Skipping bandit, this code is shell script-like so it has security" "issues on purpose.") command = "bandit" check_command_exists(command) command = "bandit -r" command = prepinform_simple(command) execute(*(command.split(" "))) return "bandit succeeded"
def do_liccheck() -> str: """ Make an explicit decision about license of referenced packages """ check_command_exists("liccheck") if not os.path.exists(f"{settings.CONFIG_FOLDER}/requirements.txt"): inform("No requirements.txt file, assuming we have no external deps") return "Skipping, not requirements.txt" command = ("liccheck " f"-r {settings.CONFIG_FOLDER}/requirements.txt " f"-s {settings.CONFIG_FOLDER}/.license_rules " "-l paranoid") command = prepinform_simple(command, no_project=True) execute(*(command.split(" "))) return "liccheck succeeded"
def do_register_scripts() -> None: """ Without this console_scripts in the entrypoints section of setup.py aren't available :return: """ check_command_exists("pip") # This doesn't work, it can't tell if "install -e ." has already run if dist_is_editable(): inform("console_scripts already registered") return # install in "editable" mode command_text = f"{VENV_SHELL} pip install -e ." inform(command_text) command_text = command_text.strip().replace(" ", " ") command = shlex.split(command_text) execute(*command)
def do_formatting(check: str, state: Dict[str, bool]) -> None: """ Format with black - this will not modify code if check is --check """ # check & format should be merged & use an arg # global FORMATTING_CHECK_DONE if state["check_already_done"]: inform("Formatting check says black will not reformat, so no need to repeat") return if sys.version_info < (3, 6): inform("Black doesn't work on python 2") return check_command_exists("black") command_text = f"{VENV_SHELL} black {PROJECT_NAME} {check}".strip().replace( " ", " " ) inform(command_text) command = shlex.split(command_text) if check: _ = execute(*command) state["check_already_done"] = True return result = execute_get_text(command, env=config_pythonpath()) assert result changed = [] for line in result.split("\n"): if "reformatted " in line: file = line[len("reformatted ") :].strip() changed.append(file) if not IS_GITLAB: if not is_git_repo("."): # don't need to git add anything because this isn't a git repo return for change in changed: if is_windows(): change = change.replace("\\", "/") command_text = f"git add {change}" inform(command_text) command = shlex.split(command_text) execute(*command)
def do_precommit(is_interactive: bool) -> None: """ Build time execution of pre-commit checks. Modifies code so run before linter. """ if not is_interactive: inform("Not running precommit because it changes files") return check_command_exists("pre-commit") if is_git_repo("."): # don't try to install because it isn't a git repo command_text = f"{VENV_SHELL} pre-commit install".strip().replace(" ", " ") inform(command_text) command = shlex.split(command_text) execute(*command) command_text = f"{VENV_SHELL} pre-commit run --all-files".strip().replace(" ", " ") inform(command_text) command = shlex.split(command_text) result = execute_get_text(command, ignore_error=True, env=config_pythonpath()) assert result changed = [] for line in result.split("\n"): if "changed " in line: file = line[len("reformatted ") :].strip() changed.append(file) if "FAILED" in result: inform(result) say_and_exit("Pre-commit Failed", "pre-commit") sys.exit(-1) if is_interactive: if not is_git_repo("."): # don't need to git add anything because this isn't a git repo return for change in changed: command_text = f"git add {change}" inform(command_text) # this breaks on windows! # command = shlex.split(command_text) execute(*command_text.split())
def do_pyupgrade(is_interactive: bool, minimum_python: str) -> str: """Update syntax to most recent variety.""" if not is_interactive: inform( "Not an interactive session, skipping pyupgrade wihch changes files." ) return "Skipping" command = "pyupgrade" check_command_exists(command) all_files = " ".join( f for f in glob.glob(f"{PROJECT_NAME}/**/*.py", recursive=True)) # as of 2021, still doesn't appear to support recursive globs natively. command = (f"{VENV_SHELL} pyupgrade " f"--{minimum_python}-plus " f"--exit-zero-even-if-changed {all_files}".strip().replace( " ", " ")) inform(command) execute(*(command.split(" "))) return f"{command} succeeded"
def do_tox() -> str: """ See if everything works with python 3.8 and upcoming libraries """ # If tox fails the build with 3.8 or some future library, that means # we can't migrate to 3.8 yet, or that we should stay with currently pinned # libraries. We should fail the overall build. # # Because we control our python version we don't have to support cross ver # compatibility, i.e. we are not supporting 2.7 & 3.x! if settings.TOX_ACTIVE: # this happens when testing the build script itself. return "tox already, not nesting" command_name = "tox" check_command_exists(command_name) command_text = f"{VENV_SHELL} {command_name}".strip().replace(" ", " ") inform(command_text) command = shlex.split(command_text) execute(*command) return "tox succeeded"
def do_dependency_installs(pipenv: bool, poetry: bool, pip: bool) -> None: """Catch up with missing deps""" if pipenv: check_command_exists("pipenv") command_text = "pipenv install --dev --skip-lock" inform(command_text) command = shlex.split(command_text) execute(*command) elif poetry: check_command_exists("poetry") command_text = "poetry install" inform(command_text) command = shlex.split(command_text) execute(*command) elif pip: # TODO: move code to deprecated section? # TODO: Check for poetry. if os.path.exists("Pipfile"): raise TypeError("Found Pipfile, settings imply we aren't using Pipenv.") if os.path.exists("requirements.txt"): command_text = "pip install -r requirements.txt" inform(command_text) command = shlex.split(command_text) execute(*command) else: inform("no requirements.txt file yet, can't install dependencies") if os.path.exists("requirements-dev.txt"): command_text = "pip install -r requirements-dev.txt" inform(command_text) command = shlex.split(command_text) execute(*command) else: inform("no requirements-dev.txt file yet, can't install dependencies") else: inform("VENV not previously activated, won't attempt to catch up on installs")
def check_package() -> None: """ Run twine check """ check_command_exists("twine") execute(*(f"{VENV_SHELL} twine check dist/*".strip().split(" ")))
def do_package() -> None: """ don't do anything that is potentially really slow or that modifies files. """ if not os.path.exists("setup.py") and not os.path.exists("setup.cfg"): inform("setup.py doesn't exists, not packaging.") return "Nope" check_command_exists("twine") for folder in ["build", "dist", PROJECT_NAME + ".egg-info"]: if os.path.exists(folder): shutil.rmtree(folder) original_umask = os.umask(0) try: try: os.makedirs(folder, 0o770) except PermissionError: execute("cmd", "mkdir", folder) finally: os.umask(original_umask) # command = f"{PYTHON} setup.py sdist --formats=gztar,zip" # bdist_wheel command_text = f"{PYTHON} setup.py sdist --formats=gztar,zip" command_text = prepinform_simple(command_text, no_project=True) command = shlex.split(command_text) result = execute_get_text(command, env=config_pythonpath()).replace("\r", "") error_count = 0 for row in result.split("\n"): check_row = str(row).lower() if check_row.startswith("adding") or check_row.startswith("copying"): # adding a file named error/warning isn't a problem continue if "no previously-included files found matching" in check_row: # excluding a file that already doesn't exist is wonderful! # why this is a warning boggles the mind. continue if "lib2to3" in check_row: # python 3.9 has deprecated lib2to3, which shows up as a warning which # causes the build to fail. Seems ignorable as we don continue # sometimes to avoid pyc getting out of sync with .py, on # dev workstations you PYTHONDONTWRITEBYTECODE=1 which just disables # pyc altogether. Why wheel cares, I don't know. has_error = any( value in check_row for value in ["Errno", "Error", "failed", "error", "warning"]) if has_error and "byte-compiling is disabled" not in check_row: inform(row) error_count += 1 if error_count > 0: say_and_exit("Package failed", "setup.py") sys.exit(-1) # pylint: disable=broad-except try: # Twine check must run after package creation. Supersedes setup.py check command_text = f"{VENV_SHELL} twine check dist/*".strip().replace( " ", " ") inform(command_text) command = shlex.split(command_text) execute(*command) except Exception as ex: inform(ex) command_text = (f"{VENV_SHELL} setup.py " "sdist " "--formats=gztar,zip".strip().replace(" ", " ")) command = shlex.split(command_text) execute(*command) def list_files(startpath: str) -> None: """ List all files, handy for remote build servers """ for root, _, files in os.walk(startpath): level = root.replace(startpath, "").count(os.sep) indent = " " * 4 * level inform("{}{}/".format(indent, os.path.basename(root))) subindent = " " * 4 * (level + 1) for file in files: inform(f"{subindent}{file}") inform("skipping twine check until I figure out what is up") list_files(startpath=".") return "Ok"