def do_jiggle_version( is_interactive: bool, target_branch: str = "", increase_version_on_all_branches: bool = False, ) -> str: """ Increase version number, but only the last number in a semantic version triplet """ if not is_interactive: inform( "Not an interactive session, skipping jiggle_version, which changes files." ) return "Skipping" # rorepo is a Repo instance pointing to the git-python repository. # For all you know, the first argument to Repo is a path to the repository # you want to work with try: repo = Repo(".") active_branch = str(repo.active_branch) except InvalidGitRepositoryError: inform("Can't detect what branch we are on. Is this a git repo?") active_branch = "don't know what branch we are on" if active_branch == target_branch or increase_version_on_all_branches: check_command_exists("jiggle_version") command = f"jiggle_version here --module={PROJECT_NAME}" parts = shlex.split(command) execute(*parts) else: inform("Not master branch, not incrementing version") return "ok"
def do_upload_package() -> None: """ Send to private package repo """ # devpi use http://localhost:3141 # login with root... # devpi login root --password= # get indexes # devpi use -l # make an index # devpi index -c dev bases=root/pypi # devpi use root/dev # Must register (may go away with newer version of devpi), must be 1 file! # twine register --config-file .pypirc -r devpi-root -u root # -p PASSWORD dist/search_service-0.1.0.zip # can be all files! # twine upload --config-file .pypirc -r devpi-root -u root -p PASSWORD dist/* # which is installable using... # pip install search-service --index-url=http://localhost:3141/root/dev/ check_command_exists("devpi") password = os.environ["DEVPI_PASSWORD"] any_zip = [file for file in os.listdir("dist") if file.endswith(".zip")][0] register_command = ( "twine register --config-file .pypirc -r devpi-root -u root" f" -p {password} dist/{any_zip}") upload_command = ("twine upload --config-file .pypirc -r devpi-root " f"-u root -p {password} dist/*") execute(*(register_command.strip().split(" "))) execute(*(upload_command.strip().split(" ")))
def do_vulture() -> str: """ This also finds code you are working on today! """ check_command_exists("vulture") # TODO: check if whitelist.py exists? command_text = f"{VENV_SHELL} vulture {PROJECT_NAME} whitelist.py" command_text = command_text.strip().replace(" ", " ") inform(command_text) command = shlex.split(command_text) output_file_name = f"{PROBLEMS_FOLDER}/dead_code.txt" with open(output_file_name, "w") as outfile: env = config_pythonpath() subprocess.call(command, stdout=outfile, env=env) if total_loc() > SMALL_CODE_BASE_CUTOFF: cutoff = MAXIMUM_DEAD_CODE else: cutoff = 0 with open(output_file_name) as file_handle: num_lines = sum(1 for line in file_handle if line) if num_lines > cutoff: say_and_exit( f"Too many lines of dead code : {num_lines}, max {cutoff}", "vulture" ) sys.exit(-1) return "dead-code (vulture) succeeded"
def convert_pipenv_to_requirements(pipenv: bool) -> None: """ Create requirement*.txt """ if not pipenv: raise TypeError( "Can't pin dependencies this way, we are only converting " "pipfile to requirements.txt") check_command_exists("pipenv_to_requirements") execute(*(f"{VENV_SHELL} pipenv_to_requirements " f"--dev-output {settings.CONFIG_FOLDER}/requirements-dev.txt " f"--output {settings.CONFIG_FOLDER}/requirements.txt".strip(). split(" "))) if not os.path.exists(f"{settings.CONFIG_FOLDER}/requirements.txt"): inform( "Warning: no requirements.txt found, assuming it is because there are" "no external dependencies yet") else: with open(f"{settings.CONFIG_FOLDER}/requirements.txt", "r+") as file: lines = file.readlines() file.seek(0) for line in lines: if line.find("-e .") == -1: file.write(line) file.truncate() with open(f"{settings.CONFIG_FOLDER}/requirements-dev.txt", "r+") as file: lines = file.readlines() file.seek(0) for line in lines: if line.find("-e .") == -1: file.write(line) file.truncate()
def do_count_lines_of_code() -> None: """ Scale failure cut offs based on Lines of Code """ command_name = "pygount" check_command_exists(command_name) command_text = prepinform_simple(command_name) # keep out of src tree, causes extraneous change detections if not os.path.exists(f"{REPORTS_FOLDER}"): os.makedirs(f"{REPORTS_FOLDER}") output_file_name = f"{REPORTS_FOLDER}/line_counts.txt" command = shlex.split(command_text) with open(output_file_name, "w") as outfile: subprocess.call(command, stdout=outfile) with open(output_file_name) as file_handle: lines = sum( int(line.split("\t")[0]) for line in file_handle if line != "\n") total_loc_local = lines if not os.path.exists(f"{settings.CONFIG_FOLDER}/.build_state"): os.makedirs(f"{settings.CONFIG_FOLDER}/.build_state") with open(f"{settings.CONFIG_FOLDER}/.build_state/pygount_total_loc.txt", "w+") as state_file: state_file.write(str(total_loc_local)) inform(f"Lines of code: {total_loc_local}") if total_loc_local == 0: say_and_exit( "No code found to build or package. Maybe the PROJECT_NAME is wrong?", "lines of code", )
def do_flake8() -> str: """ Flake8 Checks """ command = "flake8" check_command_exists(command) command_text = f"flake8 --config {settings.CONFIG_FOLDER}/.flake8" command_text = prepinform_simple(command_text) execute(*(command_text.split(" "))) return "flake 8 succeeded"
def do_detect_secrets() -> str: """ Call detect-secrets tool I think this is the problem: # Code expects to stream output to file and then expects # interactive person, so code hangs. But also hangs in git-bash detect-secrets scan test_data/config.env > foo.txt detect-secrets audit foo.txt """ inform("Detect secrets broken ... can't figure out why") return "nope" # pylint: disable=unreachable check_command_exists("detect-secrets") errors_file = f"{PROBLEMS_FOLDER}/detect-secrets-results.txt" command_text = ( f"{VENV_SHELL} detect-secrets scan " "--base64-limit 4 " # f"--exclude-files .idea|.min.js|.html|.xsd|" # f"lock.json|.scss|Pipfile.lock|.secrets.baseline|" # f"{PROBLEMS_FOLDER}/lint.txt|{errors_file}".strip().replace(" ", " ") ) inform(command_text) command = shlex.split(command_text) with open(errors_file, "w") as outfile: env = config_pythonpath() output = execute_get_text(command, ignore_error=False, env=env) outfile.write(output) # subprocess.call(command, stdout=outfile, env=env) with open(errors_file, "w+") as file_handle: text = file_handle.read() if not text: say_and_exit("Failed to check for secrets", "detect-secrets") sys.exit(-1) file_handle.write(text) try: with open(errors_file) as json_file: data = json.load(json_file) if data["results"]: for result in data["results"]: inform(result) say_and_exit( "detect-secrets has discovered high entropy strings, " "possibly passwords?", "detect-secrets", ) except json.JSONDecodeError: pass return "Detect secrets completed."
def do_docs() -> str: """ Generate docs based on rst. """ check_command_exists("make") my_env = config_pythonpath() command = f"{VENV_SHELL} make html".strip().replace(" ", " ") inform(command) execute_with_environment(command, env=my_env) return "Docs generated"
def do_yamllint() -> str: """ Check yaml files for problems """ command = "yamllint" check_command_exists(command) command = f"{VENV_SHELL} yamllint {PROJECT_NAME}".strip().replace( " ", " ") inform(command) execute(*(command.split(" "))) return "yamllint succeeded"
def do_isort() -> str: """Sort the imports to discover import order bugs and prevent import order bugs""" # This must run before black. black doesn't change import order but it wins # any arguments about formatting. # isort MUST be installed with pipx! It is not compatible with pylint in the same # venv. Maybe someday, but it just isn't worth the effort. check_command_exists("isort") command = "isort --profile black" command = prepinform_simple(command) execute(*(command.split(" "))) return "isort succeeded"
def do_safety() -> str: """ Check free database for vulnerabilities in pinned libraries. """ requirements_file_name = f"{settings.CONFIG_FOLDER}/requirements_for_safety.txt" with open(requirements_file_name, "w+") as out: subprocess.run(["pip", "freeze"], stdout=out, stderr=out, check=True) check_command_exists("safety") # ignore 38414 until aws fixes awscli execute("safety", "check", "--ignore", "38414", "--file", requirements_file_name) return "Package safety checked"
def do_git_secrets() -> str: """ Install git secrets if possible. """ if is_cmd_exe(): inform("git secrets is a bash script, only works in bash (or maybe PS") return "skipped git secrets, this is cmd.exe shell" # not sure how to check for a git subcommand if not is_git_repo("."): inform("This is not a git repo, won't run git-secrets") return "Not a git repo, skipped" check_command_exists("git") if check_is_aws(): # no easy way to install git secrets on ubuntu. return "This is AWS, not doing git-secrets" if IS_GITLAB: inform("Nothing is edited on gitlab build server") return "This is gitlab, not doing git-secrets" try: # check to see if secrets even is a git command commands = ["git secrets --install", "git secrets --register-aws"] for command in commands: command_parts = shlex.split(command) command_process = subprocess.run( command_parts, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, ) for stream in [command_process.stdout, command_process.stderr]: if stream: for line in stream.decode().split("\n"): inform("*" + line) except subprocess.CalledProcessError as cpe: inform(cpe) installed = False for stream in [cpe.stdout, cpe.stderr]: if stream: for line in stream.decode().split("\n"): inform("-" + line) if "commit-msg already exists" in line: inform("git secrets installed.") installed = True break if not installed: raise command_text = "git secrets --scan -r ./".strip().replace(" ", " ") command_parts = shlex.split(command_text) execute(*command_parts) return "git-secrets succeeded"
def do_pyroma_regardless() -> str: """ Check package goodness (essentially lints setup.py) """ if not os.path.exists("setup.py") and not os.path.exists("setup.cfg"): inform("setup.py doesn't exists, not packaging.") return "Nope" command = "pyroma" check_command_exists(command) command = f"{VENV_SHELL} pyroma --directory --min=8 .".strip().replace(" ", " ") inform(command) execute(*(command.split(" "))) return "pyroma succeeded"
def call_check_manifest_command(output_file_name: str, env: Dict[str, str]) -> None: """ To allow for checking in multiple passes """ check_command_exists("check-manifest") command_text = f"{VENV_SHELL} check-manifest".strip().replace(" ", " ") with open(output_file_name, "w") as outfile: inform(command_text) command = shlex.split(command_text) subprocess.call(command, stdout=outfile, env=env)
def do_mccabe() -> str: """ Complexity Checker """ check_command_exists("flake8") # yes, flake8, this is a plug in. # mccabe doesn't have a direct way to run it command_text = (f"flake8 --max-complexity {COMPLEXITY_CUT_OFF} " f"--config {settings.CONFIG_FOLDER}/.flake8") command_text = prepinform_simple(command_text) command = shlex.split(command_text) execute(*command) return "mccabe succeeded"
def do_python_taint() -> str: """ Security Checks """ if sys.version_info.major == 3 and sys.version_info.minor > 8: # pyt only works with python <3.8 return "skipping python taint" command = "pyt" check_command_exists(command) command = "pyt -r" command = prepinform_simple(command) execute(*(command.split(" "))) return "ok"
def do_gitchangelog() -> None: """ Extract commit comments from git to a report. Makes for a lousy CHANGELOG.md """ # TODO: this app has lots of features for cleaning up comments command_name = "gitchangelog" check_command_exists(command_name) command_text = f"{VENV_SHELL} {command_name}".strip().replace(" ", " ") inform(command_text) command = shlex.split(command_text) with open("ChangeLog", "w+") as change_log: result = execute_get_text(command, env=config_pythonpath()).replace("\r", "") change_log.write(result)
def do_jake() -> str: """ Check free database for vulnerabilities in active venv. """ # TODO: get an API key and start using # .oss-index-config command_name = "jake" check_command_exists(command_name) command_text = f"{VENV_SHELL} {command_name} ddt".strip().replace( " ", " ") inform(command_text) command = shlex.split(command_text) execute(*command) return "jake succeeded"
def do_liccheck() -> str: """ Make an explicit decision about license of referenced packages """ check_command_exists("liccheck") if not os.path.exists(f"{settings.CONFIG_FOLDER}/requirements.txt"): inform("No requirements.txt file, assuming we have no external deps") return "Skipping, not requirements.txt" command = ("liccheck " f"-r {settings.CONFIG_FOLDER}/requirements.txt " f"-s {settings.CONFIG_FOLDER}/.license_rules " "-l paranoid") command = prepinform_simple(command, no_project=True) execute(*(command.split(" "))) return "liccheck succeeded"
def do_bandit(is_shell_script_like: bool) -> str: """ Security Checks Generally returns a small number of problems to fix. """ if is_shell_script_like: return ( "Skipping bandit, this code is shell script-like so it has security" "issues on purpose.") command = "bandit" check_command_exists(command) command = "bandit -r" command = prepinform_simple(command) execute(*(command.split(" "))) return "bandit succeeded"
def do_register_scripts() -> None: """ Without this console_scripts in the entrypoints section of setup.py aren't available :return: """ check_command_exists("pip") # This doesn't work, it can't tell if "install -e ." has already run if dist_is_editable(): inform("console_scripts already registered") return # install in "editable" mode command_text = f"{VENV_SHELL} pip install -e ." inform(command_text) command_text = command_text.strip().replace(" ", " ") command = shlex.split(command_text) execute(*command)
def say_and_exit(message: str, source: str) -> None: """ Audibly notify the developer that the build is done so that long builds wouldn't cause an attention problem """ inform(f"{source}:{message}") if ( settings.SPEAK_WHEN_BUILD_FAILS and settings.IS_INTERACTIVE and not check_is_aws() ): # TODO: check a profile option or something. if check_command_exists("say", throw_on_missing=False, exit_on_missing=False): subprocess.call(["say", message]) elif check_command_exists( "wsay.exe", throw_on_missing=False, exit_on_missing=False ): subprocess.call(["wsay", message]) sys.exit(-1)
def do_formatting(check: str, state: Dict[str, bool]) -> None: """ Format with black - this will not modify code if check is --check """ # check & format should be merged & use an arg # global FORMATTING_CHECK_DONE if state["check_already_done"]: inform("Formatting check says black will not reformat, so no need to repeat") return if sys.version_info < (3, 6): inform("Black doesn't work on python 2") return check_command_exists("black") command_text = f"{VENV_SHELL} black {PROJECT_NAME} {check}".strip().replace( " ", " " ) inform(command_text) command = shlex.split(command_text) if check: _ = execute(*command) state["check_already_done"] = True return result = execute_get_text(command, env=config_pythonpath()) assert result changed = [] for line in result.split("\n"): if "reformatted " in line: file = line[len("reformatted ") :].strip() changed.append(file) if not IS_GITLAB: if not is_git_repo("."): # don't need to git add anything because this isn't a git repo return for change in changed: if is_windows(): change = change.replace("\\", "/") command_text = f"git add {change}" inform(command_text) command = shlex.split(command_text) execute(*command)
def do_mypy() -> str: """ Are types ok? """ check_command_exists("mypy") if sys.version_info < (3, 4): inform("Mypy doesn't work on python < 3.4") return "command is missing" command = (f"{VENV_SHELL} mypy {PROJECT_NAME} " "--ignore-missing-imports " "--strict".strip().replace(" ", " ")) inform(command) bash_process = subprocess.Popen(command.split(" "), stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _ = bash_process.communicate() # wait mypy_file = f"{PROBLEMS_FOLDER}/all_mypy_errors.txt" with open(mypy_file, "w", encoding="utf-8") as out_file: out_file.write(out.decode()) return mypy_file
def do_precommit(is_interactive: bool) -> None: """ Build time execution of pre-commit checks. Modifies code so run before linter. """ if not is_interactive: inform("Not running precommit because it changes files") return check_command_exists("pre-commit") if is_git_repo("."): # don't try to install because it isn't a git repo command_text = f"{VENV_SHELL} pre-commit install".strip().replace(" ", " ") inform(command_text) command = shlex.split(command_text) execute(*command) command_text = f"{VENV_SHELL} pre-commit run --all-files".strip().replace(" ", " ") inform(command_text) command = shlex.split(command_text) result = execute_get_text(command, ignore_error=True, env=config_pythonpath()) assert result changed = [] for line in result.split("\n"): if "changed " in line: file = line[len("reformatted ") :].strip() changed.append(file) if "FAILED" in result: inform(result) say_and_exit("Pre-commit Failed", "pre-commit") sys.exit(-1) if is_interactive: if not is_git_repo("."): # don't need to git add anything because this isn't a git repo return for change in changed: command_text = f"git add {change}" inform(command_text) # this breaks on windows! # command = shlex.split(command_text) execute(*command_text.split())
def do_pyupgrade(is_interactive: bool, minimum_python: str) -> str: """Update syntax to most recent variety.""" if not is_interactive: inform( "Not an interactive session, skipping pyupgrade wihch changes files." ) return "Skipping" command = "pyupgrade" check_command_exists(command) all_files = " ".join( f for f in glob.glob(f"{PROJECT_NAME}/**/*.py", recursive=True)) # as of 2021, still doesn't appear to support recursive globs natively. command = (f"{VENV_SHELL} pyupgrade " f"--{minimum_python}-plus " f"--exit-zero-even-if-changed {all_files}".strip().replace( " ", " ")) inform(command) execute(*(command.split(" "))) return f"{command} succeeded"
def do_tox() -> str: """ See if everything works with python 3.8 and upcoming libraries """ # If tox fails the build with 3.8 or some future library, that means # we can't migrate to 3.8 yet, or that we should stay with currently pinned # libraries. We should fail the overall build. # # Because we control our python version we don't have to support cross ver # compatibility, i.e. we are not supporting 2.7 & 3.x! if settings.TOX_ACTIVE: # this happens when testing the build script itself. return "tox already, not nesting" command_name = "tox" check_command_exists(command_name) command_text = f"{VENV_SHELL} {command_name}".strip().replace(" ", " ") inform(command_text) command = shlex.split(command_text) execute(*command) return "tox succeeded"
def do_pytest() -> None: """ Pytest and coverage, which replaces nose tests """ check_command_exists("pytest") # Somedays VPN just isn't there. if IS_INTERNAL_NETWORK or RUN_ALL_TESTS_REGARDLESS_TO_NETWORK: fast_only = False else: fast_only = True if fast_only: test_folder = "test/test_fast" minimum_coverage = MINIMUM_TEST_COVERAGE_FOR_FAST_TESTS else: test_folder = "test" minimum_coverage = MINIMUM_TEST_COVERAGE my_env = config_pythonpath() command = ( f"{VENV_SHELL} pytest {test_folder} -v " f"--junitxml={REPORTS_FOLDER}/sonar-unit-test-results.xml " "--cov-report xml " f"--cov={PROJECT_NAME} " f"--cov-fail-under {minimum_coverage}".strip().replace(" ", " ") + " --quiet" # 15000 pages of call stack don't help anyone ) # when it works, it is FAST. when it doesn't, we get lots of timeouts. # if not IS_GITLAB: # command += f" -n {multiprocessing.cpu_count()} " if not IS_GITLAB: command += " -n 2 " inform(command) execute_with_environment(command, my_env) inform( "Tests will not be re-run until code changes. Run pynt reset to force." )
def do_lint(folder_type: str) -> str: """ Execute pylint """ # pylint: disable=too-many-locals check_command_exists("pylint") if folder_type == PROJECT_NAME: pylintrc = f"{settings.CONFIG_FOLDER}/.pylintrc" lint_output_file_name = f"{PROBLEMS_FOLDER}/lint.txt" else: pylintrc = f"{settings.CONFIG_FOLDER}/.pylintrc_{folder_type}" lint_output_file_name = f"{PROBLEMS_FOLDER}/lint_{folder_type}.txt" if os.path.isfile(lint_output_file_name): os.remove(lint_output_file_name) if IS_DJANGO: django_bits = "--load-plugins pylint_django " else: django_bits = "" # pylint: disable=pointless-string-statement command_text = (f"{VENV_SHELL} pylint {django_bits} " f"--rcfile={pylintrc} {folder_type} ") command_text += " " "--msg-template={path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" "".strip().replace(" ", " ") inform(command_text) command = shlex.split(command_text) with open(lint_output_file_name, "w") as outfile: env = config_pythonpath() subprocess.call(command, stdout=outfile, env=env) return lint_output_file_name
def do_dependency_installs(pipenv: bool, poetry: bool, pip: bool) -> None: """Catch up with missing deps""" if pipenv: check_command_exists("pipenv") command_text = "pipenv install --dev --skip-lock" inform(command_text) command = shlex.split(command_text) execute(*command) elif poetry: check_command_exists("poetry") command_text = "poetry install" inform(command_text) command = shlex.split(command_text) execute(*command) elif pip: # TODO: move code to deprecated section? # TODO: Check for poetry. if os.path.exists("Pipfile"): raise TypeError("Found Pipfile, settings imply we aren't using Pipenv.") if os.path.exists("requirements.txt"): command_text = "pip install -r requirements.txt" inform(command_text) command = shlex.split(command_text) execute(*command) else: inform("no requirements.txt file yet, can't install dependencies") if os.path.exists("requirements-dev.txt"): command_text = "pip install -r requirements-dev.txt" inform(command_text) command = shlex.split(command_text) execute(*command) else: inform("no requirements-dev.txt file yet, can't install dependencies") else: inform("VENV not previously activated, won't attempt to catch up on installs")