def pip_check(): """ Are packages ok? """ execute("pip", "check") if PIPENV and not IS_GITHUB_CI: execute("pipenv", "check")
def check_setup_py(): with safe_cd(SRC): if IS_TRAVIS: execute(PYTHON, *("setup.py check -r -s".split(" "))) else: execute(*("{0} {1} setup.py check -r -s".format( PIPENV, PYTHON).strip().split(" ")))
def pin_dependencies(): """ Create requirement*.txt """ with safe_cd(SRC): execute( *("{0} pipenv_to_requirements".format(PIPENV).strip().split(" ")))
def git_secrets(): """ Install git secrets if possible. """ if check_is_aws(): # no easy way to install git secrets on ubuntu. return if IS_TRAVIS: # nothing is edited on travis return try: commands = ["git secrets --install", "git secrets --register-aws"] for command in commands: cp = subprocess.run(command.split(" "), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, check=True) for stream in [cp.stdout, cp.stderr]: if stream: for line in stream.decode().split("\n"): print("*" + line) except subprocess.CalledProcessError as cpe: print(cpe) installed = False for stream in [cpe.stdout, cpe.stderr]: if stream: for line in stream.decode().split("\n"): print("-" + line) if "commit-msg already exists" in line: print("git secrets installed.") installed = True break if not installed: raise execute(*("git secrets --scan".strip().split(" ")))
def package(): with safe_cd(SRC): for folder in ["build", "dist", PROJECT_NAME + ".egg-info"]: execute("rm", "-rf", folder) with safe_cd(SRC): execute(PYTHON, "-m", "build") os.system('say "package complete."')
def prospector(): """ Catch a few things with a non-strict propector run """ with safe_cd(SRC): command = "{0} prospector {1} --profile {1}_style --pylint-config-file=pylintrc.ini --profile-path=.prospector".format( PIPENV, PROJECT_NAME).strip().replace(" ", " ") print(command) execute(*(command.split(" ")))
def lint(): """ Lint """ with safe_cd(SRC): if os.path.isfile("lint.txt"): execute("rm", "lint.txt") with safe_cd(SRC): command = ( "{0} pylint --rcfile=pylintrc.ini {2}".format( PIPENV, PROJECT_NAME ) .strip() .replace(" ", " ") ) print(command) command = command.split(" ") # keep out of src tree, causes extraneous change detections lint_output_file_name = "lint.txt" with open(lint_output_file_name, "w") as outfile: env = config_pythonpath() subprocess.call(command, stdout=outfile, env=env) fatal_errors = sum( 1 for line in open(lint_output_file_name) if "no-member" in line or "no-name-in-module" in line or "import-error" in line ) if fatal_errors > 0: for line in open(lint_output_file_name): if ( "no-member" in line or "no-name-in-module" in line or "import-error" in line ): print(line) print("Fatal lint errors : {0}".format(fatal_errors)) exit(-1) cutoff = 100 num_lines = sum( 1 for line in open(lint_output_file_name) if "*************" not in line and "---------------------" not in line and "Your code has been rated at" not in line ) if num_lines > cutoff: raise TypeError( "Too many lines of lint : {0}, max {1}".format(num_lines, cutoff) )
def upload_package(): """ Upload """ with safe_cd(SRC): if IS_GITHUB_CI: pass else: execute( *("{0} twine upload dist/*".format(PIPENV).strip().split(" ")))
def package(): """ package, but don't upload """ with safe_cd(SRC): for folder in ["build", "dist", PROJECT_NAME + ".egg-info"]: execute("rm", "-rf", folder) with safe_cd(SRC): execute(PYTHON, "setup.py", "sdist", "--formats=gztar,zip")
def check_setup_py(): # if # ValueError: ZIP does not support timestamps before 1980 # then run this to ID # find . -mtime +13700 -ls with safe_cd(SRC): if IS_TRAVIS: execute(PYTHON, *("setup.py check -r -s".split(" "))) else: execute(*("{0} {1} setup.py check -r -s".format( PIPENV, PYTHON).strip().split(" ")))
def upload_package(): """ Upload """ with safe_cd(SRC): if IS_GITHUB_CI: pass else: execute( *("{0} {1} setup.py upload".format(PIPENV, PYTHON).strip().split(" ")) )
def create_venv(local='y', test='y', general='y'): """Create virtualenv w/requirements. Specify y/n for local/test/general to control installation.""" if not path.isdir(project_paths.venv): execute('virtualenv', '--distribute', '--no-site-packages', project_paths.venv) project.execute_python('-m', 'easy_install', 'pip') if local.lower() == 'y' and project_paths.local_requirements: project.execute_pip('install', '--upgrade', '-r', project_paths.local_requirements) if test.lower() == 'y' and project_paths.test_requirements: project.execute_pip('install', '--upgrade', '-r', project_paths.test_requirements) if general.lower() == 'y' and project_paths.requirements_txt: project.execute_pip('install', '--upgrade', '-r', project_paths.requirements_txt)
def compile_mark_down(): """ Convert MD to RST """ with safe_cd(SRC): if IS_TRAVIS: command = "pandoc --from=markdown --to=rst --output=README.rst README.md".strip( ).split(" ") else: command = "{0} pandoc --from=markdown --to=rst --output=README.rst README.md".format( PIPENV).strip().split(" ") execute(*(command))
def test_tox(*args, **kwargs): """Runs all tests for all environments w/pynt args/kwargs. Flush previous with pynt test_tox[flush=y]""" args = ['tox'] + list(args) # Python 2.7.x does not allow a named arg after variable length args (3 does but we have to support both) flush = kwargs.get('flush', 'n') if 'flush' in kwargs.keys(): del kwargs['flush'] if flush.lower() == 'y': args.append('-r') execute(*args, **kwargs)
def compile_mark_down(): """ Convert MD to RST """ # print("Not compiling README.md because moderately complex MD makes pypi rst parser puke.") with safe_cd(SRC): if IS_TRAVIS: command = "pandoc --from=markdown --to=rst --output=README.rst README.md".strip( ).split(" ") else: command = "{0} pandoc --from=markdown --to=rst --output=README.rst README.md".format( PIPENV).strip().split(" ") execute(*(command))
def check_setup_py(): """ Setup.py checks package things including README.rst """ with safe_cd(SRC): if IS_GITHUB_CI: execute(PYTHON, *("setup.py check -r -s".split(" "))) else: execute( *( "{0} {1} setup.py check -r -s".format(PIPENV, PYTHON) .strip() .split(" ") ) )
def pip_check(): """ Are packages ok? """ execute("pip", "check") execute("twine", "check") if PIPENV and not IS_TRAVIS: execute("pipenv", "check") execute("safety", "check", "-r", "requirements_dev.txt")
def formatting(): with safe_cd(SRC): if sys.version_info < (3, 6): print("Black doesn't work on python 2") return command = "{0} black {1}".format(PIPENV, PROJECT_NAME).strip() print(command) result = execute_get_text(command) assert result changed = [] for line in result.split("\n"): if "reformatted " in line: file = line[len("reformatted "):].strip() changed.append(file) for change in changed: command = "git add {0}".format(change) print(command) execute(*(command.split(" ")))
def update_pip_and_pipenv(): # env = config_pythonpath() execute("pip", "install", "--upgrade", "pip")
def pwd_ls(dir="/tmp"): '''Run "pwd followed by "ls" on a given directory''' with safe_cd(dir): execute('pwd') execute("ls",dir)
def jiggle_version(): with safe_cd(SRC): command = "{0} jiggle_version here --module={1}".format( PIPENV, PROJECT_NAME).strip() execute(*(command.split(" ")))
def jiggle_version(): """ Bump version. Not dogfooding 'cause it would create a circular dependency. """ # This is the primordial version, not the script/library version execute(PYTHON, "jiggle_version_self.py")
def clean_state(): with safe_cd(".build_state"): # wild cards don't expand by default for file in os.listdir("."): if file.startswith("last") and file.endswith(".txt"): execute("rm", "-f", str(file))
def compile_py(): """ Catch on the worst syntax errors """ with safe_cd(SRC): execute(PYTHON, "-m", "compileall", PROJECT_NAME)
def delete_venv(): """Deletes the venv. Uses a max size check for added safety.""" safe_size_check(project_paths.venv, "Aborting venv removal for safety.") execute('rm', '-rf', project_paths.venv)
def compile_py(): with safe_cd(SRC): execute(PYTHON, "-m", "compileall", PROJECT_NAME)
def clean(): with safe_cd(SRC): execute("rm", "-rf", ".mypy_cache", ".build_state", "dist", "build", PROJECT_NAME + ".egg-info", "dead_code.txt", "mypy_errors.txt", "detect-secrets-results.txt", "lint.txt")
def pwd_ls(dir="/tmp"): '''Run "pwd followed by "ls" on a given directory''' with safe_cd(dir): execute('pwd') execute("ls", dir)