def pip_check():
"""
Are packages ok?
"""
execute("pip", "check")
if PIPENV and not IS_GITHUB_CI:
execute("pipenv", "check")
def check_setup_py():
with safe_cd(SRC):
if IS_TRAVIS:
execute(PYTHON, *("setup.py check -r -s".split(" ")))
else:
execute(*("{0} {1} setup.py check -r -s".format(
PIPENV, PYTHON).strip().split(" ")))
def pin_dependencies():
"""
Create requirement*.txt
"""
with safe_cd(SRC):
execute(
*("{0} pipenv_to_requirements".format(PIPENV).strip().split(" ")))
def git_secrets():
"""
Install git secrets if possible.
"""
if check_is_aws():
# no easy way to install git secrets on ubuntu.
return
if IS_TRAVIS:
# nothing is edited on travis
return
try:
commands = ["git secrets --install", "git secrets --register-aws"]
for command in commands:
cp = subprocess.run(command.split(" "),
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
shell=False, check=True)
for stream in [cp.stdout, cp.stderr]:
if stream:
for line in stream.decode().split("\n"):
print("*" + line)
except subprocess.CalledProcessError as cpe:
print(cpe)
installed = False
for stream in [cpe.stdout, cpe.stderr]:
if stream:
for line in stream.decode().split("\n"):
print("-" + line)
if "commit-msg already exists" in line:
print("git secrets installed.")
installed = True
break
if not installed:
raise
execute(*("git secrets --scan".strip().split(" ")))
def package():
with safe_cd(SRC):
for folder in ["build", "dist", PROJECT_NAME + ".egg-info"]:
execute("rm", "-rf", folder)
with safe_cd(SRC):
execute(PYTHON, "-m", "build")
os.system('say "package complete."')
def prospector():
"""
Catch a few things with a non-strict propector run
"""
with safe_cd(SRC):
command = "{0} prospector {1} --profile {1}_style --pylint-config-file=pylintrc.ini --profile-path=.prospector".format(
PIPENV, PROJECT_NAME).strip().replace(" ", " ")
print(command)
execute(*(command.split(" ")))
def lint():
"""
Lint
"""
with safe_cd(SRC):
if os.path.isfile("lint.txt"):
execute("rm", "lint.txt")
with safe_cd(SRC):
command = (
"{0} pylint --rcfile=pylintrc.ini {2}".format(
PIPENV, PROJECT_NAME
)
.strip()
.replace(" ", " ")
)
print(command)
command = command.split(" ")
# keep out of src tree, causes extraneous change detections
lint_output_file_name = "lint.txt"
with open(lint_output_file_name, "w") as outfile:
env = config_pythonpath()
subprocess.call(command, stdout=outfile, env=env)
fatal_errors = sum(
1
for line in open(lint_output_file_name)
if "no-member" in line
or "no-name-in-module" in line
or "import-error" in line
)
if fatal_errors > 0:
for line in open(lint_output_file_name):
if (
"no-member" in line
or "no-name-in-module" in line
or "import-error" in line
):
print(line)
print("Fatal lint errors : {0}".format(fatal_errors))
exit(-1)
cutoff = 100
num_lines = sum(
1
for line in open(lint_output_file_name)
if "*************" not in line
and "---------------------" not in line
and "Your code has been rated at" not in line
)
if num_lines > cutoff:
raise TypeError(
"Too many lines of lint : {0}, max {1}".format(num_lines, cutoff)
)
def upload_package():
"""
Upload
"""
with safe_cd(SRC):
if IS_GITHUB_CI:
pass
else:
execute(
*("{0} twine upload dist/*".format(PIPENV).strip().split(" ")))
def package():
"""
package, but don't upload
"""
with safe_cd(SRC):
for folder in ["build", "dist", PROJECT_NAME + ".egg-info"]:
execute("rm", "-rf", folder)
with safe_cd(SRC):
execute(PYTHON, "setup.py", "sdist", "--formats=gztar,zip")
def check_setup_py():
# if
# ValueError: ZIP does not support timestamps before 1980
# then run this to ID
# find . -mtime +13700 -ls
with safe_cd(SRC):
if IS_TRAVIS:
execute(PYTHON, *("setup.py check -r -s".split(" ")))
else:
execute(*("{0} {1} setup.py check -r -s".format(
PIPENV, PYTHON).strip().split(" ")))
def upload_package():
"""
Upload
"""
with safe_cd(SRC):
if IS_GITHUB_CI:
pass
else:
execute(
*("{0} {1} setup.py upload".format(PIPENV, PYTHON).strip().split(" "))
)
def create_venv(local='y', test='y', general='y'):
"""Create virtualenv w/requirements. Specify y/n for local/test/general to control installation."""
if not path.isdir(project_paths.venv):
execute('virtualenv', '--distribute', '--no-site-packages', project_paths.venv)
project.execute_python('-m', 'easy_install', 'pip')
if local.lower() == 'y' and project_paths.local_requirements:
project.execute_pip('install', '--upgrade', '-r', project_paths.local_requirements)
if test.lower() == 'y' and project_paths.test_requirements:
project.execute_pip('install', '--upgrade', '-r', project_paths.test_requirements)
if general.lower() == 'y' and project_paths.requirements_txt:
project.execute_pip('install', '--upgrade', '-r', project_paths.requirements_txt)
def compile_mark_down():
"""
Convert MD to RST
"""
with safe_cd(SRC):
if IS_TRAVIS:
command = "pandoc --from=markdown --to=rst --output=README.rst README.md".strip(
).split(" ")
else:
command = "{0} pandoc --from=markdown --to=rst --output=README.rst README.md".format(
PIPENV).strip().split(" ")
execute(*(command))
def test_tox(*args, **kwargs):
"""Runs all tests for all environments w/pynt args/kwargs. Flush previous with pynt test_tox[flush=y]"""
args = ['tox'] + list(args)
# Python 2.7.x does not allow a named arg after variable length args (3 does but we have to support both)
flush = kwargs.get('flush', 'n')
if 'flush' in kwargs.keys():
del kwargs['flush']
if flush.lower() == 'y':
args.append('-r')
execute(*args, **kwargs)
def compile_mark_down():
"""
Convert MD to RST
"""
# print("Not compiling README.md because moderately complex MD makes pypi rst parser puke.")
with safe_cd(SRC):
if IS_TRAVIS:
command = "pandoc --from=markdown --to=rst --output=README.rst README.md".strip(
).split(" ")
else:
command = "{0} pandoc --from=markdown --to=rst --output=README.rst README.md".format(
PIPENV).strip().split(" ")
execute(*(command))
def check_setup_py():
"""
Setup.py checks package things including README.rst
"""
with safe_cd(SRC):
if IS_GITHUB_CI:
execute(PYTHON, *("setup.py check -r -s".split(" ")))
else:
execute(
*(
"{0} {1} setup.py check -r -s".format(PIPENV, PYTHON)
.strip()
.split(" ")
)
)
def pip_check():
"""
Are packages ok?
"""
execute("pip", "check")
execute("twine", "check")
if PIPENV and not IS_TRAVIS:
execute("pipenv", "check")
execute("safety", "check", "-r", "requirements_dev.txt")
def formatting():
with safe_cd(SRC):
if sys.version_info < (3, 6):
print("Black doesn't work on python 2")
return
command = "{0} black {1}".format(PIPENV, PROJECT_NAME).strip()
print(command)
result = execute_get_text(command)
assert result
changed = []
for line in result.split("\n"):
if "reformatted " in line:
file = line[len("reformatted "):].strip()
changed.append(file)
for change in changed:
command = "git add {0}".format(change)
print(command)
execute(*(command.split(" ")))
def update_pip_and_pipenv():
# env = config_pythonpath()
execute("pip", "install", "--upgrade", "pip")
def pwd_ls(dir="/tmp"):
'''Run "pwd followed by "ls" on a given directory'''
with safe_cd(dir):
execute('pwd')
execute("ls",dir)
def jiggle_version():
with safe_cd(SRC):
command = "{0} jiggle_version here --module={1}".format(
PIPENV, PROJECT_NAME).strip()
execute(*(command.split(" ")))
def jiggle_version():
"""
Bump version. Not dogfooding 'cause it would create a circular dependency.
"""
# This is the primordial version, not the script/library version
execute(PYTHON, "jiggle_version_self.py")
def clean_state():
with safe_cd(".build_state"):
# wild cards don't expand by default
for file in os.listdir("."):
if file.startswith("last") and file.endswith(".txt"):
execute("rm", "-f", str(file))
def compile_py():
"""
Catch on the worst syntax errors
"""
with safe_cd(SRC):
execute(PYTHON, "-m", "compileall", PROJECT_NAME)
def delete_venv():
"""Deletes the venv. Uses a max size check for added safety."""
safe_size_check(project_paths.venv, "Aborting venv removal for safety.")
execute('rm', '-rf', project_paths.venv)
def compile_py():
with safe_cd(SRC):
execute(PYTHON, "-m", "compileall", PROJECT_NAME)
def clean():
with safe_cd(SRC):
execute("rm", "-rf", ".mypy_cache", ".build_state", "dist", "build",
PROJECT_NAME + ".egg-info", "dead_code.txt", "mypy_errors.txt",
"detect-secrets-results.txt", "lint.txt")
def pwd_ls(dir="/tmp"):
'''Run "pwd followed by "ls" on a given directory'''
with safe_cd(dir):
execute('pwd')
execute("ls", dir)
