def black(ctx, loc="local", check=False, debug=False, verbose=0, tests=False):
"""
Run black code formatter
Usage: inv ci.black
"""
env = get_compose_env(ctx, loc=loc)
# Only display result
ctx.config["run"]["echo"] = True
# Override run commands env variables one key at a time
for k, v in env.items():
ctx.config["run"]["env"][k] = v
_cmd = "poetry run black "
if check:
_cmd += "--check "
if verbose >= 3:
_cmd += "--verbose "
if tests:
_cmd += "tests tasks "
_cmd += "app"
if verbose >= 1:
msg = "[black] bout to run command: \n"
click.secho(msg, fg="green")
click.secho(_cmd, fg="green")
ctx.run(_cmd)
def gunzip_save(ctx, loc="docker", image=CI_IMAGE, verbose=0):
"""
take contents of dockerfile and cache it locally into $HOME/.cache/docker
Usage: inv travis.gunzip-save
"""
env = get_compose_env(ctx, loc=loc)
# Override run commands' env variables one key at a time
for k, v in env.items():
ctx.config["run"]["env"][k] = v
if verbose >= 1:
click.secho(" [gunzip-save] travis 'install:' section",
fg=COLOR_SUCCESS)
click.secho(
" [gunzip-save] pull docker cache from docker hub if available",
fg=COLOR_SUCCESS,
)
_cmd = r"""
mkdir -p $CACHE_DIR
if [ ! -f ${CACHE_FILE_BASE} ]; then docker save $REPO_NAME:base | gzip > ${CACHE_FILE_BASE} || true; fi
if [ ! -f ${CACHE_FILE_RUNTIME} ]; then docker save $REPO_NAME:runtime-image | gzip > ${CACHE_FILE_RUNTIME} || true; fi
"""
if verbose >= 1:
msg = "[build-gunzip-before-install] cmd is: "
click.secho(msg, fg=COLOR_SUCCESS)
msg = "{}".format(_cmd)
click.secho(msg, fg=COLOR_SUCCESS)
ctx.run(_cmd)
def coverage_clean(ctx, loc="local", verbose=0, cleanup=False):
"""
clean coverage files
Usage: inv ci.coverage-clean
"""
env = get_compose_env(ctx, loc=loc)
# Override run commands' env variables one key at a time
for k, v in env.items():
ctx.config["run"]["env"][k] = v
_cmd = r"""
find . -name '*.pyc' -exec rm -fv {} +
find . -name '*.pyo' -exec rm -fv {} +
find . -name '__pycache__' -exec rm -frv {} +
rm -f .coverage
rm -rf htmlcov/*
rm -rf cov_annotate/*
rm -f cov.xml
"""
if verbose >= 1:
msg = "{}".format(_cmd)
click.secho(msg, fg=COLOR_SUCCESS)
ctx.run(_cmd)
def build_gunzip_before_install(ctx, loc="docker", image=CI_IMAGE, verbose=0):
"""
docker container/image from gnzip tar file in $HOME/.cache/docker
Usage: inv travis.build_gunzip_before_install
"""
env = get_compose_env(ctx, loc=loc)
# Override run commands' env variables one key at a time
for k, v in env.items():
ctx.config["run"]["env"][k] = v
if verbose >= 1:
msg = "[build-gunzip-before-install] Building using docker-compose"
click.secho(msg, fg=COLOR_SUCCESS)
_cmd = r"""
if [ -f ${CACHE_FILE_BASE} ]; then gunzip -c ${CACHE_FILE_BASE} | docker load || true; fi
if [ -f ${CACHE_FILE_RUNTIME} ]; then gunzip -c ${CACHE_FILE_RUNTIME} | docker load || true; fi
"""
if verbose >= 1:
msg = "[build-gunzip-before-install] cmd is: "
click.secho(msg, fg=COLOR_SUCCESS)
msg = "{}".format(_cmd)
click.secho(msg, fg=COLOR_SUCCESS)
ctx.run(_cmd)
def pip_deps(ctx, loc="local", verbose=0, cleanup=False):
"""
lock flask pip dependencies
Usage: inv local.pip_deps
"""
env = get_compose_env(ctx, loc=loc)
# Override run commands' env variables one key at a time
for k, v in env.items():
ctx.config["run"]["env"][k] = v
if verbose >= 1:
msg = "[pip-deps] Create virtual environment, initialize it, install packages, and remind user to activate after make is done"
click.secho(msg, fg=COLOR_SUCCESS)
_cmd = r"""
pip-compile --output-file requirements.txt requirements.in --upgrade
"""
if verbose >= 1:
msg = "[pip-deps] Install dependencies: "
click.secho(msg, fg=COLOR_SUCCESS)
msg = "{}".format(_cmd)
click.secho(msg, fg=COLOR_SUCCESS)
ctx.run(_cmd)
def black(ctx, loc="local", check=True, debug=False, verbose=0):
"""
Run black code formatter
Usage: inv ci.black
"""
env = get_compose_env(ctx, loc=loc)
# Only display result
ctx.config["run"]["echo"] = True
# Override run commands env variables one key at a time
for k, v in env.items():
ctx.config["run"]["env"][k] = v
_black_excludes = r"/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|_build|buck-out|dist|venv*)/"
_cmd = ""
if check:
_cmd = "black --check --exclude=venv* --verbose main.py application config.py"
else:
if verbose >= 1:
msg = "[black] check mode disabled"
click.secho(msg, fg="green")
_cmd = r"black --exclude='{}' --verbose main.py application config.py".format(_black_excludes)
ctx.run(_cmd)
def pylint(ctx, loc="local", tests=False, everything=False, specific=""):
"""
pylint fake-medium-fastapi folder
Usage: inv ci.pylint
"""
env = get_compose_env(ctx, loc=loc)
# Only display result
ctx.config["run"]["echo"] = True
# Override run commands env variables one key at a time
for k, v in env.items():
ctx.config["run"]["env"][k] = v
if tests:
ctx.run(
"poetry run pylint --disable=all --enable=F,E --rcfile ./lint-configs-python/python/pylintrc tests"
)
elif everything:
ctx.run(
"poetry run pylint --rcfile ./lint-configs-python/python/pylintrc tests app"
)
elif specific:
ctx.run(
f"poetry run pylint --disable=all --enable={specific} --rcfile ./lint-configs-python/python/pylintrc tests app"
)
else:
ctx.run(
"poetry run pylint --disable=all --enable=F,E --rcfile ./lint-configs-python/python/pylintrc app"
)
def postman(ctx, loc="local", dry_run=False, verbose=0):
"""
Run integration tests against active api server.
"""
env = get_compose_env(ctx, loc=loc)
# Only display result
ctx.config["run"]["echo"] = True
# Override run commands env variables one key at a time
for k, v in env.items():
ctx.config["run"]["env"][k] = v
_cmd = "poetry run ./postman/run-api-tests.sh"
if verbose >= 1:
msg = "{}".format(_cmd)
click.secho(msg, fg=COLOR_SUCCESS)
if dry_run:
click.secho(
"[postman] DRY RUN mode enabled, not executing command: {}".format(
_cmd),
fg=COLOR_CAUTION,
)
else:
ctx.run(_cmd)
def detect_os(ctx, loc="local", verbose=0):
"""
detect what type of os we are using
Usage: inv local.detect-os
"""
env = get_compose_env(ctx, loc=loc)
# Override run commands' env variables one key at a time
for k, v in env.items():
ctx.config["run"]["env"][k] = v
res_os = ctx.run("uname -s")
ctx.config["run"]["env"]["OS"] = "{}".format(res_os.stdout)
if ctx.config["run"]["env"]["OS"] == "Windows_NT":
ctx.config["run"]["env"]["DETECTED_OS"] = "Windows"
else:
ctx.config["run"]["env"]["DETECTED_OS"] = ctx.config["run"]["env"][
"OS"]
if verbose >= 1:
msg = "[detect-os] Detected: {}".format(
ctx.config["run"]["env"]["DETECTED_OS"])
click.secho(msg, fg=COLOR_SUCCESS)
if ctx.config["run"]["env"]["DETECTED_OS"] == "Darwin":
ctx.config["run"]["env"]["ARCHFLAGS"] = "-arch x86_64"
ctx.config["run"]["env"][
"PKG_CONFIG_PATH"] = "/usr/local/opt/libffi/lib/pkgconfig"
ctx.config["run"]["env"]["LDFLAGS"] = "-L/usr/local/opt/openssl/lib"
ctx.config["run"]["env"]["CFLAGS"] = "-I/usr/local/opt/openssl/include"
def travis(ctx, loc="local", check=True, debug=False, verbose=0):
"""
Run travis
Usage: inv ci.travis
"""
env = get_compose_env(ctx, loc=loc)
# Only display result
ctx.config["run"]["echo"] = True
# Override run commands env variables one key at a time
for k, v in env.items():
ctx.config["run"]["env"][k] = v
if verbose >= 1:
msg = "[travis] check mode disabled"
click.secho(msg, fg="green")
# _cmd = r"""
# mv -f .coverage .coverage.tests || true
# coverage combine
# """
_cmd = r"""
pytest
"""
ctx.run(_cmd)
def preflight(ctx, loc="docker", image=CI_IMAGE, verbose=0):
"""
Configure env for travis builds
Usage: inv travis.env-set
"""
env = get_compose_env(ctx, loc=loc)
# Override run commands' env variables one key at a time
for k, v in env.items():
ctx.config["run"]["env"][k] = v
_VALIDATE_ENVS = [
"CI_IMAGE",
"CACHE_DIR",
"CACHE_FILE_BASE",
"CACHE_FILE_RUNTIME",
"IS_CI_ENVIRONMENT",
"CONTAINER_UID",
"CONTAINER_GID",
"TAG",
"STANDARD_CACHE_DIR",
"WHEELHOUSE",
"PIP_WHEEL_DIR",
]
# Verify everything has been set that we care about.
# TODO: Move this into its own private task, so it can be called from anywhere
for v in _VALIDATE_ENVS:
assert ctx.config["run"]["env"][v]
if verbose >= 1:
msg = "[preflight] {}={}".format(v, ctx.config["run"]["env"][v])
click.secho(msg, fg=COLOR_SUCCESS)
def setup_jupyter(ctx, loc="local", verbose=0, cleanup=False):
"""
Setup jupyter notebook
Usage: inv local.setup-jupyter
"""
env = get_compose_env(ctx, loc=loc)
# Override run commands' env variables one key at a time
for k, v in env.items():
ctx.config["run"]["env"][k] = v
msg = "[setup-jupyter] installing dependencies"
click.secho(msg, fg=COLOR_SUCCESS)
_cmd = r"""
pip install jupyter
pip install ipython-sql cython
python -m ipykernel install --user
pip install jupyter_nbextensions_configurator jupyter_contrib_nbextensions
jupyter contrib nbextension install --user
jupyter nbextensions_configurator enable --user
pip install ipywidgets
jupyter nbextension enable --py widgetsnbextension --sys-prefix
"""
if verbose >= 1:
msg = "{}".format(_cmd)
click.secho(msg, fg=COLOR_SUCCESS)
ctx.run(_cmd)
def contrib(ctx, loc="local", verbose=0, cleanup=False):
"""
Install contrib files in correct places
Usage: inv local.contrib
"""
env = get_compose_env(ctx, loc=loc)
# Override run commands' env variables one key at a time
for k, v in env.items():
ctx.config["run"]["env"][k] = v
if verbose >= 1:
msg = "[contrib] Create virtual environment, initialize it, install packages, and remind user to activate after make is done"
click.secho(msg, fg=COLOR_SUCCESS)
_cmd = r"""
cp -fv ./contrib/.pdbrc ~/.pdbrc
cp -fv ./contrib/.pdbrc.py ~/.pdbrc.py
mkdir -p ~/ptpython/ || true
cp -fv ./contrib/.ptpython_config.py ~/ptpython/config.py
"""
if verbose >= 1:
msg = "[contrib] Install configs: "
click.secho(msg, fg=COLOR_SUCCESS)
msg = "{}".format(_cmd)
click.secho(msg, fg=COLOR_SUCCESS)
ctx.run(_cmd)
def dev(ctx, loc="local", verbose=0, cleanup=False):
"""
install dev version of application
Usage: inv local.dev
"""
env = get_compose_env(ctx, loc=loc)
# Override run commands' env variables one key at a time
for k, v in env.items():
ctx.config["run"]["env"][k] = v
_cmd = r"""
pip install -e .
"""
if verbose >= 1:
msg = "{}".format(_cmd)
click.secho(msg, fg=COLOR_SUCCESS)
ctx.run(_cmd)
msg = "[dev] testing cli works"
click.secho(msg, fg=COLOR_SUCCESS)
ctx.run("ultronctl --help")
def pr_sha(ctx, loc="local", quiet=False, verbose=0):
"""
Return `git rev-parse HEAD` for project.
Usage: inv docker.lint-test or inv local.lint-test
"""
env = get_compose_env(ctx, loc=loc)
# Only display result
ctx.config["run"]["echo"] = False
# Override run commands env variables one key at a time
for k, v in env.items():
ctx.config["run"]["env"][k] = v
res = ctx.run("git rev-parse HEAD")
# override CI_IMAGE value
ctx.config["run"]["env"]["PR_SHA"] = "{}".format(res.stdout)
ctx.config["run"]["env"]["REPO_NAME"] = "bossjones/fake-medium-fastapi-ci"
ctx.config["run"]["env"]["IMAGE_TAG"] = "{}:{}".format(
ctx.config["run"]["env"]["REPO_NAME"],
ctx.config["run"]["env"]["PR_SHA"])
ctx.config["run"]["env"]["TAG"] = ctx.config["run"]["env"]["IMAGE_TAG"]
if verbose >= 1:
msg = "[PR_SHA] {}".format(ctx.config["run"]["env"]["PR_SHA"])
click.secho(msg, fg="green")
def pstree(ctx, loc="local", verbose=0, cleanup=False):
"""
Run jupyter notebook
Usage: inv local.pstree
"""
env = get_compose_env(ctx, loc=loc)
# Override run commands' env variables one key at a time
for k, v in env.items():
ctx.config["run"]["env"][k] = v
# SOURCE: https://wilsonmar.github.io/ports-open/
# lsof -nP +c 15 | grep LISTEN
_cmd = r"""
ps aux | grep 'python fake-medium-fastapi/web.py' | awk '{print $2}' | head -1
"""
if verbose >= 1:
msg = "{}".format(_cmd)
click.secho(msg, fg=COLOR_SUCCESS)
pid_res = ctx.run(_cmd)
_cmd = r"""
pstree -p {}
""".format(pid_res.stdout)
if verbose >= 1:
msg = "{}".format(_cmd)
click.secho(msg, fg=COLOR_SUCCESS)
ctx.run(_cmd)
def jupyter(ctx, loc="local", verbose=0, cleanup=False):
"""
Run jupyter notebook
Usage: inv local.jupyter
"""
env = get_compose_env(ctx, loc=loc)
# Override run commands' env variables one key at a time
for k, v in env.items():
ctx.config["run"]["env"][k] = v
_cmd = r"""
pip install -e .
"""
if verbose >= 1:
msg = "{}".format(_cmd)
click.secho(msg, fg=COLOR_SUCCESS)
ctx.run(_cmd)
msg = "[jupyter] start up notebook"
click.secho(msg, fg=COLOR_SUCCESS)
msg = "[jupyter] Great guide to jupyter here: https://www.datacamp.com/community/tutorials/tutorial-jupyter-notebook"
click.secho(msg, fg=COLOR_SUCCESS)
ctx.run("jupyter notebook")
def clean_pyi(ctx, loc="local", verbose=0, dry_run=False):
"""
Clean all stub files
To clean stubs:
Usage: inv ci.clean-pyi -vvv
To clean stubs(dry run):
Usage: inv ci.clean-pyi -vvv --dry-run
"""
env = get_compose_env(ctx, loc=loc)
# Only display result
ctx.config["run"]["echo"] = True
# Override run commands' env variables one key at a time
for k, v in env.items():
ctx.config["run"]["env"][k] = v
# NOTE: https://monkeytype.readthedocs.io/en/stable/faq.html#why-did-my-test-coverage-measurement-stop-working
_cmd = r"""find . -name '*.pyi' -exec rm -fv {} +"""
if verbose >= 1:
msg = "{}".format(_cmd)
click.secho(msg, fg=COLOR_SUCCESS)
if dry_run:
click.secho(
"[monkeytype] DRY RUN mode enabled, not executing command: {}".
format(_cmd),
fg=COLOR_CAUTION,
)
else:
ctx.run(_cmd)
def bootstrap(ctx, loc="local", verbose=0, cleanup=False):
"""
start up flask application
Usage: inv local.serve
"""
env = get_compose_env(ctx, loc=loc)
# Override run commands' env variables one key at a time
for k, v in env.items():
ctx.config["run"]["env"][k] = v
if verbose >= 1:
msg = "[install] Create virtual environment, initialize it, install packages, and remind user to activate after make is done"
click.secho(msg, fg=COLOR_SUCCESS)
# pip install pre-commit
# pre-commit install -f --install-hooks
_cmd = r"""
pip install -r requirements.txt
"""
if verbose >= 1:
msg = "[install] Install dependencies: "
click.secho(msg, fg=COLOR_SUCCESS)
msg = "{}".format(_cmd)
click.secho(msg, fg=COLOR_SUCCESS)
ctx.run(_cmd)
def serve_daemon(ctx):
"""
Start up fastapi server (daemonized)
Usage: inv docker.serve-daemon or inv local.serve-daemon
"""
env = get_compose_env(ctx)
ctx.run(f"bash script/serve-daemon", env=env)
def initial_data(ctx):
"""
Seed db data
Usage: inv docker.initial-data or inv local.initial-data
"""
env = get_compose_env(ctx)
ctx.run(f"python ultron8/initial_data.py", env=env)
def backend_pre_start(ctx):
"""
Docker backend steps
Usage: inv docker.backend-pre-start or inv local.backend-pre-start
"""
env = get_compose_env(ctx)
ctx.run(f"python ultron8/api/backend_pre_start.py", env=env)
def serve(ctx):
"""
Start up fastapi server
Usage: inv docker.serve or inv local.serve
"""
env = get_compose_env(ctx)
ctx.run(f"bash script/serve", env=env)
def setup_test(ctx):
"""
Setup python tests
Usage: inv docker.setup-test or inv local.setup-test
"""
env = get_compose_env(ctx)
ctx.run(f"python setup.py test", env=env)
def autoflake(
ctx,
loc="local",
verbose=0,
check=False,
dry_run=False,
in_place=False,
remove_all_unused_imports=False,
):
"""
Use autoflake to remove unused imports, recursively, remove unused variables, and exclude __init__.py
To run autoflake in check only mode:
Usage: inv ci.autoflake --check -vvv
To run autoflake in check only mode(dry-run):
Usage: inv ci.autoflake --check -vvv --dry-run
To run autoflake inplace w/(dry-run):
Usage: inv ci.autoflake --in-place -vvv --dry-run
"""
env = get_compose_env(ctx, loc=loc)
# Only display result
ctx.config["run"]["echo"] = True
# Override run commands' env variables one key at a time
for k, v in env.items():
ctx.config["run"]["env"][k] = v
# To remove all unused imports (whether or not they are from the standard library), use the --remove-all-unused-imports option.
_cmd = "poetry run autoflake"
_cmd += " --recursive --remove-unused-variables"
if remove_all_unused_imports:
_cmd += " --remove-all-unused-imports "
if check:
_cmd += " --check"
if in_place:
_cmd += " --in-place"
_cmd += " --exclude=__init__.py"
_cmd += " app"
_cmd += " tests"
_cmd += " tasks"
if verbose >= 1:
msg = "{}".format(_cmd)
click.secho(msg, fg=COLOR_SUCCESS)
if dry_run:
click.secho(
"[autoflake] DRY RUN mode enabled, not executing command: {}".
format(_cmd),
fg=COLOR_CAUTION,
)
else:
ctx.run(_cmd)
def lint_tests(ctx):
"""
Check Python code style
Usage: inv docker.lint-test or inv local.lint-test
"""
env = get_compose_env(ctx)
ctx.run(
f"/usr/local/bin/flake8 --ignore=E501,W503 src/ tasks/ tests/ scripts/ migrations/",
env=env,
)
def coverage_run(ctx):
"""
Check Python code coverage
Usage: inv docker.coverage-run or inv local.coverage-run
"""
env = get_compose_env(ctx)
ctx.run(
f"coverage run --source=ultron8/ setup.py tests; coverage report --show-missing; coverage html",
env=env,
)
def test(ctx):
"""
Run pytest
Usage: inv docker.test or inv local.test
"""
env = get_compose_env(ctx)
ctx.run(
f"py.test --cov-config .coveragerc --verbose --cov-report term --cov-report xml --cov=ultron8 tests",
env=env,
)
def run_pytest(ctx):
"""
Pytest Runner w/ coverage
Usage: inv docker.run-pytest or inv local.run-pytest
"""
env = get_compose_env(ctx)
ctx.run(
f"pytest -s --tb short --cov-config .coveragerc --cov ultron8 tests --cov-report term-missing --cov-report xml:cov.xml --cov-report html:htmlcov --cov-report annotate:cov_annotate",
env=env,
)
def test_pdb(ctx):
"""
Run pytest with pdb enabled
Usage: inv docker.test-pdb or inv local.test-pdb
"""
env = get_compose_env(ctx)
ctx.run(
f"py.test --cov-config .coveragerc --verbose --pdb --showlocals --cov-report term --cov-report xml --cov=ultron8 tests",
env=env,
)
