def black(ctx, loc="local", check=False, debug=False, verbose=0, tests=False): """ Run black code formatter Usage: inv ci.black """ env = get_compose_env(ctx, loc=loc) # Only display result ctx.config["run"]["echo"] = True # Override run commands env variables one key at a time for k, v in env.items(): ctx.config["run"]["env"][k] = v _cmd = "poetry run black " if check: _cmd += "--check " if verbose >= 3: _cmd += "--verbose " if tests: _cmd += "tests tasks " _cmd += "app" if verbose >= 1: msg = "[black] bout to run command: \n" click.secho(msg, fg="green") click.secho(_cmd, fg="green") ctx.run(_cmd)
def gunzip_save(ctx, loc="docker", image=CI_IMAGE, verbose=0): """ take contents of dockerfile and cache it locally into $HOME/.cache/docker Usage: inv travis.gunzip-save """ env = get_compose_env(ctx, loc=loc) # Override run commands' env variables one key at a time for k, v in env.items(): ctx.config["run"]["env"][k] = v if verbose >= 1: click.secho(" [gunzip-save] travis 'install:' section", fg=COLOR_SUCCESS) click.secho( " [gunzip-save] pull docker cache from docker hub if available", fg=COLOR_SUCCESS, ) _cmd = r""" mkdir -p $CACHE_DIR if [ ! -f ${CACHE_FILE_BASE} ]; then docker save $REPO_NAME:base | gzip > ${CACHE_FILE_BASE} || true; fi if [ ! -f ${CACHE_FILE_RUNTIME} ]; then docker save $REPO_NAME:runtime-image | gzip > ${CACHE_FILE_RUNTIME} || true; fi """ if verbose >= 1: msg = "[build-gunzip-before-install] cmd is: " click.secho(msg, fg=COLOR_SUCCESS) msg = "{}".format(_cmd) click.secho(msg, fg=COLOR_SUCCESS) ctx.run(_cmd)
def coverage_clean(ctx, loc="local", verbose=0, cleanup=False): """ clean coverage files Usage: inv ci.coverage-clean """ env = get_compose_env(ctx, loc=loc) # Override run commands' env variables one key at a time for k, v in env.items(): ctx.config["run"]["env"][k] = v _cmd = r""" find . -name '*.pyc' -exec rm -fv {} + find . -name '*.pyo' -exec rm -fv {} + find . -name '__pycache__' -exec rm -frv {} + rm -f .coverage rm -rf htmlcov/* rm -rf cov_annotate/* rm -f cov.xml """ if verbose >= 1: msg = "{}".format(_cmd) click.secho(msg, fg=COLOR_SUCCESS) ctx.run(_cmd)
def build_gunzip_before_install(ctx, loc="docker", image=CI_IMAGE, verbose=0): """ docker container/image from gnzip tar file in $HOME/.cache/docker Usage: inv travis.build_gunzip_before_install """ env = get_compose_env(ctx, loc=loc) # Override run commands' env variables one key at a time for k, v in env.items(): ctx.config["run"]["env"][k] = v if verbose >= 1: msg = "[build-gunzip-before-install] Building using docker-compose" click.secho(msg, fg=COLOR_SUCCESS) _cmd = r""" if [ -f ${CACHE_FILE_BASE} ]; then gunzip -c ${CACHE_FILE_BASE} | docker load || true; fi if [ -f ${CACHE_FILE_RUNTIME} ]; then gunzip -c ${CACHE_FILE_RUNTIME} | docker load || true; fi """ if verbose >= 1: msg = "[build-gunzip-before-install] cmd is: " click.secho(msg, fg=COLOR_SUCCESS) msg = "{}".format(_cmd) click.secho(msg, fg=COLOR_SUCCESS) ctx.run(_cmd)
def pip_deps(ctx, loc="local", verbose=0, cleanup=False): """ lock flask pip dependencies Usage: inv local.pip_deps """ env = get_compose_env(ctx, loc=loc) # Override run commands' env variables one key at a time for k, v in env.items(): ctx.config["run"]["env"][k] = v if verbose >= 1: msg = "[pip-deps] Create virtual environment, initialize it, install packages, and remind user to activate after make is done" click.secho(msg, fg=COLOR_SUCCESS) _cmd = r""" pip-compile --output-file requirements.txt requirements.in --upgrade """ if verbose >= 1: msg = "[pip-deps] Install dependencies: " click.secho(msg, fg=COLOR_SUCCESS) msg = "{}".format(_cmd) click.secho(msg, fg=COLOR_SUCCESS) ctx.run(_cmd)
def black(ctx, loc="local", check=True, debug=False, verbose=0): """ Run black code formatter Usage: inv ci.black """ env = get_compose_env(ctx, loc=loc) # Only display result ctx.config["run"]["echo"] = True # Override run commands env variables one key at a time for k, v in env.items(): ctx.config["run"]["env"][k] = v _black_excludes = r"/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|_build|buck-out|dist|venv*)/" _cmd = "" if check: _cmd = "black --check --exclude=venv* --verbose main.py application config.py" else: if verbose >= 1: msg = "[black] check mode disabled" click.secho(msg, fg="green") _cmd = r"black --exclude='{}' --verbose main.py application config.py".format(_black_excludes) ctx.run(_cmd)
def pylint(ctx, loc="local", tests=False, everything=False, specific=""): """ pylint fake-medium-fastapi folder Usage: inv ci.pylint """ env = get_compose_env(ctx, loc=loc) # Only display result ctx.config["run"]["echo"] = True # Override run commands env variables one key at a time for k, v in env.items(): ctx.config["run"]["env"][k] = v if tests: ctx.run( "poetry run pylint --disable=all --enable=F,E --rcfile ./lint-configs-python/python/pylintrc tests" ) elif everything: ctx.run( "poetry run pylint --rcfile ./lint-configs-python/python/pylintrc tests app" ) elif specific: ctx.run( f"poetry run pylint --disable=all --enable={specific} --rcfile ./lint-configs-python/python/pylintrc tests app" ) else: ctx.run( "poetry run pylint --disable=all --enable=F,E --rcfile ./lint-configs-python/python/pylintrc app" )
def postman(ctx, loc="local", dry_run=False, verbose=0): """ Run integration tests against active api server. """ env = get_compose_env(ctx, loc=loc) # Only display result ctx.config["run"]["echo"] = True # Override run commands env variables one key at a time for k, v in env.items(): ctx.config["run"]["env"][k] = v _cmd = "poetry run ./postman/run-api-tests.sh" if verbose >= 1: msg = "{}".format(_cmd) click.secho(msg, fg=COLOR_SUCCESS) if dry_run: click.secho( "[postman] DRY RUN mode enabled, not executing command: {}".format( _cmd), fg=COLOR_CAUTION, ) else: ctx.run(_cmd)
def detect_os(ctx, loc="local", verbose=0): """ detect what type of os we are using Usage: inv local.detect-os """ env = get_compose_env(ctx, loc=loc) # Override run commands' env variables one key at a time for k, v in env.items(): ctx.config["run"]["env"][k] = v res_os = ctx.run("uname -s") ctx.config["run"]["env"]["OS"] = "{}".format(res_os.stdout) if ctx.config["run"]["env"]["OS"] == "Windows_NT": ctx.config["run"]["env"]["DETECTED_OS"] = "Windows" else: ctx.config["run"]["env"]["DETECTED_OS"] = ctx.config["run"]["env"][ "OS"] if verbose >= 1: msg = "[detect-os] Detected: {}".format( ctx.config["run"]["env"]["DETECTED_OS"]) click.secho(msg, fg=COLOR_SUCCESS) if ctx.config["run"]["env"]["DETECTED_OS"] == "Darwin": ctx.config["run"]["env"]["ARCHFLAGS"] = "-arch x86_64" ctx.config["run"]["env"][ "PKG_CONFIG_PATH"] = "/usr/local/opt/libffi/lib/pkgconfig" ctx.config["run"]["env"]["LDFLAGS"] = "-L/usr/local/opt/openssl/lib" ctx.config["run"]["env"]["CFLAGS"] = "-I/usr/local/opt/openssl/include"
def travis(ctx, loc="local", check=True, debug=False, verbose=0): """ Run travis Usage: inv ci.travis """ env = get_compose_env(ctx, loc=loc) # Only display result ctx.config["run"]["echo"] = True # Override run commands env variables one key at a time for k, v in env.items(): ctx.config["run"]["env"][k] = v if verbose >= 1: msg = "[travis] check mode disabled" click.secho(msg, fg="green") # _cmd = r""" # mv -f .coverage .coverage.tests || true # coverage combine # """ _cmd = r""" pytest """ ctx.run(_cmd)
def preflight(ctx, loc="docker", image=CI_IMAGE, verbose=0): """ Configure env for travis builds Usage: inv travis.env-set """ env = get_compose_env(ctx, loc=loc) # Override run commands' env variables one key at a time for k, v in env.items(): ctx.config["run"]["env"][k] = v _VALIDATE_ENVS = [ "CI_IMAGE", "CACHE_DIR", "CACHE_FILE_BASE", "CACHE_FILE_RUNTIME", "IS_CI_ENVIRONMENT", "CONTAINER_UID", "CONTAINER_GID", "TAG", "STANDARD_CACHE_DIR", "WHEELHOUSE", "PIP_WHEEL_DIR", ] # Verify everything has been set that we care about. # TODO: Move this into its own private task, so it can be called from anywhere for v in _VALIDATE_ENVS: assert ctx.config["run"]["env"][v] if verbose >= 1: msg = "[preflight] {}={}".format(v, ctx.config["run"]["env"][v]) click.secho(msg, fg=COLOR_SUCCESS)
def setup_jupyter(ctx, loc="local", verbose=0, cleanup=False): """ Setup jupyter notebook Usage: inv local.setup-jupyter """ env = get_compose_env(ctx, loc=loc) # Override run commands' env variables one key at a time for k, v in env.items(): ctx.config["run"]["env"][k] = v msg = "[setup-jupyter] installing dependencies" click.secho(msg, fg=COLOR_SUCCESS) _cmd = r""" pip install jupyter pip install ipython-sql cython python -m ipykernel install --user pip install jupyter_nbextensions_configurator jupyter_contrib_nbextensions jupyter contrib nbextension install --user jupyter nbextensions_configurator enable --user pip install ipywidgets jupyter nbextension enable --py widgetsnbextension --sys-prefix """ if verbose >= 1: msg = "{}".format(_cmd) click.secho(msg, fg=COLOR_SUCCESS) ctx.run(_cmd)
def contrib(ctx, loc="local", verbose=0, cleanup=False): """ Install contrib files in correct places Usage: inv local.contrib """ env = get_compose_env(ctx, loc=loc) # Override run commands' env variables one key at a time for k, v in env.items(): ctx.config["run"]["env"][k] = v if verbose >= 1: msg = "[contrib] Create virtual environment, initialize it, install packages, and remind user to activate after make is done" click.secho(msg, fg=COLOR_SUCCESS) _cmd = r""" cp -fv ./contrib/.pdbrc ~/.pdbrc cp -fv ./contrib/.pdbrc.py ~/.pdbrc.py mkdir -p ~/ptpython/ || true cp -fv ./contrib/.ptpython_config.py ~/ptpython/config.py """ if verbose >= 1: msg = "[contrib] Install configs: " click.secho(msg, fg=COLOR_SUCCESS) msg = "{}".format(_cmd) click.secho(msg, fg=COLOR_SUCCESS) ctx.run(_cmd)
def dev(ctx, loc="local", verbose=0, cleanup=False): """ install dev version of application Usage: inv local.dev """ env = get_compose_env(ctx, loc=loc) # Override run commands' env variables one key at a time for k, v in env.items(): ctx.config["run"]["env"][k] = v _cmd = r""" pip install -e . """ if verbose >= 1: msg = "{}".format(_cmd) click.secho(msg, fg=COLOR_SUCCESS) ctx.run(_cmd) msg = "[dev] testing cli works" click.secho(msg, fg=COLOR_SUCCESS) ctx.run("ultronctl --help")
def pr_sha(ctx, loc="local", quiet=False, verbose=0): """ Return `git rev-parse HEAD` for project. Usage: inv docker.lint-test or inv local.lint-test """ env = get_compose_env(ctx, loc=loc) # Only display result ctx.config["run"]["echo"] = False # Override run commands env variables one key at a time for k, v in env.items(): ctx.config["run"]["env"][k] = v res = ctx.run("git rev-parse HEAD") # override CI_IMAGE value ctx.config["run"]["env"]["PR_SHA"] = "{}".format(res.stdout) ctx.config["run"]["env"]["REPO_NAME"] = "bossjones/fake-medium-fastapi-ci" ctx.config["run"]["env"]["IMAGE_TAG"] = "{}:{}".format( ctx.config["run"]["env"]["REPO_NAME"], ctx.config["run"]["env"]["PR_SHA"]) ctx.config["run"]["env"]["TAG"] = ctx.config["run"]["env"]["IMAGE_TAG"] if verbose >= 1: msg = "[PR_SHA] {}".format(ctx.config["run"]["env"]["PR_SHA"]) click.secho(msg, fg="green")
def pstree(ctx, loc="local", verbose=0, cleanup=False): """ Run jupyter notebook Usage: inv local.pstree """ env = get_compose_env(ctx, loc=loc) # Override run commands' env variables one key at a time for k, v in env.items(): ctx.config["run"]["env"][k] = v # SOURCE: https://wilsonmar.github.io/ports-open/ # lsof -nP +c 15 | grep LISTEN _cmd = r""" ps aux | grep 'python fake-medium-fastapi/web.py' | awk '{print $2}' | head -1 """ if verbose >= 1: msg = "{}".format(_cmd) click.secho(msg, fg=COLOR_SUCCESS) pid_res = ctx.run(_cmd) _cmd = r""" pstree -p {} """.format(pid_res.stdout) if verbose >= 1: msg = "{}".format(_cmd) click.secho(msg, fg=COLOR_SUCCESS) ctx.run(_cmd)
def jupyter(ctx, loc="local", verbose=0, cleanup=False): """ Run jupyter notebook Usage: inv local.jupyter """ env = get_compose_env(ctx, loc=loc) # Override run commands' env variables one key at a time for k, v in env.items(): ctx.config["run"]["env"][k] = v _cmd = r""" pip install -e . """ if verbose >= 1: msg = "{}".format(_cmd) click.secho(msg, fg=COLOR_SUCCESS) ctx.run(_cmd) msg = "[jupyter] start up notebook" click.secho(msg, fg=COLOR_SUCCESS) msg = "[jupyter] Great guide to jupyter here: https://www.datacamp.com/community/tutorials/tutorial-jupyter-notebook" click.secho(msg, fg=COLOR_SUCCESS) ctx.run("jupyter notebook")
def clean_pyi(ctx, loc="local", verbose=0, dry_run=False): """ Clean all stub files To clean stubs: Usage: inv ci.clean-pyi -vvv To clean stubs(dry run): Usage: inv ci.clean-pyi -vvv --dry-run """ env = get_compose_env(ctx, loc=loc) # Only display result ctx.config["run"]["echo"] = True # Override run commands' env variables one key at a time for k, v in env.items(): ctx.config["run"]["env"][k] = v # NOTE: https://monkeytype.readthedocs.io/en/stable/faq.html#why-did-my-test-coverage-measurement-stop-working _cmd = r"""find . -name '*.pyi' -exec rm -fv {} +""" if verbose >= 1: msg = "{}".format(_cmd) click.secho(msg, fg=COLOR_SUCCESS) if dry_run: click.secho( "[monkeytype] DRY RUN mode enabled, not executing command: {}". format(_cmd), fg=COLOR_CAUTION, ) else: ctx.run(_cmd)
def bootstrap(ctx, loc="local", verbose=0, cleanup=False): """ start up flask application Usage: inv local.serve """ env = get_compose_env(ctx, loc=loc) # Override run commands' env variables one key at a time for k, v in env.items(): ctx.config["run"]["env"][k] = v if verbose >= 1: msg = "[install] Create virtual environment, initialize it, install packages, and remind user to activate after make is done" click.secho(msg, fg=COLOR_SUCCESS) # pip install pre-commit # pre-commit install -f --install-hooks _cmd = r""" pip install -r requirements.txt """ if verbose >= 1: msg = "[install] Install dependencies: " click.secho(msg, fg=COLOR_SUCCESS) msg = "{}".format(_cmd) click.secho(msg, fg=COLOR_SUCCESS) ctx.run(_cmd)
def serve_daemon(ctx): """ Start up fastapi server (daemonized) Usage: inv docker.serve-daemon or inv local.serve-daemon """ env = get_compose_env(ctx) ctx.run(f"bash script/serve-daemon", env=env)
def initial_data(ctx): """ Seed db data Usage: inv docker.initial-data or inv local.initial-data """ env = get_compose_env(ctx) ctx.run(f"python ultron8/initial_data.py", env=env)
def backend_pre_start(ctx): """ Docker backend steps Usage: inv docker.backend-pre-start or inv local.backend-pre-start """ env = get_compose_env(ctx) ctx.run(f"python ultron8/api/backend_pre_start.py", env=env)
def serve(ctx): """ Start up fastapi server Usage: inv docker.serve or inv local.serve """ env = get_compose_env(ctx) ctx.run(f"bash script/serve", env=env)
def setup_test(ctx): """ Setup python tests Usage: inv docker.setup-test or inv local.setup-test """ env = get_compose_env(ctx) ctx.run(f"python setup.py test", env=env)
def autoflake( ctx, loc="local", verbose=0, check=False, dry_run=False, in_place=False, remove_all_unused_imports=False, ): """ Use autoflake to remove unused imports, recursively, remove unused variables, and exclude __init__.py To run autoflake in check only mode: Usage: inv ci.autoflake --check -vvv To run autoflake in check only mode(dry-run): Usage: inv ci.autoflake --check -vvv --dry-run To run autoflake inplace w/(dry-run): Usage: inv ci.autoflake --in-place -vvv --dry-run """ env = get_compose_env(ctx, loc=loc) # Only display result ctx.config["run"]["echo"] = True # Override run commands' env variables one key at a time for k, v in env.items(): ctx.config["run"]["env"][k] = v # To remove all unused imports (whether or not they are from the standard library), use the --remove-all-unused-imports option. _cmd = "poetry run autoflake" _cmd += " --recursive --remove-unused-variables" if remove_all_unused_imports: _cmd += " --remove-all-unused-imports " if check: _cmd += " --check" if in_place: _cmd += " --in-place" _cmd += " --exclude=__init__.py" _cmd += " app" _cmd += " tests" _cmd += " tasks" if verbose >= 1: msg = "{}".format(_cmd) click.secho(msg, fg=COLOR_SUCCESS) if dry_run: click.secho( "[autoflake] DRY RUN mode enabled, not executing command: {}". format(_cmd), fg=COLOR_CAUTION, ) else: ctx.run(_cmd)
def lint_tests(ctx): """ Check Python code style Usage: inv docker.lint-test or inv local.lint-test """ env = get_compose_env(ctx) ctx.run( f"/usr/local/bin/flake8 --ignore=E501,W503 src/ tasks/ tests/ scripts/ migrations/", env=env, )
def coverage_run(ctx): """ Check Python code coverage Usage: inv docker.coverage-run or inv local.coverage-run """ env = get_compose_env(ctx) ctx.run( f"coverage run --source=ultron8/ setup.py tests; coverage report --show-missing; coverage html", env=env, )
def test(ctx): """ Run pytest Usage: inv docker.test or inv local.test """ env = get_compose_env(ctx) ctx.run( f"py.test --cov-config .coveragerc --verbose --cov-report term --cov-report xml --cov=ultron8 tests", env=env, )
def run_pytest(ctx): """ Pytest Runner w/ coverage Usage: inv docker.run-pytest or inv local.run-pytest """ env = get_compose_env(ctx) ctx.run( f"pytest -s --tb short --cov-config .coveragerc --cov ultron8 tests --cov-report term-missing --cov-report xml:cov.xml --cov-report html:htmlcov --cov-report annotate:cov_annotate", env=env, )
def test_pdb(ctx): """ Run pytest with pdb enabled Usage: inv docker.test-pdb or inv local.test-pdb """ env = get_compose_env(ctx) ctx.run( f"py.test --cov-config .coveragerc --verbose --pdb --showlocals --cov-report term --cov-report xml --cov=ultron8 tests", env=env, )