def run_format(ctx, check=False, quiet=False):
"""Run black code formatter on source code.
:param [invoke.Context] ctx: Invoke context.
:param [bool] check: Only check formatting, don't reformat the code.
:param [bool] quiet: Don't show progress. Only errors.
:raises UnexpectedExit: On formatter failure.
"""
print_header("Formatting code", icon="🖤")
flags = []
if check:
flags.append("--check")
if quiet:
flags.append("--quiet")
dirs = f"{PROJECT_INFO.source_directory} {PROJECT_INFO.tests_directory} {PROJECT_INFO.tasks_directory}"
cmd = f"black {dirs} " + " ".join(flags)
result: Result = ctx.run(cmd, pty=True, warn=True)
if result.return_code == 1 and check:
cprint(
"Code was not formatted before commit. Try following:\n"
" * Enable pre-commit hook by running `pre-commit install` in the repository.\n"
" * Run formatter manually with `pipenv run inv format` before committing code.",
color="red",
)
raise UnexpectedExit(result)
if result.return_code > 1:
raise UnexpectedExit(result)
def _lint_pycodestyle(ctx: Context, path: Optional[List[str]] = None):
"""Run PEP8 checking on code; this includes primary code (source) and secondary code (tests, tasks, etc.).
PEP8 checking is done via pycodestyle.
Args:
ctx: Context
path: Path override. Run tests only on given paths.
"""
# Why pycodestyle and pylint? So far, pylint does not check against every convention in PEP8. As pylint's
# functionality grows, we should move all PEP8 checking to pylint and remove pycodestyle
print_header("code style (PEP8)", level=2)
ensure_reports_dir()
paths = to_pathlib_path(path, [
PROJECT_INFO.source_directory, PROJECT_INFO.tests_directory,
PROJECT_INFO.tasks_directory
])
try:
ctx.run(
f"pycodestyle --ignore=E501,W503,E231 --exclude=.svn,CVS,.bzr,.hg,.git,__pycache__,.tox,*_config_parser.py "
f"{paths_to_str(paths)} > {REPORT_PYCODESTYLE_FPATH}")
# Ignores explained:
# - E501: Line length is checked by PyLint
# - W503: Disable checking of "Line break before binary operator". PEP8 recently (~2019) switched to
# "line break before the operator" style, so we should permit this usage.
# - E231: "missing whitespace after ','" is a false positive. Handled by black formatter.
finally:
if os.path.exists(REPORT_PYCODESTYLE_FPATH):
format_messages(read_contents(REPORT_PYCODESTYLE_FPATH))
def _lint_docstyle(ctx: Context, path: Optional[List[str]] = None):
"""Run docstring linting on source code.
Docstring linting is done via pydocstyle. The pydocstyle config can be found in the `.pydocstyle` file.
This ensures compliance with PEP 257, with a few exceptions. Note that pylint also carries out additional
docstyle checks.
Args:
ctx: Context
path: Path override. Run tests only on given paths.
"""
print_header("documentation style", level=2)
ensure_reports_dir()
paths = to_pathlib_path(path, [
PROJECT_INFO.source_directory, PROJECT_INFO.tests_directory,
PROJECT_INFO.tasks_directory
])
try:
ctx.run(
f"pydocstyle {paths_to_str(paths)} > {REPORT_PYDOCSTYLE_FPATH}")
finally:
if os.path.exists(REPORT_PYDOCSTYLE_FPATH):
format_messages(read_contents(REPORT_PYDOCSTYLE_FPATH))
def _lint_pylint(ctx: Context, path: Optional[List[str]] = None):
"""Run pylint on code; this includes primary code (source) and secondary code (tests, tasks, etc.).
The bulk of our code conventions are enforced via pylint. The pylint config can be found in the `.pylintrc` file.
Args:
ctx: Context
path: Path override. Run tests only on given paths.
"""
print_header("pylint", level=2)
paths = to_pathlib_path(path, [
PROJECT_INFO.source_directory, PROJECT_INFO.tests_directory,
PROJECT_INFO.tasks_directory
])
src = PROJECT_INFO.source_directory
grouped_paths = groupby(
paths, lambda current_path: src in current_path.parents or current_path
== src)
for is_source, group in grouped_paths:
if is_source:
run_pylint(ctx, list(group), REPORT_PYLINT_SOURCE_FPATH,
PYLINT_CONFIG_SOURCE_FPATH)
else:
run_pylint(ctx, list(group), REPORT_PYLINT_TESTS_FPATH,
PYLINT_CONFIG_TESTS_FPATH)
def coverage_report(ctx):
"""Analyse coverage and generate a report to term and HTML; from combined unit and integration tests."""
print_header("Generating coverage report", icon="📃")
ensure_reports_dir()
coverage_files = [] # we'll make a copy because `combine` will erase them
for test_type in TestType.__members__.values():
test_conf = test_type.value
if not test_conf.coverage_dat.exists():
cprint(
f"Could not find coverage dat file for {test_conf.name} tests: {test_conf.coverage_dat}",
"yellow")
else:
print(
f"{test_conf.name.title()} test coverage: {get_total_coverage(ctx, test_conf.coverage_dat)}"
)
temp_copy = test_conf.coverage_dat.with_name(
test_conf.coverage_dat.name.replace(".dat", "-copy.dat"))
shutil.copy(test_conf.coverage_dat, temp_copy)
coverage_files.append(str(temp_copy))
ctx.run(f"""
export COVERAGE_FILE="{_COVERAGE_DAT_COMBINED}"
coverage combine {" ".join(coverage_files)}
coverage html -d {_COVERAGE_HTML}
""")
print(
f"Total coverage: {get_total_coverage(ctx, _COVERAGE_DAT_COMBINED)}\n")
print(
f"Refer to coverage report for full analysis in '{_COVERAGE_HTML}/index.html'\n"
f"Or open the report in your default browser with:\n"
f" pipenv run inv coverage-open")
def lint(ctx, path=None):
"""Run linting on the entire code base (source code, tasks, tests).
Args:
ctx (invoke.Context): Context
path (Optional[List[str]]): Path override. Run tests only on given paths.
"""
print_header("Linting", icon="🔎")
# These cannot be pre/post tasks because arguments cannot be passed through.
_lint_pylint(ctx, path)
_lint_pycodestyle(ctx, path)
_lint_docstyle(ctx, path)
def build(ctx):
print_header("Running build", icon="🔨")
for package in os.listdir(PROJECT_INFO.namespace_directory):
print_header(f"Building '{package}' package", level=2)
print("Cleanup the 'build' directory")
shutil.rmtree("build", ignore_errors=True)
ctx.run(
f"python setup.py bdist_wheel --package {package}",
env={"PYTHONPATH": PROJECT_INFO.source_directory},
pty=True,
)
def switch_python_version(ctx, version):
"""Switches the local Python virtual environment to a different Python version.
Use this to test the sub-packages with a different Python version. CI pipeline always
checks all supported versions automatically.
Notes:
This task calls ``deactivate`` as a precaution for cases when the task is called
from an active virtual environment.
Args:
ctx (invoke.Context): Context
version (str): Desired Python version. You can use only MAJOR.MINOR (for example 3.6).
"""
print_header(f"Switching to Python {version}", icon="🐍")
ctx.run(f"deactivate; git clean -fxd .venv && pipenv sync --python {version} -d", pty=True)
def upload(ctx):
print_header("Uploading library to pypi", icon="⬆")
pypi_api_token = os.getenv("PYPI_API_TOKEN")
if not pypi_api_token:
cprint("PYPI_API_TOKEN environment variable to authenticate with PyPI is not set.", color="red")
raise RuntimeError()
if not pypi_api_token.startswith("pypi-"):
pypi_api_token = f"pypi-{pypi_api_token}"
ctx.run(
"twine upload --skip-existing --non-interactive dist/*",
pty=True,
# See https://pypi.org/help/#apitoken
env={"TWINE_USERNAME": "******", "TWINE_PASSWORD": pypi_api_token},
)
def run_pylint(ctx, source_dirs: List[Path], report_path: Path,
pylintrc_fpath: Path):
"""Run pylint with a given configuration on a given code tree and output to a given report file."""
print_header(paths_to_str(source_dirs, ", "), level=3)
ensure_reports_dir()
try:
# pylint won't lint all `.py` files unless they're in a package (`__init__.py` must exist in the same dir)
# see https://github.com/PyCQA/pylint/issues/352
# instead of calling pylint directly, here we use `find` to search for all `py` files, regardless of being in
# # a package
ctx.run(f"export PYTHONPATH={PROJECT_INFO.source_directory}\n"
f'find {paths_to_str(source_dirs)} -type f -name "*.py" | '
f"xargs pylint --rcfile {pylintrc_fpath} > {report_path}")
finally:
if os.path.exists(str(report_path)):
format_messages(read_contents(report_path),
"^.*rated at 10.00/10.*$")
def _run_tests(ctx, test_type: TestType, path: Optional[List[str]] = None):
"""Execute the tests for a given test type."""
print_header(f"️Running {test_type.name} tests️", icon="🔎🐛")
ensure_reports_dir()
paths = to_pathlib_path(path, [test_type.value.directory])
ctx.run(
f"""
{test_type.value.exports}
export PYTHONPATH="$PYTHONPATH:{PROJECT_INFO.source_directory}"
pytest \
--cov={PROJECT_INFO.source_directory} --cov-report="" --cov-branch \
--junitxml={test_type.value.test_report_xml} -vv \
{paths_to_str(paths)}
""",
pty=True,
)
def _typecheck(ctx, paths: List[Path], force_typing=False):
print_header(("Forced" if force_typing else "Optional") + " typing",
level=2)
common_flags = [
"--show-column-numbers",
"--show-error-codes",
"--color-output",
"--warn-unused-config",
"--warn-unused-ignores",
"--follow-imports silent",
f"--junit-xml {_REPORTS_DIR}",
*(["--strict", "--allow-untyped-decorators"] if force_typing else []),
# Untyped decorators are allowed because they may be third party decorators
]
ctx.run(
f"set -o pipefail; mypy {' '.join(common_flags)} {paths_to_str(paths)}",
pty=True)
def install_subpackage_dependencies(ctx, name=None, force=False):
"""Replaces top-level Pipfile dependencies with sub-package dependencies.
Use in CI to install only single sub-package dependencies.
Use without the ``name`` in local development to get dependencies from all sub-packages.
Args:
ctx (invoke.Context): Context
name (Optional[str]): Name of sub-package for which to collect and install dependencies.
If not specified, all sub-packages will be used.
force (bool): Forces reinstall.
"""
print_header("Sub-packages", icon="📦")
print_header("Collecting dependencies", level=2, icon="🛒")
packages = os.listdir(PROJECT_INFO.namespace_directory) if name is None else [name]
all_requirements = ""
libraries_to_sub_packages = defaultdict(list)
for package in packages:
print(f" * Collecting '{package}' package")
with open(PROJECT_INFO.namespace_directory / package / "requirements.txt") as file_pointer:
requirements = file_pointer.read()
all_requirements += requirements
for library in _list_libraries(requirements):
libraries_to_sub_packages[library].append(package)
if not force and _requirements_checksum_not_changed(all_requirements):
return
print_header("Uninstalling previous dependencies", level=2, icon="🔽")
ctx.run("pipenv clean", pty=True)
print_header("Installing new dependencies", level=2, icon="🔼")
conflicts: List[Match] = []
for package in packages:
print_header(package, level=3)
requirements_file_path = PROJECT_INFO.namespace_directory / package / "requirements.txt"
result = ctx.run(f"pipenv run pip install -r {requirements_file_path}", echo=True)
conflicts.extend(match for match in _RE_PIP_CONFLICT.finditer(result.stdout))
_print_conflicting_packages(libraries_to_sub_packages, conflicts)
def typecheck(ctx, path=None):
"""Run type checking on source code.
A non-zero return code from this task indicates invalid types were discovered.
Args:
ctx (invoke.Context): Invoke context.
path (Optional[List[str]]): Path override. Run tests only on given paths.
"""
print_header("RUNNING TYPE CHECKER")
ensure_reports_dir()
src = PROJECT_INFO.source_directory
paths = to_pathlib_path(
path,
[src, PROJECT_INFO.tests_directory, PROJECT_INFO.tasks_directory])
grouped_paths = groupby(
paths, lambda current_path: src in current_path.parents or current_path
== src)
for force_typing, group in grouped_paths:
_typecheck(ctx, list(group), force_typing)
def build(ctx):
print_header("Running build", icon="🔨")
print("Cleanup the 'build' directory")
shutil.rmtree("build", ignore_errors=True)
print_header("Generate requirements.txt file", icon="âš™", level=2)
parsed_pipfile = Pipfile.load("Pipfile")
requirements_file = "\n".join([
"# DON'T CHANGE THIS FILE! It is generated by `inv build`.\n",
*list(parsed_pipfile.data["default"].keys())
])
print(requirements_file)
with open("requirements.txt", "w") as fd:
fd.write(requirements_file)
print_header("Build packages", icon="🔨", level=2)
ctx.run("python setup.py sdist bdist_wheel",
pty=True,
env={"PYTHONPATH": PROJECT_INFO.source_directory})
print_header("Check PyPI description", icon="👀", level=2)
ctx.run("twine check dist/*")
