def py(session: nox.sessions.Session) -> None:
"""Runs py.test for a sample using the specified version of Python."""
if session.python in TESTED_VERSIONS:
_session_tests(session)
else:
session.skip("SKIPPED: {} tests are disabled for this sample.".format(
session.python))
def precommit(session: nox.sessions.Session):
"""
Perform pre-commit hooks of iris codebase.
Parameters
----------
session: object
A `nox.sessions.Session` object.
"""
import yaml
# Pip install the session requirements.
session.install("pre-commit")
# Load the pre-commit configuration YAML file.
with open(".pre-commit-config.yaml", "r") as fi:
config = yaml.load(fi, Loader=yaml.FullLoader)
# List of pre-commit hook ids that we don't want to run.
excluded = ["no-commit-to-branch"]
# Enumerate the ids of pre-commit hooks we do want to run.
ids = [
hook["id"] for entry in config["repos"] for hook in entry["hooks"]
if hook["id"] not in excluded
]
# Execute the pre-commit hooks.
[session.run("pre-commit", "run", "--all-files", id) for id in ids]
def prepare_venv(session: nox.sessions.Session) -> None:
"""
Create and cache the nox session conda environment, and additionally
provide conda environment package details and info.
Note that, iris is installed into the environment using pip.
Parameters
----------
session: object
A `nox.sessions.Session` object.
Notes
-----
See
- https://github.com/theacodes/nox/issues/346
- https://github.com/theacodes/nox/issues/260
"""
lockfile = session_lockfile(session)
venv_dir = session.virtualenv.location_name
if not venv_populated(session):
# environment has been created but packages not yet installed
# populate the environment from the lockfile
logger.debug(f"Populating conda env at {venv_dir}")
session.conda_install("--file", str(lockfile))
cache_venv(session)
elif venv_changed(session):
# destroy the environment and rebuild it
logger.debug(f"Lockfile changed. Re-creating conda env at {venv_dir}")
_re_orig = session.virtualenv.reuse_existing
session.virtualenv.reuse_existing = False
session.virtualenv.create()
session.conda_install("--file", str(lockfile))
session.virtualenv.reuse_existing = _re_orig
cache_venv(session)
logger.debug(f"Environment {venv_dir} up to date")
cache_cartopy(session)
# Determine whether verbose diagnostics have been requested
# from the command line.
verbose = "-v" in session.posargs or "--verbose" in session.posargs
if verbose:
session.run("conda", "info")
session.run("conda", "list", f"--prefix={venv_dir}")
session.run(
"conda",
"list",
f"--prefix={venv_dir}",
"--explicit",
)
def blacken(session: nox.sessions.Session) -> None:
session.install("black")
black_files = []
for target in BLACK_PATHS:
black_files += [
f"{target}/{path}" for path in os.listdir(target)
if path.endswith(".py")
]
session.run("black", *black_files)
def lint(session: nox.sessions.Session) -> None:
session.install("autoflake", "black", "flake8", "isort",
"seed-isort-config")
session.run("autoflake", "--in-place", "--recursive", *source_files)
session.run("seed-isort-config", "--application-directories=ddtrace_asgi")
session.run("isort", "--project=ddtrace_asgi", "--recursive", "--apply",
*source_files)
session.run("black", "--target-version=py36", *source_files)
check(session)
def coverage(session: nox.sessions.Session) -> None:
"""Report test coverage."""
args = session.posargs or [
"-s",
"--cov=package_name",
"--cov-report=term-missing",
"--cov-report=xml",
"tests/",
]
session.run("poetry", "install", "--no-dev", external=True)
install_with_constraints(session, "aresponses", "pytest", "pytest-aiohttp",
"pytest-cov")
session.run("pytest", *args)
def _session_tests(
session: nox.sessions.Session, post_install: Callable = None
) -> None:
"""Runs py.test for a particular project."""
if os.path.exists("requirements.txt"):
session.install("-r", "requirements.txt")
if os.path.exists("requirements-test.txt"):
session.install("-r", "requirements-test.txt")
if INSTALL_LIBRARY_FROM_SOURCE:
session.install("-e", _get_repo_root())
if post_install:
post_install(session)
session.run(
"pytest",
*(PYTEST_COMMON_ARGS + session.posargs),
# Pytest will return 5 when no tests are collected. This can happen
# on travis where slow and flaky tests are excluded.
# See http://doc.pytest.org/en/latest/_modules/_pytest/main.html
success_codes=[0, 5],
env=get_pytest_env_vars()
)
def install_with_constraints(session: nox.sessions.Session, *args: str,
**kwargs: Any):
"""Install PyPI packages based on pyproject.toml constraints."""
with tempfile.NamedTemporaryFile() as requirements:
session.run(
"poetry",
"export",
"--dev",
"--without-hashes",
"--format=requirements.txt",
f"--output={requirements.name}",
external=True,
)
session.install(f"--constraint={requirements.name}", *args, **kwargs)
def isort(session: nox.sessions.Session):
"""
Perform isort import checking of iris codebase.
Parameters
----------
session: object
A `nox.sessions.Session` object.
"""
# Pip install the session requirements.
session.install("isort")
# Execute the isort import checker.
session.run("isort", "--check", ".")
def check(session: nox.sessions.Session) -> None:
session.install("black", "flake8", "flake8-bugbear",
"flake8-comprehensions", "isort", "mypy")
session.run("black", "--check", "--diff", "--target-version=py36",
*source_files)
session.run("flake8", *source_files)
session.run("mypy", *source_files)
session.run(
"isort",
"--check",
"--diff",
"--project=ddtrace_asgi",
"--recursive",
*source_files,
)
def cache_cartopy(session: nox.sessions.Session) -> None:
"""
Determine whether to cache the cartopy natural earth shapefiles.
Parameters
----------
session: object
A `nox.sessions.Session` object.
"""
if not CARTOPY_CACHE_DIR.is_dir():
session.run(
"python",
"-c",
"import cartopy; cartopy.io.shapereader.natural_earth()",
)
def _install_and_cache_venv(session: nox.sessions.Session) -> None:
"""
Cache the nox session environment.
This consists of saving a hexdigest (sha256) of the associated
Conda lock file.
Parameters
----------
session: object
A `nox.sessions.Session` object.
"""
lockfile = _session_lockfile(session)
session.conda_install(f"--file={lockfile}")
with _session_cachefile(session).open("w") as cachefile:
cachefile.write(_file_content(lockfile))
def gallery(session: nox.sessions.Session):
"""
Perform iris gallery doc-tests.
Parameters
----------
session: object
A `nox.sessions.Session` object.
"""
prepare_venv(session)
session.install("--no-deps", "--editable", ".")
session.run(
"python",
"-m",
"iris.tests.runner",
"--gallery-tests",
)
def tests(session: nox.sessions.Session):
"""
Perform iris system, integration and unit tests.
Parameters
----------
session: object
A `nox.sessions.Session` object.
"""
prepare_venv(session)
session.install("--no-deps", "--editable", ".")
session.run(
"python",
"-m",
"iris.tests.runner",
"--default-tests",
"--system-tests",
)
def test(session: nox.sessions.Session, parser_backend):
session.run(
"poetry",
"install",
"-v",
"-E",
"test",
*(("-E",
parser_backend) if parser_backend == "lark-parser" else tuple()),
"--no-dev",
external=True,
)
if "standalone":
if not lark_parser_path.exists():
build_lark_parser()
else:
if lark_parser_path.exists():
lark_parser_path.unlink()
session.run("pytest", "-vv", "--cov=jsonpath", "--cov-append")
def tests(session: nox.sessions.Session):
"""
Perform esmf-regrid integration and unit tests.
Parameters
----------
session: object
A `nox.sessions.Session` object.
"""
_prepare_env(session)
# Install the esmf-regrid source in develop mode.
session.install("--no-deps", "--editable", ".")
if COVERAGE:
# Execute the tests with code coverage.
session.conda_install("--channel=conda-forge", *COVERAGE_PACKAGES)
session.run("pytest", "--cov-report=xml", "--cov")
session.run("codecov")
else:
# Execute the tests.
session.run("pytest")
def lint(session: nox.sessions.Session) -> None:
if not TEST_CONFIG['enforce_type_hints']:
session.install("flake8", "flake8-import-order")
else:
session.install("flake8", "flake8-import-order", "flake8-annotations")
local_names = _determine_local_import_names(".")
args = FLAKE8_COMMON_ARGS + [
"--application-import-names", ",".join(local_names), "."
]
session.run("flake8", *args)
def readmegen(session: nox.sessions.Session, path: str) -> None:
"""(Re-)generates the readme for a sample."""
session.install("jinja2", "pyyaml")
dir_ = os.path.dirname(path)
if os.path.exists(os.path.join(dir_, "requirements.txt")):
session.install("-r", os.path.join(dir_, "requirements.txt"))
in_file = os.path.join(dir_, "README.rst.in")
session.run("python",
_get_repo_root() + "/scripts/readme-gen/readme_gen.py",
in_file)
def format(session: nox.sessions.Session) -> None:
"""
Run isort to sort imports. Then run black
to format code to uniform standard.
"""
session.install(BLACK_VERSION, ISORT_VERSION)
python_files = [path for path in os.listdir(".") if path.endswith(".py")]
# Use the --fss option to sort imports using strict alphabetical order.
# See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections
session.run("isort", "--fss", *python_files)
session.run("black", *python_files)
def flake8(session: nox.sessions.Session):
"""
Perform flake8 linting of the code-base.
Parameters
----------
session: object
A `nox.sessions.Session` object.
"""
# Pip install the session requirements.
session.install("flake8", "flake8-docstrings", "flake8-import-order")
# Execute the flake8 linter on the package.
session.run("flake8", PACKAGE)
# Execute the flake8 linter on this file.
session.run("flake8", __file__)
def black(session: nox.sessions.Session):
"""
Perform black format checking of iris.
Parameters
----------
session: object
A `nox.sessions.Session` object.
"""
# Pip install the session requirements.
session.install("black==20.8b1")
# Execute the black format checker on the package.
session.run("black", "--check", PACKAGE)
# Execute the black format checker on this file.
session.run("black", "--check", __file__)
def flake8(session: nox.sessions.Session):
"""
Perform flake8 linting of iris.
Parameters
----------
session: object
A `nox.sessions.Session` object.
"""
# Pip install the session requirements.
session.install("flake8")
# Execute the flake8 linter on the package.
session.run("flake8", PACKAGE)
# Execute the flake8 linter on this file.
session.run("flake8", __file__)
def benchmarks(session: nox.sessions.Session, ci_mode: bool, gh_pages: bool):
"""
Perform esmf-regrid performance benchmarks (using Airspeed Velocity).
Parameters
----------
session: object
A `nox.sessions.Session` object.
ci_mode: bool
Run a cut-down selection of benchmarks, comparing the current commit to
the last commit for performance regressions.
gh_pages: bool
Run ``asv gh-pages --rewrite`` once finished.
Notes
-----
ASV is set up to use ``nox --session=tests --install-only`` to prepare
the benchmarking environment.
"""
session.install("asv", "nox", "pyyaml")
session.cd("benchmarks")
# Skip over setup questions for a new machine.
session.run("asv", "machine", "--yes")
def asv_exec(*sub_args: str) -> None:
run_args = ["asv", *sub_args]
help_output = session.run(*run_args, "--help", silent=True)
if "--python" in help_output:
# Not all asv commands accept the --python kwarg.
run_args.append(f"--python={session.python}")
session.run(*run_args)
if ci_mode:
# If on a PR: compare to the base (target) branch.
# Else: compare to previous commit.
previous_commit = os.environ.get("CIRRUS_BASE_SHA", "HEAD^1")
try:
asv_exec("continuous", previous_commit, "HEAD", "--bench=ci")
finally:
asv_exec("compare", previous_commit, "HEAD")
else:
# f32f23a5 = first supporting commit for nox_asv_plugin.py .
asv_exec("run", "f32f23a5..HEAD")
if gh_pages:
asv_exec("gh-pages", "--rewrite")
def benchmarks(session: nox.sessions.Session, ci_mode: bool):
"""
Perform esmf-regrid performance benchmarks (using Airspeed Velocity).
Parameters
----------
session: object
A `nox.sessions.Session` object.
ci_mode: bool
Run a cut-down selection of benchmarks, comparing the current commit to
the last commit for performance regressions.
Notes
-----
ASV is set up to use ``nox --session=tests --install-only`` to prepare
the benchmarking environment. This session environment must use a Python
version that is also available for ``--session=tests``.
"""
session.install("asv", "nox")
session.cd("benchmarks")
# Skip over setup questions for a new machine.
session.run("asv", "machine", "--yes")
def asv_exec(*sub_args: str) -> None:
run_args = ["asv", *sub_args]
session.run(*run_args)
if ci_mode:
# If on a PR: compare to the base (target) branch.
# Else: compare to previous commit.
previous_commit = os.environ.get("PR_BASE_SHA", "HEAD^1")
try:
asv_exec(
"continuous",
"--factor=1.2",
previous_commit,
"HEAD",
"--attribute",
"rounds=4",
)
finally:
asv_exec("compare", previous_commit, "HEAD")
else:
# f5ceb808 = first commit supporting nox --install-only .
asv_exec("run", "f5ceb808..HEAD")
def linkcheck(session: nox.sessions.Session):
"""
Perform iris doc link check.
Parameters
----------
session: object
A `nox.sessions.Session` object.
"""
prepare_venv(session)
session.install("--no-deps", "--editable", ".")
session.cd("docs")
session.run(
"make",
"clean",
"html",
external=True,
)
session.run(
"make",
"linkcheck",
external=True,
)
def _session_tests(session: nox.sessions.Session,
post_install: Callable = None) -> None:
# check for presence of tests
test_list = glob.glob("*_test.py") + glob.glob("test_*.py")
test_list.extend(glob.glob("tests"))
if len(test_list) == 0:
print("No tests found, skipping directory.")
return
if TEST_CONFIG["pip_version_override"]:
pip_version = TEST_CONFIG["pip_version_override"]
session.install(f"pip=={pip_version}")
"""Runs py.test for a particular project."""
concurrent_args = []
if os.path.exists("requirements.txt"):
if os.path.exists("constraints.txt"):
session.install("-r", "requirements.txt", "-c", "constraints.txt")
else:
session.install("-r", "requirements.txt")
with open("requirements.txt") as rfile:
packages = rfile.read()
if os.path.exists("requirements-test.txt"):
if os.path.exists("constraints-test.txt"):
session.install("-r", "requirements-test.txt", "-c",
"constraints-test.txt")
else:
session.install("-r", "requirements-test.txt")
with open("requirements-test.txt") as rtfile:
packages += rtfile.read()
if INSTALL_LIBRARY_FROM_SOURCE:
session.install("-e", _get_repo_root())
if post_install:
post_install(session)
if "pytest-parallel" in packages:
concurrent_args.extend(
['--workers', 'auto', '--tests-per-worker', 'auto'])
elif "pytest-xdist" in packages:
concurrent_args.extend(['-n', 'auto'])
session.run(
"pytest",
*(PYTEST_COMMON_ARGS + session.posargs + concurrent_args),
# Pytest will return 5 when no tests are collected. This can happen
# on travis where slow and flaky tests are excluded.
# See http://doc.pytest.org/en/latest/_modules/_pytest/main.html
success_codes=[0, 5],
env=get_pytest_env_vars(),
)
def blacken(session: nox.sessions.Session) -> None:
"""Run black. Format code to uniform standard."""
session.install(BLACK_VERSION)
python_files = [path for path in os.listdir(".") if path.endswith(".py")]
session.run("black", *python_files)
def blacken(session: nox.sessions.Session) -> None:
session.install(BLACK_VERSION)
python_files = [path for path in os.listdir(".") if path.endswith(".py")]
session.run("black", *python_files)
def lint(session: nox.sessions.Session) -> None:
session.install("flake8==3.7.9", "black==19.10b0", "mypy==0.770")
session.run("flake8", *PACKAGE_FILES)
session.run("black", "--check", *PACKAGE_FILES)
def format(session: nox.sessions.Session) -> None:
session.install("black==19.10b0", "isort==4.3.21")
session.run("isort", "--recursive", *PACKAGE_FILES)
session.run("black", *PACKAGE_FILES)
