def py(session: nox.sessions.Session) -> None: """Runs py.test for a sample using the specified version of Python.""" if session.python in TESTED_VERSIONS: _session_tests(session) else: session.skip("SKIPPED: {} tests are disabled for this sample.".format( session.python))
def precommit(session: nox.sessions.Session): """ Perform pre-commit hooks of iris codebase. Parameters ---------- session: object A `nox.sessions.Session` object. """ import yaml # Pip install the session requirements. session.install("pre-commit") # Load the pre-commit configuration YAML file. with open(".pre-commit-config.yaml", "r") as fi: config = yaml.load(fi, Loader=yaml.FullLoader) # List of pre-commit hook ids that we don't want to run. excluded = ["no-commit-to-branch"] # Enumerate the ids of pre-commit hooks we do want to run. ids = [ hook["id"] for entry in config["repos"] for hook in entry["hooks"] if hook["id"] not in excluded ] # Execute the pre-commit hooks. [session.run("pre-commit", "run", "--all-files", id) for id in ids]
def prepare_venv(session: nox.sessions.Session) -> None: """ Create and cache the nox session conda environment, and additionally provide conda environment package details and info. Note that, iris is installed into the environment using pip. Parameters ---------- session: object A `nox.sessions.Session` object. Notes ----- See - https://github.com/theacodes/nox/issues/346 - https://github.com/theacodes/nox/issues/260 """ lockfile = session_lockfile(session) venv_dir = session.virtualenv.location_name if not venv_populated(session): # environment has been created but packages not yet installed # populate the environment from the lockfile logger.debug(f"Populating conda env at {venv_dir}") session.conda_install("--file", str(lockfile)) cache_venv(session) elif venv_changed(session): # destroy the environment and rebuild it logger.debug(f"Lockfile changed. Re-creating conda env at {venv_dir}") _re_orig = session.virtualenv.reuse_existing session.virtualenv.reuse_existing = False session.virtualenv.create() session.conda_install("--file", str(lockfile)) session.virtualenv.reuse_existing = _re_orig cache_venv(session) logger.debug(f"Environment {venv_dir} up to date") cache_cartopy(session) # Determine whether verbose diagnostics have been requested # from the command line. verbose = "-v" in session.posargs or "--verbose" in session.posargs if verbose: session.run("conda", "info") session.run("conda", "list", f"--prefix={venv_dir}") session.run( "conda", "list", f"--prefix={venv_dir}", "--explicit", )
def blacken(session: nox.sessions.Session) -> None: session.install("black") black_files = [] for target in BLACK_PATHS: black_files += [ f"{target}/{path}" for path in os.listdir(target) if path.endswith(".py") ] session.run("black", *black_files)
def lint(session: nox.sessions.Session) -> None: session.install("autoflake", "black", "flake8", "isort", "seed-isort-config") session.run("autoflake", "--in-place", "--recursive", *source_files) session.run("seed-isort-config", "--application-directories=ddtrace_asgi") session.run("isort", "--project=ddtrace_asgi", "--recursive", "--apply", *source_files) session.run("black", "--target-version=py36", *source_files) check(session)
def coverage(session: nox.sessions.Session) -> None: """Report test coverage.""" args = session.posargs or [ "-s", "--cov=package_name", "--cov-report=term-missing", "--cov-report=xml", "tests/", ] session.run("poetry", "install", "--no-dev", external=True) install_with_constraints(session, "aresponses", "pytest", "pytest-aiohttp", "pytest-cov") session.run("pytest", *args)
def _session_tests( session: nox.sessions.Session, post_install: Callable = None ) -> None: """Runs py.test for a particular project.""" if os.path.exists("requirements.txt"): session.install("-r", "requirements.txt") if os.path.exists("requirements-test.txt"): session.install("-r", "requirements-test.txt") if INSTALL_LIBRARY_FROM_SOURCE: session.install("-e", _get_repo_root()) if post_install: post_install(session) session.run( "pytest", *(PYTEST_COMMON_ARGS + session.posargs), # Pytest will return 5 when no tests are collected. This can happen # on travis where slow and flaky tests are excluded. # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html success_codes=[0, 5], env=get_pytest_env_vars() )
def install_with_constraints(session: nox.sessions.Session, *args: str, **kwargs: Any): """Install PyPI packages based on pyproject.toml constraints.""" with tempfile.NamedTemporaryFile() as requirements: session.run( "poetry", "export", "--dev", "--without-hashes", "--format=requirements.txt", f"--output={requirements.name}", external=True, ) session.install(f"--constraint={requirements.name}", *args, **kwargs)
def isort(session: nox.sessions.Session): """ Perform isort import checking of iris codebase. Parameters ---------- session: object A `nox.sessions.Session` object. """ # Pip install the session requirements. session.install("isort") # Execute the isort import checker. session.run("isort", "--check", ".")
def check(session: nox.sessions.Session) -> None: session.install("black", "flake8", "flake8-bugbear", "flake8-comprehensions", "isort", "mypy") session.run("black", "--check", "--diff", "--target-version=py36", *source_files) session.run("flake8", *source_files) session.run("mypy", *source_files) session.run( "isort", "--check", "--diff", "--project=ddtrace_asgi", "--recursive", *source_files, )
def cache_cartopy(session: nox.sessions.Session) -> None: """ Determine whether to cache the cartopy natural earth shapefiles. Parameters ---------- session: object A `nox.sessions.Session` object. """ if not CARTOPY_CACHE_DIR.is_dir(): session.run( "python", "-c", "import cartopy; cartopy.io.shapereader.natural_earth()", )
def _install_and_cache_venv(session: nox.sessions.Session) -> None: """ Cache the nox session environment. This consists of saving a hexdigest (sha256) of the associated Conda lock file. Parameters ---------- session: object A `nox.sessions.Session` object. """ lockfile = _session_lockfile(session) session.conda_install(f"--file={lockfile}") with _session_cachefile(session).open("w") as cachefile: cachefile.write(_file_content(lockfile))
def gallery(session: nox.sessions.Session): """ Perform iris gallery doc-tests. Parameters ---------- session: object A `nox.sessions.Session` object. """ prepare_venv(session) session.install("--no-deps", "--editable", ".") session.run( "python", "-m", "iris.tests.runner", "--gallery-tests", )
def tests(session: nox.sessions.Session): """ Perform iris system, integration and unit tests. Parameters ---------- session: object A `nox.sessions.Session` object. """ prepare_venv(session) session.install("--no-deps", "--editable", ".") session.run( "python", "-m", "iris.tests.runner", "--default-tests", "--system-tests", )
def test(session: nox.sessions.Session, parser_backend): session.run( "poetry", "install", "-v", "-E", "test", *(("-E", parser_backend) if parser_backend == "lark-parser" else tuple()), "--no-dev", external=True, ) if "standalone": if not lark_parser_path.exists(): build_lark_parser() else: if lark_parser_path.exists(): lark_parser_path.unlink() session.run("pytest", "-vv", "--cov=jsonpath", "--cov-append")
def tests(session: nox.sessions.Session): """ Perform esmf-regrid integration and unit tests. Parameters ---------- session: object A `nox.sessions.Session` object. """ _prepare_env(session) # Install the esmf-regrid source in develop mode. session.install("--no-deps", "--editable", ".") if COVERAGE: # Execute the tests with code coverage. session.conda_install("--channel=conda-forge", *COVERAGE_PACKAGES) session.run("pytest", "--cov-report=xml", "--cov") session.run("codecov") else: # Execute the tests. session.run("pytest")
def lint(session: nox.sessions.Session) -> None: if not TEST_CONFIG['enforce_type_hints']: session.install("flake8", "flake8-import-order") else: session.install("flake8", "flake8-import-order", "flake8-annotations") local_names = _determine_local_import_names(".") args = FLAKE8_COMMON_ARGS + [ "--application-import-names", ",".join(local_names), "." ] session.run("flake8", *args)
def readmegen(session: nox.sessions.Session, path: str) -> None: """(Re-)generates the readme for a sample.""" session.install("jinja2", "pyyaml") dir_ = os.path.dirname(path) if os.path.exists(os.path.join(dir_, "requirements.txt")): session.install("-r", os.path.join(dir_, "requirements.txt")) in_file = os.path.join(dir_, "README.rst.in") session.run("python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file)
def format(session: nox.sessions.Session) -> None: """ Run isort to sort imports. Then run black to format code to uniform standard. """ session.install(BLACK_VERSION, ISORT_VERSION) python_files = [path for path in os.listdir(".") if path.endswith(".py")] # Use the --fss option to sort imports using strict alphabetical order. # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections session.run("isort", "--fss", *python_files) session.run("black", *python_files)
def flake8(session: nox.sessions.Session): """ Perform flake8 linting of the code-base. Parameters ---------- session: object A `nox.sessions.Session` object. """ # Pip install the session requirements. session.install("flake8", "flake8-docstrings", "flake8-import-order") # Execute the flake8 linter on the package. session.run("flake8", PACKAGE) # Execute the flake8 linter on this file. session.run("flake8", __file__)
def black(session: nox.sessions.Session): """ Perform black format checking of iris. Parameters ---------- session: object A `nox.sessions.Session` object. """ # Pip install the session requirements. session.install("black==20.8b1") # Execute the black format checker on the package. session.run("black", "--check", PACKAGE) # Execute the black format checker on this file. session.run("black", "--check", __file__)
def flake8(session: nox.sessions.Session): """ Perform flake8 linting of iris. Parameters ---------- session: object A `nox.sessions.Session` object. """ # Pip install the session requirements. session.install("flake8") # Execute the flake8 linter on the package. session.run("flake8", PACKAGE) # Execute the flake8 linter on this file. session.run("flake8", __file__)
def benchmarks(session: nox.sessions.Session, ci_mode: bool, gh_pages: bool): """ Perform esmf-regrid performance benchmarks (using Airspeed Velocity). Parameters ---------- session: object A `nox.sessions.Session` object. ci_mode: bool Run a cut-down selection of benchmarks, comparing the current commit to the last commit for performance regressions. gh_pages: bool Run ``asv gh-pages --rewrite`` once finished. Notes ----- ASV is set up to use ``nox --session=tests --install-only`` to prepare the benchmarking environment. """ session.install("asv", "nox", "pyyaml") session.cd("benchmarks") # Skip over setup questions for a new machine. session.run("asv", "machine", "--yes") def asv_exec(*sub_args: str) -> None: run_args = ["asv", *sub_args] help_output = session.run(*run_args, "--help", silent=True) if "--python" in help_output: # Not all asv commands accept the --python kwarg. run_args.append(f"--python={session.python}") session.run(*run_args) if ci_mode: # If on a PR: compare to the base (target) branch. # Else: compare to previous commit. previous_commit = os.environ.get("CIRRUS_BASE_SHA", "HEAD^1") try: asv_exec("continuous", previous_commit, "HEAD", "--bench=ci") finally: asv_exec("compare", previous_commit, "HEAD") else: # f32f23a5 = first supporting commit for nox_asv_plugin.py . asv_exec("run", "f32f23a5..HEAD") if gh_pages: asv_exec("gh-pages", "--rewrite")
def benchmarks(session: nox.sessions.Session, ci_mode: bool): """ Perform esmf-regrid performance benchmarks (using Airspeed Velocity). Parameters ---------- session: object A `nox.sessions.Session` object. ci_mode: bool Run a cut-down selection of benchmarks, comparing the current commit to the last commit for performance regressions. Notes ----- ASV is set up to use ``nox --session=tests --install-only`` to prepare the benchmarking environment. This session environment must use a Python version that is also available for ``--session=tests``. """ session.install("asv", "nox") session.cd("benchmarks") # Skip over setup questions for a new machine. session.run("asv", "machine", "--yes") def asv_exec(*sub_args: str) -> None: run_args = ["asv", *sub_args] session.run(*run_args) if ci_mode: # If on a PR: compare to the base (target) branch. # Else: compare to previous commit. previous_commit = os.environ.get("PR_BASE_SHA", "HEAD^1") try: asv_exec( "continuous", "--factor=1.2", previous_commit, "HEAD", "--attribute", "rounds=4", ) finally: asv_exec("compare", previous_commit, "HEAD") else: # f5ceb808 = first commit supporting nox --install-only . asv_exec("run", "f5ceb808..HEAD")
def linkcheck(session: nox.sessions.Session): """ Perform iris doc link check. Parameters ---------- session: object A `nox.sessions.Session` object. """ prepare_venv(session) session.install("--no-deps", "--editable", ".") session.cd("docs") session.run( "make", "clean", "html", external=True, ) session.run( "make", "linkcheck", external=True, )
def _session_tests(session: nox.sessions.Session, post_install: Callable = None) -> None: # check for presence of tests test_list = glob.glob("*_test.py") + glob.glob("test_*.py") test_list.extend(glob.glob("tests")) if len(test_list) == 0: print("No tests found, skipping directory.") return if TEST_CONFIG["pip_version_override"]: pip_version = TEST_CONFIG["pip_version_override"] session.install(f"pip=={pip_version}") """Runs py.test for a particular project.""" concurrent_args = [] if os.path.exists("requirements.txt"): if os.path.exists("constraints.txt"): session.install("-r", "requirements.txt", "-c", "constraints.txt") else: session.install("-r", "requirements.txt") with open("requirements.txt") as rfile: packages = rfile.read() if os.path.exists("requirements-test.txt"): if os.path.exists("constraints-test.txt"): session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") else: session.install("-r", "requirements-test.txt") with open("requirements-test.txt") as rtfile: packages += rtfile.read() if INSTALL_LIBRARY_FROM_SOURCE: session.install("-e", _get_repo_root()) if post_install: post_install(session) if "pytest-parallel" in packages: concurrent_args.extend( ['--workers', 'auto', '--tests-per-worker', 'auto']) elif "pytest-xdist" in packages: concurrent_args.extend(['-n', 'auto']) session.run( "pytest", *(PYTEST_COMMON_ARGS + session.posargs + concurrent_args), # Pytest will return 5 when no tests are collected. This can happen # on travis where slow and flaky tests are excluded. # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html success_codes=[0, 5], env=get_pytest_env_vars(), )
def blacken(session: nox.sessions.Session) -> None: """Run black. Format code to uniform standard.""" session.install(BLACK_VERSION) python_files = [path for path in os.listdir(".") if path.endswith(".py")] session.run("black", *python_files)
def blacken(session: nox.sessions.Session) -> None: session.install(BLACK_VERSION) python_files = [path for path in os.listdir(".") if path.endswith(".py")] session.run("black", *python_files)
def lint(session: nox.sessions.Session) -> None: session.install("flake8==3.7.9", "black==19.10b0", "mypy==0.770") session.run("flake8", *PACKAGE_FILES) session.run("black", "--check", *PACKAGE_FILES)
def format(session: nox.sessions.Session) -> None: session.install("black==19.10b0", "isort==4.3.21") session.run("isort", "--recursive", *PACKAGE_FILES) session.run("black", *PACKAGE_FILES)