def doctest(session: nox.sessions.Session):
"""
Perform iris doctests and gallery.
Parameters
----------
session: object
A `nox.sessions.Session` object.
"""
prepare_venv(session)
session.install("--no-deps", "--editable", ".")
session.cd("docs")
session.run(
"make",
"clean",
"html",
external=True,
)
session.run(
"make",
"doctest",
external=True,
)
session.cd("..")
session.run(
"python",
"-m",
"iris.tests.runner",
"--gallery-tests",
)
def _prepare_env(session: nox.sessions.Session) -> None:
lockfile = _session_lockfile(session)
venv_dir = session.virtualenv.location_name
if not _venv_populated(session):
# Environment has been created but packages not yet installed.
# Populate the environment from the lockfile.
logger.debug(f"Populating conda env: {venv_dir}")
session.conda_install(f"--file={lockfile}")
_cache_venv(session)
elif _venv_changed(session):
# Destroy the environment and rebuild it.
logger.debug(f"Lockfile changed. Recreating conda env: {venv_dir}")
_reuse_original = session.virtualenv.reuse_existing
session.virtualenv.reuse_existing = False
session.virtualenv.create()
session.conda_install(f"--file={lockfile}")
session.virtualenv.reuse_existing = _reuse_original
_cache_venv(session)
logger.debug(f"Environment up to date: {venv_dir}")
iris_artifact = _get_iris_github_artifact(session)
if iris_artifact:
# Install the iris source in develop mode.
tmp_dir = Path(session.create_tmp())
iris_dir = tmp_dir / "iris"
cwd = Path.cwd()
if not iris_dir.is_dir():
session.run_always("git",
"clone",
IRIS_GITHUB,
str(iris_dir),
external=True)
session.cd(str(iris_dir))
session.run_always("git", "fetch", "origin", external=True)
session.run_always("git", "checkout", iris_artifact, external=True)
session.cd(str(cwd))
session.install("--no-deps", "--editable", str(iris_dir))
# Determine whether verbose diagnostics have been requested
# from the command line.
verbose = "-v" in session.posargs or "--verbose" in session.posargs
if verbose:
session.run_always("conda", "info")
session.run_always("conda", "list", f"--prefix={venv_dir}")
session.run_always(
"conda",
"list",
f"--prefix={venv_dir}",
"--explicit",
)
def benchmarks(session: nox.sessions.Session, ci_mode: bool, gh_pages: bool):
"""
Perform esmf-regrid performance benchmarks (using Airspeed Velocity).
Parameters
----------
session: object
A `nox.sessions.Session` object.
ci_mode: bool
Run a cut-down selection of benchmarks, comparing the current commit to
the last commit for performance regressions.
gh_pages: bool
Run ``asv gh-pages --rewrite`` once finished.
Notes
-----
ASV is set up to use ``nox --session=tests --install-only`` to prepare
the benchmarking environment.
"""
session.install("asv", "nox", "pyyaml")
session.cd("benchmarks")
# Skip over setup questions for a new machine.
session.run("asv", "machine", "--yes")
def asv_exec(*sub_args: str) -> None:
run_args = ["asv", *sub_args]
help_output = session.run(*run_args, "--help", silent=True)
if "--python" in help_output:
# Not all asv commands accept the --python kwarg.
run_args.append(f"--python={session.python}")
session.run(*run_args)
if ci_mode:
# If on a PR: compare to the base (target) branch.
# Else: compare to previous commit.
previous_commit = os.environ.get("CIRRUS_BASE_SHA", "HEAD^1")
try:
asv_exec("continuous", previous_commit, "HEAD", "--bench=ci")
finally:
asv_exec("compare", previous_commit, "HEAD")
else:
# f32f23a5 = first supporting commit for nox_asv_plugin.py .
asv_exec("run", "f32f23a5..HEAD")
if gh_pages:
asv_exec("gh-pages", "--rewrite")
def benchmarks(session: nox.sessions.Session, ci_mode: bool):
"""
Perform esmf-regrid performance benchmarks (using Airspeed Velocity).
Parameters
----------
session: object
A `nox.sessions.Session` object.
ci_mode: bool
Run a cut-down selection of benchmarks, comparing the current commit to
the last commit for performance regressions.
Notes
-----
ASV is set up to use ``nox --session=tests --install-only`` to prepare
the benchmarking environment. This session environment must use a Python
version that is also available for ``--session=tests``.
"""
session.install("asv", "nox")
session.cd("benchmarks")
# Skip over setup questions for a new machine.
session.run("asv", "machine", "--yes")
def asv_exec(*sub_args: str) -> None:
run_args = ["asv", *sub_args]
session.run(*run_args)
if ci_mode:
# If on a PR: compare to the base (target) branch.
# Else: compare to previous commit.
previous_commit = os.environ.get("PR_BASE_SHA", "HEAD^1")
try:
asv_exec(
"continuous",
"--factor=1.2",
previous_commit,
"HEAD",
"--attribute",
"rounds=4",
)
finally:
asv_exec("compare", previous_commit, "HEAD")
else:
# f5ceb808 = first commit supporting nox --install-only .
asv_exec("run", "f5ceb808..HEAD")
def linkcheck(session: nox.sessions.Session):
"""
Perform iris doc link check.
Parameters
----------
session: object
A `nox.sessions.Session` object.
"""
prepare_venv(session)
session.install("--no-deps", "--editable", ".")
session.cd("docs")
session.run(
"make",
"clean",
"html",
external=True,
)
session.run(
"make",
"linkcheck",
external=True,
)
def benchmarks(session: nox.sessions.Session, ci_mode: bool, long_mode: bool,
gh_pages: bool):
"""
Perform esmf-regrid performance benchmarks (using Airspeed Velocity).
Parameters
----------
session: object
A `nox.sessions.Session` object.
ci_mode: bool
Run a cut-down selection of benchmarks, comparing the current commit to
the last commit for performance regressions.
long_mode: bool
Run the long running benchmarks at the current head of the repo.
gh_pages: bool
Run ``asv gh-pages --rewrite`` once finished.
Notes
-----
ASV is set up to use ``nox --session=tests --install-only`` to prepare
the benchmarking environment.
"""
session.install("asv", "nox", "pyyaml")
if "DATA_GEN_PYTHON" in os.environ:
print("Using existing data generation environment.")
else:
print("Setting up the data generation environment...")
session.run("nox", "--session=tests", "--install-only",
f"--python={session.python}")
data_gen_python = next(
Path(".nox").rglob(
f"tests*/bin/python{session.python}")).resolve()
session.env["DATA_GEN_PYTHON"] = data_gen_python
print("Running ASV...")
session.cd("benchmarks")
# Skip over setup questions for a new machine.
session.run("asv", "machine", "--yes")
def asv_exec(*sub_args: str) -> None:
run_args = ["asv", *sub_args]
help_output = session.run(*run_args, "--help", silent=True)
if "--python" in help_output:
# Not all asv commands accept the --python kwarg.
run_args.append(f"--python={session.python}")
session.run(*run_args)
if ci_mode:
# If on a PR: compare to the base (target) branch.
# Else: compare to previous commit.
previous_commit = os.environ.get("CIRRUS_BASE_SHA", "HEAD^1")
try:
asv_exec("continuous", previous_commit, "HEAD", "--bench=ci",
"--factor=2")
finally:
asv_exec("compare", previous_commit, "HEAD", "--factor=2")
elif long_mode:
asv_exec("run", "HEAD^!", "--bench=long")
else:
# f32f23a5 = first supporting commit for nox_asv_plugin.py .
asv_exec("run", "f32f23a5..HEAD")
if gh_pages:
asv_exec("gh-pages", "--rewrite")
def benchmarks(
session: nox.sessions.Session,
run_type: Literal["overnight", "branch", "custom"],
):
"""
Perform Iris performance benchmarks (using Airspeed Velocity).
All run types require a single Nox positional argument (e.g.
``nox --session="foo" -- my_pos_arg``) - detailed in the parameters
section - and can optionally accept a series of further arguments that will
be added to session's ASV command.
Parameters
----------
session: object
A `nox.sessions.Session` object.
run_type: {"overnight", "branch", "custom"}
* ``overnight``: benchmarks all commits between the input **first
commit** to ``HEAD``, comparing each to its parent for performance
shifts. If a commit causes shifts, the output is saved to a file:
``.asv/performance-shifts/<commit-sha>``. Designed for checking the
previous 24 hours' commits, typically in a scheduled script.
* ``branch``: Performs the same operations as ``overnight``, but always
on two commits only - ``HEAD``, and ``HEAD``'s merge-base with the
input **base branch**. Output from this run is never saved to a file.
Designed for testing if the active branch's changes cause performance
shifts - anticipating what would be caught by ``overnight`` once
merged.
**For maximum accuracy, avoid using the machine that is running this
session. Run time could be >1 hour for the full benchmark suite.**
* ``custom``: run ASV with the input **ASV sub-command**, without any
preset arguments - must all be supplied by the user. So just like
running ASV manually, with the convenience of re-using the session's
scripted setup steps.
Examples
--------
* ``nox --session="benchmarks(overnight)" -- a1b23d4``
* ``nox --session="benchmarks(branch)" -- upstream/main``
* ``nox --session="benchmarks(branch)" -- upstream/mesh-data-model``
* ``nox --session="benchmarks(branch)" -- upstream/main --bench=regridding``
* ``nox --session="benchmarks(custom)" -- continuous a1b23d4 HEAD --quick``
"""
# The threshold beyond which shifts are 'notable'. See `asv compare`` docs
# for more.
COMPARE_FACTOR = 1.2
session.install("asv", "nox")
data_gen_var = "DATA_GEN_PYTHON"
if data_gen_var in os.environ:
print("Using existing data generation environment.")
else:
print("Setting up the data generation environment...")
# Get Nox to build an environment for the `tests` session, but don't
# run the session. Will re-use a cached environment if appropriate.
session.run_always(
"nox",
"--session=tests",
"--install-only",
f"--python={_PY_VERSION_LATEST}",
)
# Find the environment built above, set it to be the data generation
# environment.
data_gen_python = next(
Path(".nox").rglob(
f"tests*/bin/python{_PY_VERSION_LATEST}")).resolve()
session.env[data_gen_var] = data_gen_python
mule_dir = data_gen_python.parents[1] / "resources" / "mule"
if not mule_dir.is_dir():
print("Installing Mule into data generation environment...")
session.run_always(
"git",
"clone",
"https://github.com/metomi/mule.git",
str(mule_dir),
external=True,
)
session.run_always(
str(data_gen_python),
"-m",
"pip",
"install",
str(mule_dir / "mule"),
external=True,
)
print("Running ASV...")
session.cd("benchmarks")
# Skip over setup questions for a new machine.
session.run("asv", "machine", "--yes")
# All run types require one Nox posarg.
run_type_arg = {
"overnight": "first commit",
"branch": "base branch",
"custom": "ASV sub-command",
}
if run_type not in run_type_arg.keys():
message = f"Unsupported run-type: {run_type}"
raise NotImplementedError(message)
if not session.posargs:
message = (f"Missing mandatory first Nox session posarg: "
f"{run_type_arg[run_type]}")
raise ValueError(message)
first_arg = session.posargs[0]
# Optional extra arguments to be passed down to ASV.
asv_args = session.posargs[1:]
def asv_compare(*commits):
"""Run through a list of commits comparing each one to the next."""
commits = [commit[:8] for commit in commits]
shifts_dir = Path(".asv") / "performance-shifts"
for i in range(len(commits) - 1):
before = commits[i]
after = commits[i + 1]
asv_command_ = f"asv compare {before} {after} --factor={COMPARE_FACTOR} --split"
session.run(*asv_command_.split(" "))
if run_type == "overnight":
# Record performance shifts.
# Run the command again but limited to only showing performance
# shifts.
shifts = session.run(*asv_command_.split(" "),
"--only-changed",
silent=True)
if shifts:
# Write the shifts report to a file.
# Dir is used by .github/workflows/benchmarks.yml,
# but not cached - intended to be discarded after run.
shifts_dir.mkdir(exist_ok=True, parents=True)
shifts_path = (shifts_dir / after).with_suffix(".txt")
with shifts_path.open("w") as shifts_file:
shifts_file.write(shifts)
# Common ASV arguments used for both `overnight` and `bench` run_types.
asv_harness = "asv run {posargs} --attribute rounds=4 --interleave-rounds --strict --show-stderr"
if run_type == "overnight":
first_commit = first_arg
commit_range = f"{first_commit}^^.."
asv_command = asv_harness.format(posargs=commit_range)
session.run(*asv_command.split(" "), *asv_args)
# git rev-list --first-parent is the command ASV uses.
git_command = f"git rev-list --first-parent {commit_range}"
commit_string = session.run(*git_command.split(" "),
silent=True,
external=True)
commit_list = commit_string.rstrip().split("\n")
asv_compare(*reversed(commit_list))
elif run_type == "branch":
base_branch = first_arg
git_command = f"git merge-base HEAD {base_branch}"
merge_base = session.run(*git_command.split(" "),
silent=True,
external=True)[:8]
with NamedTemporaryFile("w") as hashfile:
hashfile.writelines([merge_base, "\n", "HEAD"])
hashfile.flush()
commit_range = f"HASHFILE:{hashfile.name}"
asv_command = asv_harness.format(posargs=commit_range)
session.run(*asv_command.split(" "), *asv_args)
asv_compare(merge_base, "HEAD")
else:
asv_subcommand = first_arg
assert run_type == "custom"
session.run("asv", asv_subcommand, *asv_args)
