def doctest(session: nox.sessions.Session): """ Perform iris doctests and gallery. Parameters ---------- session: object A `nox.sessions.Session` object. """ prepare_venv(session) session.install("--no-deps", "--editable", ".") session.cd("docs") session.run( "make", "clean", "html", external=True, ) session.run( "make", "doctest", external=True, ) session.cd("..") session.run( "python", "-m", "iris.tests.runner", "--gallery-tests", )
def _prepare_env(session: nox.sessions.Session) -> None: lockfile = _session_lockfile(session) venv_dir = session.virtualenv.location_name if not _venv_populated(session): # Environment has been created but packages not yet installed. # Populate the environment from the lockfile. logger.debug(f"Populating conda env: {venv_dir}") session.conda_install(f"--file={lockfile}") _cache_venv(session) elif _venv_changed(session): # Destroy the environment and rebuild it. logger.debug(f"Lockfile changed. Recreating conda env: {venv_dir}") _reuse_original = session.virtualenv.reuse_existing session.virtualenv.reuse_existing = False session.virtualenv.create() session.conda_install(f"--file={lockfile}") session.virtualenv.reuse_existing = _reuse_original _cache_venv(session) logger.debug(f"Environment up to date: {venv_dir}") iris_artifact = _get_iris_github_artifact(session) if iris_artifact: # Install the iris source in develop mode. tmp_dir = Path(session.create_tmp()) iris_dir = tmp_dir / "iris" cwd = Path.cwd() if not iris_dir.is_dir(): session.run_always("git", "clone", IRIS_GITHUB, str(iris_dir), external=True) session.cd(str(iris_dir)) session.run_always("git", "fetch", "origin", external=True) session.run_always("git", "checkout", iris_artifact, external=True) session.cd(str(cwd)) session.install("--no-deps", "--editable", str(iris_dir)) # Determine whether verbose diagnostics have been requested # from the command line. verbose = "-v" in session.posargs or "--verbose" in session.posargs if verbose: session.run_always("conda", "info") session.run_always("conda", "list", f"--prefix={venv_dir}") session.run_always( "conda", "list", f"--prefix={venv_dir}", "--explicit", )
def benchmarks(session: nox.sessions.Session, ci_mode: bool, gh_pages: bool): """ Perform esmf-regrid performance benchmarks (using Airspeed Velocity). Parameters ---------- session: object A `nox.sessions.Session` object. ci_mode: bool Run a cut-down selection of benchmarks, comparing the current commit to the last commit for performance regressions. gh_pages: bool Run ``asv gh-pages --rewrite`` once finished. Notes ----- ASV is set up to use ``nox --session=tests --install-only`` to prepare the benchmarking environment. """ session.install("asv", "nox", "pyyaml") session.cd("benchmarks") # Skip over setup questions for a new machine. session.run("asv", "machine", "--yes") def asv_exec(*sub_args: str) -> None: run_args = ["asv", *sub_args] help_output = session.run(*run_args, "--help", silent=True) if "--python" in help_output: # Not all asv commands accept the --python kwarg. run_args.append(f"--python={session.python}") session.run(*run_args) if ci_mode: # If on a PR: compare to the base (target) branch. # Else: compare to previous commit. previous_commit = os.environ.get("CIRRUS_BASE_SHA", "HEAD^1") try: asv_exec("continuous", previous_commit, "HEAD", "--bench=ci") finally: asv_exec("compare", previous_commit, "HEAD") else: # f32f23a5 = first supporting commit for nox_asv_plugin.py . asv_exec("run", "f32f23a5..HEAD") if gh_pages: asv_exec("gh-pages", "--rewrite")
def benchmarks(session: nox.sessions.Session, ci_mode: bool): """ Perform esmf-regrid performance benchmarks (using Airspeed Velocity). Parameters ---------- session: object A `nox.sessions.Session` object. ci_mode: bool Run a cut-down selection of benchmarks, comparing the current commit to the last commit for performance regressions. Notes ----- ASV is set up to use ``nox --session=tests --install-only`` to prepare the benchmarking environment. This session environment must use a Python version that is also available for ``--session=tests``. """ session.install("asv", "nox") session.cd("benchmarks") # Skip over setup questions for a new machine. session.run("asv", "machine", "--yes") def asv_exec(*sub_args: str) -> None: run_args = ["asv", *sub_args] session.run(*run_args) if ci_mode: # If on a PR: compare to the base (target) branch. # Else: compare to previous commit. previous_commit = os.environ.get("PR_BASE_SHA", "HEAD^1") try: asv_exec( "continuous", "--factor=1.2", previous_commit, "HEAD", "--attribute", "rounds=4", ) finally: asv_exec("compare", previous_commit, "HEAD") else: # f5ceb808 = first commit supporting nox --install-only . asv_exec("run", "f5ceb808..HEAD")
def linkcheck(session: nox.sessions.Session): """ Perform iris doc link check. Parameters ---------- session: object A `nox.sessions.Session` object. """ prepare_venv(session) session.install("--no-deps", "--editable", ".") session.cd("docs") session.run( "make", "clean", "html", external=True, ) session.run( "make", "linkcheck", external=True, )
def benchmarks(session: nox.sessions.Session, ci_mode: bool, long_mode: bool, gh_pages: bool): """ Perform esmf-regrid performance benchmarks (using Airspeed Velocity). Parameters ---------- session: object A `nox.sessions.Session` object. ci_mode: bool Run a cut-down selection of benchmarks, comparing the current commit to the last commit for performance regressions. long_mode: bool Run the long running benchmarks at the current head of the repo. gh_pages: bool Run ``asv gh-pages --rewrite`` once finished. Notes ----- ASV is set up to use ``nox --session=tests --install-only`` to prepare the benchmarking environment. """ session.install("asv", "nox", "pyyaml") if "DATA_GEN_PYTHON" in os.environ: print("Using existing data generation environment.") else: print("Setting up the data generation environment...") session.run("nox", "--session=tests", "--install-only", f"--python={session.python}") data_gen_python = next( Path(".nox").rglob( f"tests*/bin/python{session.python}")).resolve() session.env["DATA_GEN_PYTHON"] = data_gen_python print("Running ASV...") session.cd("benchmarks") # Skip over setup questions for a new machine. session.run("asv", "machine", "--yes") def asv_exec(*sub_args: str) -> None: run_args = ["asv", *sub_args] help_output = session.run(*run_args, "--help", silent=True) if "--python" in help_output: # Not all asv commands accept the --python kwarg. run_args.append(f"--python={session.python}") session.run(*run_args) if ci_mode: # If on a PR: compare to the base (target) branch. # Else: compare to previous commit. previous_commit = os.environ.get("CIRRUS_BASE_SHA", "HEAD^1") try: asv_exec("continuous", previous_commit, "HEAD", "--bench=ci", "--factor=2") finally: asv_exec("compare", previous_commit, "HEAD", "--factor=2") elif long_mode: asv_exec("run", "HEAD^!", "--bench=long") else: # f32f23a5 = first supporting commit for nox_asv_plugin.py . asv_exec("run", "f32f23a5..HEAD") if gh_pages: asv_exec("gh-pages", "--rewrite")
def benchmarks( session: nox.sessions.Session, run_type: Literal["overnight", "branch", "custom"], ): """ Perform Iris performance benchmarks (using Airspeed Velocity). All run types require a single Nox positional argument (e.g. ``nox --session="foo" -- my_pos_arg``) - detailed in the parameters section - and can optionally accept a series of further arguments that will be added to session's ASV command. Parameters ---------- session: object A `nox.sessions.Session` object. run_type: {"overnight", "branch", "custom"} * ``overnight``: benchmarks all commits between the input **first commit** to ``HEAD``, comparing each to its parent for performance shifts. If a commit causes shifts, the output is saved to a file: ``.asv/performance-shifts/<commit-sha>``. Designed for checking the previous 24 hours' commits, typically in a scheduled script. * ``branch``: Performs the same operations as ``overnight``, but always on two commits only - ``HEAD``, and ``HEAD``'s merge-base with the input **base branch**. Output from this run is never saved to a file. Designed for testing if the active branch's changes cause performance shifts - anticipating what would be caught by ``overnight`` once merged. **For maximum accuracy, avoid using the machine that is running this session. Run time could be >1 hour for the full benchmark suite.** * ``custom``: run ASV with the input **ASV sub-command**, without any preset arguments - must all be supplied by the user. So just like running ASV manually, with the convenience of re-using the session's scripted setup steps. Examples -------- * ``nox --session="benchmarks(overnight)" -- a1b23d4`` * ``nox --session="benchmarks(branch)" -- upstream/main`` * ``nox --session="benchmarks(branch)" -- upstream/mesh-data-model`` * ``nox --session="benchmarks(branch)" -- upstream/main --bench=regridding`` * ``nox --session="benchmarks(custom)" -- continuous a1b23d4 HEAD --quick`` """ # The threshold beyond which shifts are 'notable'. See `asv compare`` docs # for more. COMPARE_FACTOR = 1.2 session.install("asv", "nox") data_gen_var = "DATA_GEN_PYTHON" if data_gen_var in os.environ: print("Using existing data generation environment.") else: print("Setting up the data generation environment...") # Get Nox to build an environment for the `tests` session, but don't # run the session. Will re-use a cached environment if appropriate. session.run_always( "nox", "--session=tests", "--install-only", f"--python={_PY_VERSION_LATEST}", ) # Find the environment built above, set it to be the data generation # environment. data_gen_python = next( Path(".nox").rglob( f"tests*/bin/python{_PY_VERSION_LATEST}")).resolve() session.env[data_gen_var] = data_gen_python mule_dir = data_gen_python.parents[1] / "resources" / "mule" if not mule_dir.is_dir(): print("Installing Mule into data generation environment...") session.run_always( "git", "clone", "https://github.com/metomi/mule.git", str(mule_dir), external=True, ) session.run_always( str(data_gen_python), "-m", "pip", "install", str(mule_dir / "mule"), external=True, ) print("Running ASV...") session.cd("benchmarks") # Skip over setup questions for a new machine. session.run("asv", "machine", "--yes") # All run types require one Nox posarg. run_type_arg = { "overnight": "first commit", "branch": "base branch", "custom": "ASV sub-command", } if run_type not in run_type_arg.keys(): message = f"Unsupported run-type: {run_type}" raise NotImplementedError(message) if not session.posargs: message = (f"Missing mandatory first Nox session posarg: " f"{run_type_arg[run_type]}") raise ValueError(message) first_arg = session.posargs[0] # Optional extra arguments to be passed down to ASV. asv_args = session.posargs[1:] def asv_compare(*commits): """Run through a list of commits comparing each one to the next.""" commits = [commit[:8] for commit in commits] shifts_dir = Path(".asv") / "performance-shifts" for i in range(len(commits) - 1): before = commits[i] after = commits[i + 1] asv_command_ = f"asv compare {before} {after} --factor={COMPARE_FACTOR} --split" session.run(*asv_command_.split(" ")) if run_type == "overnight": # Record performance shifts. # Run the command again but limited to only showing performance # shifts. shifts = session.run(*asv_command_.split(" "), "--only-changed", silent=True) if shifts: # Write the shifts report to a file. # Dir is used by .github/workflows/benchmarks.yml, # but not cached - intended to be discarded after run. shifts_dir.mkdir(exist_ok=True, parents=True) shifts_path = (shifts_dir / after).with_suffix(".txt") with shifts_path.open("w") as shifts_file: shifts_file.write(shifts) # Common ASV arguments used for both `overnight` and `bench` run_types. asv_harness = "asv run {posargs} --attribute rounds=4 --interleave-rounds --strict --show-stderr" if run_type == "overnight": first_commit = first_arg commit_range = f"{first_commit}^^.." asv_command = asv_harness.format(posargs=commit_range) session.run(*asv_command.split(" "), *asv_args) # git rev-list --first-parent is the command ASV uses. git_command = f"git rev-list --first-parent {commit_range}" commit_string = session.run(*git_command.split(" "), silent=True, external=True) commit_list = commit_string.rstrip().split("\n") asv_compare(*reversed(commit_list)) elif run_type == "branch": base_branch = first_arg git_command = f"git merge-base HEAD {base_branch}" merge_base = session.run(*git_command.split(" "), silent=True, external=True)[:8] with NamedTemporaryFile("w") as hashfile: hashfile.writelines([merge_base, "\n", "HEAD"]) hashfile.flush() commit_range = f"HASHFILE:{hashfile.name}" asv_command = asv_harness.format(posargs=commit_range) session.run(*asv_command.split(" "), *asv_args) asv_compare(merge_base, "HEAD") else: asv_subcommand = first_arg assert run_type == "custom" session.run("asv", asv_subcommand, *asv_args)