def setup_env(session: nox.Session) -> None: """Setup a basic (virtual) environment for manual testing.""" env_dir = THIS_DIR / "venv" bin_dir = env_dir / ("Scripts" if WINDOWS else "bin") wipe(session, env_dir) session.run(sys.executable, "-m", "virtualenv", str(env_dir), silent=True) session.run(bin_dir / "python", "-m", "pip", "install", "flit", silent=True) session.run(bin_dir / "python", "-m", "flit", "install", _FLIT_EDITABLE, silent=True) session.run(bin_dir / "python", "-m", "pip", "install", "black", silent=True) session.log( "Virtual environment at project root named `venv` ready to go!")
def cleanup(session: nox.Session) -> None: """Cleanup any temporary files made in this project by its nox tasks.""" import shutil # Remove directories for raw_path in ["./site", "./.nox"]: path = pathlib.Path(raw_path) try: shutil.rmtree(str(path.absolute())) except Exception as exc: session.warn(f"[ FAIL ] Failed to remove '{raw_path}': {exc!s}") else: session.log(f"[ OK ] Removed '{raw_path}'") # Remove individual files for raw_path in ["./.coverage", "./coverage_html.xml"]: path = pathlib.Path(raw_path) try: path.unlink() except Exception as exc: session.warn(f"[ FAIL ] Failed to remove '{raw_path}': {exc!s}") else: session.log(f"[ OK ] Removed '{raw_path}'")
def wipe(session: nox.Session, path: Union[str, Path], once: bool = False) -> None: if "--install-only" in sys.argv: return if isinstance(path, str): path = Path.cwd() / path normalized = path.relative_to(Path.cwd()) if not path.exists(): return if once: if not any(path == entry for entry in _no_wipe): _no_wipe.append(path) else: return if path.is_file(): session.log(f"Deleting `{normalized}` file.") path.unlink() elif path.is_dir(): session.log(f"Deleting `{normalized}` directory.") shutil.rmtree(path)
def generate_docs(session: nox.Session) -> None: """Generate docs for this project using Pdoc.""" install_requirements(session, ".[docs]", "--use-feature=in-tree-build") session.log("Building docs into ./docs") output_directory = _try_find_option(session, "-o", "--output") or "./docs" session.run("pdoc", "--docformat", "numpy", "-o", output_directory, "./sake", "-t", "./templates") session.log("Docs generated: %s", pathlib.Path("./docs/index.html").absolute()) if not _try_find_option(session, "-j", "--json", when_empty="true"): return import httpx # Note: this can be linked to a specific hash by adding it between raw and {file.name} as another route segment. code = httpx.get( "https://gist.githubusercontent.com/FasterSpeeding/19a6d3f44cdd0a1f3b2437a8c5eef07a/raw/json_index_docs.py" ).read() # This is saved to a temporary file to avoid the source showing up in any of the output. # A try, finally is used to delete the file rather than relying on delete=True behaviour # as on Windows the file cannot be accessed by other processes if delete is True. file = tempfile.NamedTemporaryFile(delete=False) try: with file: file.write(code) session.run("python", file.name, "sake", "-o", str(pathlib.Path(output_directory) / "search.json")) finally: pathlib.Path(file.name).unlink(missing_ok=False)
def set_up_vscode(session: nox.Session) -> None: """ Helper function that will set VSCode's workspace settings to use the auto-created virtual environment and enable pytest support. If called, this function will only do anything if there aren't already VSCode workspace settings defined. Args: session (nox.Session): The enclosing nox session. """ if not VSCODE_DIR.exists(): session.log("Setting up VSCode Workspace.") VSCODE_DIR.mkdir(parents=True) SETTINGS_JSON.touch() settings = { "python.defaultInterpreterPath": PYTHON, "python.testing.pytestEnabled": True, "python.testing.pytestArgs": [PROJECT_TESTS.name], } with open(SETTINGS_JSON, mode="w", encoding="utf-8") as f: json.dump(settings, f, sort_keys=True, indent=4)
def build_release(session: nox.Session) -> None: version = release.get_version_from_arguments(session) if not version: session.error("Usage: nox -s build-release -- YY.N[.P]") session.log("# Ensure no files in dist/") if release.have_files_in_folder("dist"): session.error( "There are files in dist/. Remove them and try again. " "You can use `git clean -fxdi -- dist` command to do this") session.log("# Install dependencies") session.install("setuptools", "wheel", "twine") with release.isolated_temporary_checkout(session, version) as build_dir: session.log( "# Start the build in an isolated, " f"temporary Git checkout at {build_dir!s}", ) with release.workdir(session, build_dir): tmp_dists = build_dists(session) tmp_dist_paths = (build_dir / p for p in tmp_dists) session.log(f"# Copying dists from {build_dir}") os.makedirs("dist", exist_ok=True) for dist, final in zip(tmp_dist_paths, tmp_dists): session.log(f"# Copying {dist} to {final}") shutil.copy(dist, final)
def bump(session: nox.Session) -> None: """ Bump the major/minor/patch version (if nothing given, just shows the version). """ session.install("tomli") output = session.run( "python", "-c", "import tomli, pathlib; p = pathlib.Path('pyproject.toml'); print(tomli.loads(p.read_text())['project']['version'])", silent=True, ) current_version = output.strip() if not session.posargs: session.log(f"Current version: {current_version}") return new_version = session.posargs[0] session.log(f"Bumping from {current_version} to {new_version}") replace_version( Path("src/uhi/__init__.py"), '__version__ = "{version}"', current_version, new_version, ) replace_version(Path("pyproject.toml"), 'version = "{version}"', current_version, new_version) print(f"git switch -c chore/bump/{new_version}") print("git add -u src/uhi/__init__.py pyproject.toml") print(f"git commit -m 'chore: bump version to {new_version}'") print("gh pr create --fill")
def docker_qa_compiler(session: nox.Session, version: str) -> None: """Build and run the docker container for a specific GCC version.""" session_id = "docker_qa_build_compiler({})".format(version) session.log(f"Notify session {session_id}") session.notify(session_id) session_id = f"docker_qa_run_compiler({version})" session.log(f"Notify session {session_id}") session.notify(session_id)
def test(session: nox.Session) -> None: # Get the common wheels. if should_update_common_wheels(): # fmt: off run_with_protected_pip( session, "wheel", "-w", LOCATIONS["common-wheels"], "-r", REQUIREMENTS["common-wheels"], ) # fmt: on else: msg = f"Re-using existing common-wheels at {LOCATIONS['common-wheels']}." session.log(msg) # Build source distribution sdist_dir = os.path.join(session.virtualenv.location, "sdist") if os.path.exists(sdist_dir): shutil.rmtree(sdist_dir, ignore_errors=True) # fmt: off session.run( "python", "setup.py", "sdist", "--formats=zip", "--dist-dir", sdist_dir, silent=True, ) # fmt: on generated_files = os.listdir(sdist_dir) assert len(generated_files) == 1 generated_sdist = os.path.join(sdist_dir, generated_files[0]) # Install source distribution run_with_protected_pip(session, "install", generated_sdist) # Install test dependencies run_with_protected_pip(session, "install", "-r", REQUIREMENTS["tests"]) # Parallelize tests as much as possible, by default. arguments = session.posargs or ["-n", "auto"] # Run the tests # LC_CTYPE is set to get UTF-8 output inside of the subprocesses that our # tests use. session.run( "pytest", *arguments, env={ "LC_CTYPE": "en_US.UTF-8", "SETUPTOOLS_USE_DISTUTILS": "stdlib", }, )
def release(session: nox.Session) -> None: """ Kicks off the automated release process by creating and pushing a new tag. Invokes bump2version with the posarg setting the version. Usage: $ nox -s release -- [major|minor|patch] """ # Little known Nox fact: Passing silent=True captures the output status = session.run( "git", "status", "--porcelain", silent=True, external=True ).strip() if len(status) > 1: session.error("All changes must be committed or removed before release") branch = session.run( "git", "rev-parse", "--abbrev-ref", "HEAD", silent=True, external=True ).strip() if branch != DEFAULT_BRANCH: session.error( f"Must be on {DEFAULT_BRANCH!r} branch. Currently on {branch!r} branch" ) parser = argparse.ArgumentParser(description="Release a new semantic version.") parser.add_argument( "version", type=str, nargs=1, help="The type of semver release to make.", choices={"major", "minor", "patch"}, ) args: argparse.Namespace = parser.parse_args(args=session.posargs) version: str = args.version.pop() # If we get here, we should be good to go # Let's do a final check for safety confirm = input( f"You are about to bump the {version!r} version. Are you sure? [y/n]: " ) # Abort on anything other than 'y' if confirm.lower().strip() != "y": session.error(f"You said no when prompted to bump the {version!r} version.") session.install("--upgrade", "pip", "setuptools", "wheel") session.install("bump2version") session.log(f"Bumping the {version!r} version") session.run("bump2version", version) session.log("Pushing the new tag") session.run("git", "push", external=True) session.run("git", "push", "--tags", external=True)
def docs(session: nox.Session) -> None: """invoke sphinx-build to build the HTML docs""" session.install("-r", "documentation/requirements.txt") session.install("-e", ".") outdir = session.cache_dir / "docs_out" session.run("sphinx-build", "./documentation/source", str(outdir), "--color", "-b", "html") index = (outdir / "index.html").resolve().as_uri() session.log(f"Documentation is available at {index}")
def dev_coverage_combine(session: nox.Session) -> None: """Combine coverage from previous dev_* runs into a .coverage file.""" session.install(*coverage_deps) coverage_files = glob.glob("**/.coverage.test.*", recursive=True) session.run("coverage", "combine", *coverage_files) session.log( "Wrote combined coverage database for all tests to '.coverage'.")
def dev_coverage_report(session: nox.Session) -> None: """Report coverage results.""" session.install(*coverage_report_deps) session.log("Python coverage") session.run("coverage", "report") session.log("Library coverage") session.run("gcovr", "--print-summary", "--txt")
def bump(session: nox.Session) -> None: """ Set to a new version, use -- <version>, otherwise will use the latest version. """ parser = argparse.ArgumentParser(description="Process some integers.") parser.add_argument("--commit", action="store_true", help="Make a branch and commit.") parser.add_argument("version", nargs="?", help="The version to process - leave off for latest.") args = parser.parse_args(session.posargs) if args.version is None: session.install("lastversion") version = session.run("lastversion", "kitware/cmake", log=False, silent=True).strip() else: version = args.version session.install("requests") extra = ["--quiet"] if args.commit else [] session.run("python", "scripts/update_cmake_version.py", version, *extra) if args.commit: session.run("git", "switch", "-c", f"update-to-cmake-{version}", external=True) files = ( "CMakeUrls.cmake", "docs/index.rst", "README.rst", "tests/test_distribution.py", "docs/update_cmake_version.rst", ) session.run( "git", "add", "-u", *files, external=True, ) session.run("git", "commit", "-m", f"Update to CMake {version}", external=True) session.log( 'Complete! Now run: gh pr create --fill --body "Created by running `nox -s bump -- --commit`"' )
def release(session: nox.Session) -> None: """ Kicks off the automated release process by creating and pushing a new tag. Invokes bump2version with the posarg setting the version. Usage: $ nox -s release -- [major|minor|patch] """ enforce_branch_no_changes(session) allowed_args: Set[str] = {"major", "minor", "patch"} n_args: int = len(session.posargs) if n_args != 1: session.error( f"Only 1 session arg allowed, got {n_args}. Pass one of: {allowed_args}" ) # If we get here, we know there's only 1 posarg version = session.posargs.pop() if version not in allowed_args: session.error( f"Invalid argument: got {version!r}, expected one of: {allowed_args}" ) # If we get here, we should be good to go # Let's do a final check for safety confirm = input( f"You are about to bump the {version!r} version. Are you sure? [y/n]: " ) # Abort on anything other than 'y' if confirm.lower().strip() != "y": session.error( f"You said no when prompted to bump the {version!r} version.") update_seeds(session) session.install("bump2version") session.log(f"Bumping the {version!r} version") session.run("bump2version", version) session.log("Pushing the new tag") session.run("git", "push", external=True) session.run("git", "push", "--tags", external=True)
def build(session: nox.Session) -> None: """ Build SDists and wheels. """ session.install("build") session.log("Building normal files") session.run("python", "-m", "build", *session.posargs) session.log("Building pybind11-global files (PYBIND11_GLOBAL_SDIST=1)") session.run("python", "-m", "build", *session.posargs, env={"PYBIND11_GLOBAL_SDIST": "1"})
def dev_test_sim( session: nox.Session, sim: Optional[str], toplevel_lang: Optional[str], gpi_interface: Optional[str], ) -> None: """Test a development version of cocotb against a simulator.""" session.env["CFLAGS"] = "-Werror -Wno-deprecated-declarations -g --coverage" session.env["COCOTB_LIBRARY_COVERAGE"] = "1" session.env["CXXFLAGS"] = "-Werror" session.env["LDFLAGS"] = "--coverage" session.install(*test_deps, *coverage_deps) session.install("-e", ".") env = env_vars_for_test(sim, toplevel_lang, gpi_interface) config_str = stringify_dict(env) # Remove a potentially existing coverage file from a previous run for the # same test configuration. coverage_file = Path(f".coverage.test.sim-{sim}-{toplevel_lang}-{gpi_interface}") with suppress(FileNotFoundError): coverage_file.unlink() session.log(f"Running 'make test' against a simulator {config_str}") session.run("make", "test", external=True, env=env) session.log(f"Running simulator-specific tests against a simulator {config_str}") session.run( "pytest", "-v", "--cov=cocotb", "--cov-branch", # Don't display coverage report here "--cov-report=", "-k", "simulator_required", ) Path(".coverage").rename(".coverage.pytest") session.log(f"All tests passed with configuration {config_str}!") # Combine coverage produced during the test runs, and place it in a file # with a name specific to this invocation of dev_test_sim(). coverage_files = glob.glob("**/.coverage.cocotb", recursive=True) if not coverage_files: session.error( "No coverage files found. Something went wrong during the test execution." ) coverage_files.append(".coverage.pytest") session.run("coverage", "combine", "--append", *coverage_files) Path(".coverage").rename(coverage_file) session.log(f"Stored Python coverage for this test run in {coverage_file}.") # Combine coverage from all nox sessions as last step after all sessions # have completed. session.notify("dev_coverage_combine")
def tests_compiler(session: nox.Session, version: str) -> None: """Run the test with a specifiv GCC version.""" session.install( "jinja2", "lxml", "pygments==2.7.4", "pytest", "pytest-timeout", "cmake", "yaxmldiff", ) if platform.system() == "Windows": session.install("pywin32") coverage_args = [] if os.environ.get("USE_COVERAGE") == "true": session.install("pytest-cov") coverage_args = ["--cov=gcovr", "--cov-branch"] session.install("-e", ".") set_environment(session, version) session.log("Print tool versions") session.run("python", "--version") # Use full path to executable session.env["CC"] = shutil.which(session.env["CC"]).replace( os.path.sep, "/") session.run(session.env["CC"], "--version", external=True) session.env["CXX"] = shutil.which(session.env["CXX"]).replace( os.path.sep, "/") session.run(session.env["CXX"], "--version", external=True) session.env["GCOV"] = shutil.which(session.env["CC"].replace( "clang", "llvm-cov").replace("gcc", "gcov")).replace(os.path.sep, "/") session.run(session.env["GCOV"], "--version", external=True) if "llvm-cov" in session.env["GCOV"]: session.env["GCOV"] += " gcov" session.chdir("gcovr/tests") session.run("make", "--silent", "clean", external=True) session.chdir("../..") args = ["-m", "pytest"] args += coverage_args args += session.posargs # For docker tests if "NOX_POSARGS" in os.environ: args += shlex.split(os.environ["NOX_POSARGS"]) if "--" not in args: args += ["--"] + DEFAULT_TEST_DIRECTORIES session.run("python", *args)
def release(session: nox.Session) -> None: """ Kicks off the automated release process by creating and pushing a new tag. Invokes bump2version with the posarg setting the version. Usage: $ nox -s release -- [major|minor|patch] """ enforce_branch_no_changes(session) parser = argparse.ArgumentParser( description="Release a new semantic version.") parser.add_argument( "version", type=str, nargs=1, help="The type of semver release to make.", choices={"major", "minor", "patch"}, ) args: argparse.Namespace = parser.parse_args(args=session.posargs) version: str = args.version.pop() # If we get here, we should be good to go # Let's do a final check for safety confirm = input( f"You are about to bump the {version!r} version. Are you sure? [y/n]: " ) # Abort on anything other than 'y' if confirm.lower().strip() != "y": session.error( f"You said no when prompted to bump the {version!r} version.") update_seeds(session) session.install("bump2version") session.log(f"Bumping the {version!r} version") session.run("bump2version", version) session.log("Pushing the new tag") session.run("git", "push", external=True) session.run("git", "push", "--tags", external=True)
def docs(session: nox.Session) -> None: """ Build the docs. Pass "serve" to serve. """ session.chdir("docs") session.install("-r", "requirements.txt") session.run("sphinx-build", "-M", "html", ".", "_build") if session.posargs: if "serve" in session.posargs: session.log( "Launching docs at http://localhost:8000/ - use Ctrl-C to quit" ) session.run("python", "-m", "http.server", "8000", "-d", "_build/html") else: session.error("Unsupported argument to docs")
def do_docs(session: nox.Session, *, live: bool) -> None: session.install(".") install_requirement(session, "docs-base") if live: install_requirement(session, "docs-live") session.cd("docs") wipe(session, "_build") cmd: Tuple[str, ...] if not live: cmd = ("sphinx-build", ".", "_build", "-W", "--keep-going", "-n") else: cmd = ("sphinx-autobuild", ".", "_build", "-a", "-n") session.run(*cmd, *session.posargs) if not live: uri = (THIS_DIR / "docs" / "_build" / "index.html").as_uri() session.log(f"Built docs: {uri}")
def lint(session: nox.Session) -> None: """Run the lint (flake8 and black).""" session.install("flake8", "flake8-print") # Black installs under Pypy but doesn't necessarily run (cf psf/black#2559). if platform.python_implementation() == "CPython": session.install(BLACK_PINNED_VERSION) if session.posargs: args = session.posargs else: args = DEFAULT_LINT_ARGUMENTS session.run("flake8", *args) if platform.python_implementation() == "CPython": session.run("python", "-m", "black", "--diff", "--check", *args) else: session.log( f"Skip black because of platform {platform.python_implementation()}." )
def build(session: nox.Session) -> str: """ Make an SDist and a wheel. Only runs once. """ global built if not built: session.log( "The files produced locally by this job are not intended to be redistributable" ) session.install("build") tmpdir = session.create_tmp() session.run("python", "-m", "build", "--outdir", tmpdir, env=BUILD_ENV) (wheel_path, ) = Path(tmpdir).glob("*.whl") (sdist_path, ) = Path(tmpdir).glob("*.tar.gz") Path("dist").mkdir(exist_ok=True) wheel_path.rename(f"dist/{wheel_path.name}") sdist_path.rename(f"dist/{sdist_path.name}") built = wheel_path.name return built
def upload_release(session: nox.Session) -> None: version = release.get_version_from_arguments(session) if not version: session.error("Usage: nox -s upload-release -- YY.N[.P]") session.log("# Install dependencies") session.install("twine") distribution_files = glob.glob("dist/*") session.log(f"# Distribution files: {distribution_files}") # Sanity check: Make sure there's 2 distribution files. count = len(distribution_files) if count != 2: session.error( f"Expected 2 distribution files for upload, got {count}. " f"Remove dist/ and run 'nox -s build-release -- {version}'") # Sanity check: Make sure the files are correctly named. distfile_names = (os.path.basename(fn) for fn in distribution_files) expected_distribution_files = [ f"pip-{version}-py3-none-any.whl", f"pip-{version}.tar.gz", ] if sorted(distfile_names) != sorted(expected_distribution_files): session.error( f"Distribution files do not seem to be for {version} release.") session.log("# Upload distributions") session.run("twine", "upload", *distribution_files)
def do_release(session: nox.Session) -> None: """Do a release to PyPI.""" # TODO: maybe add version validation if modified_files_in_git() or modified_files_in_git("--staged"): session.error( "Repository not clean, please remove, unstage, or commit your changes" ) if not len(session.posargs): session.error("Usage: nox -s publish -- <version> [publish-args]") else: version = session.posargs[0] update_version(session, version, MAIN_MODULE) update_changelog(session, version, CHANGELOG) commit_files(session, [MAIN_MODULE, CHANGELOG], message=f"Prepare for release {version}") create_git_tag(session, version, message=f"Release {version}") with isolated_temporary_checkout(session) as workdir: session.chdir(workdir) session.install("flit") session.run("flit", "publish", *session.posargs[1:]) session.chdir(THIS_DIR) update_version(session, next_development_version(version), MAIN_MODULE) update_changelog(session, "<unreleased>", CHANGELOG) commit_files(session, [MAIN_MODULE, CHANGELOG], message="Let's get back to development") session.log( "Alright, just do a push to GitHub and everything should be done") session.log("If you're paranoid, go ahead, verify that things are fine") session.log("If not, sit back and relax, you just did a release 🎉")
def pc_bump(session: nox.Session) -> None: session.install("lastversion") style = Path("pages/developers/style.md") txt = style.read_text() old_versions = {m[1]: m[2].strip('"') for m in PC_VERS.finditer(txt)} for proj, old_version in old_versions.items(): new_version = session.run("lastversion", proj, silent=True).strip() if old_version.lstrip("v") == new_version: continue if old_version.startswith("v"): new_version = f"v{new_version}" before = PC_REPL_LINE.format(proj, old_version) after = PC_REPL_LINE.format(proj, new_version) session.log(f"Bump: {old_version} -> {new_version}") txt = txt.replace(before, after) style.write_text(txt)
def dev_test_nosim(session: nox.Session) -> None: """Run the simulator-agnostic tests against a cocotb development version.""" session.install(*test_deps, *coverage_deps) session.install("-e", ".") # Remove a potentially existing coverage file from a previous run for the # same test configuration. coverage_file = Path(".coverage.test.pytest") with suppress(FileNotFoundError): coverage_file.unlink() # Run pytest with the default configuration in setup.cfg. session.log("Running simulator-agnostic tests with pytest") session.run( "pytest", "-v", "--cov=cocotb", "--cov-branch", # Don't display coverage report here "--cov-report=", "-k", "not simulator_required", ) # Run pytest for files which can only be tested in the source tree, not in # the installed binary (otherwise we get an "import file mismatch" error # from pytest). session.log("Running simulator-agnostic tests in the source tree with pytest") pytest_sourcetree = [ "cocotb/utils.py", "cocotb/binary.py", "cocotb/types/", "cocotb/_sim_versions.py", ] session.run( "pytest", "-v", "--doctest-modules", "--cov=cocotb", "--cov-branch", # Don't display coverage report here "--cov-report=", # Append to the .coverage file created in the previous pytest # invocation in this session. "--cov-append", "-k", "not simulator_required", *pytest_sourcetree, ) session.log("All tests passed!") # Rename the .coverage file to make it unique for the Path(".coverage").rename(coverage_file) session.notify("dev_coverage_combine")
def qa(session: nox.Session) -> None: """Run the quality tests.""" session_id = "lint" session.log(f"Notify session {session_id}") session.notify(session_id, []) session_id = "doc" session.log(f"Notify session {session_id}") session.notify(session_id, []) session_id = f"tests_compiler({GCC_VERSION2USE})" session.log(f"Notify session {session_id}") session.notify(session_id)
def qa_compiler(session: nox.Session, version: str) -> None: """Run the quality tests for a specific GCC version.""" session_id = "lint" session.log(f"Notify session {session_id}") session.notify(session_id, []) session_id = "doc" session.log(f"Notify session {session_id}") session.notify(session_id, []) session_id = f"tests_compiler({version})" session.log(f"Notify session {session_id}") session.notify(session_id)
def build_dists(session: nox.Session) -> List[str]: """Return dists with valid metadata.""" session.log( "# Check if there's any Git-untracked files before building the wheel", ) has_forbidden_git_untracked_files = any( # Don't report the environment this session is running in not untracked_file.startswith(".nox/build-release/") for untracked_file in release.get_git_untracked_files()) if has_forbidden_git_untracked_files: session.error( "There are untracked files in the working directory. " "Remove them and try again", ) session.log("# Build distributions") session.run("python", "setup.py", "sdist", "bdist_wheel", silent=True) produced_dists = glob.glob("dist/*") session.log(f"# Verify distributions: {', '.join(produced_dists)}") session.run("twine", "check", *produced_dists, silent=True) return produced_dists