def remove_trailing_whitespaces_for_file(file: str, session: nox.Session,
check_only: bool) -> bool:
try:
with open(file, "rb") as fp:
lines = fp.readlines()
new_lines = lines[:]
for i in range(len(new_lines)):
line = lines[i].rstrip(b"\n\r \t")
line += b"\n"
new_lines[i] = line
if lines == new_lines:
return False
if check_only:
session.log(f"Trailing whitespaces found in {file}")
return True
session.log(f"Removing trailing whitespaces present in {file}")
with open(file, "wb") as fp:
fp.writelines(new_lines)
if GIT is not None:
result = subprocess.check_call([GIT, "add", file, "-vf"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=None)
assert result == 0, f"`git add {file} -v' exited with code {result}"
return True
except Exception as ex:
print("Failed to check", file, "because", type(ex).__name__, ex)
return False
def pytest(session: nox.Session) -> None:
"""Run unit tests and measure code coverage.
Coverage can be disabled with the `--skip-coverage` flag.
"""
session.install("-r", "requirements.txt", "-r", "dev-requirements.txt")
_pytest(session)
def _pytest(session: nox.Session, *py_flags: str) -> None:
try:
os.remove(".coverage")
except:
# Ignore errors
pass
session.run("python", *py_flags, "-m", "pytest", *FLAGS, *session.posargs, config.TEST_PACKAGE)
def pytest_speedups(session: nox.Session) -> None:
"""Run unit tests and measure code coverage, using speedup modules.
Coverage can be disabled with the `--skip-coverage` flag.
"""
session.install("-r", "requirements.txt", "-r", "speedup-requirements.txt",
"-r", "dev-requirements.txt")
_pytest(session, "-OO")
def reformat_code(session: nox.Session) -> None:
"""Remove trailing whitespace in source, run isort and then run black code formatter."""
session.install("-r", "dev-requirements.txt")
remove_trailing_whitespaces()
session.run("isort", *REFORMATING_PATHS)
session.run("black", *REFORMATING_PATHS)
def _pytest(session: nox.Session, *py_flags: str) -> None:
if "--skip-coverage" in session.posargs:
session.posargs.remove("--skip-coverage")
flags = RUN_FLAGS
else:
flags = [*RUN_FLAGS, *COVERAGE_FLAGS]
session.run("python", *py_flags, "-m", "pytest", *flags, *session.posargs,
config.TEST_PACKAGE)
def mypy(session: nox.Session) -> None:
"""Perform static type analysis on Python source code."""
session.install("-r", "requirements.txt", "-r", "dev-requirements.txt")
session.run(
"mypy",
"-p",
config.MAIN_PACKAGE,
"--config",
config.MYPY_INI,
)
def verify_types(session: nox.Session) -> None:
"""Verify the "type completeness" of types exported by the library using Pyright."""
session.install("-r", "dev-requirements.txt")
session.install(".")
# session.env["PYRIGHT_PYTHON_GLOBAL_NODE"] = "off"
session.env["PYRIGHT_PYTHON_FORCE_VERSION"] = config.PYRIGHT_VERSION
session.run("python", "-m", "pyright", "--version")
session.run("python", "-m", "pyright", "--verifytypes", "hikari", "--ignoreexternal")
def flake8(session: nox.Session) -> None:
"""Run code linting, SAST, and analysis."""
session.install("-r", "requirements.txt", "-r", "flake8-requirements.txt")
session.run(
"flake8",
"--statistics",
"--show-source",
"--benchmark",
"--tee",
config.MAIN_PACKAGE,
config.TEST_PACKAGE,
)
def flake8_html(session: nox.Session) -> None:
"""Run code linting, SAST, and analysis and generate an HTML report."""
session.install("-r", "requirements.txt", "-r", "flake8-requirements.txt")
session.run(
"flake8",
"--format=html",
f"--htmldir={config.FLAKE8_REPORT}",
"--statistics",
"--show-source",
"--benchmark",
"--tee",
config.MAIN_PACKAGE,
config.TEST_PACKAGE,
)
def _generate_stubs(session: nox.Session) -> None:
session.run("stubgen", *STUBGEN_GENERATE, "-o", ".", "--include-private", "--no-import")
stub_paths = [path + "i" for path in STUBGEN_GENERATE]
session.run("isort", *stub_paths)
session.run("black", *stub_paths)
for stub_path in stub_paths:
with open(stub_path, "r") as fp:
content = fp.read()
with open(stub_path, "w") as fp:
fp.write("# DO NOT MANUALLY EDIT THIS FILE!\n")
fp.write("# This file was automatically generated by `nox -s generate-stubs`\n\n")
fp.write(content)
def reformat_code(session: nox.Session) -> None:
"""Remove trailing whitespace in source, run isort and then run black code formatter."""
remove_trailing_whitespaces()
# isort
session.install(f"isort=={ISORT_VERSION}")
session.run("isort", *REFORMATING_PATHS)
# black
session.install(f"black=={BLACK_VERSION}")
session.run("black", *REFORMATING_PATHS)
def pdoc3(session: nox.Session) -> None:
"""Generate documentation with pdoc."""
session.install("-r", "requirements.txt", "-r", "dev-requirements.txt")
session.env["PDOC3_GENERATING"] = "1"
session.run(
"python",
"docs/patched_pdoc.py",
config.MAIN_PACKAGE,
"--html",
"--output-dir",
config.ARTIFACT_DIRECTORY,
"--template-dir",
config.DOCUMENTATION_DIRECTORY,
"--force",
)
shutil.copyfile(
os.path.join(config.DOCUMENTATION_DIRECTORY, config.LOGO_SOURCE),
os.path.join(config.ARTIFACT_DIRECTORY, config.LOGO_SOURCE),
)
def reformat_code(session: nox.Session) -> None:
"""Remove trailing whitespace in source, run isort, codespell and then run black code formatter."""
session.install("-r", "dev-requirements.txt")
remove_trailing_whitespaces(session)
session.run("isort", *config.PYTHON_REFORMATTING_PATHS)
session.run("black", *config.PYTHON_REFORMATTING_PATHS)
session.run("codespell", "-w", *config.FULL_REFORMATTING_PATHS)
def mypy(session: nox.Session) -> None:
"""Perform static type analysis on Python source code."""
session.install("-r", "requirements.txt", "-r", "speedup-requirements.txt",
"-r", "dev-requirements.txt")
_generate_stubs(session)
session.run("mypy", "-p", config.MAIN_PACKAGE, "--config",
config.PYPROJECT_TOML)
session.run("mypy", "-p", config.EXAMPLE_SCRIPTS, "--config",
config.PYPROJECT_TOML)
def remove_trailing_whitespaces(session: nox.Session,
check_only: bool = False) -> None:
session.log(
f"Searching for stray trailing whitespaces in files ending in {config.REFORMATTING_FILE_EXTS}"
)
count = 0
total = 0
start = time.perf_counter()
for path in config.FULL_REFORMATTING_PATHS:
if os.path.isfile(path):
total += 1
count += remove_trailing_whitespaces_for_file(
path, session, check_only)
for root, dirs, files in os.walk(path, topdown=True,
followlinks=False):
for file in files:
if file.casefold().endswith(config.REFORMATTING_FILE_EXTS):
total += 1
count += remove_trailing_whitespaces_for_file(
os.path.join(root, file), session, check_only)
i = len(dirs) - 1
while i >= 0:
if dirs[i] == "__pycache__":
del dirs[i]
i -= 1
end = time.perf_counter()
remark = "Good job! " if not count else ""
message = "Had to fix" if not check_only else "Found issues in"
session.log(
f"{message} {count} file(s). "
f"{remark}Took {1_000 * (end - start):.2f}ms to check {total} files in this project.",
)
if check_only and count:
session.error(
"Trailing whitespaces found. Try running 'nox -s reformat-code' to fix them"
)
def pytest(session: nox.Session) -> None:
"""Run unit tests and measure code coverage."""
session.install("-r", "requirements.txt", "-r", "dev-requirements.txt")
_pytest(session)
def slotscheck(session: nox.Session) -> None:
"""Check for common slotting mistakes."""
session.install(".", "-r", "dev-requirements.txt")
session.run("slotscheck", "-m", config.MAIN_PACKAGE)
def coveralls(session: nox.Session) -> None:
"""Run coveralls. This has little effect outside TravisCI."""
session.install("-U", "python-coveralls")
session.run("coveralls")
def _generate_stubs(session: nox.Session) -> None:
session.run("stubgen", *STUBGEN_GENERATE, "-o", ".", "--include-private",
"--no-import")
def pages(session: nox.Session) -> None:
"""Generate website pages."""
if not os.path.exists(config.ARTIFACT_DIRECTORY):
os.mkdir(config.ARTIFACT_DIRECTORY)
# Static
print("Copying static objects...")
copy_from_in(config.PAGES_DIRECTORY, config.ARTIFACT_DIRECTORY)
# Documentation
session.install("-r", "requirements.txt", "-r", "dev-requirements.txt")
session.env["PDOC3_GENERATING"] = "1"
print("Building documentation...")
session.run(
"python",
"docs/patched_pdoc.py",
config.MAIN_PACKAGE,
"--html",
"--output-dir",
config.ARTIFACT_DIRECTORY,
"--template-dir",
config.DOCUMENTATION_DIRECTORY,
"--force",
)
# Rename `hikari` into `documentation`
# print("Renaming output dir...")
# print(f"{config.ARTIFACT_DIRECTORY}/{config.MAIN_PACKAGE} -> {config.ARTIFACT_DIRECTORY}/documentation")
# shutil.rmtree(f"{config.ARTIFACT_DIRECTORY}/documentation", ignore_errors=True)
# shutil.move(f"{config.ARTIFACT_DIRECTORY}/{config.MAIN_PACKAGE}", f"{config.ARTIFACT_DIRECTORY}/documentation")
# Pre-generated indexes
if shutil.which("npm") is None:
message = "'npm' not installed, can't prebuild index"
if "CI" in os.environ:
session.error(message)
session.skip(message)
print("Prebuilding index...")
session.run("npm", "install", "[email protected]", external=True)
session.run(
"node",
"scripts/prebuild_index.js",
f"{config.ARTIFACT_DIRECTORY}/hikari/index.json",
f"{config.ARTIFACT_DIRECTORY}/hikari/prebuilt_index.json",
external=True,
)
def generate_stubs(session: nox.Session) -> None:
"""Generate the stubs for the package."""
session.install("-r", "dev-requirements.txt")
_generate_stubs(session)
def safety(session: nox.Session) -> None:
"""Perform dependency scanning."""
# Temporary addition to avoid safety erroring due to https://github.com/pypa/pip/pull/9827
session.install("--upgrade", "pip")
session.install("-r", "requirements.txt", "-r", "dev-requirements.txt")
session.run("safety", "check", "--full-report")
def verify_types(session: nox.Session) -> None:
"""Verify the "type completeness" of types exported by the library using Pyright."""
session.install("-r", "dev-requirements.txt")
session.install(".")
session.run("python", "-m", "pyright", "--verifytypes", config.MAIN_PACKAGE, "--ignoreexternal")
def codespell(session: nox.Session) -> None:
"""Run codespell to check for spelling mistakes."""
session.install("-r", "dev-requirements.txt")
session.run("codespell", *config.FULL_REFORMATTING_PATHS)
def pytest_speedups(session: nox.Session) -> None:
"""Run unit tests and measure code coverage, using speedup modules."""
session.install("-r", "requirements.txt", "-r", "speedup-requirements.txt", "-r", "dev-requirements.txt")
_pytest(session, "-OO")
def _pytest(session: nox.Session, *py_flags: str) -> None:
shutil.rmtree(".coverage", ignore_errors=True)
session.run("python", *py_flags, "-m", "pytest", *FLAGS, *session.posargs, config.TEST_PACKAGE)
def twemoji_test(session: nox.Session):
"""Brute-force test all possible Twemoji mappings for Discord unicode emojis."""
session.install("-U", "-e", ".", "requests")
session.run("python", "scripts/test_twemoji_mapping.py")
def safety(session: nox.Session) -> None:
"""Perform dependency scanning."""
session.install("-r", "requirements.txt", "-r", "dev-requirements.txt")
session.run("safety", "check", "--full-report")
