def task_checklinks():
"""check whether links in built docs are valid"""
key = "check_links"
args = [
"pytest-check-links",
"-o",
"junit_suite_name=checklinks",
"--check-anchors",
"--check-links-cache",
"--check-links-cache-name=build/check_links/cache",
# a few days seems reasonable
f"--check-links-cache-expire-after={60 * 60 * 24 * 3}",
# might be able to relax this, eventually
"-k",
"not (master or carousel)",
]
return dict(
uptodate=[config_changed(dict(args=args))],
actions=[
U.okit(key, remove=True),
lambda: (P.BUILD / "check_links/cache").mkdir(parents=True,
exist_ok=True),
[
*args,
P.DOCS_OUT,
],
U.okit(key),
],
file_dep=[*P.ALL_SPELL_DOCS()],
targets=[P.OK / key],
)
def task_nbtest():
"""smoke test all notebooks with nbconvert"""
env = dict(os.environ)
env.update(WXYZ_WIDGET_LOG_OUT=str(P.WIDGET_LOG_OUT))
return dict(
file_dep=[*P.ALL_SRC_PY, *P.ALL_IPYNB, P.OK / "setup_py"],
targets=[P.OK / "nbtest"],
actions=[
lambda: [
P.WIDGET_LOG_OUT.exists() or P.WIDGET_LOG_OUT.mkdir(), None
][-1],
U.okit("nbtest", True),
lambda: U.call(
[
*P.PYM,
"pytest",
"-vv",
"-n",
"auto",
"-o",
f"junit_suite_name=nbtest_{P.OS}_{P.PY_VER}",
*os.environ.get("WXYZ_PYTEST_ARGS", "").split(" "),
],
cwd=P.PY_SRC / "wxyz_notebooks",
env=env,
) == 0,
U.okit("nbtest"),
],
)
def _make_spell(path):
rel = path.relative_to(P.DOCS_OUT)
spell_key = "spell_" + str(rel.as_posix()).replace("/", "_").replace(
".", "/")
args = [
"hunspell", "-d", P.SPELL_LANGS, "-p", P.DICTIONARY, "-l", "-H", path
]
def _spell():
misspelled = [
line.strip() for line in subprocess.check_output(args).decode(
"utf-8").splitlines() if line.strip()
]
if misspelled:
print(">> misspelled words in ", path)
print("\n".join(sorted(set(misspelled))))
return False
return True
return dict(
name=spell_key,
file_dep=[path, P.DICTIONARY, P.README],
actions=[U.okit(spell_key, remove=True), _spell,
U.okit(spell_key)],
targets=[P.OK / spell_key],
)
def task_setup_py_dev():
"""ensure local packages are installed and editable"""
def write_reqs_txt():
"""write out a requirements file so everything can be installed in one go"""
P.BUILD.exists() or P.BUILD.mkdir()
P.PY_DEV_REQS.write_text("\n".join(
[f"-e {p.parent.relative_to(P.ROOT)}" for p in P.PY_SETUP]))
yield dict(
name="reqs_txt",
targets=[P.PY_DEV_REQS],
file_dep=[*P.ALL_SETUP_CFG, *P.PY_SETUP],
actions=[write_reqs_txt],
)
yield dict(
name="pip",
file_dep=[
P.PY_DEV_REQS,
*[
p.parent / "labextension" / "package.json"
for p in P.WXYZ_LAB_EXTENSIONS
],
],
targets=[P.OK / "setup_py"],
actions=[
U.okit("setup_py", remove=True),
[
*P.PIP,
"install",
"--no-deps",
"--ignore-installed",
"-r",
P.PY_DEV_REQS,
],
[*P.PIP, "freeze"],
[*P.PIP, "check"],
U.okit("setup_py"),
],
)
yield dict(
name="lab",
file_dep=[P.PY_DEV_REQS, P.OK / "setup_py"],
targets=[P.OK / "setup_lab"],
actions=[
U.okit("setup_lab", remove=True),
*[(_make_develop, [p.parent]) for p in P.WXYZ_LAB_EXTENSIONS],
["jupyter", "labextension", "list"],
U.okit("setup_lab"),
],
)
def task_integrity():
"""check various sources of version and documentation issues"""
return dict(
file_dep=[
*P.ALL_SRC_PY,
*P.ALL_MD,
*P.ALL_SETUP_CFG,
P.POSTBUILD,
P.SCRIPTS / "_integrity.py",
],
actions=[
U.okit("integrity", remove=True),
[*P.PYM, "_scripts._integrity"],
U.okit("integrity"),
],
targets=[P.OK / "integrity"],
)
def _make_linters(label, files):
prev = [P.OK / "setup_py"]
next_prev = []
for i, cmd_group in enumerate(P.PY_LINT_CMDS):
for linter, cmd in cmd_group.items():
ok = f"lint_{label}_{i}_{linter}"
next_prev += [P.OK / ok]
yield dict(
name=f"{label}:{linter}",
file_dep=[*files, *prev]
if prev else [*files, P.OK / "setup_py"],
actions=[
U.okit(ok, remove=True),
*(cmd(files) if callable(cmd) else [cmd + files]),
U.okit(ok),
],
targets=[P.OK / ok],
)
prev = next_prev
next_prev = []
def task_setup_py_ci():
"""CI: setup python packages from wheels"""
return dict(
file_dep=[*P.WHEELS.values()],
targets=[P.OK / "setup_py", P.OK / "setup_lab"],
actions=[
U.okit("setup_py", remove=True),
U.okit("setup_lab", remove=True),
[
*P.PIP,
"install",
"--no-deps",
"--ignore-installed",
*P.WHEELS.values(),
],
[*P.PIP, "freeze"],
[*P.PIP, "check"],
U.okit("setup_py"),
["jupyter", "labextension", "list"],
U.okit("setup_lab"),
],
)
def task_robot():
"""test in browser with robot framework"""
file_dep = [
*P.ALL_ROBOT,
*P.ALL_SRC_PY,
*P.ATEST_PY,
*P.ALL_TS,
*P.ALL_IPYNB,
P.SCRIPTS / "_atest.py",
P.OK / "setup_lab",
]
if not P.RUNNING_IN_CI:
file_dep += [P.OK / "robot_lint"]
return dict(
file_dep=sorted(file_dep),
actions=[U.okit("robot", remove=True), [*ATEST],
U.okit("robot")],
targets=[P.OK / "robot"],
)
def task_release():
"""run all tasks, except re-locking and docs"""
return dict(
file_dep=[
*sum(
[[
P.OK / f"lint_{group}_1_pylint",
P.OK / f"lint_{group}_1_flake8"
] for group in P.LINT_GROUPS],
[],
),
P.SHA256SUMS,
P.OK / "integrity",
P.OK / "nbtest",
P.OK / "robot",
],
targets=[P.OK / "release"],
actions=[
U.okit("release", remove=True),
lambda: print("OK to release"),
U.okit("release"),
],
)
def task_lint():
"""detect and (hopefully) correct code style/formatting"""
for label, files in P.LINT_GROUPS.items():
for linter in _make_linters(label, files):
yield linter
yield dict(
name="prettier:core",
uptodate=[config_changed(P.README.read_text(encoding="utf-8"))],
file_dep=[P.YARN_INTEGRITY, P.YARN_LOCK],
actions=[[
"jlpm", "prettier", "--write", "--list-different", P.README
]],
targets=[P.README],
)
yield dict(
name="prettier:rest",
file_dep=[P.YARN_INTEGRITY, P.YARN_LOCK, *P.ALL_PRETTIER],
targets=[P.OK / "prettier"],
actions=[
U.okit("prettier", remove=True),
["jlpm", "lint:prettier"],
U.okit("prettier"),
],
)
yield dict(
name="eslint",
file_dep=[
P.YARN_INTEGRITY,
P.YARN_LOCK,
P.OK / "prettier",
*sum([[*p.rglob("*.ts")] for p in P.TS_SRC], []),
],
targets=[P.OK / "eslint"],
actions=[
U.okit("eslint", remove=True),
["jlpm", "lint:eslint"],
U.okit("eslint"),
],
)
yield dict(
name="robot",
file_dep=[*P.ALL_ROBOT, *P.ATEST_PY],
targets=[P.OK / "robot_lint"],
actions=[
U.okit("robot_dry_run", remove=True),
[*P.PYM, "robot.tidy", "--inplace", *P.ALL_ROBOT],
[*ATEST, "--dryrun"],
U.okit("robot_lint"),
],
)