def test_text_classifier_tutorial(tmp_path):
notebook_path = Path(TUTORIALS_PATH) / "Training_a_text_classifier.ipynb"
# adapt notebook to CI (make its execution quicker + comment lines)
notebook = load_notebook(str(notebook_path))
for cell in notebook["cells"]:
if cell["source"].startswith("!pip install"):
cell["source"] = re.sub(r"!pip install", r"#!pip install",
cell["source"])
if cell["source"].startswith("trainer_config ="):
cell["source"] = re.sub(r"num_epochs=[0-9][0-9]?", r"num_epochs=1",
cell["source"])
if cell["source"].startswith("pl.train("):
cell["source"] = re.sub(r"training=train_ds", r"training=valid_ds",
cell["source"])
if cell["source"].startswith("pl_trained.explore"):
cell["source"] = re.sub(r"pl_trained.explore",
r"#pl_trained.explore", cell["source"])
# dump adapted notebook
mod_notebook_path = tmp_path / notebook_path.name
with mod_notebook_path.open("w") as file:
file.write(str(dump_notebook(notebook)))
# test adapted notebook
fixture = NBRegressionFixture(exec_timeout=100)
fixture.check(str(mod_notebook_path))
def test_jupyter_example():
""" Test for climate_observations jupyter notebook """
fixture = NBRegressionFixture(
diff_ignore=(
"/metadata/language_info", # Python version depends on testing
"/cells/*/outputs/",
),
force_regen=True,
)
fixture.check(EXAMPLE_DIR / "climate_observations.ipynb")
def test_regression_coverage():
"""Test a regression that will fail."""
fixture = NBRegressionFixture()
fixture.diff_ignore = ("/metadata/language_info/version", )
fixture.coverage = True
result = fixture.check(
os.path.join(PATH, "raw_files", "coverage_test", "call_package.ipynb"))
assert COVERAGE_KEY in result.process_resources
assert "!coverage.py:" in result.process_resources[COVERAGE_KEY]
assert "package.py" in result.process_resources[COVERAGE_KEY]
assert "[1,2,3]" in result.process_resources[COVERAGE_KEY]
def test_regression_diff_ignore_pass():
"""Test a regression that will succeed by ignoring certain notebook paths."""
fixture = NBRegressionFixture()
fixture.diff_ignore = (
"/metadata/language_info/version",
"/cells/*/execution_count",
"/cells/*/outputs/*/traceback",
"/cells/*/outputs/*/execution_count",
"/cells/12/outputs/0/data/text/latex",
"/cells/9/outputs/0/metadata/application/json",
)
fixture.check(os.path.join(PATH, "raw_files", "different_outputs.ipynb"))
def test_slot_filling_tutorial(tmp_path):
notebook_path = (Path(TUTORIALS_PATH) /
"Training_a_sequence_tagger_for_Slot_Filling.ipynb")
# adapt notebook to CI (make its execution quicker + comment lines)
notebook = load_notebook(str(notebook_path))
for cell in notebook["cells"]:
if cell["source"].startswith("!pip install"):
cell["source"] = re.sub(r"!pip install", r"#!pip install",
cell["source"])
if cell["source"].startswith(
"from biome.text.configuration import FeaturesConfiguration"):
cell["source"] = re.sub(
r"https://dl.fbaipublicfiles.com/fasttext/vectors-english/wiki-news-300d-1M.vec.zip",
r"https://biome-tutorials-data.s3-eu-west-1.amazonaws.com/token_classifier/wiki-news-300d-1M.head.vec",
cell["source"],
)
if cell["source"].startswith("trainer_config ="):
cell["source"] = re.sub(
r"TrainerConfiguration\(\)",
r"TrainerConfiguration(num_epochs=1)",
cell["source"],
)
if cell["source"].startswith("pl.train("):
cell["source"] = re.sub(
r"pl.train",
r"from biome.text.configuration import TrainerConfiguration\npl.train",
cell["source"],
)
cell["source"] = re.sub(
r"training=train_ds",
r"training=valid_ds",
cell["source"],
)
cell["source"] = re.sub(
r"test=test_ds,",
r"test=test_ds, trainer=TrainerConfiguration(num_epochs=1)",
cell["source"],
)
# dump adapted notebook
mod_notebook_path = tmp_path / notebook_path.name
with mod_notebook_path.open("w") as file:
file.write(str(dump_notebook(notebook)))
# test adapted notebook
fixture = NBRegressionFixture(exec_timeout=200)
fixture.check(str(mod_notebook_path))
def test_nb_regression_cmndline_setting_init(testdir):
"""Test the nb_regression fixture is initialised with the commandline settings."""
testdir.makepyfile("""
import attr
def test_nb(nb_regression):
assert attr.asdict(nb_regression) == {config}
""".format(config=attr.asdict(
NBRegressionFixture(
**{
"exec_allow_errors": True,
"exec_timeout": 90,
"force_regen": True,
# the following are the defaults for pytest-cov
"cov_source": (),
"cov_config": ".coveragerc",
}))))
result = testdir.runpytest("-vv", "--nb-exec-timeout", "90",
"--nb-exec-errors", "--nb-force-regen")
# fnmatch_lines does an assertion internally
result.stdout.fnmatch_lines(["*::test_nb PASSED*"])
# make sure that that we get a '0' exit code for the testsuite
assert result.ret == 0
def nb_tester():
"""Test notebooks using pytest-notebook"""
nb_regression = NBRegressionFixture(
diff_ignore=(
"/metadata/language_info",
"/cells/*/execution_count",
"/cells/*/outputs/*",
),
exec_timeout=1800,
)
return nb_regression
def test_regression_regex_replace_pass():
"""Test a regression that will succeed by regex replacing certain paths."""
fixture = NBRegressionFixture()
fixture.diff_ignore = (
"/metadata/language_info/version",
"/cells/*/execution_count",
"/cells/*/outputs/*/execution_count",
"/cells/12/outputs/0/data/text/latex",
"/cells/9/outputs/0/metadata/application/json",
)
fixture.diff_replace = (
("/cells/*/outputs/*/traceback", r"\<module\>.*\n", "<module>\n"),
("/cells/*/outputs/*/traceback", r"[\-]+", "-"),
(
"/cells/*/outputs/*/traceback",
r"\s*Traceback \(most recent call last\)",
" Traceback (most recent call last)",
),
(
"/cells/*/outputs/*/traceback",
r"\<ipython\-input\-[\-0-9a-zA-Z]*\>",
"<ipython-input-XXX>",
),
)
fixture.check(os.path.join(PATH, "raw_files", "different_outputs.ipynb"))
def test_notebook1(nb_regression: nb.NBRegressionFixture, notebook):
## Generate from the un-run notebook
nb_regression.force_regen = True
nb_regression.check(notebook, False)
## Run notebook against generated
## ignore output for now
nb_regression.diff_ignore = ("/cells/*/outputs/*/text", )
nb_regression.force_regen = False
nb_regression.check(notebook)
def test_jupyter_notebook():
"""Test that the `RmqThreadCommunicator` can be used in a Jupyter notebook."""
from pytest_notebook.nb_regression import NBRegressionFixture
fixture = NBRegressionFixture(exec_timeout=50)
fixture.diff_color_words = False
fixture.diff_ignore = ('/metadata/language_info/version', )
my_dir = pathlib.Path(__file__).parent
with open(my_dir / pathlib.Path('notebooks/communicator.ipynb')) as handle:
fixture.check(handle)
def test_notebook1(nb_regression: nb.NBRegressionFixture, notebook):
## Generate from the un-run notebook
nb_regression.force_regen = True
try:
nb_regression.check(notebook, False)
except TimeoutError as te:
assert (
False
), f"pynotebook `{NOTEBOOK_NAME}` timed out after {nb_regression.exec_timeout}s during test: {te}.\nFor more details see: https://jupyterbook.org/content/execute.html#setting-execution-timeout"
## Run notebook against generated
## ignore output for now
nb_regression.diff_ignore = ("/cells/*/outputs/*/text",)
nb_regression.force_regen = False
nb_regression.check(notebook)
from pytest_notebook.nb_regression import NBRegressionFixture
EXEC_CWD = str(pathlib.Path(__file__).resolve().parent.parent)
fixture = NBRegressionFixture(
exec_timeout=120,
exec_cwd=EXEC_CWD,
diff_color_words=True,
diff_ignore=(
"/cells/*/outputs/*/data/text/plain",
"/cells/*/outputs/*/data/image/svg+xml",
"/cells/*/outputs/*/text",
"/cells/*/outputs/",
"/cells/*/outputs/*/data/image/png",
"/cells/*/outputs/*/data/text/html",
"/cells/*/outputs/*/output/data/"
"/cells/*/outputs/*/data/application/vnd.plotly.v1+json",
"/cells/*/outputs/*/execution_count",
"/cells/*/execution_count",
"/cells/1/outputs/",
"/cells/*/execution_count/",
"/cells/*/metadata/",
"/cells/*/outputs/metadata/",
"/metadata/",
),
)
def test_cluster_analysis_notebook():
notebook = os.path.join(EXEC_CWD, "examples", "Cluster_Analysis.ipynb")
fixture.check(notebook, raise_errors=True)
def test_init_fixture():
"""Test initialisation of NBRegressionFixture."""
fixture = NBRegressionFixture(exec_timeout=10)
assert fixture.exec_timeout == 10
def runtest(self):
"""Run the test."""
kwargs, other_args = gather_config_options(self.config)
fixture = NBRegressionFixture(**kwargs)
fixture.check(self.fspath)
def nb_regression(pytestconfig):
"""Fixture to execute a Jupyter Notebook, and test its output is as expected."""
kwargs, other_args = gather_config_options(pytestconfig)
return NBRegressionFixture(**kwargs)
from pathlib import Path
import pytest
from pytest_notebook.nb_regression import NBRegressionFixture
EXAMPLE_DIR = Path(__file__).parent.parent.parent / "example"
FIXTURE = NBRegressionFixture(
diff_ignore=(
"/metadata/language_info", # Python version depends on testing
"/cells/*/outputs/",
),
force_regen=True,
)
@pytest.mark.slow
def test_jupyter_example():
""" Test for climate_observations jupyter notebook """
FIXTURE.check(EXAMPLE_DIR / "climate_observations.ipynb")
def test_regression_fail():
"""Test a regression that will fail."""
fixture = NBRegressionFixture()
with pytest.raises(NBRegressionError):
fixture.check(os.path.join(PATH, "raw_files", "different_outputs.ipynb"))
def test_notebook(pytestconfig, db_test_app, filename):
"""Execute Jupyter Notebook, using a clean AiiDA database/profile, and test its output is as expected.
Can be executed by: ``pytest --cry17-nb-tests --log-cli-level=info``
"""
from pytest_notebook.nb_regression import NBRegressionFixture
from pytest_notebook.plugin import gather_config_options
kwargs, other_args = gather_config_options(pytestconfig)
nb_regression = NBRegressionFixture(**kwargs)
nb_regression.diff_replace = (
(
"/cells/*/outputs/*/text",
"\\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\\b",
"<UUID>",
),
(
"/cells/*/outputs/*/data/text",
"\\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\\b",
"<UUID>",
),
("/cells/*/outputs/*/text", "[-/|\\\\]\b", ""),
(
"/cells/*/outputs/*/text",
"postgres:\\s+Connected as.*\n",
"postgres: Connected as <ADDRESS>",
),
("/cells/*/outputs/*/text", "repository:\\s+/.+\n",
"repository: <DIRPATH>"),
(
"/cells/*/outputs/*/text",
"Daemon is running as PID.+\n",
"Daemon is running as <PID>\n",
),
("/cells/*/outputs/*/text", "\\d+s\\sago", "XXs ago"),
(
"/cells/*/outputs/*/text",
"(ctime|mtime)\\s+\\d{2,4}-\\d{1,2}-\\d{1,2}.+\n",
"(c/m)time <DATETIME>\n",
),
(
"/cells/*/outputs/*/text",
"time an entry changed state\\:.+\n",
"time an entry changed state: <TIME>\n",
),
(
"/cells/*/outputs/*/text",
"\\d{2,4}-\\d{1,2}-\\d{1,2}\\s\\d{1,2}:\\d{1,2}:\\d{1,2}",
"<DATETIME>",
),
(
"/cells/*/outputs/*/data/text",
"\\'remote_workdir\\'\\:\\s*\\'[\\_\\/a-zA-Z0-9]+\\'",
"'remote_workdir': '<DIRPATH>'",
),
(
"/cells/*/outputs/*/data/text",
"\\'\\_aiida_hash\\'\\:\\s*\\'[a-z0-9]+\\'",
"'_aiida_hash': '<HASH>'",
),
(
"/cells/*/outputs/*/data/text",
"\\<graphviz.dot.Digraph at .+\\>",
"<graphviz.dot.Digraph>",
),
(
"/cells/*/outputs/*/data/image/svg+xml",
"\\<\\!\\-\\-\\sGenerated\\sby\\sgraphviz\\sversion.*\\-\\-\\>",
"<!-- Generated by graphviz version XXX -->",
),
)
from aiida.cmdline.utils.common import get_env_with_venv_bin
# This environmental variable propagates in the jupyter kernel,
# so that the test aiida database/profile is used.
os.environ["AIIDA_PATH"] = db_test_app.environment.config_dir
# We don't actually need to start a daemon, because ``aiida.engine.run`` creates its own.
# However, for `verdi status` and `verdi process list`, we then get warning messages.
curr_env = get_env_with_venv_bin()
output = subprocess.check_output(["verdi", "daemon", "start"],
env=curr_env,
stderr=subprocess.STDOUT)
logger.info(output)
try:
source_dir = os.path.abspath(os.path.dirname(__file__))
with io.open(os.path.join(source_dir, filename), "r+") as handle:
nb_regression.check(handle)
finally:
output = subprocess.check_output(["verdi", "daemon", "stop"],
env=curr_env,
stderr=subprocess.STDOUT)
logger.info(output)
def test_nb_regression_ini_setting_init(testdir):
"""Test the nb_regression fixture is initialised with the config file settings."""
testdir.makeini(r"""
[pytest]
nb_exec_cwd = {path}
nb_exec_allow_errors = True
nb_exec_timeout = 100
nb_diff_use_color = True
nb_diff_color_words = True
nb_diff_ignore =
/metadata/language_info/version
/cells/*/execution_count
/cells/*/outputs/*/traceback
/cells/*/outputs/*/execution_count
/cells/12/outputs/0/data/text/latex
/cells/9/outputs/0/metadata/application/json
nb_post_processors =
nb_diff_replace =
/cells/*/outputs/*/traceback \<ipython\-input\-[\-0-9a-zA-Z]*\> "< >"
""".format(path=os.path.join(PATH, "raw_files")))
testdir.makepyfile("""
import attr
def test_nb(nb_regression):
assert attr.asdict(nb_regression) == {config}
""".format(config=attr.asdict(
NBRegressionFixture(
**{
"exec_cwd":
os.path.join(PATH, "raw_files"),
"exec_allow_errors":
True,
"exec_timeout":
100,
"post_processors": (),
"diff_ignore": (
"/metadata/language_info/version",
"/cells/*/execution_count",
"/cells/*/outputs/*/traceback",
"/cells/*/outputs/*/execution_count",
"/cells/12/outputs/0/data/text/latex",
"/cells/9/outputs/0/metadata/application/json",
),
"diff_replace": ((
"/cells/*/outputs/*/traceback",
"\\<ipython\\-input\\-[\\-0-9a-zA-Z]*\\>",
"< >",
), ),
"diff_use_color":
True,
"diff_color_words":
True,
# the following are the defaults for pytest-cov
"cov_source": (),
"cov_config":
".coveragerc",
}))))
result = testdir.runpytest("-vv")
# fnmatch_lines does an assertion internally
result.stdout.fnmatch_lines(["*::test_nb PASSED*"])
# make sure that that we get a '0' exit code for the testsuite
assert result.ret == 0
import importlib_resources
from pytest_notebook.nb_regression import NBRegressionFixture
import notebooks
fixture = NBRegressionFixture(exec_timeout=300)
fixture.diff_color_words = False
fixture.diff_replace = (("/cells/*/outputs", "\\r", ""),)
# ignoring some output cells,
# because pd.DataFrame.value_counts' return value is inconsistent
fixture.diff_ignore = (
"/cells/*/execution_count",
"/metadata/language_info/version",
"/cells/68/outputs/0/text",
"/cells/69/outputs/0/text",
"/cells/78/outputs/0/text",
"/cells/134/outputs/0/text",
"/cells/135/outputs/0/text",
"/cells/136/outputs/0/text",
)
def cli_notebook_output():
with importlib_resources.path(notebooks, "CLI.ipynb") as path:
fixture.check(str(path))
