def test_regression_regex_replace_pass():
"""Test a regression that will succeed by regex replacing certain paths."""
fixture = NBRegressionFixture()
fixture.diff_ignore = (
"/metadata/language_info/version",
"/cells/*/execution_count",
"/cells/*/outputs/*/execution_count",
"/cells/12/outputs/0/data/text/latex",
"/cells/9/outputs/0/metadata/application/json",
)
fixture.diff_replace = (
("/cells/*/outputs/*/traceback", r"\<module\>.*\n", "<module>\n"),
("/cells/*/outputs/*/traceback", r"[\-]+", "-"),
(
"/cells/*/outputs/*/traceback",
r"\s*Traceback \(most recent call last\)",
" Traceback (most recent call last)",
),
(
"/cells/*/outputs/*/traceback",
r"\<ipython\-input\-[\-0-9a-zA-Z]*\>",
"<ipython-input-XXX>",
),
)
fixture.check(os.path.join(PATH, "raw_files", "different_outputs.ipynb"))
def test_text_classifier_tutorial(tmp_path):
notebook_path = Path(TUTORIALS_PATH) / "Training_a_text_classifier.ipynb"
# adapt notebook to CI (make its execution quicker + comment lines)
notebook = load_notebook(str(notebook_path))
for cell in notebook["cells"]:
if cell["source"].startswith("!pip install"):
cell["source"] = re.sub(r"!pip install", r"#!pip install",
cell["source"])
if cell["source"].startswith("trainer_config ="):
cell["source"] = re.sub(r"num_epochs=[0-9][0-9]?", r"num_epochs=1",
cell["source"])
if cell["source"].startswith("pl.train("):
cell["source"] = re.sub(r"training=train_ds", r"training=valid_ds",
cell["source"])
if cell["source"].startswith("pl_trained.explore"):
cell["source"] = re.sub(r"pl_trained.explore",
r"#pl_trained.explore", cell["source"])
# dump adapted notebook
mod_notebook_path = tmp_path / notebook_path.name
with mod_notebook_path.open("w") as file:
file.write(str(dump_notebook(notebook)))
# test adapted notebook
fixture = NBRegressionFixture(exec_timeout=100)
fixture.check(str(mod_notebook_path))
def test_jupyter_example():
""" Test for climate_observations jupyter notebook """
fixture = NBRegressionFixture(
diff_ignore=(
"/metadata/language_info", # Python version depends on testing
"/cells/*/outputs/",
),
force_regen=True,
)
fixture.check(EXAMPLE_DIR / "climate_observations.ipynb")
def test_notebook1(nb_regression: nb.NBRegressionFixture, notebook):
## Generate from the un-run notebook
nb_regression.force_regen = True
nb_regression.check(notebook, False)
## Run notebook against generated
## ignore output for now
nb_regression.diff_ignore = ("/cells/*/outputs/*/text", )
nb_regression.force_regen = False
nb_regression.check(notebook)
def test_jupyter_notebook():
"""Test that the `RmqThreadCommunicator` can be used in a Jupyter notebook."""
from pytest_notebook.nb_regression import NBRegressionFixture
fixture = NBRegressionFixture(exec_timeout=50)
fixture.diff_color_words = False
fixture.diff_ignore = ('/metadata/language_info/version', )
my_dir = pathlib.Path(__file__).parent
with open(my_dir / pathlib.Path('notebooks/communicator.ipynb')) as handle:
fixture.check(handle)
def test_regression_diff_ignore_pass():
"""Test a regression that will succeed by ignoring certain notebook paths."""
fixture = NBRegressionFixture()
fixture.diff_ignore = (
"/metadata/language_info/version",
"/cells/*/execution_count",
"/cells/*/outputs/*/traceback",
"/cells/*/outputs/*/execution_count",
"/cells/12/outputs/0/data/text/latex",
"/cells/9/outputs/0/metadata/application/json",
)
fixture.check(os.path.join(PATH, "raw_files", "different_outputs.ipynb"))
def test_notebook1(nb_regression: nb.NBRegressionFixture, notebook):
## Generate from the un-run notebook
nb_regression.force_regen = True
try:
nb_regression.check(notebook, False)
except TimeoutError as te:
assert (
False
), f"pynotebook `{NOTEBOOK_NAME}` timed out after {nb_regression.exec_timeout}s during test: {te}.\nFor more details see: https://jupyterbook.org/content/execute.html#setting-execution-timeout"
## Run notebook against generated
## ignore output for now
nb_regression.diff_ignore = ("/cells/*/outputs/*/text",)
nb_regression.force_regen = False
nb_regression.check(notebook)
def test_slot_filling_tutorial(tmp_path):
notebook_path = (Path(TUTORIALS_PATH) /
"Training_a_sequence_tagger_for_Slot_Filling.ipynb")
# adapt notebook to CI (make its execution quicker + comment lines)
notebook = load_notebook(str(notebook_path))
for cell in notebook["cells"]:
if cell["source"].startswith("!pip install"):
cell["source"] = re.sub(r"!pip install", r"#!pip install",
cell["source"])
if cell["source"].startswith(
"from biome.text.configuration import FeaturesConfiguration"):
cell["source"] = re.sub(
r"https://dl.fbaipublicfiles.com/fasttext/vectors-english/wiki-news-300d-1M.vec.zip",
r"https://biome-tutorials-data.s3-eu-west-1.amazonaws.com/token_classifier/wiki-news-300d-1M.head.vec",
cell["source"],
)
if cell["source"].startswith("trainer_config ="):
cell["source"] = re.sub(
r"TrainerConfiguration\(\)",
r"TrainerConfiguration(num_epochs=1)",
cell["source"],
)
if cell["source"].startswith("pl.train("):
cell["source"] = re.sub(
r"pl.train",
r"from biome.text.configuration import TrainerConfiguration\npl.train",
cell["source"],
)
cell["source"] = re.sub(
r"training=train_ds",
r"training=valid_ds",
cell["source"],
)
cell["source"] = re.sub(
r"test=test_ds,",
r"test=test_ds, trainer=TrainerConfiguration(num_epochs=1)",
cell["source"],
)
# dump adapted notebook
mod_notebook_path = tmp_path / notebook_path.name
with mod_notebook_path.open("w") as file:
file.write(str(dump_notebook(notebook)))
# test adapted notebook
fixture = NBRegressionFixture(exec_timeout=200)
fixture.check(str(mod_notebook_path))
def test_regression_coverage():
"""Test a regression that will fail."""
fixture = NBRegressionFixture()
fixture.diff_ignore = ("/metadata/language_info/version", )
fixture.coverage = True
result = fixture.check(
os.path.join(PATH, "raw_files", "coverage_test", "call_package.ipynb"))
assert COVERAGE_KEY in result.process_resources
assert "!coverage.py:" in result.process_resources[COVERAGE_KEY]
assert "package.py" in result.process_resources[COVERAGE_KEY]
assert "[1,2,3]" in result.process_resources[COVERAGE_KEY]
def runtest(self):
"""Run the test."""
kwargs, other_args = gather_config_options(self.config)
fixture = NBRegressionFixture(**kwargs)
fixture.check(self.fspath)
def test_notebook(pytestconfig, db_test_app, filename):
"""Execute Jupyter Notebook, using a clean AiiDA database/profile, and test its output is as expected.
Can be executed by: ``pytest --cry17-nb-tests --log-cli-level=info``
"""
from pytest_notebook.nb_regression import NBRegressionFixture
from pytest_notebook.plugin import gather_config_options
kwargs, other_args = gather_config_options(pytestconfig)
nb_regression = NBRegressionFixture(**kwargs)
nb_regression.diff_replace = (
(
"/cells/*/outputs/*/text",
"\\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\\b",
"<UUID>",
),
(
"/cells/*/outputs/*/data/text",
"\\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\\b",
"<UUID>",
),
("/cells/*/outputs/*/text", "[-/|\\\\]\b", ""),
(
"/cells/*/outputs/*/text",
"postgres:\\s+Connected as.*\n",
"postgres: Connected as <ADDRESS>",
),
("/cells/*/outputs/*/text", "repository:\\s+/.+\n",
"repository: <DIRPATH>"),
(
"/cells/*/outputs/*/text",
"Daemon is running as PID.+\n",
"Daemon is running as <PID>\n",
),
("/cells/*/outputs/*/text", "\\d+s\\sago", "XXs ago"),
(
"/cells/*/outputs/*/text",
"(ctime|mtime)\\s+\\d{2,4}-\\d{1,2}-\\d{1,2}.+\n",
"(c/m)time <DATETIME>\n",
),
(
"/cells/*/outputs/*/text",
"time an entry changed state\\:.+\n",
"time an entry changed state: <TIME>\n",
),
(
"/cells/*/outputs/*/text",
"\\d{2,4}-\\d{1,2}-\\d{1,2}\\s\\d{1,2}:\\d{1,2}:\\d{1,2}",
"<DATETIME>",
),
(
"/cells/*/outputs/*/data/text",
"\\'remote_workdir\\'\\:\\s*\\'[\\_\\/a-zA-Z0-9]+\\'",
"'remote_workdir': '<DIRPATH>'",
),
(
"/cells/*/outputs/*/data/text",
"\\'\\_aiida_hash\\'\\:\\s*\\'[a-z0-9]+\\'",
"'_aiida_hash': '<HASH>'",
),
(
"/cells/*/outputs/*/data/text",
"\\<graphviz.dot.Digraph at .+\\>",
"<graphviz.dot.Digraph>",
),
(
"/cells/*/outputs/*/data/image/svg+xml",
"\\<\\!\\-\\-\\sGenerated\\sby\\sgraphviz\\sversion.*\\-\\-\\>",
"<!-- Generated by graphviz version XXX -->",
),
)
from aiida.cmdline.utils.common import get_env_with_venv_bin
# This environmental variable propagates in the jupyter kernel,
# so that the test aiida database/profile is used.
os.environ["AIIDA_PATH"] = db_test_app.environment.config_dir
# We don't actually need to start a daemon, because ``aiida.engine.run`` creates its own.
# However, for `verdi status` and `verdi process list`, we then get warning messages.
curr_env = get_env_with_venv_bin()
output = subprocess.check_output(["verdi", "daemon", "start"],
env=curr_env,
stderr=subprocess.STDOUT)
logger.info(output)
try:
source_dir = os.path.abspath(os.path.dirname(__file__))
with io.open(os.path.join(source_dir, filename), "r+") as handle:
nb_regression.check(handle)
finally:
output = subprocess.check_output(["verdi", "daemon", "stop"],
env=curr_env,
stderr=subprocess.STDOUT)
logger.info(output)
def test_regression_fail():
"""Test a regression that will fail."""
fixture = NBRegressionFixture()
with pytest.raises(NBRegressionError):
fixture.check(os.path.join(PATH, "raw_files", "different_outputs.ipynb"))