def test_regression_regex_replace_pass(): """Test a regression that will succeed by regex replacing certain paths.""" fixture = NBRegressionFixture() fixture.diff_ignore = ( "/metadata/language_info/version", "/cells/*/execution_count", "/cells/*/outputs/*/execution_count", "/cells/12/outputs/0/data/text/latex", "/cells/9/outputs/0/metadata/application/json", ) fixture.diff_replace = ( ("/cells/*/outputs/*/traceback", r"\<module\>.*\n", "<module>\n"), ("/cells/*/outputs/*/traceback", r"[\-]+", "-"), ( "/cells/*/outputs/*/traceback", r"\s*Traceback \(most recent call last\)", " Traceback (most recent call last)", ), ( "/cells/*/outputs/*/traceback", r"\<ipython\-input\-[\-0-9a-zA-Z]*\>", "<ipython-input-XXX>", ), ) fixture.check(os.path.join(PATH, "raw_files", "different_outputs.ipynb"))
def test_text_classifier_tutorial(tmp_path): notebook_path = Path(TUTORIALS_PATH) / "Training_a_text_classifier.ipynb" # adapt notebook to CI (make its execution quicker + comment lines) notebook = load_notebook(str(notebook_path)) for cell in notebook["cells"]: if cell["source"].startswith("!pip install"): cell["source"] = re.sub(r"!pip install", r"#!pip install", cell["source"]) if cell["source"].startswith("trainer_config ="): cell["source"] = re.sub(r"num_epochs=[0-9][0-9]?", r"num_epochs=1", cell["source"]) if cell["source"].startswith("pl.train("): cell["source"] = re.sub(r"training=train_ds", r"training=valid_ds", cell["source"]) if cell["source"].startswith("pl_trained.explore"): cell["source"] = re.sub(r"pl_trained.explore", r"#pl_trained.explore", cell["source"]) # dump adapted notebook mod_notebook_path = tmp_path / notebook_path.name with mod_notebook_path.open("w") as file: file.write(str(dump_notebook(notebook))) # test adapted notebook fixture = NBRegressionFixture(exec_timeout=100) fixture.check(str(mod_notebook_path))
def test_jupyter_example(): """ Test for climate_observations jupyter notebook """ fixture = NBRegressionFixture( diff_ignore=( "/metadata/language_info", # Python version depends on testing "/cells/*/outputs/", ), force_regen=True, ) fixture.check(EXAMPLE_DIR / "climate_observations.ipynb")
def test_notebook1(nb_regression: nb.NBRegressionFixture, notebook): ## Generate from the un-run notebook nb_regression.force_regen = True nb_regression.check(notebook, False) ## Run notebook against generated ## ignore output for now nb_regression.diff_ignore = ("/cells/*/outputs/*/text", ) nb_regression.force_regen = False nb_regression.check(notebook)
def test_jupyter_notebook(): """Test that the `RmqThreadCommunicator` can be used in a Jupyter notebook.""" from pytest_notebook.nb_regression import NBRegressionFixture fixture = NBRegressionFixture(exec_timeout=50) fixture.diff_color_words = False fixture.diff_ignore = ('/metadata/language_info/version', ) my_dir = pathlib.Path(__file__).parent with open(my_dir / pathlib.Path('notebooks/communicator.ipynb')) as handle: fixture.check(handle)
def test_regression_diff_ignore_pass(): """Test a regression that will succeed by ignoring certain notebook paths.""" fixture = NBRegressionFixture() fixture.diff_ignore = ( "/metadata/language_info/version", "/cells/*/execution_count", "/cells/*/outputs/*/traceback", "/cells/*/outputs/*/execution_count", "/cells/12/outputs/0/data/text/latex", "/cells/9/outputs/0/metadata/application/json", ) fixture.check(os.path.join(PATH, "raw_files", "different_outputs.ipynb"))
def test_notebook1(nb_regression: nb.NBRegressionFixture, notebook): ## Generate from the un-run notebook nb_regression.force_regen = True try: nb_regression.check(notebook, False) except TimeoutError as te: assert ( False ), f"pynotebook `{NOTEBOOK_NAME}` timed out after {nb_regression.exec_timeout}s during test: {te}.\nFor more details see: https://jupyterbook.org/content/execute.html#setting-execution-timeout" ## Run notebook against generated ## ignore output for now nb_regression.diff_ignore = ("/cells/*/outputs/*/text",) nb_regression.force_regen = False nb_regression.check(notebook)
def test_slot_filling_tutorial(tmp_path): notebook_path = (Path(TUTORIALS_PATH) / "Training_a_sequence_tagger_for_Slot_Filling.ipynb") # adapt notebook to CI (make its execution quicker + comment lines) notebook = load_notebook(str(notebook_path)) for cell in notebook["cells"]: if cell["source"].startswith("!pip install"): cell["source"] = re.sub(r"!pip install", r"#!pip install", cell["source"]) if cell["source"].startswith( "from biome.text.configuration import FeaturesConfiguration"): cell["source"] = re.sub( r"https://dl.fbaipublicfiles.com/fasttext/vectors-english/wiki-news-300d-1M.vec.zip", r"https://biome-tutorials-data.s3-eu-west-1.amazonaws.com/token_classifier/wiki-news-300d-1M.head.vec", cell["source"], ) if cell["source"].startswith("trainer_config ="): cell["source"] = re.sub( r"TrainerConfiguration\(\)", r"TrainerConfiguration(num_epochs=1)", cell["source"], ) if cell["source"].startswith("pl.train("): cell["source"] = re.sub( r"pl.train", r"from biome.text.configuration import TrainerConfiguration\npl.train", cell["source"], ) cell["source"] = re.sub( r"training=train_ds", r"training=valid_ds", cell["source"], ) cell["source"] = re.sub( r"test=test_ds,", r"test=test_ds, trainer=TrainerConfiguration(num_epochs=1)", cell["source"], ) # dump adapted notebook mod_notebook_path = tmp_path / notebook_path.name with mod_notebook_path.open("w") as file: file.write(str(dump_notebook(notebook))) # test adapted notebook fixture = NBRegressionFixture(exec_timeout=200) fixture.check(str(mod_notebook_path))
def test_regression_coverage(): """Test a regression that will fail.""" fixture = NBRegressionFixture() fixture.diff_ignore = ("/metadata/language_info/version", ) fixture.coverage = True result = fixture.check( os.path.join(PATH, "raw_files", "coverage_test", "call_package.ipynb")) assert COVERAGE_KEY in result.process_resources assert "!coverage.py:" in result.process_resources[COVERAGE_KEY] assert "package.py" in result.process_resources[COVERAGE_KEY] assert "[1,2,3]" in result.process_resources[COVERAGE_KEY]
def runtest(self): """Run the test.""" kwargs, other_args = gather_config_options(self.config) fixture = NBRegressionFixture(**kwargs) fixture.check(self.fspath)
def test_notebook(pytestconfig, db_test_app, filename): """Execute Jupyter Notebook, using a clean AiiDA database/profile, and test its output is as expected. Can be executed by: ``pytest --cry17-nb-tests --log-cli-level=info`` """ from pytest_notebook.nb_regression import NBRegressionFixture from pytest_notebook.plugin import gather_config_options kwargs, other_args = gather_config_options(pytestconfig) nb_regression = NBRegressionFixture(**kwargs) nb_regression.diff_replace = ( ( "/cells/*/outputs/*/text", "\\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\\b", "<UUID>", ), ( "/cells/*/outputs/*/data/text", "\\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\\b", "<UUID>", ), ("/cells/*/outputs/*/text", "[-/|\\\\]\b", ""), ( "/cells/*/outputs/*/text", "postgres:\\s+Connected as.*\n", "postgres: Connected as <ADDRESS>", ), ("/cells/*/outputs/*/text", "repository:\\s+/.+\n", "repository: <DIRPATH>"), ( "/cells/*/outputs/*/text", "Daemon is running as PID.+\n", "Daemon is running as <PID>\n", ), ("/cells/*/outputs/*/text", "\\d+s\\sago", "XXs ago"), ( "/cells/*/outputs/*/text", "(ctime|mtime)\\s+\\d{2,4}-\\d{1,2}-\\d{1,2}.+\n", "(c/m)time <DATETIME>\n", ), ( "/cells/*/outputs/*/text", "time an entry changed state\\:.+\n", "time an entry changed state: <TIME>\n", ), ( "/cells/*/outputs/*/text", "\\d{2,4}-\\d{1,2}-\\d{1,2}\\s\\d{1,2}:\\d{1,2}:\\d{1,2}", "<DATETIME>", ), ( "/cells/*/outputs/*/data/text", "\\'remote_workdir\\'\\:\\s*\\'[\\_\\/a-zA-Z0-9]+\\'", "'remote_workdir': '<DIRPATH>'", ), ( "/cells/*/outputs/*/data/text", "\\'\\_aiida_hash\\'\\:\\s*\\'[a-z0-9]+\\'", "'_aiida_hash': '<HASH>'", ), ( "/cells/*/outputs/*/data/text", "\\<graphviz.dot.Digraph at .+\\>", "<graphviz.dot.Digraph>", ), ( "/cells/*/outputs/*/data/image/svg+xml", "\\<\\!\\-\\-\\sGenerated\\sby\\sgraphviz\\sversion.*\\-\\-\\>", "<!-- Generated by graphviz version XXX -->", ), ) from aiida.cmdline.utils.common import get_env_with_venv_bin # This environmental variable propagates in the jupyter kernel, # so that the test aiida database/profile is used. os.environ["AIIDA_PATH"] = db_test_app.environment.config_dir # We don't actually need to start a daemon, because ``aiida.engine.run`` creates its own. # However, for `verdi status` and `verdi process list`, we then get warning messages. curr_env = get_env_with_venv_bin() output = subprocess.check_output(["verdi", "daemon", "start"], env=curr_env, stderr=subprocess.STDOUT) logger.info(output) try: source_dir = os.path.abspath(os.path.dirname(__file__)) with io.open(os.path.join(source_dir, filename), "r+") as handle: nb_regression.check(handle) finally: output = subprocess.check_output(["verdi", "daemon", "stop"], env=curr_env, stderr=subprocess.STDOUT) logger.info(output)
def test_regression_fail(): """Test a regression that will fail.""" fixture = NBRegressionFixture() with pytest.raises(NBRegressionError): fixture.check(os.path.join(PATH, "raw_files", "different_outputs.ipynb"))