def test_run_example_notebooks(nb_path):
"""Smoke test ensuring that example notebooks run without error.
The `pytest_notebook` package also includes regression test functionality against
saved notebook outputs, if we want to check that later.
"""
nb = notebook.load_notebook(nb_path)
execution.execute_notebook(nb, cwd=EXAMPLES_DIR, timeout=120)
def test_execute_notebook_fail():
"""Test executing a notebook that should fail."""
notebook = nbformat.read(os.path.join(PATH, "raw_files",
"different_outputs.ipynb"),
as_version=4)
exec_results = execute_notebook(notebook)
assert exec_results.exec_error
def test_execute_notebook_success():
"""Test executing a notebook that should succeed."""
notebook = nbformat.read(os.path.join(PATH, "raw_files",
"different_outputs.ipynb"),
as_version=4)
exec_results = execute_notebook(notebook,
cwd=os.path.join(PATH, "raw_files"))
if exec_results.exec_error:
raise exec_results.exec_error
assert isinstance(exec_results.notebook, nbformat.NotebookNode)
def test_execute_notebook_with_coverage():
"""Test executing a notebook with coverage."""
notebook = create_notebook()
notebook.cells = [
create_cell(
dedent("""\
from pytest_notebook.notebook import create_notebook
create_notebook()
import nbformat
"""))
]
exec_results = execute_notebook(notebook,
cwd=os.path.join(PATH, "raw_files"),
with_coverage=True)
if exec_results.exec_error:
raise exec_results.exec_error
assert isinstance(exec_results.notebook, nbformat.NotebookNode)
assert "!coverage.py" in exec_results.resources[COVERAGE_KEY]
def check(
self, path: Union[TextIO, str], raise_errors: bool = True
) -> NBRegressionResult:
"""Execute the Notebook and compare its initial vs. final contents.
if ``force_regen`` is True, the new notebook will be written to ``path``
if ``raise_errors`` is True:
:raise nbconvert.preprocessors.CellExecutionError: if error in execution
:raise NBConfigValidationError: if the notebook metadata is invalid
:raise NBRegressionError: if diffs present
:rtype: NBRegressionResult
"""
__tracebackhide__ = True
if hasattr(path, "name"):
abspath = os.path.abspath(path.name)
else:
abspath = os.path.abspath(str(path))
logger.debug(f"Checking file: {abspath}")
nb_initial, nb_config = load_notebook_with_config(path)
resources = copy.deepcopy(self.process_resources)
if not self.exec_cwd:
self.exec_cwd = os.path.dirname(abspath)
if self.exec_notebook:
logger.debug("Executing notebook.")
exec_results = execute_notebook(
nb_initial,
resources=resources,
cwd=self.exec_cwd,
timeout=self.exec_timeout,
allow_errors=self.exec_allow_errors,
with_coverage=self.coverage,
cov_config_file=self.cov_config,
cov_source=self.cov_source,
)
exec_error = exec_results.exec_error
nb_final = exec_results.notebook
resources = exec_results.resources
else:
exec_error = None
nb_final = nb_initial
# TODO merge on fail option (using pytest-cov --no-cov-on-fail)
if self.cov_merge and exec_results.has_coverage:
logger.info(f"Merging coverage.")
self.cov_merge.data.update(
exec_results.coverage_data(self.cov_merge.debug),
aliases=_get_coverage_aliases(self.cov_merge),
)
# we also take this opportunity to remove ''
# from the unmatched source packages, which is caused by using `--cov=`
self.cov_merge.source_pkgs_unmatched = [
p for p in self.cov_merge.source_pkgs_unmatched if p
]
for proc_name in self.post_processors:
logger.debug(f"Applying post processor: {proc_name}")
post_proc = load_processor(proc_name)
nb_final, resources = post_proc(nb_final, resources)
regex_replace = list(self.diff_replace) + list(nb_config.diff_replace)
if regex_replace:
logger.debug(f"Applying replacements: {regex_replace}")
nb_initial_replace = regex_replace_nb(nb_initial, regex_replace)
nb_final_replace = regex_replace_nb(nb_final, regex_replace)
else:
nb_initial_replace = nb_initial
nb_final_replace = nb_final
full_diff = diff_notebooks(nb_initial_replace, nb_final_replace)
diff_ignore = copy.deepcopy(nb_config.diff_ignore)
diff_ignore.update(self.diff_ignore)
logger.debug(f"filtering diff by ignoring: {diff_ignore}")
filtered_diff = filter_diff(full_diff, diff_ignore)
diff_string = diff_to_string(
nb_initial_replace,
filtered_diff,
use_color=self.diff_use_color,
color_words=self.diff_color_words,
)
# TODO optionally write diff to file
regen_exc = None
if filtered_diff and self.force_regen and not exec_error:
if hasattr(path, "close") and hasattr(path, "name"):
path.close()
with open(path.name, "w") as handle:
nbformat.write(nb_final, handle)
else:
nbformat.write(nb_final, str(path))
regen_exc = NBRegressionError(
f"Files differ and --nb-force-regen set, "
f"regenerating file at:\n- {abspath}"
)
if not raise_errors:
pass
elif exec_error:
print("Diff up to exception:\n" + diff_string, file=sys.stderr)
raise exec_error
elif regen_exc:
print("Diff before regeneration:\n" + diff_string, file=sys.stderr)
raise regen_exc
elif filtered_diff:
raise NBRegressionError(diff_string)
return NBRegressionResult(
nb_initial, nb_final, full_diff, filtered_diff, diff_string, resources
)