def test_artifacts(tmp_path):
cache = JupyterCacheBase(str(tmp_path))
with pytest.raises(IOError):
cache.cache_notebook_file(
path=os.path.join(NB_PATH, "basic.ipynb"),
uri="basic.ipynb",
artifacts=(os.path.join(NB_PATH),),
check_validity=False,
)
cache.cache_notebook_file(
path=os.path.join(NB_PATH, "basic.ipynb"),
uri="basic.ipynb",
artifacts=(os.path.join(NB_PATH, "artifact_folder", "artifact.txt"),),
check_validity=False,
)
hashkey = cache.get_cache_record(1).hashkey
assert {
str(p.relative_to(tmp_path)) for p in tmp_path.glob("**/*") if p.is_file()
} == {
"global.db",
f"executed/{hashkey}/base.ipynb",
f"executed/{hashkey}/artifacts/artifact_folder/artifact.txt",
}
bundle = cache.get_cache_bundle(1)
assert {str(p) for p in bundle.artifacts.relative_paths} == {
"artifact_folder/artifact.txt"
}
text = list(h.read().decode() for r, h in bundle.artifacts)[0]
assert text.rstrip() == "An artifact"
with cache.cache_artefacts_temppath(1) as path:
assert path.joinpath("artifact_folder").exists()
def test_list_caches(tmp_path):
db = JupyterCacheBase(str(tmp_path))
db.cache_notebook_file(
path=os.path.join(NB_PATH, "basic.ipynb"),
uri="basic.ipynb",
check_validity=False,
)
runner = CliRunner()
result = runner.invoke(cmd_cache.list_caches, ["-p", tmp_path])
assert result.exception is None, result.output
assert result.exit_code == 0, result.output
assert "basic.ipynb" in result.output.strip(), result.output
def test_show_staged(tmp_path):
db = JupyterCacheBase(str(tmp_path))
db.cache_notebook_file(
path=os.path.join(NB_PATH, "basic.ipynb"), check_validity=False
)
db.stage_notebook_file(path=os.path.join(NB_PATH, "basic.ipynb"))
runner = CliRunner()
result = runner.invoke(cmd_stage.show_staged, ["-p", tmp_path, "1"])
assert result.exception is None, result.output
assert result.exit_code == 0, result.output
assert "basic.ipynb" in result.output.strip(), result.output
def test_merge_match_into_notebook(tmp_path):
cache = JupyterCacheBase(str(tmp_path))
cache.cache_notebook_file(
path=os.path.join(NB_PATH, "basic.ipynb"), check_validity=False
)
nb = nbf.read(os.path.join(NB_PATH, "basic_unrun.ipynb"), 4)
pk, merged = cache.merge_match_into_notebook(nb)
assert merged.cells[1] == {
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [{"name": "stdout", "output_type": "stream", "text": "1\n"}],
"source": "a=1\nprint(a)",
}
def test_list_caches_latest_only(tmp_path):
db = JupyterCacheBase(str(tmp_path))
db.cache_notebook_file(
path=os.path.join(NB_PATH, "basic.ipynb"),
uri="basic.ipynb",
check_validity=False,
)
db.cache_notebook_file(
path=os.path.join(NB_PATH, "complex_outputs.ipynb"),
uri="basic.ipynb",
check_validity=False,
)
runner = CliRunner()
result = runner.invoke(cmd_cache.list_caches, ["-p", tmp_path])
assert result.exception is None, result.output
assert result.exit_code == 0, result.output
assert len(result.output.strip().splitlines()) == 4, result.output
result = runner.invoke(cmd_cache.list_caches, ["-p", tmp_path, "--latest-only"])
assert result.exception is None, result.output
assert result.exit_code == 0, result.output
assert len(result.output.strip().splitlines()) == 3, result.output
def test_diff_nbs(tmp_path):
db = JupyterCacheBase(str(tmp_path))
path = os.path.join(NB_PATH, "basic.ipynb")
path2 = os.path.join(NB_PATH, "basic_failing.ipynb")
db.cache_notebook_file(path, check_validity=False)
# nb_bundle = db.get_cache_bundle(1)
# nb_bundle.nb.cells[0].source = "# New Title"
# db.stage_notebook_bundle(nb_bundle)
runner = CliRunner()
result = runner.invoke(cmd_cache.diff_nb, ["-p", tmp_path, "1", path2])
assert result.exception is None, result.output
assert result.exit_code == 0, result.output
print(result.output.splitlines()[2:])
assert result.output.splitlines()[1:] == [
"--- cached pk=1",
f"+++ other: {path2}",
"## inserted before nb/cells/0:",
"+ code cell:",
"+ source:",
"+ raise Exception('oopsie!')",
"",
"## deleted nb/cells/0:",
"- code cell:",
"- execution_count: 2",
"- source:",
"- a=1",
"- print(a)",
"- outputs:",
"- output 0:",
"- output_type: stream",
"- name: stdout",
"- text:",
"- 1",
"",
"",
"Success!",
]
def test_v4_2_to_v4_5(tmp_path):
"""Test that caching a v4.2 notebook can be recovered,
if the notebook is updated to v4.5 (adding cell ids).
"""
cache = JupyterCacheBase(str(tmp_path))
cache_record = cache.cache_notebook_file(
path=os.path.join(NB_PATH, "basic.ipynb"),
uri="basic.ipynb",
check_validity=False,
)
(pk, nb) = cache.merge_match_into_notebook(
nbf.read(os.path.join(NB_PATH, "basic_v4-5.ipynb"), nbf.NO_CONVERT))
assert cache_record.pk == pk
assert nb.nbformat_minor == 5, nb
assert "id" in nb.cells[1], nb
def test_artifacts_skip_patterns(tmp_path):
cache = JupyterCacheBase(str(tmp_path))
with pytest.raises(IOError):
cache.cache_notebook_file(
path=os.path.join(NB_PATH, "basic.ipynb"),
uri="basic.ipynb",
artifacts=(os.path.join(NB_PATH), ),
check_validity=False,
)
cache.cache_notebook_file(
path=os.path.join(NB_PATH, "basic.ipynb"),
uri="basic.ipynb",
artifacts=(
os.path.join(NB_PATH, "artifact_folder", "artifact.txt"),
os.path.join(NB_PATH, "artifact_folder", "__pycache__"),
),
check_validity=False,
)
# __pycache__ is ignored and not saved as artifact in cache
bundle = cache.get_cache_bundle(1)
assert {str(p)
for p in bundle.artifacts.relative_paths
} == {"artifact_folder/artifact.txt"}
def test_basic_workflow(tmp_path):
cache = JupyterCacheBase(str(tmp_path))
with pytest.raises(NbValidityError):
cache.cache_notebook_file(path=os.path.join(NB_PATH, "basic.ipynb"))
cache.cache_notebook_file(
path=os.path.join(NB_PATH, "basic.ipynb"),
uri="basic.ipynb",
check_validity=False,
)
assert cache.list_cache_records()[0].uri == "basic.ipynb"
pk = cache.match_cache_file(path=os.path.join(NB_PATH, "basic.ipynb")).pk
nb_bundle = cache.get_cache_bundle(pk)
assert nb_bundle.nb.metadata["kernelspec"] == {
"display_name": "Python 3",
"language": "python",
"name": "python3",
}
assert set(nb_bundle.record.to_dict().keys()) == {
"pk",
"hashkey",
"uri",
"data",
"created",
"accessed",
"description",
}
# assert cache.get_cache_codecell(pk, 0).source == "a=1\nprint(a)"
path = os.path.join(NB_PATH, "basic_failing.ipynb")
diff = cache.diff_nbfile_with_cache(pk, path, as_str=True, use_color=False)
assert diff == dedent(
f"""\
nbdiff
--- cached pk=1
+++ other: {path}
## inserted before nb/cells/0:
+ code cell:
+ source:
+ raise Exception('oopsie!')
## deleted nb/cells/0:
- code cell:
- execution_count: 2
- source:
- a=1
- print(a)
- outputs:
- output 0:
- output_type: stream
- name: stdout
- text:
- 1
"""
)
cache.remove_cache(pk)
assert cache.list_cache_records() == []
cache.cache_notebook_file(
path=os.path.join(NB_PATH, "basic.ipynb"),
uri="basic.ipynb",
check_validity=False,
)
with pytest.raises(ValueError):
cache.stage_notebook_file(os.path.join(NB_PATH, "basic.ipynb"), assets=[""])
cache.stage_notebook_file(
os.path.join(NB_PATH, "basic.ipynb"),
assets=[os.path.join(NB_PATH, "basic.ipynb")],
)
assert [r.pk for r in cache.list_staged_records()] == [1]
assert [r.pk for r in cache.list_staged_unexecuted()] == []
cache.stage_notebook_file(os.path.join(NB_PATH, "basic_failing.ipynb"))
assert [r.pk for r in cache.list_staged_records()] == [1, 2]
assert [r.pk for r in cache.list_staged_unexecuted()] == [2]
bundle = cache.get_staged_notebook(os.path.join(NB_PATH, "basic_failing.ipynb"))
assert bundle.nb.metadata
cache.clear_cache()
assert cache.list_cache_records() == []