def test_clear_cache(tmp_path):
JupyterCacheBase(str(tmp_path))
runner = CliRunner()
result = runner.invoke(cmd_main.clear_cache, ["-p", tmp_path], input="y")
assert result.exception is None, result.output
assert result.exit_code == 0, result.output
assert "Cache cleared!" in result.output.strip(), result.output
def test_artifacts(tmp_path):
cache = JupyterCacheBase(str(tmp_path))
with pytest.raises(IOError):
cache.cache_notebook_file(
path=os.path.join(NB_PATH, "basic.ipynb"),
uri="basic.ipynb",
artifacts=(os.path.join(NB_PATH),),
check_validity=False,
)
cache.cache_notebook_file(
path=os.path.join(NB_PATH, "basic.ipynb"),
uri="basic.ipynb",
artifacts=(os.path.join(NB_PATH, "artifact_folder", "artifact.txt"),),
check_validity=False,
)
hashkey = cache.get_cache_record(1).hashkey
assert {
str(p.relative_to(tmp_path)) for p in tmp_path.glob("**/*") if p.is_file()
} == {
"global.db",
f"executed/{hashkey}/base.ipynb",
f"executed/{hashkey}/artifacts/artifact_folder/artifact.txt",
}
bundle = cache.get_cache_bundle(1)
assert {str(p) for p in bundle.artifacts.relative_paths} == {
"artifact_folder/artifact.txt"
}
text = list(h.read().decode() for r, h in bundle.artifacts)[0]
assert text.rstrip() == "An artifact"
with cache.cache_artefacts_temppath(1) as path:
assert path.joinpath("artifact_folder").exists()
def test_execution_timeout_config(tmp_path):
""" tests the timeout value passed to the executor"""
from jupyter_cache.executors import load_executor
db = JupyterCacheBase(str(tmp_path))
db.stage_notebook_file(
path=os.path.join(NB_PATH, "complex_outputs_unrun.ipynb"))
executor = load_executor("basic", db)
result = executor.run_and_cache(timeout=60)
assert result == {
"succeeded": [os.path.join(NB_PATH, "complex_outputs_unrun.ipynb")],
"excepted": [],
"errored": [],
}
db.clear_cache()
db.stage_notebook_file(
path=os.path.join(NB_PATH, "complex_outputs_unrun.ipynb"))
executor = load_executor("basic", db)
result = executor.run_and_cache(timeout=1)
assert result == {
"succeeded": [],
"excepted": [os.path.join(NB_PATH, "complex_outputs_unrun.ipynb")],
"errored": [],
}
def test_cache_nbs(tmp_path):
db = JupyterCacheBase(str(tmp_path))
path = os.path.join(NB_PATH, "basic.ipynb")
runner = CliRunner()
result = runner.invoke(cmd_cache.cache_nbs, ["-p", tmp_path, "--no-validate", path])
assert result.exception is None, result.output
assert result.exit_code == 0, result.output
assert "basic.ipynb" in result.output.strip(), result.output
assert db.list_cache_records()[0].uri == path
def clear_cache(cache_path):
"""Clear the cache completely."""
from jupyter_cache.cache.main import JupyterCacheBase
db = JupyterCacheBase(cache_path)
click.confirm("Are you sure you want to permanently clear the cache!?",
abort=True)
db.clear_cache()
click.secho("Cache cleared!", fg="green")
def test_unstage_nbs(tmp_path):
db = JupyterCacheBase(str(tmp_path))
path = os.path.join(NB_PATH, "basic.ipynb")
runner = CliRunner()
result = runner.invoke(cmd_stage.stage_nbs, ["-p", tmp_path, path])
result = runner.invoke(cmd_stage.unstage_nbs_uri, ["-p", tmp_path, path])
assert result.exception is None, result.output
assert result.exit_code == 0, result.output
assert "basic.ipynb" in result.output.strip(), result.output
assert db.list_staged_records() == []
def test_list_caches(tmp_path):
db = JupyterCacheBase(str(tmp_path))
db.cache_notebook_file(
path=os.path.join(NB_PATH, "basic.ipynb"),
uri="basic.ipynb",
check_validity=False,
)
runner = CliRunner()
result = runner.invoke(cmd_cache.list_caches, ["-p", tmp_path])
assert result.exception is None, result.output
assert result.exit_code == 0, result.output
assert "basic.ipynb" in result.output.strip(), result.output
def test_show_staged(tmp_path):
db = JupyterCacheBase(str(tmp_path))
db.cache_notebook_file(
path=os.path.join(NB_PATH, "basic.ipynb"), check_validity=False
)
db.stage_notebook_file(path=os.path.join(NB_PATH, "basic.ipynb"))
runner = CliRunner()
result = runner.invoke(cmd_stage.show_staged, ["-p", tmp_path, "1"])
assert result.exception is None, result.output
assert result.exit_code == 0, result.output
assert "basic.ipynb" in result.output.strip(), result.output
def test_run_in_temp_false(tmp_path):
""" tests the timeout value passed to the executor"""
from jupyter_cache.executors import load_executor
db = JupyterCacheBase(str(tmp_path))
db.stage_notebook_file(path=os.path.join(NB_PATH, "basic.ipynb"))
executor = load_executor("basic", db)
result = executor.run_and_cache(run_in_temp=False)
assert result == {
"succeeded": [os.path.join(NB_PATH, "basic.ipynb")],
"excepted": [],
"errored": [],
}
def test_merge_match_into_notebook(tmp_path):
cache = JupyterCacheBase(str(tmp_path))
cache.cache_notebook_file(
path=os.path.join(NB_PATH, "basic.ipynb"), check_validity=False
)
nb = nbf.read(os.path.join(NB_PATH, "basic_unrun.ipynb"), 4)
pk, merged = cache.merge_match_into_notebook(nb)
assert merged.cells[1] == {
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [{"name": "stdout", "output_type": "stream", "text": "1\n"}],
"source": "a=1\nprint(a)",
}
def test_execution_timeout_metadata(tmp_path):
""" tests the timeout metadata key in notebooks"""
from jupyter_cache.executors import load_executor
db = JupyterCacheBase(str(tmp_path))
db.stage_notebook_file(
path=os.path.join(NB_PATH, "sleep_2_timeout_1.ipynb"))
executor = load_executor("basic", db)
result = executor.run_and_cache()
assert result == {
"succeeded": [],
"excepted": [os.path.join(NB_PATH, "sleep_2_timeout_1.ipynb")],
"errored": [],
}
def test_v4_2_to_v4_5(tmp_path):
"""Test that caching a v4.2 notebook can be recovered,
if the notebook is updated to v4.5 (adding cell ids).
"""
cache = JupyterCacheBase(str(tmp_path))
cache_record = cache.cache_notebook_file(
path=os.path.join(NB_PATH, "basic.ipynb"),
uri="basic.ipynb",
check_validity=False,
)
(pk, nb) = cache.merge_match_into_notebook(
nbf.read(os.path.join(NB_PATH, "basic_v4-5.ipynb"), nbf.NO_CONVERT))
assert cache_record.pk == pk
assert nb.nbformat_minor == 5, nb
assert "id" in nb.cells[1], nb
def test_execution(tmp_path):
from jupyter_cache.executors import load_executor
db = JupyterCacheBase(str(tmp_path))
db.stage_notebook_file(path=os.path.join(NB_PATH, "basic_unrun.ipynb"))
db.stage_notebook_file(path=os.path.join(NB_PATH, "basic_failing.ipynb"))
db.stage_notebook_file(
path=os.path.join(NB_PATH, "external_output.ipynb"),
assets=(os.path.join(NB_PATH, "basic.ipynb"), ),
)
executor = load_executor("basic", db)
result = executor.run_and_cache()
print(result)
assert result == {
"succeeded": [
os.path.join(NB_PATH, "basic_unrun.ipynb"),
os.path.join(NB_PATH, "external_output.ipynb"),
],
"excepted": [os.path.join(NB_PATH, "basic_failing.ipynb")],
"errored": [],
}
assert len(db.list_cache_records()) == 2
bundle = db.get_cache_bundle(1)
assert bundle.nb.cells[0] == {
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [{
"name": "stdout",
"output_type": "stream",
"text": "1\n"
}],
"source": "a=1\nprint(a)",
}
assert "execution_seconds" in bundle.record.data
with db.cache_artefacts_temppath(2) as path:
paths = [
str(p.relative_to(path)) for p in path.glob("**/*") if p.is_file()
]
assert paths == ["artifact.txt"]
assert path.joinpath("artifact.txt").read_text(encoding="utf8") == "hi"
stage_record = db.get_staged_record(2)
assert stage_record.traceback is not None
assert "Exception: oopsie!" in stage_record.traceback
def test_cache_with_artifact(tmp_path):
JupyterCacheBase(str(tmp_path))
nb_path = os.path.join(NB_PATH, "basic.ipynb")
a_path = os.path.join(NB_PATH, "artifact_folder", "artifact.txt")
runner = CliRunner()
result = runner.invoke(
cmd_cache.cache_nb, ["-p", tmp_path, "--no-validate", "-nb", nb_path, a_path]
)
assert result.exception is None, result.output
assert result.exit_code == 0, result.output
assert "basic.ipynb" in result.output.strip(), result.output
result = runner.invoke(cmd_cache.show_cache, ["-p", tmp_path, "1"])
assert result.exception is None, result.output
assert result.exit_code == 0, result.output
assert "- artifact_folder/artifact.txt" in result.output.strip(), result.output
result = runner.invoke(
cmd_cache.cat_artifact, ["-p", tmp_path, "1", "artifact_folder/artifact.txt"]
)
assert result.exception is None, result.output
assert result.exit_code == 0, result.output
assert "An artifact" in result.output.strip(), result.output
def test_list_caches_latest_only(tmp_path):
db = JupyterCacheBase(str(tmp_path))
db.cache_notebook_file(
path=os.path.join(NB_PATH, "basic.ipynb"),
uri="basic.ipynb",
check_validity=False,
)
db.cache_notebook_file(
path=os.path.join(NB_PATH, "complex_outputs.ipynb"),
uri="basic.ipynb",
check_validity=False,
)
runner = CliRunner()
result = runner.invoke(cmd_cache.list_caches, ["-p", tmp_path])
assert result.exception is None, result.output
assert result.exit_code == 0, result.output
assert len(result.output.strip().splitlines()) == 4, result.output
result = runner.invoke(cmd_cache.list_caches, ["-p", tmp_path, "--latest-only"])
assert result.exception is None, result.output
assert result.exit_code == 0, result.output
assert len(result.output.strip().splitlines()) == 3, result.output
def test_diff_nbs(tmp_path):
db = JupyterCacheBase(str(tmp_path))
path = os.path.join(NB_PATH, "basic.ipynb")
path2 = os.path.join(NB_PATH, "basic_failing.ipynb")
db.cache_notebook_file(path, check_validity=False)
# nb_bundle = db.get_cache_bundle(1)
# nb_bundle.nb.cells[0].source = "# New Title"
# db.stage_notebook_bundle(nb_bundle)
runner = CliRunner()
result = runner.invoke(cmd_cache.diff_nb, ["-p", tmp_path, "1", path2])
assert result.exception is None, result.output
assert result.exit_code == 0, result.output
print(result.output.splitlines()[2:])
assert result.output.splitlines()[1:] == [
"--- cached pk=1",
f"+++ other: {path2}",
"## inserted before nb/cells/0:",
"+ code cell:",
"+ source:",
"+ raise Exception('oopsie!')",
"",
"## deleted nb/cells/0:",
"- code cell:",
"- execution_count: 2",
"- source:",
"- a=1",
"- print(a)",
"- outputs:",
"- output 0:",
"- output_type: stream",
"- name: stdout",
"- text:",
"- 1",
"",
"",
"Success!",
]
def test_artifacts_skip_patterns(tmp_path):
cache = JupyterCacheBase(str(tmp_path))
with pytest.raises(IOError):
cache.cache_notebook_file(
path=os.path.join(NB_PATH, "basic.ipynb"),
uri="basic.ipynb",
artifacts=(os.path.join(NB_PATH), ),
check_validity=False,
)
cache.cache_notebook_file(
path=os.path.join(NB_PATH, "basic.ipynb"),
uri="basic.ipynb",
artifacts=(
os.path.join(NB_PATH, "artifact_folder", "artifact.txt"),
os.path.join(NB_PATH, "artifact_folder", "__pycache__"),
),
check_validity=False,
)
# __pycache__ is ignored and not saved as artifact in cache
bundle = cache.get_cache_bundle(1)
assert {str(p)
for p in bundle.artifacts.relative_paths
} == {"artifact_folder/artifact.txt"}
def test_basic_workflow(tmp_path):
cache = JupyterCacheBase(str(tmp_path))
with pytest.raises(NbValidityError):
cache.cache_notebook_file(path=os.path.join(NB_PATH, "basic.ipynb"))
cache.cache_notebook_file(
path=os.path.join(NB_PATH, "basic.ipynb"),
uri="basic.ipynb",
check_validity=False,
)
assert cache.list_cache_records()[0].uri == "basic.ipynb"
pk = cache.match_cache_file(path=os.path.join(NB_PATH, "basic.ipynb")).pk
nb_bundle = cache.get_cache_bundle(pk)
assert nb_bundle.nb.metadata["kernelspec"] == {
"display_name": "Python 3",
"language": "python",
"name": "python3",
}
assert set(nb_bundle.record.to_dict().keys()) == {
"pk",
"hashkey",
"uri",
"data",
"created",
"accessed",
"description",
}
# assert cache.get_cache_codecell(pk, 0).source == "a=1\nprint(a)"
path = os.path.join(NB_PATH, "basic_failing.ipynb")
diff = cache.diff_nbfile_with_cache(pk, path, as_str=True, use_color=False)
assert diff == dedent(
f"""\
nbdiff
--- cached pk=1
+++ other: {path}
## inserted before nb/cells/0:
+ code cell:
+ source:
+ raise Exception('oopsie!')
## deleted nb/cells/0:
- code cell:
- execution_count: 2
- source:
- a=1
- print(a)
- outputs:
- output 0:
- output_type: stream
- name: stdout
- text:
- 1
"""
)
cache.remove_cache(pk)
assert cache.list_cache_records() == []
cache.cache_notebook_file(
path=os.path.join(NB_PATH, "basic.ipynb"),
uri="basic.ipynb",
check_validity=False,
)
with pytest.raises(ValueError):
cache.stage_notebook_file(os.path.join(NB_PATH, "basic.ipynb"), assets=[""])
cache.stage_notebook_file(
os.path.join(NB_PATH, "basic.ipynb"),
assets=[os.path.join(NB_PATH, "basic.ipynb")],
)
assert [r.pk for r in cache.list_staged_records()] == [1]
assert [r.pk for r in cache.list_staged_unexecuted()] == []
cache.stage_notebook_file(os.path.join(NB_PATH, "basic_failing.ipynb"))
assert [r.pk for r in cache.list_staged_records()] == [1, 2]
assert [r.pk for r in cache.list_staged_unexecuted()] == [2]
bundle = cache.get_staged_notebook(os.path.join(NB_PATH, "basic_failing.ipynb"))
assert bundle.nb.metadata
cache.clear_cache()
assert cache.list_cache_records() == []