def test_file_hash_detects_change(tmp_path):
file_path = tmp_path / "value.pkl"
counter = Counter()
add_one_counter = call_count(counter)(add_one)
n1 = ComputeNode(add_one,
file_path=file_path,
save=save_pickle,
deps=["x"])
n2 = ComputeNode(add_one_counter,
file_path=file_path,
load=load_pickle,
deps=["x"])
out = QueueRunner(n1)
computed_val = out.compute(x=1)
# Must recompute because file changed, which means hashes won't match
save_pickle(file_path, 3)
out = QueueRunner(n2)
assert out.compute(x=1) == computed_val
assert counter.count == 1
def test_load_save_different_x(tmp_path):
file_path = tmp_path / "save.pkl"
n = ComputeNode(add_one,
file_path=file_path,
save=save_pickle,
load=load_pickle,
deps=["x"])
out = QueueRunner(n)
value = out.compute(x=65)
counter = Counter()
add_one_counter = call_count(counter)(add_one)
n = ComputeNode(
add_one_counter,
file_path=file_path,
save=save_pickle,
load=load_pickle,
deps=["x"],
)(x=66)
out = QueueRunner(n)
assert out.value != value
assert counter.count == 1
def test_save(tmp_path):
file_path = tmp_path / "save.pkl"
n = ComputeNode(add_one, file_path=file_path, save=save_pickle, deps=["x"])
out = QueueRunner(n)
computed_value = out.compute(x=1013)
saved_value = load_pickle(file_path)
assert computed_value == saved_value
def test_file_hash_works_for_int(tmp_path):
file_path = tmp_path / "value.pkl"
counter = Counter()
add_one_counter = call_count(counter)(add_one)
n1 = ComputeNode(add_one,
file_path=file_path,
save=save_pickle,
deps=["x"])
n2 = ComputeNode(add_one_counter,
file_path=file_path,
load=load_pickle,
deps=["x"])
out = QueueRunner(n1)
computed_val = out.compute(x=1)
out = QueueRunner(n2)
assert out.compute(x=1) == computed_val
assert counter.count == 0
def test_load_file_not_found_no_exception(tmp_path):
file_path = tmp_path / "bad.out"
n1 = ComputeNode(add_one,
file_path=file_path,
save=save_pickle,
load=load_pickle,
deps=["x"])
n2 = ComputeNode(add_one,
file_path=file_path,
save=save_pickle,
load=load_pickle,
deps=["x"])
out = QueueRunner(n1)
out.compute(x=1)
os.remove(file_path)
# Expect no exception
out = QueueRunner(n2)
out.compute(x=1)
def test_bad_load_raises(tmp_path):
def bad_load(path):
raise Exception("Failed to load.")
file_path = tmp_path / "bad.out"
n1 = ComputeNode(add_one,
file_path=file_path,
save=save_pickle,
load=bad_load,
deps=["x"])
n2 = ComputeNode(add_one,
file_path=file_path,
save=save_pickle,
load=bad_load,
deps=["x"])
out = QueueRunner(n1)
out.compute(x=1)
with pytest.raises(Exception):
out = QueueRunner(n2)
out.compute(x=1)
def test_bad_save_raises(tmp_path):
def bad_save(path, value):
raise Exception("Failed to save.")
file_path = tmp_path / "bad.out"
# Expect not to raise
with pytest.raises(Exception):
n = ComputeNode(add_one,
file_path=file_path,
save=bad_save,
deps=["x"])
out = QueueRunner(n)
out.compute(x=1)
def test_manual_save_success(tmp_path):
file_path = tmp_path / "bad.out"
n1 = ComputeNode(add_one,
file_path=file_path,
save=save_pickle,
deps=["x"])
out = QueueRunner(n1)
computed_value = out.compute(x=1)
n1.save()
# Pass nop for transform to ensure load is used (not recomputing)
n2 = ComputeNode(lambda: None,
file_path=file_path,
load=load_pickle,
deps=["x"])
n2.bind_all({"x": 1})
out = QueueRunner(n2)
assert out.value == computed_value
def test_load_save(tmp_path):
def get_file_path(params):
fn = "x%s_z%s.pkl" % (params["x"], params["z"])
return tmp_path / fn
@create_node(file_path=get_file_path, load=load_pickle, save=save_pickle)
def x2z3_load_save(x, z):
return x * 2 + z * 3
out = x2z3_load_save("x", "z")(x=1, z=2)
out = QueueRunner(out)
assert out.value == 8
out2 = ComputeNode(lambda: None,
file_path=get_file_path,
load=load_pickle,
deps=["x", "z"])
out2 = out2(x=1, z=2)
assert out2.value == 8
def test_save_missing_raises():
n = ComputeNode(add_one, deps=["x"])
with pytest.raises(ValueError):
n.save()
def test_manual_save_before_compute_raises(tmp_path):
file_path = tmp_path / "bad.out"
n = ComputeNode(add_one, file_path=file_path, save=save_pickle, deps=["x"])
with pytest.raises(ValueError):
n.save()
def test_compute_node():
def foobar(x):
return x
assert foobar.__name__ in str(ComputeNode(foobar))