def test_file_hash_detects_change(tmp_path): file_path = tmp_path / "value.pkl" counter = Counter() add_one_counter = call_count(counter)(add_one) n1 = ComputeNode(add_one, file_path=file_path, save=save_pickle, deps=["x"]) n2 = ComputeNode(add_one_counter, file_path=file_path, load=load_pickle, deps=["x"]) out = QueueRunner(n1) computed_val = out.compute(x=1) # Must recompute because file changed, which means hashes won't match save_pickle(file_path, 3) out = QueueRunner(n2) assert out.compute(x=1) == computed_val assert counter.count == 1
def test_load_save_different_x(tmp_path): file_path = tmp_path / "save.pkl" n = ComputeNode(add_one, file_path=file_path, save=save_pickle, load=load_pickle, deps=["x"]) out = QueueRunner(n) value = out.compute(x=65) counter = Counter() add_one_counter = call_count(counter)(add_one) n = ComputeNode( add_one_counter, file_path=file_path, save=save_pickle, load=load_pickle, deps=["x"], )(x=66) out = QueueRunner(n) assert out.value != value assert counter.count == 1
def test_save(tmp_path): file_path = tmp_path / "save.pkl" n = ComputeNode(add_one, file_path=file_path, save=save_pickle, deps=["x"]) out = QueueRunner(n) computed_value = out.compute(x=1013) saved_value = load_pickle(file_path) assert computed_value == saved_value
def test_file_hash_works_for_int(tmp_path): file_path = tmp_path / "value.pkl" counter = Counter() add_one_counter = call_count(counter)(add_one) n1 = ComputeNode(add_one, file_path=file_path, save=save_pickle, deps=["x"]) n2 = ComputeNode(add_one_counter, file_path=file_path, load=load_pickle, deps=["x"]) out = QueueRunner(n1) computed_val = out.compute(x=1) out = QueueRunner(n2) assert out.compute(x=1) == computed_val assert counter.count == 0
def test_manual_save_success(tmp_path): file_path = tmp_path / "bad.out" n1 = ComputeNode(add_one, file_path=file_path, save=save_pickle, deps=["x"]) out = QueueRunner(n1) computed_value = out.compute(x=1) n1.save() # Pass nop for transform to ensure load is used (not recomputing) n2 = ComputeNode(lambda: None, file_path=file_path, load=load_pickle, deps=["x"]) n2.bind_all({"x": 1}) out = QueueRunner(n2) assert out.value == computed_value
def test_load_file_not_found_no_exception(tmp_path): file_path = tmp_path / "bad.out" n1 = ComputeNode(add_one, file_path=file_path, save=save_pickle, load=load_pickle, deps=["x"]) n2 = ComputeNode(add_one, file_path=file_path, save=save_pickle, load=load_pickle, deps=["x"]) out = QueueRunner(n1) out.compute(x=1) os.remove(file_path) # Expect no exception out = QueueRunner(n2) out.compute(x=1)
def test_bad_load_raises(tmp_path): def bad_load(path): raise Exception("Failed to load.") file_path = tmp_path / "bad.out" n1 = ComputeNode(add_one, file_path=file_path, save=save_pickle, load=bad_load, deps=["x"]) n2 = ComputeNode(add_one, file_path=file_path, save=save_pickle, load=bad_load, deps=["x"]) out = QueueRunner(n1) out.compute(x=1) with pytest.raises(Exception): out = QueueRunner(n2) out.compute(x=1)
def test_bad_save_raises(tmp_path): def bad_save(path, value): raise Exception("Failed to save.") file_path = tmp_path / "bad.out" # Expect not to raise with pytest.raises(Exception): n = ComputeNode(add_one, file_path=file_path, save=bad_save, deps=["x"]) out = QueueRunner(n) out.compute(x=1)
def test_load_save(tmp_path): def get_file_path(params): fn = "x%s_z%s.pkl" % (params["x"], params["z"]) return tmp_path / fn @create_node(file_path=get_file_path, load=load_pickle, save=save_pickle) def x2z3_load_save(x, z): return x * 2 + z * 3 out = x2z3_load_save("x", "z")(x=1, z=2) out = QueueRunner(out) assert out.value == 8 out2 = ComputeNode(lambda: None, file_path=get_file_path, load=load_pickle, deps=["x", "z"]) out2 = out2(x=1, z=2) assert out2.value == 8
def test_save_missing_raises(): n = ComputeNode(add_one, deps=["x"]) with pytest.raises(ValueError): n.save()
def test_manual_save_before_compute_raises(tmp_path): file_path = tmp_path / "bad.out" n = ComputeNode(add_one, file_path=file_path, save=save_pickle, deps=["x"]) with pytest.raises(ValueError): n.save()
def test_compute_node(): def foobar(x): return x assert foobar.__name__ in str(ComputeNode(foobar))