def test_load_save_different_x(tmp_path):
file_path = tmp_path / "save.pkl"
n = ComputeNode(add_one,
file_path=file_path,
save=save_pickle,
load=load_pickle,
deps=["x"])
out = QueueRunner(n)
value = out.compute(x=65)
counter = Counter()
add_one_counter = call_count(counter)(add_one)
n = ComputeNode(
add_one_counter,
file_path=file_path,
save=save_pickle,
load=load_pickle,
deps=["x"],
)(x=66)
out = QueueRunner(n)
assert out.value != value
assert counter.count == 1
def test_load_save_foreach(tmp_path):
def file_path(params):
return tmp_path / f"save-{params['xs']}.pkl"
n = ForeachComputeNode(
add_one,
file_path=file_path,
save=save_pickle,
load=load_pickle,
deps=["xs"],
foreach="xs",
)
out = QueueRunner(n)
xs = [65, 70, 1]
value = out.compute(xs=xs)
counter = Counter()
add_one_counter = call_count(counter)(add_one)
n = ForeachComputeNode(
add_one_counter,
file_path=file_path,
save=save_pickle,
load=load_pickle,
deps=["xs"],
foreach="xs",
)(xs=xs)
out = QueueRunner(n)
assert out.value == value
assert counter.count == 0
def test_memcache_disable():
ao = add_one("x")
out = QueueRunner(ao)
assert out.compute(x=123) == 124
assert not ao._mem_cache
assert ao._value is None
def test_reusing_pipeline():
ao = add_one("x")
out = ao(x=2)
out = QueueRunner(out)
assert out.value == 3
assert out.compute(x=3) == 4
def test_reusing_pipeline_nested():
ao = add_one("x")
mt = multiply_two(ao)
out = mt(x=2)
out = QueueRunner(out)
assert out.value == 6
assert out.compute(x=3) == 8
def test_shared_binding():
ao = add_one("x")
st = sub_two("y")
s = sum_all(ao, st)
out = s(x=2)
out = QueueRunner(out)
assert out.compute(y=3) == 4
assert out.compute(y=5) == 6
def test_save(tmp_path):
file_path = tmp_path / "save.pkl"
n = ComputeNode(add_one, file_path=file_path, save=save_pickle, deps=["x"])
out = QueueRunner(n)
computed_value = out.compute(x=1013)
saved_value = load_pickle(file_path)
assert computed_value == saved_value
def test_force_compute():
o1 = add_one("x")
o2 = sub_two("y")
out = sum_all(o1, o2)
out = sum_all(o1, out)
out = multiply_two(out)
out = out(y=4)
out = QueueRunner(out)
assert out.compute(x=2) == 16
def test_reusing_pipeline_binding():
ao = add_one("x")
mt = multiply_two(ao)
out = mt(x=2)
out = QueueRunner(out)
assert out.value == 6
out = mt(x=3)
out = QueueRunner(out)
assert out.value == 8
def test_full_pipeline():
o1 = add_one("x")
o2 = sub_two("y")
out = sum_all(o1, o2)
out = sum_all(o1, out)
out = multiply_two(out)
out = out(x=1, y=4)
o1 = QueueRunner(o1)
out = QueueRunner(out)
assert o1.value == 2
assert out.value == 12
def test_bad_save_raises(tmp_path):
def bad_save(path, value):
raise Exception("Failed to save.")
file_path = tmp_path / "bad.out"
# Expect not to raise
with pytest.raises(Exception):
n = ComputeNode(add_one,
file_path=file_path,
save=bad_save,
deps=["x"])
out = QueueRunner(n)
out.compute(x=1)
def test_transform_calls_compute_foreach(call_counter):
ao = add_one_foreach("xs")
out = QueueRunner(ao)
assert out.compute(xs=[1, 2, 3]) == [2, 3, 4]
assert call_counter.count == 3
call_counter.reset()
assert out.compute(xs=[1, 2, 3]) == [2, 3, 4]
assert call_counter.count == 0
call_counter.reset()
assert out.compute(xs=[2, 3, 4]) == [3, 4, 5]
assert call_counter.count == 3
def test_one_node_pipeline():
out = add_one("x")
out = out(x=1)
out = QueueRunner(out)
assert out.value == 2
def test_two_node_pipeline():
out = add_one("x")
out = multiply_two(out)
out = out(x=2)
out = QueueRunner(out)
assert out.value == 6
def test_two_node_pipeline():
"""
1 -> add_one -> 2 -> multiply_two -> 4
"""
out = multiply_two(x=1)
out = QueueRunner(out)
assert out.value == 4
def test_save_foreach(tmp_path):
def file_path(params):
return tmp_path / f"save_{params['xs']}.pkl"
n = ForeachComputeNode(add_one,
file_path=file_path,
save=save_pickle,
deps=["xs"],
foreach="xs")
out = QueueRunner(n)
xs = [1013, 1, -5]
computed_values = out.compute(xs=xs)
for val, computed_val in zip(xs, computed_values):
saved_val = load_pickle(file_path({"xs": val}))
assert computed_val == saved_val
def test_mixed_deps():
out = add_one("x")
out = sum_all(1, out)
out = out(x=2)
out = QueueRunner(out)
assert out.value == 4
def test_multi_dep_pipeline():
o1 = add_one("x")
o2 = sub_two("y")
out = sum_all(o1, o2)
out = out(x=1, y=4)
out = QueueRunner(out)
assert out.value == 4
def test_dep_and_input():
"""
5 -> add_one -> 6 -> remainder -> 2
4 ->
"""
out = remainder(x=5, y=4)
out = QueueRunner(out)
assert out.value
def test_one_node_pipeline():
"""
1 -> add_one -> 2
"""
a = add_one("x")
out = a(x=1)
out = QueueRunner(out)
assert out.value == 2
def test_multiple_inputs_source():
"""
1 3 5 -> sum_all -> 9
"""
s = sum_all("x", "y", "z")
out = s(x=1, y=3, z=5)
out = QueueRunner(out)
assert out.value == 9
def test_missing_graph_inputs_throws():
out = add_one("x")
out = multiply_two(out)
out = out()
out = QueueRunner(out)
with pytest.raises(ValueError):
out.value
def test_multi_dep_pipeline():
"""
2 -> add_one -> 3 -> add_together -> 5
4 -> subtract_one -> 2 ->
"""
out = add_together(x=2, y=4)
out = QueueRunner(out)
assert out.value == 4 + 1
def test_same_source_multi_dep_pipeline():
"""
1 -> add_one -> 2 -> multiply_two -> 4 -> subtract_together -> 3
-> divide_two -> 1 ->
"""
out = subtract_together(x=1)
out = QueueRunner(out)
assert out.value == 3
def test_transform_calls_compute_force_queue(call_counter):
ao1 = add_one("x")
st = sub_two(ao1)
ao2 = add_one(st)
s = sum_(st, ao2)
out = QueueRunner(s)
assert out.compute(x=4, force=True) == 7
assert call_counter.count == 4
call_counter.reset()
assert out.compute(x=4, force=True) == 7
assert call_counter.count == 4
call_counter.reset()
assert out.compute(x=4, force=False) == 7
assert call_counter.count == 0
def test_vars_dict():
o1 = add_one("x")
o2 = sub_two("y")
out = sum_all(o1, o2)
out = sum_all(o1, out)
out = multiply_two(out)
out = out({"x": 1, "y": 4})
out = QueueRunner(out)
assert out.value == 12
def test_file_hash_detects_change(tmp_path):
file_path = tmp_path / "value.pkl"
counter = Counter()
add_one_counter = call_count(counter)(add_one)
n1 = ComputeNode(add_one,
file_path=file_path,
save=save_pickle,
deps=["x"])
n2 = ComputeNode(add_one_counter,
file_path=file_path,
load=load_pickle,
deps=["x"])
out = QueueRunner(n1)
computed_val = out.compute(x=1)
# Must recompute because file changed, which means hashes won't match
save_pickle(file_path, 3)
out = QueueRunner(n2)
assert out.compute(x=1) == computed_val
assert counter.count == 1
def test_transform_calls_one_node_value(call_counter):
ao = add_one("x")
out = ao(x=1)
out = QueueRunner(out)
assert out.value == 2
assert call_counter.count == 1
call_counter.reset()
assert out.value == 2
assert call_counter.count == 0
def test_manual_save_success(tmp_path):
file_path = tmp_path / "bad.out"
n1 = ComputeNode(add_one,
file_path=file_path,
save=save_pickle,
deps=["x"])
out = QueueRunner(n1)
computed_value = out.compute(x=1)
n1.save()
# Pass nop for transform to ensure load is used (not recomputing)
n2 = ComputeNode(lambda: None,
file_path=file_path,
load=load_pickle,
deps=["x"])
n2.bind_all({"x": 1})
out = QueueRunner(n2)
assert out.value == computed_value
def test_foreach_pipeline():
xs = list(range(10))
ao = add_one("xs", foreach="xs")
mt = multiply_two(ao, foreach=0)
s = sum_list(mt)
print(s.text_graph())
out = s(xs=xs)
print(s.text_graph())
out = QueueRunner(out)
correct_val = _sum_list([_multiply_two(_add_one(v)) for v in xs])
assert out.value == correct_val