def test_node_arguments(self):
result = fungraph.fun(
operator.add,
fungraph.fun(lambda: 2),
fungraph.fun(lambda: 3),
).cachedcompute()
self.assertEqual(result, 5)
def test_nested_delayed_function_and_arguments(self):
node = fungraph.fun(
delayed(operator.add),
delayed(operator.mul)(1, 2),
fungraph.fun(operator.mul, 3, delayed(4)),
)
self.assertEqual(node.cachedcompute(), 14)
def test_node_mixedarguments(self):
result = fungraph.fun(
_add_xy,
fungraph.fun(lambda: 2),
y=fungraph.fun(lambda: 3),
).cachedcompute()
self.assertEqual(result, 5)
def test_clone(self):
node = fungraph.fun(
operator.add,
fungraph.fun(lambda: 2),
fungraph.fun(lambda: 3),
)
clone = node.clone()
self.assertEqual(node.cachedcompute(), clone.cachedcompute())
def main():
args = [
fungraph.fun(slowadd, x=np.random.uniform(), y=np.random.uniform())
for _ in range(8)
]
f = fungraph.fun(addmany, *args)
# print(timeit(f.compute)) # would take about 8 seconds
with localclient():
print(timeit(lambda: f(), number=1)) # should take about 1 second
def test_cache_new_object(self):
cachedir = tempfile.mkdtemp()
node1 = fungraph.fun(_slow_identity, 5, waitseconds=1)
f1 = lambda: node1.cachedcompute(cache=cachedir)
t1 = timeonce(f1)
node2 = fungraph.fun(_slow_identity, 5, waitseconds=1)
f2 = lambda: node2.cachedcompute(cache=cachedir)
t2 = timeonce(f2)
self.assertGreater(t1, 0.5)
self.assertLess(t2, 0.5)
def test_scan_pathargument(self):
node = fungraph.fun(
_add_xy,
x=fungraph.fun(_mul_xy, x=1, y=2),
y=fungraph.fun(_mul_xy, x=3, y=4),
)
scan = node.scan({"x/x": [1, 2, 3], "y/x": [1, 2, 3]})
self.assertEqual(node.cachedcompute(), 2 + 3 * 4)
self.assertEqual(scan.cachedcompute(),
(1 * 2 + 1 * 4, 2 * 2 + 2 * 4, 3 * 2 + 3 * 4))
def test_modify_clone(self):
node = fungraph.fun(
operator.add,
fungraph.fun(lambda: 2),
fungraph.fun(lambda: 3),
)
clone = node.clone()
clone[1] = fungraph.fun(lambda: 4)
self.assertEqual(node.cachedcompute(), 5)
self.assertEqual(clone.cachedcompute(), 6)
def test_modify_nodearguments(self):
node = fungraph.fun(
operator.add,
fungraph.fun(lambda: 2),
fungraph.fun(lambda: 3),
)
result1 = node.cachedcompute()
node[1] = fungraph.fun(lambda: 4)
result2 = node.cachedcompute()
self.assertEqual(result1, 5)
self.assertEqual(result2, 6)
def test_mixed_named_unnamed_graph(self):
node = fungraph.fun(
operator.add,
fungraph.fun(lambda: 1),
fungraph.named("b", lambda: 2),
)
b = node["b"]
self.assertEqual(node.cachedcompute(), 3)
self.assertEqual(b.cachedcompute(), 2)
self.assertEqual(b.name, "b")
return
def _test_add():
cachedir = os.sep.join(
(tempfile.gettempdir(), "fungraphtestmultiprocessfixeddir"))
t = _timenodeonce(
fungraph.fun(
_slow_add,
x=fungraph.fun(lambda: 2),
y=fungraph.fun(lambda: 3),
),
cachedir=cachedir,
)
print(t)
return t
def test_path_arguments(self):
node = fungraph.fun(
_add_xy,
fungraph.fun(_mul_xy, 1, 2),
fungraph.fun(_mul_xy, 3, 4),
)
self.assertEqual(node[0][0], 1)
self.assertEqual(node[0][1], 2)
self.assertEqual(node[1][0], 3)
self.assertEqual(node[1][1], 4)
self.assertEqual(node["0/0"], 1)
self.assertEqual(node["0/1"], 2)
self.assertEqual(node["1/0"], 3)
self.assertEqual(node["1/1"], 4)
def test_path_kwarguments(self):
node = fungraph.fun(
_add_xy,
x=fungraph.fun(_mul_xy, x=1, y=2),
y=fungraph.fun(_mul_xy, x=3, y=4),
)
self.assertEqual(node["x"]["x"], 1)
self.assertEqual(node["x"]["y"], 2)
self.assertEqual(node["y"]["x"], 3)
self.assertEqual(node["y"]["y"], 4)
self.assertEqual(node["x/x"], 1)
self.assertEqual(node["x/y"], 2)
self.assertEqual(node["y/x"], 3)
self.assertEqual(node["y/y"], 4)
def test_clone_reuses_cache(self):
cachedir = tempfile.mkdtemp()
node = fungraph.fun(
operator.add,
fungraph.fun(_slow_identity, 2, waitseconds=1),
fungraph.fun(_slow_identity, 3, waitseconds=1),
)
clone = node.clone()
nodefun = lambda: node.cachedcompute(cache=cachedir)
clonefun = lambda: clone.cachedcompute(cache=cachedir)
tn1 = timeonce(nodefun)
tn2 = timeonce(nodefun)
tc1 = timeonce(clonefun)
self.assertGreater(tn1, 0.5)
self.assertLess(tn2, 0.5)
self.assertLess(tc1, 0.5)
def test_paralell(self):
cluster = dask.distributed.LocalCluster(
n_workers=8, processes=True, threads_per_worker=1, memory_limit="auto"
)
with dask.distributed.Client(address=cluster):
def slowfunc(loc):
time.sleep(1)
return random.gauss(loc, 1.0)
N = 8
args = [fungraph.fun(slowfunc, random.uniform(0.0, 1.0)) for _ in range(N)]
jobs = fungraph.fun(lambda *args: sum(args), *args)
t1 = timeonce(lambda: slowfunc(1.0))
tn = timeonce(jobs.cachedcompute)
self.assertLess(tn, (t1 * N) / 2.0)
def test_shelveable(self):
node1 = fungraph.fun(_add_xy, x=2, y=3)
with shelve.open("testshelf.shelf.db") as s:
s["test_node"] = node1
with shelve.open("testshelf.shelf.db") as s:
node2 = s["test_node"]
self.assertEqual(node1.cachedcompute(), node2.cachedcompute())
def test_modify_keywordarguments(self):
node = fungraph.fun(_add_xy, x=2, y=3)
result1 = node.cachedcompute()
node["y"] = 4
result2 = node.cachedcompute()
self.assertEqual(result1, 5)
self.assertEqual(result2, 6)
def test_modify_arguments(self):
node = fungraph.fun(operator.add, 2, 3)
result1 = node.cachedcompute()
node[1] = 4
result2 = node.cachedcompute()
self.assertEqual(result1, 5)
self.assertEqual(result2, 6)
def test_cache_compute_dict_speed(self):
f = fungraph.fun(_slow_func, 1, 2)
cache = MemoryCache()
t1 = timeonce(lambda: f.cachedcompute(cache=cache))
t2 = timeonce(lambda: f.cachedcompute(cache=cache))
self.assertGreater(t1, 0.5)
self.assertLess(t2, 0.5)
def _getcompositenamedtuplenode():
return fungraph.fun(
lambda x: x,
CompositeNamedTuple3(
TestNamedTuple3([1, 2, 3], {"a": 1, "b": 2}),
TestNamedTuple3([4, 5, 6], {"c": 4, "d": 5}),
),
)
def test_cache(self):
cachedir = tempfile.mkdtemp()
node = fungraph.fun(_slow_identity, 5, waitseconds=1)
f = lambda: node.cachedcompute(cache=cachedir)
t1 = timeonce(f)
t2 = timeonce(f)
self.assertGreater(t1, 0.5)
self.assertLess(t2, 0.5)
def test_modify_path_arguments(self):
node = fungraph.fun(
_add_xy,
fungraph.fun(_mul_xy, 1, y=2),
y=fungraph.fun(_mul_xy, 3, y=4),
)
node["0/0"] = 10
node["0/y"] = 20
node["y/0"] = 30
node["y/y"] = 40
self.assertEqual(node[0][0], 10)
self.assertEqual(node[0]["y"], 20)
self.assertEqual(node["y"][0], 30)
self.assertEqual(node["y"]["y"], 40)
self.assertEqual(node["0/0"], 10)
self.assertEqual(node["0/y"], 20)
self.assertEqual(node["y/0"], 30)
self.assertEqual(node["y/y"], 40)
def test_paralell_uses_cache(self):
cluster = dask.distributed.LocalCluster(
n_workers=8, processes=True, threads_per_worker=1, memory_limit="auto"
)
with dask.distributed.Client(address=cluster):
def slowfunc(loc):
time.sleep(1)
return random.gauss(loc, 1.0)
N = 8
args = [fungraph.fun(slowfunc, random.uniform(0.0, 1.0)) for _ in range(N)]
jobs = fungraph.fun(lambda *args: sum(args), *args)
with TemporaryDirectory() as d:
with fungraph.cachecontext(d):
t1 = timeonce(jobs.compute)
t2 = timeonce(jobs.compute)
self.assertGreater(t1, 1.0)
self.assertLess(t2, 0.5)
def test_set_nameclash_with_kwargument(self):
node = fungraph.fun(
_add_xy,
x=fungraph.named("y", lambda: 1),
y=fungraph.named("x", lambda: 2),
)
node["x"] = fungraph.named("z", lambda: 3)
# prefer arguments over named
self.assertEqual(node.cachedcompute(), 5)
return
def test_cache_multiprocessing(self):
with tempfile.TemporaryDirectory() as cachedir:
node = fungraph.fun(_slow_add, x=2, y=3, waitseconds=1)
t1 = _timenodeonce(node, cachedir)
with Pool() as p:
tothers = p.starmap(_timenodeonce, [(node.clone(), cachedir)
for _ in range(10)])
self.assertGreater(t1, 0.5)
for t2 in tothers:
self.assertLess(t2, 0.5)
def test_set_nameclash_with_named(self):
node = fungraph.fun(
operator.add,
fungraph.named("x", lambda: 1),
fungraph.named("x", lambda: 2),
)
node["x"] = fungraph.named("x", lambda: 3)
# set first found result
self.assertEqual(node.cachedcompute(), 5)
return
def test_get_nameclash_with_named(self):
node = fungraph.fun(
operator.add,
fungraph.named("x", lambda: 1),
fungraph.named("x", lambda: 2),
)
x = node["x"]
# return first found result
self.assertEqual(node.cachedcompute(), 3)
self.assertEqual(x.cachedcompute(), 1)
self.assertEqual(x.name, "x")
return
def test_get_nameclash_with_kwargument(self):
node = fungraph.fun(
_add_xy,
x=fungraph.named("y", lambda: 1),
y=fungraph.named("x", lambda: 2),
)
x = node["x"]
# prefer arguments over named
self.assertEqual(node.cachedcompute(), 3)
self.assertEqual(x.cachedcompute(), 1)
self.assertEqual(x.name, "y")
return
def test_get_nameclash_with_kwargument_explicit(self):
node = fungraph.fun(
_add_xy,
x=fungraph.named("y", lambda: 1),
y=fungraph.named("x", lambda: 2),
)
x = node[fungraph.Name("x")]
y = node[fungraph.KeywordArgument("x")]
self.assertEqual(x.cachedcompute(), 2)
self.assertEqual(x.name, "x")
self.assertEqual(y.cachedcompute(), 1)
self.assertEqual(y.name, "y")
return
def test_set_nameclash_with_kwargument_explicit(self):
node = fungraph.fun(
_add_xy,
x=fungraph.named("y", lambda: 1),
y=fungraph.named("x", lambda: 2),
)
node[fungraph.Name("x")] = fungraph.named("z", lambda: 3)
node[fungraph.KeywordArgument("x")] = fungraph.named("w", lambda: 4)
self.assertEqual(node["x"].cachedcompute(), 4)
self.assertEqual(node["x"].name, "w")
self.assertEqual(node["y"].cachedcompute(), 3)
self.assertEqual(node["y"].name, "z")
return
