def test_node_arguments(self): result = fungraph.fun( operator.add, fungraph.fun(lambda: 2), fungraph.fun(lambda: 3), ).cachedcompute() self.assertEqual(result, 5)
def test_nested_delayed_function_and_arguments(self): node = fungraph.fun( delayed(operator.add), delayed(operator.mul)(1, 2), fungraph.fun(operator.mul, 3, delayed(4)), ) self.assertEqual(node.cachedcompute(), 14)
def test_node_mixedarguments(self): result = fungraph.fun( _add_xy, fungraph.fun(lambda: 2), y=fungraph.fun(lambda: 3), ).cachedcompute() self.assertEqual(result, 5)
def test_clone(self): node = fungraph.fun( operator.add, fungraph.fun(lambda: 2), fungraph.fun(lambda: 3), ) clone = node.clone() self.assertEqual(node.cachedcompute(), clone.cachedcompute())
def main(): args = [ fungraph.fun(slowadd, x=np.random.uniform(), y=np.random.uniform()) for _ in range(8) ] f = fungraph.fun(addmany, *args) # print(timeit(f.compute)) # would take about 8 seconds with localclient(): print(timeit(lambda: f(), number=1)) # should take about 1 second
def test_cache_new_object(self): cachedir = tempfile.mkdtemp() node1 = fungraph.fun(_slow_identity, 5, waitseconds=1) f1 = lambda: node1.cachedcompute(cache=cachedir) t1 = timeonce(f1) node2 = fungraph.fun(_slow_identity, 5, waitseconds=1) f2 = lambda: node2.cachedcompute(cache=cachedir) t2 = timeonce(f2) self.assertGreater(t1, 0.5) self.assertLess(t2, 0.5)
def test_scan_pathargument(self): node = fungraph.fun( _add_xy, x=fungraph.fun(_mul_xy, x=1, y=2), y=fungraph.fun(_mul_xy, x=3, y=4), ) scan = node.scan({"x/x": [1, 2, 3], "y/x": [1, 2, 3]}) self.assertEqual(node.cachedcompute(), 2 + 3 * 4) self.assertEqual(scan.cachedcompute(), (1 * 2 + 1 * 4, 2 * 2 + 2 * 4, 3 * 2 + 3 * 4))
def test_modify_clone(self): node = fungraph.fun( operator.add, fungraph.fun(lambda: 2), fungraph.fun(lambda: 3), ) clone = node.clone() clone[1] = fungraph.fun(lambda: 4) self.assertEqual(node.cachedcompute(), 5) self.assertEqual(clone.cachedcompute(), 6)
def test_modify_nodearguments(self): node = fungraph.fun( operator.add, fungraph.fun(lambda: 2), fungraph.fun(lambda: 3), ) result1 = node.cachedcompute() node[1] = fungraph.fun(lambda: 4) result2 = node.cachedcompute() self.assertEqual(result1, 5) self.assertEqual(result2, 6)
def test_mixed_named_unnamed_graph(self): node = fungraph.fun( operator.add, fungraph.fun(lambda: 1), fungraph.named("b", lambda: 2), ) b = node["b"] self.assertEqual(node.cachedcompute(), 3) self.assertEqual(b.cachedcompute(), 2) self.assertEqual(b.name, "b") return
def _test_add(): cachedir = os.sep.join( (tempfile.gettempdir(), "fungraphtestmultiprocessfixeddir")) t = _timenodeonce( fungraph.fun( _slow_add, x=fungraph.fun(lambda: 2), y=fungraph.fun(lambda: 3), ), cachedir=cachedir, ) print(t) return t
def test_path_arguments(self): node = fungraph.fun( _add_xy, fungraph.fun(_mul_xy, 1, 2), fungraph.fun(_mul_xy, 3, 4), ) self.assertEqual(node[0][0], 1) self.assertEqual(node[0][1], 2) self.assertEqual(node[1][0], 3) self.assertEqual(node[1][1], 4) self.assertEqual(node["0/0"], 1) self.assertEqual(node["0/1"], 2) self.assertEqual(node["1/0"], 3) self.assertEqual(node["1/1"], 4)
def test_path_kwarguments(self): node = fungraph.fun( _add_xy, x=fungraph.fun(_mul_xy, x=1, y=2), y=fungraph.fun(_mul_xy, x=3, y=4), ) self.assertEqual(node["x"]["x"], 1) self.assertEqual(node["x"]["y"], 2) self.assertEqual(node["y"]["x"], 3) self.assertEqual(node["y"]["y"], 4) self.assertEqual(node["x/x"], 1) self.assertEqual(node["x/y"], 2) self.assertEqual(node["y/x"], 3) self.assertEqual(node["y/y"], 4)
def test_clone_reuses_cache(self): cachedir = tempfile.mkdtemp() node = fungraph.fun( operator.add, fungraph.fun(_slow_identity, 2, waitseconds=1), fungraph.fun(_slow_identity, 3, waitseconds=1), ) clone = node.clone() nodefun = lambda: node.cachedcompute(cache=cachedir) clonefun = lambda: clone.cachedcompute(cache=cachedir) tn1 = timeonce(nodefun) tn2 = timeonce(nodefun) tc1 = timeonce(clonefun) self.assertGreater(tn1, 0.5) self.assertLess(tn2, 0.5) self.assertLess(tc1, 0.5)
def test_paralell(self): cluster = dask.distributed.LocalCluster( n_workers=8, processes=True, threads_per_worker=1, memory_limit="auto" ) with dask.distributed.Client(address=cluster): def slowfunc(loc): time.sleep(1) return random.gauss(loc, 1.0) N = 8 args = [fungraph.fun(slowfunc, random.uniform(0.0, 1.0)) for _ in range(N)] jobs = fungraph.fun(lambda *args: sum(args), *args) t1 = timeonce(lambda: slowfunc(1.0)) tn = timeonce(jobs.cachedcompute) self.assertLess(tn, (t1 * N) / 2.0)
def test_shelveable(self): node1 = fungraph.fun(_add_xy, x=2, y=3) with shelve.open("testshelf.shelf.db") as s: s["test_node"] = node1 with shelve.open("testshelf.shelf.db") as s: node2 = s["test_node"] self.assertEqual(node1.cachedcompute(), node2.cachedcompute())
def test_modify_keywordarguments(self): node = fungraph.fun(_add_xy, x=2, y=3) result1 = node.cachedcompute() node["y"] = 4 result2 = node.cachedcompute() self.assertEqual(result1, 5) self.assertEqual(result2, 6)
def test_modify_arguments(self): node = fungraph.fun(operator.add, 2, 3) result1 = node.cachedcompute() node[1] = 4 result2 = node.cachedcompute() self.assertEqual(result1, 5) self.assertEqual(result2, 6)
def test_cache_compute_dict_speed(self): f = fungraph.fun(_slow_func, 1, 2) cache = MemoryCache() t1 = timeonce(lambda: f.cachedcompute(cache=cache)) t2 = timeonce(lambda: f.cachedcompute(cache=cache)) self.assertGreater(t1, 0.5) self.assertLess(t2, 0.5)
def _getcompositenamedtuplenode(): return fungraph.fun( lambda x: x, CompositeNamedTuple3( TestNamedTuple3([1, 2, 3], {"a": 1, "b": 2}), TestNamedTuple3([4, 5, 6], {"c": 4, "d": 5}), ), )
def test_cache(self): cachedir = tempfile.mkdtemp() node = fungraph.fun(_slow_identity, 5, waitseconds=1) f = lambda: node.cachedcompute(cache=cachedir) t1 = timeonce(f) t2 = timeonce(f) self.assertGreater(t1, 0.5) self.assertLess(t2, 0.5)
def test_modify_path_arguments(self): node = fungraph.fun( _add_xy, fungraph.fun(_mul_xy, 1, y=2), y=fungraph.fun(_mul_xy, 3, y=4), ) node["0/0"] = 10 node["0/y"] = 20 node["y/0"] = 30 node["y/y"] = 40 self.assertEqual(node[0][0], 10) self.assertEqual(node[0]["y"], 20) self.assertEqual(node["y"][0], 30) self.assertEqual(node["y"]["y"], 40) self.assertEqual(node["0/0"], 10) self.assertEqual(node["0/y"], 20) self.assertEqual(node["y/0"], 30) self.assertEqual(node["y/y"], 40)
def test_paralell_uses_cache(self): cluster = dask.distributed.LocalCluster( n_workers=8, processes=True, threads_per_worker=1, memory_limit="auto" ) with dask.distributed.Client(address=cluster): def slowfunc(loc): time.sleep(1) return random.gauss(loc, 1.0) N = 8 args = [fungraph.fun(slowfunc, random.uniform(0.0, 1.0)) for _ in range(N)] jobs = fungraph.fun(lambda *args: sum(args), *args) with TemporaryDirectory() as d: with fungraph.cachecontext(d): t1 = timeonce(jobs.compute) t2 = timeonce(jobs.compute) self.assertGreater(t1, 1.0) self.assertLess(t2, 0.5)
def test_set_nameclash_with_kwargument(self): node = fungraph.fun( _add_xy, x=fungraph.named("y", lambda: 1), y=fungraph.named("x", lambda: 2), ) node["x"] = fungraph.named("z", lambda: 3) # prefer arguments over named self.assertEqual(node.cachedcompute(), 5) return
def test_cache_multiprocessing(self): with tempfile.TemporaryDirectory() as cachedir: node = fungraph.fun(_slow_add, x=2, y=3, waitseconds=1) t1 = _timenodeonce(node, cachedir) with Pool() as p: tothers = p.starmap(_timenodeonce, [(node.clone(), cachedir) for _ in range(10)]) self.assertGreater(t1, 0.5) for t2 in tothers: self.assertLess(t2, 0.5)
def test_set_nameclash_with_named(self): node = fungraph.fun( operator.add, fungraph.named("x", lambda: 1), fungraph.named("x", lambda: 2), ) node["x"] = fungraph.named("x", lambda: 3) # set first found result self.assertEqual(node.cachedcompute(), 5) return
def test_get_nameclash_with_named(self): node = fungraph.fun( operator.add, fungraph.named("x", lambda: 1), fungraph.named("x", lambda: 2), ) x = node["x"] # return first found result self.assertEqual(node.cachedcompute(), 3) self.assertEqual(x.cachedcompute(), 1) self.assertEqual(x.name, "x") return
def test_get_nameclash_with_kwargument(self): node = fungraph.fun( _add_xy, x=fungraph.named("y", lambda: 1), y=fungraph.named("x", lambda: 2), ) x = node["x"] # prefer arguments over named self.assertEqual(node.cachedcompute(), 3) self.assertEqual(x.cachedcompute(), 1) self.assertEqual(x.name, "y") return
def test_get_nameclash_with_kwargument_explicit(self): node = fungraph.fun( _add_xy, x=fungraph.named("y", lambda: 1), y=fungraph.named("x", lambda: 2), ) x = node[fungraph.Name("x")] y = node[fungraph.KeywordArgument("x")] self.assertEqual(x.cachedcompute(), 2) self.assertEqual(x.name, "x") self.assertEqual(y.cachedcompute(), 1) self.assertEqual(y.name, "y") return
def test_set_nameclash_with_kwargument_explicit(self): node = fungraph.fun( _add_xy, x=fungraph.named("y", lambda: 1), y=fungraph.named("x", lambda: 2), ) node[fungraph.Name("x")] = fungraph.named("z", lambda: 3) node[fungraph.KeywordArgument("x")] = fungraph.named("w", lambda: 4) self.assertEqual(node["x"].cachedcompute(), 4) self.assertEqual(node["x"].name, "w") self.assertEqual(node["y"].cachedcompute(), 3) self.assertEqual(node["y"].name, "z") return