def setUp(self): """ R0 R1 R2 \ / | I0 I1 \ / FINAL """ result = Dataset("final_result") intermediate0 = Dataset("intermediate0") intermediate1 = Dataset("intermediate1") raw0 = Dataset("raw0") raw1 = Dataset("raw1") raw2 = Dataset("raw2") result.dependson(intermediate0, intermediate1, raw0) intermediate0.dependson(raw0, raw1) intermediate1.dependson(raw2) self.result = result self.intermediate0 = intermediate0 self.intermediate1 = intermediate1 self.raw0 = raw0 self.raw1 = raw1 self.raw2 = raw2 return
def test_graph_include(self): a = Dataset("a") b = Dataset("b") c = Dataset("c") d = Dataset("d") d.dependson(c) c.dependson(a, b) # Include only parents of `d` dot = depgraph.graphviz(d, include=lambda d1, d2: d in d1.children(0)) self.assertEqual(len(dot.split("\n")), 3) self.assertTrue('"c" -> "d"' in dot)
def test_graph_node_name(self): a = Dataset("a") b = Dataset("b") c = Dataset("c") d = Dataset("d") d.dependson(c) c.dependson(a, b) dot = depgraph.graphviz(d, node_id=lambda d: d.name.upper()) self.assertEqual(len(dot.split("\n")), 5) self.assertTrue('"C" -> "D"' in dot) self.assertTrue('"A" -> "C"' in dot) self.assertTrue('"B" -> "C"' in dot)
def test_graph(self): a = Dataset("a") b = Dataset("b") c = Dataset("c") d = Dataset("d") d.dependson(c) c.dependson(a, b) dot = depgraph.graphviz(d) self.assertEqual(len(dot.split("\n")), 5) self.assertTrue('"c" -> "d"' in dot) self.assertTrue('"a" -> "c"' in dot) self.assertTrue('"b" -> "c"' in dot)
def test_is_older3(self): """ compare a dependency group to a singular dependency """ dep1a = Dataset(fullpath("testdata/1a")) dep1b = Dataset(fullpath("testdata/1b")) dep1c = Dataset(fullpath("testdata/1c")) group1 = DatasetGroup(fullpath("testdata/1"), [dep1a, dep1b, dep1c]) for dep in group1: makefile(dep.name) time.sleep(0.05) dep2 = Dataset(fullpath("testdata/2")) makefile(dep2.name) self.assertTrue(group1.is_older_than(dep2))
def test_graph_edge_style(self): a = Dataset("violet") b = Dataset("green") c = Dataset("red") d = Dataset("blue") d.dependson(c) c.dependson(a, b) dot = depgraph.graphviz(d, style=lambda d1, d2: {"color": d2.name, "weight": 2}) self.assertEqual(len(dot.split("\n")), 5) for line in dot.split("\n"): if line.startswith('"red"'): self.assertTrue("color=blue" in line) self.assertTrue("weight=2" in line) elif line.startswith('"violent"'): self.assertTrue("color=red" in line) self.assertTrue("weight=2" in line) elif line.startswith('"green"'): self.assertTrue("color=red" in line) self.assertTrue("weight=2" in line)
def test_graph(self): a = Dataset("a") b = Dataset("b") c = Dataset("c") d = Dataset("d") d.dependson(c) c.dependson(a, b) dot = depgraph.graphviz(d) self.assertEqual(len(dot.split("\n")), 5) self.assertTrue("c -> d" in dot) self.assertTrue("a -> c" in dot) self.assertTrue("b -> c" in dot)
def test_is_older2(self): """ define two dependency groups, where files ages overlap, and so group 1 is not absolutely older than group 2 """ dep1a = Dataset(fullpath("testdata/1a")) dep1b = Dataset(fullpath("testdata/1b")) dep2c = Dataset(fullpath("testdata/2c")) for dep in (dep1a, dep1b, dep2c): makefile(dep.name) time.sleep(0.05) dep1c = Dataset(fullpath("testdata/1c")) dep2a = Dataset(fullpath("testdata/2a")) dep2b = Dataset(fullpath("testdata/2b")) for dep in (dep1c, dep2a, dep2b): makefile(dep.name) group1 = DatasetGroup(fullpath("testdata/1"), [dep1a, dep1b, dep1c]) group2 = DatasetGroup(fullpath("testdata/2"), [dep2a, dep2b, dep2c]) self.assertFalse(group1.is_older_than(group2))
def test_is_older1(self): """ define two dependency groups, where all files are older in one than in the other. """ dep1a = Dataset(fullpath("testdata/1a")) dep1b = Dataset(fullpath("testdata/1b")) dep1c = Dataset(fullpath("testdata/1c")) for dep in (dep1a, dep1b, dep1c): makefile(dep.name) time.sleep(0.05) dep2a = Dataset(fullpath("testdata/2a")) dep2b = Dataset(fullpath("testdata/2b")) dep2c = Dataset(fullpath("testdata/2c")) for dep in (dep2a, dep2b, dep2c): makefile(dep.name) group1 = DatasetGroup(fullpath("testdata/1"), [dep1a, dep1b, dep1c]) group2 = DatasetGroup(fullpath("testdata/2"), [dep2a, dep2b, dep2c]) self.assertTrue(group1.is_older_than(group2))
def test_cyclic2(self): a = Dataset("a") b = Dataset("b") c = Dataset("c") d = Dataset("d") e = Dataset("e") f = Dataset("f") a.dependson(f) f.dependson(d, e) e.dependson(b, c) d.dependson(b) c.dependson(a) b.dependson(a) self.assertFalse(depgraph.is_acyclic(f))
def test_children(self): d1a = Dataset("1a") d1b = Dataset("1b") d1c = Dataset("1c") d1d = Dataset("1d") d2a = Dataset("2a") d2b = Dataset("2b") d2c = Dataset("2c") d2a.dependson(d1a, d1b) d2b.dependson(d1c) d2c.dependson(d1d) dg = DatasetGroup("dg", [d1a, d1b, d1c]) self.assertEqual(set(dg.children()), set([d2a, d2b]))
def setUp(self): """ define a simple dependency graph that is complex enough to be interesting. R0 R1 R2 R3 [raw data] \ / | | DA0 DA1 / \ / \ / DB0 DB1 \ / | \ \ / | \ DC0 DC1 DC2 [products] """ raw0 = Dataset(fullpath("testdata/raw0"), prog="rawdata") raw1 = Dataset(fullpath("testdata/raw1"), prog="rawdata") raw2 = Dataset(fullpath("testdata/raw2"), prog="rawdata") raw3 = Dataset(fullpath("testdata/raw3"), prog="rawdata") da0 = Dataset(fullpath("testproject/da0"), prog="step1") da1 = Dataset(fullpath("testproject/da1"), prog="step2") db0 = Dataset(fullpath("testproject/db0"), prog="step3") db1 = Dataset(fullpath("testproject/db1"), prog="step4") dc0 = Dataset(fullpath("testproject/dc0"), prog="step5") dc1 = Dataset(fullpath("testproject/dc1"), prog="step6") dc2 = Dataset(fullpath("testproject/dc2"), prog="step7") da0.dependson(raw0, raw1) da1.dependson(raw2) db0.dependson(da0, da1) db1.dependson(da1, raw3) dc0.dependson(db0, db1) dc1.dependson(db1) dc2.dependson(db1) self.raw0 = raw0 self.raw1 = raw1 self.raw2 = raw2 self.raw3 = raw3 self.da0 = da0 self.da1 = da1 self.db0 = db0 self.db1 = db1 self.dc0 = dc0 self.dc1 = dc1 self.dc2 = dc2 # initialize "raw" data rawdir = fullpath("testdata") ensureisdir(rawdir) for dep in (raw0, raw1, raw2, raw3): makefile(dep.name) time.sleep(0.05) cleandir(fullpath("testproject")) return
def setUp(self): """ R0 R1 R2 \ / | I0 I1 \ / FINAL """ result = Dataset("final_result") intermediate0 = Dataset("intermediate0") intermediate1 = Dataset("intermediate1") raw0 = Dataset("raw0") raw1 = Dataset("raw1") raw2 = Dataset("raw2") result.dependson(intermediate0, intermediate1, raw0) intermediate0.dependson(raw0, raw1) intermediate1.dependson(raw2) self.result = result self.intermediate0 = intermediate0 self.intermediate1 = intermediate1 self.raw0 = raw0 self.raw1 = raw1 self.raw2 = raw2 return
def test_cyclic2(self): a = Dataset("a") b = Dataset("b") c = Dataset("c") d = Dataset("d") e = Dataset("e") f = Dataset("f") a.dependson(f) f.dependson(d, e) e.dependson(b, c) d.dependson(b) c.dependson(a) b.dependson(a) self.assertFalse(depgraph.is_acyclic(f))
def setUp(self): """ define a simple dependency graph that is complex enough to be interesting. R0 R1 R2 R3 [raw data] \ / | | DA0 DA1 / \ / \ / DB0 DB1 \ / | \ \ / | \ DC0 DC1 DC2 [products] """ raw0 = Dataset(fullpath("testdata/raw0"), prog="rawdata") raw1 = Dataset(fullpath("testdata/raw1"), prog="rawdata") raw2 = Dataset(fullpath("testdata/raw2"), prog="rawdata") raw3 = Dataset(fullpath("testdata/raw3"), prog="rawdata") da0 = Dataset(fullpath("testproject/da0"), prog="step1") da1 = Dataset(fullpath("testproject/da1"), prog="step2") db0 = Dataset(fullpath("testproject/db0"), prog="step3") db1 = Dataset(fullpath("testproject/db1"), prog="step4") dc0 = Dataset(fullpath("testproject/dc0"), prog="step5") dc1 = Dataset(fullpath("testproject/dc1"), prog="step6") dc2 = Dataset(fullpath("testproject/dc2"), prog="step7") da0.dependson(raw0, raw1) da1.dependson(raw2) db0.dependson(da0, da1) db1.dependson(da1, raw3) dc0.dependson(db0, db1) dc1.dependson(db1) dc2.dependson(db1) self.raw0 = raw0 self.raw1 = raw1 self.raw2 = raw2 self.raw3 = raw3 self.da0 = da0 self.da1 = da1 self.db0 = db0 self.db1 = db1 self.dc0 = dc0 self.dc1 = dc1 self.dc2 = dc2 # initialize "raw" data rawdir = fullpath("testdata") ensureisdir(rawdir) for dep in (raw0, raw1, raw2, raw3): makefile(dep.name) time.sleep(0.05) cleandir(fullpath("testproject")) return
from depgraph import Dataset, buildmanager # Define Datasets # Use an optional keyword `tool` to provide a key instructing our build tool # how to assemble this product. Here we've used strings, but another pattern # would be to provide a callback function R0 = Dataset("data/raw0", tool="read_csv") R1 = Dataset("data/raw1", tool="read_csv") R2 = Dataset("data/raw2", tool="database_query") R3 = Dataset("data/raw3", tool="read_hdf") DA0 = Dataset("step1/da0", tool="merge_fish_counts") DA1 = Dataset("step1/da1", tool="process_filter") DB0 = Dataset("step2/db0", tool="join_counts") DB1 = Dataset("step2/db1", tool="join_by_date") DC0 = Dataset("results/dc0", tool="merge_model_obs") DC1 = Dataset("results/dc1", tool="compute_uncertainty") DC2 = Dataset("results/dc2", tool="make_plots") # Declare dependency relationships so that depgraph and determine the order of # the build DA0.dependson(R0, R1) DA1.dependson(R2) DB0.dependson(DA0, DA1) DB1.dependson(DA1, R3) DC0.dependson(DB0, DB1) DC1.dependson(DB1) DC2.dependson(DB1)