def testPatternReuse(self): '''Test repeated use of steps that use pattern and produce different files.''' # for f in ['A1.txt', 'A2.txt', 'B1.txt', 'B1.txt.p', 'B2.txt', 'B2.txt.p']: FileTarget(f).remove('both') # # A1 <- P <- B1 # A1 <- P <- B2 # A2 # script = SoS_Script(''' [A_1] input: 'B1.txt.p', 'B2.txt.p' output: 'A1.txt' sh: touch A1.txt [A_2] sh: touch A2.txt [B1: provides='B1.txt'] sh: touch B1.txt [B2: provides='B2.txt'] sh: touch B2.txt [P: provides='{filename}.p'] input: filename sh: touch ${output} ''') # the workflow should call step K for step C_2, but not C_3 wf = script.workflow() dag = Base_Executor(wf).initialize_dag() self.assertDAG(dag, ''' strict digraph "" { "P ['B2.txt.p']"; "B1 ['B1.txt']"; "B2 ['B2.txt']"; A_2; A_1; "P ['B1.txt.p']"; "P ['B2.txt.p']" -> A_1; "B1 ['B1.txt']" -> "P ['B1.txt.p']"; "B2 ['B2.txt']" -> "P ['B2.txt.p']"; A_1 -> A_2; "P ['B1.txt.p']" -> A_1; } ''') Base_Executor(wf).run() for f in ['A1.txt', 'A2.txt', 'B1.txt', 'B1.txt.p', 'B2.txt', 'B2.txt.p']: t = FileTarget(f) self.assertTrue(t.exists()) t.remove('both')
def testPatternReuse(self): '''Test repeated use of steps that use pattern and produce different files.''' # for f in [ 'A1.txt', 'A2.txt', 'B1.txt', 'B1.txt.p', 'B2.txt', 'B2.txt.p' ]: FileTarget(f).remove('both') # # A1 <- P <- B1 # A1 <- P <- B2 # A2 # script = SoS_Script(''' [A_1] input: 'B1.txt.p', 'B2.txt.p' output: 'A1.txt' sh: touch A1.txt [A_2] sh: touch A2.txt [B1: provides='B1.txt'] sh: touch B1.txt [B2: provides='B2.txt'] sh: touch B2.txt [P: provides='{filename}.p'] input: filename sh: touch ${output} ''') # the workflow should call step K for step C_2, but not C_3 wf = script.workflow() dag = Base_Executor(wf).initialize_dag() self.assertDAG( dag, ''' strict digraph "" { "P ['B2.txt.p']"; "B1 ['B1.txt']"; "B2 ['B2.txt']"; A_2; A_1; "P ['B1.txt.p']"; "P ['B2.txt.p']" -> A_1; "B1 ['B1.txt']" -> "P ['B1.txt.p']"; "B2 ['B2.txt']" -> "P ['B2.txt.p']"; A_1 -> A_2; "P ['B1.txt.p']" -> A_1; } ''') Base_Executor(wf).run() for f in [ 'A1.txt', 'A2.txt', 'B1.txt', 'B1.txt.p', 'B2.txt', 'B2.txt.p' ]: t = FileTarget(f) self.assertTrue(t.exists()) t.remove('both')
def testTarget(self): '''Test executing only part of a workflow.''' # for f in [ 'A1.txt', 'A2.txt', 'C2.txt', 'B2.txt', 'B1.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt' ]: FileTarget(f).remove('both') # # A1 <- B1 <- B2 <- B3 # | # | # \/ # A2 <- B2 <- C1 <- C2 <- C4 # C3 # script = SoS_Script(''' [A_1] input: 'B1.txt' output: 'A1.txt' sh: touch A1.txt [A_2] depends: 'B2.txt' sh: touch A2.txt [B1: provides='B1.txt'] depends: 'B2.txt' sh: touch B1.txt [B2: provides='B2.txt'] depends: 'B3.txt', 'C1.txt' sh: touch B2.txt [B3: provides='B3.txt'] sh: touch B3.txt [C1: provides='C1.txt'] depends: 'C2.txt', 'C3.txt' sh: touch C1.txt [C2: provides='C2.txt'] depends: 'C4.txt' sh: touch C2.txt [C3: provides='C3.txt'] depends: 'C4.txt' sh: touch C3.txt [C4: provides='C4.txt'] sh: touch C4.txt ''') # the workflow should call step K for step C_2, but not C_3 wf = script.workflow() # # test 1, we only need to generate target 'B1.txt' dag = Base_Executor(wf).initialize_dag(targets=['B1.txt']) # note that A2 is no longer mentioned self.assertDAG( dag, ''' strict digraph "" { "B3 ['B3.txt']"; "C4 ['C4.txt']"; "C2 ['C2.txt']"; "C1 ['C1.txt']"; "B1 ['B1.txt']"; "B2 ['B2.txt']"; "C3 ['C3.txt']"; "B3 ['B3.txt']" -> "B2 ['B2.txt']"; "C4 ['C4.txt']" -> "C3 ['C3.txt']"; "C4 ['C4.txt']" -> "C2 ['C2.txt']"; "C2 ['C2.txt']" -> "C1 ['C1.txt']"; "C1 ['C1.txt']" -> "B2 ['B2.txt']"; "B2 ['B2.txt']" -> "B1 ['B1.txt']"; "C3 ['C3.txt']" -> "C1 ['C1.txt']"; } ''') Base_Executor(wf).run(targets=['B1.txt']) for f in ['A1.txt', 'A2.txt']: self.assertFalse(FileTarget(f).exists()) for f in [ 'C2.txt', 'B2.txt', 'B1.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt' ]: t = FileTarget(f) self.assertTrue(t.exists()) t.remove('both') # # test 2, we would like to generate two files dag = Base_Executor(wf).initialize_dag(targets=['B2.txt', 'C2.txt']) # note that A2 is no longer mentioned self.assertDAG( dag, ''' strict digraph "" { "C4 ['C4.txt']"; "B2 ['B2.txt']"; "C3 ['C3.txt']"; "B3 ['B3.txt']"; "C2 ['C2.txt']"; "C1 ['C1.txt']"; "C4 ['C4.txt']" -> "C2 ['C2.txt']"; "C4 ['C4.txt']" -> "C3 ['C3.txt']"; "C3 ['C3.txt']" -> "C1 ['C1.txt']"; "B3 ['B3.txt']" -> "B2 ['B2.txt']"; "C2 ['C2.txt']" -> "C1 ['C1.txt']"; "C1 ['C1.txt']" -> "B2 ['B2.txt']"; } ''') Base_Executor(wf).run(targets=['B2.txt', 'C2.txt']) for f in ['A1.txt', 'B1.txt', 'A2.txt']: self.assertFalse(FileTarget(f).exists()) for f in ['C2.txt', 'B2.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt']: t = FileTarget(f) self.assertTrue(t.exists()) t.remove('both') # # test 3, generate two separate trees # dag = Base_Executor(wf).initialize_dag(targets=['B3.txt', 'C2.txt']) # note that A2 is no longer mentioned self.assertDAG( dag, ''' strict digraph "" { "B3 ['B3.txt']"; "C2 ['C2.txt']"; "C4 ['C4.txt']"; "C4 ['C4.txt']" -> "C2 ['C2.txt']"; } ''') Base_Executor(wf).run(targets=['B3.txt', 'C2.txt']) for f in ['A1.txt', 'B1.txt', 'A2.txt', 'B2.txt', 'C1.txt', 'C3.txt']: self.assertFalse(FileTarget(f).exists()) for f in ['C2.txt', 'B3.txt', 'C4.txt']: t = FileTarget(f) self.assertTrue(t.exists()) t.remove('both')
def testLongChain(self): '''Test long make file style dependencies.''' # for f in [ 'A1.txt', 'A2.txt', 'C2.txt', 'B2.txt', 'B1.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt' ]: FileTarget(f).remove('both') # # A1 <- B1 <- B2 <- B3 # | # | # \/ # A2 <- B2 <- C1 <- C2 <- C4 # C3 # script = SoS_Script(''' [A_1] input: 'B1.txt' output: 'A1.txt' sh: touch A1.txt [A_2] depends: 'B2.txt' sh: touch A2.txt [B1: provides='B1.txt'] depends: 'B2.txt' sh: touch B1.txt [B2: provides='B2.txt'] depends: 'B3.txt', 'C1.txt' sh: touch B2.txt [B3: provides='B3.txt'] sh: touch B3.txt [C1: provides='C1.txt'] depends: 'C2.txt', 'C3.txt' sh: touch C1.txt [C2: provides='C2.txt'] depends: 'C4.txt' sh: touch C2.txt [C3: provides='C3.txt'] depends: 'C4.txt' sh: touch C3.txt [C4: provides='C4.txt'] sh: touch C4.txt ''') # the workflow should call step K for step C_2, but not C_3 wf = script.workflow() dag = Base_Executor(wf).initialize_dag() self.assertDAG( dag, ''' strict digraph "" { "C4 ['C4.txt']"; "B1 ['B1.txt']"; "C1 ['C1.txt']"; "C2 ['C2.txt']"; "C3 ['C3.txt']"; A_1; "B2 ['B2.txt']"; "B3 ['B3.txt']"; A_2; "C4 ['C4.txt']" -> "C2 ['C2.txt']"; "C4 ['C4.txt']" -> "C3 ['C3.txt']"; "B1 ['B1.txt']" -> A_1; "C1 ['C1.txt']" -> "B2 ['B2.txt']"; "C2 ['C2.txt']" -> "C1 ['C1.txt']"; "C3 ['C3.txt']" -> "C1 ['C1.txt']"; A_1 -> A_2; "B2 ['B2.txt']" -> "B1 ['B1.txt']"; "B2 ['B2.txt']" -> A_2; "B3 ['B3.txt']" -> "B2 ['B2.txt']"; } ''') Base_Executor(wf).run() for f in [ 'A1.txt', 'A2.txt', 'C2.txt', 'B2.txt', 'B1.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt' ]: t = FileTarget(f) self.assertTrue(t.exists()) t.remove('both')
def testTarget(self): '''Test executing only part of a workflow.''' # for f in ['A1.txt', 'A2.txt', 'C2.txt', 'B2.txt', 'B1.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt']: FileTarget(f).remove('both') # # A1 <- B1 <- B2 <- B3 # | # | # \/ # A2 <- B2 <- C1 <- C2 <- C4 # C3 # script = SoS_Script(''' [A_1] input: 'B1.txt' output: 'A1.txt' sh: touch A1.txt [A_2] depends: 'B2.txt' sh: touch A2.txt [B1: provides='B1.txt'] depends: 'B2.txt' sh: touch B1.txt [B2: provides='B2.txt'] depends: 'B3.txt', 'C1.txt' sh: touch B2.txt [B3: provides='B3.txt'] sh: touch B3.txt [C1: provides='C1.txt'] depends: 'C2.txt', 'C3.txt' sh: touch C1.txt [C2: provides='C2.txt'] depends: 'C4.txt' sh: touch C2.txt [C3: provides='C3.txt'] depends: 'C4.txt' sh: touch C3.txt [C4: provides='C4.txt'] sh: touch C4.txt ''') # the workflow should call step K for step C_2, but not C_3 wf = script.workflow() # # test 1, we only need to generate target 'B1.txt' dag = Base_Executor(wf).initialize_dag(targets=['B1.txt']) # note that A2 is no longer mentioned self.assertDAG(dag, ''' strict digraph "" { "B3 ['B3.txt']"; "C4 ['C4.txt']"; "C2 ['C2.txt']"; "C1 ['C1.txt']"; "B1 ['B1.txt']"; "B2 ['B2.txt']"; "C3 ['C3.txt']"; "B3 ['B3.txt']" -> "B2 ['B2.txt']"; "C4 ['C4.txt']" -> "C3 ['C3.txt']"; "C4 ['C4.txt']" -> "C2 ['C2.txt']"; "C2 ['C2.txt']" -> "C1 ['C1.txt']"; "C1 ['C1.txt']" -> "B2 ['B2.txt']"; "B2 ['B2.txt']" -> "B1 ['B1.txt']"; "C3 ['C3.txt']" -> "C1 ['C1.txt']"; } ''') Base_Executor(wf).run(targets=['B1.txt']) for f in ['A1.txt', 'A2.txt']: self.assertFalse(FileTarget(f).exists()) for f in ['C2.txt', 'B2.txt', 'B1.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt']: t = FileTarget(f) self.assertTrue(t.exists()) t.remove('both') # # test 2, we would like to generate two files dag = Base_Executor(wf).initialize_dag(targets=['B2.txt', 'C2.txt']) # note that A2 is no longer mentioned self.assertDAG(dag, ''' strict digraph "" { "C4 ['C4.txt']"; "B2 ['B2.txt']"; "C3 ['C3.txt']"; "B3 ['B3.txt']"; "C2 ['C2.txt']"; "C1 ['C1.txt']"; "C4 ['C4.txt']" -> "C2 ['C2.txt']"; "C4 ['C4.txt']" -> "C3 ['C3.txt']"; "C3 ['C3.txt']" -> "C1 ['C1.txt']"; "B3 ['B3.txt']" -> "B2 ['B2.txt']"; "C2 ['C2.txt']" -> "C1 ['C1.txt']"; "C1 ['C1.txt']" -> "B2 ['B2.txt']"; } ''') Base_Executor(wf).run(targets=['B2.txt', 'C2.txt']) for f in ['A1.txt', 'B1.txt', 'A2.txt']: self.assertFalse(FileTarget(f).exists()) for f in ['C2.txt', 'B2.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt']: t = FileTarget(f) self.assertTrue(t.exists()) t.remove('both') # # test 3, generate two separate trees # dag = Base_Executor(wf).initialize_dag(targets=['B3.txt', 'C2.txt']) # note that A2 is no longer mentioned self.assertDAG(dag, ''' strict digraph "" { "B3 ['B3.txt']"; "C2 ['C2.txt']"; "C4 ['C4.txt']"; "C4 ['C4.txt']" -> "C2 ['C2.txt']"; } ''') Base_Executor(wf).run(targets=['B3.txt', 'C2.txt']) for f in ['A1.txt', 'B1.txt', 'A2.txt', 'B2.txt', 'C1.txt', 'C3.txt']: self.assertFalse(FileTarget(f).exists()) for f in ['C2.txt', 'B3.txt', 'C4.txt']: t = FileTarget(f) self.assertTrue(t.exists()) t.remove('both')
def testLongChain(self): '''Test long make file style dependencies.''' # for f in ['A1.txt', 'A2.txt', 'C2.txt', 'B2.txt', 'B1.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt']: FileTarget(f).remove('both') # # A1 <- B1 <- B2 <- B3 # | # | # \/ # A2 <- B2 <- C1 <- C2 <- C4 # C3 # script = SoS_Script(''' [A_1] input: 'B1.txt' output: 'A1.txt' sh: touch A1.txt [A_2] depends: 'B2.txt' sh: touch A2.txt [B1: provides='B1.txt'] depends: 'B2.txt' sh: touch B1.txt [B2: provides='B2.txt'] depends: 'B3.txt', 'C1.txt' sh: touch B2.txt [B3: provides='B3.txt'] sh: touch B3.txt [C1: provides='C1.txt'] depends: 'C2.txt', 'C3.txt' sh: touch C1.txt [C2: provides='C2.txt'] depends: 'C4.txt' sh: touch C2.txt [C3: provides='C3.txt'] depends: 'C4.txt' sh: touch C3.txt [C4: provides='C4.txt'] sh: touch C4.txt ''') # the workflow should call step K for step C_2, but not C_3 wf = script.workflow() dag = Base_Executor(wf).initialize_dag() self.assertDAG(dag, ''' strict digraph "" { "C4 ['C4.txt']"; "B1 ['B1.txt']"; "C1 ['C1.txt']"; "C2 ['C2.txt']"; "C3 ['C3.txt']"; A_1; "B2 ['B2.txt']"; "B3 ['B3.txt']"; A_2; "C4 ['C4.txt']" -> "C2 ['C2.txt']"; "C4 ['C4.txt']" -> "C3 ['C3.txt']"; "B1 ['B1.txt']" -> A_1; "C1 ['C1.txt']" -> "B2 ['B2.txt']"; "C2 ['C2.txt']" -> "C1 ['C1.txt']"; "C3 ['C3.txt']" -> "C1 ['C1.txt']"; A_1 -> A_2; "B2 ['B2.txt']" -> "B1 ['B1.txt']"; "B2 ['B2.txt']" -> A_2; "B3 ['B3.txt']" -> "B2 ['B2.txt']"; } ''') Base_Executor(wf).run() for f in ['A1.txt', 'A2.txt', 'C2.txt', 'B2.txt', 'B1.txt', 'B3.txt', 'C1.txt', 'C3.txt', 'C4.txt']: t = FileTarget(f) self.assertTrue(t.exists()) t.remove('both')